mirror of
https://git.vectorsigma.ru/public/tubearchivist.git
synced 2026-08-09 05:09:28 +00:00
Compare commits
61 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e98d2d3590 | ||
|
|
61ccb1c47a | ||
|
|
b4b287bda4 | ||
|
|
ca8f4c9fa4 | ||
|
|
18b8a17df1 | ||
|
|
f192c39be8 | ||
|
|
f1b89345e2 | ||
|
|
f6c702fba7 | ||
|
|
f619a5f72d | ||
|
|
f7dd77ee3e | ||
|
|
0f41c54406 | ||
|
|
13d1927840 | ||
|
|
d0124c1a5c | ||
|
|
5ed80d983d | ||
|
|
24d66e33a7 | ||
|
|
786caf7bc1 | ||
|
|
e4b7258f7d | ||
|
|
90258e75e6 | ||
|
|
b48b7c7e72 | ||
|
|
454d520394 | ||
|
|
ee5f83e89f | ||
|
|
fa518c029c | ||
|
|
aa1d0b759e | ||
|
|
abfd109203 | ||
|
|
ebf53f6486 | ||
|
|
2450a4ad1d | ||
|
|
41ea0c2c58 | ||
|
|
acb24e38a6 | ||
|
|
a1fbc0b436 | ||
|
|
e052935e2d | ||
|
|
4d2bd51603 | ||
|
|
b3d3ebe9d0 | ||
|
|
d05c00aac6 | ||
|
|
0c8fcbd724 | ||
|
|
7e67ed02aa | ||
|
|
e101c4abc7 | ||
|
|
6a7a84792e | ||
|
|
2fa907c478 | ||
|
|
cc965ac473 | ||
|
|
8f5311ff04 | ||
|
|
e590d456a1 | ||
|
|
a5be5a719b | ||
|
|
b5ceb264ec | ||
|
|
31c967cc11 | ||
|
|
2818bd5d52 | ||
|
|
939cd7fb62 | ||
|
|
172ced7129 | ||
|
|
11c122cba3 | ||
|
|
456bc57df3 | ||
|
|
091024553b | ||
|
|
ad016df105 | ||
|
|
79d134cea8 | ||
|
|
fb046bed5b | ||
|
|
f6b6185fb2 | ||
|
|
5b7e3e877b | ||
|
|
3c5302336f | ||
|
|
55c9721b51 | ||
|
|
fc788c586d | ||
|
|
e912cf7fb6 | ||
|
|
c111050c8f | ||
|
|
0e8d3b76c1 |
4
.github/ISSUE_TEMPLATE/BUG-REPORT.yml
vendored
4
.github/ISSUE_TEMPLATE/BUG-REPORT.yml
vendored
@@ -11,10 +11,12 @@ body:
|
||||
- type: checkboxes
|
||||
id: latest
|
||||
attributes:
|
||||
label: Latest and Greatest
|
||||
label: "I've read the documentation"
|
||||
options:
|
||||
- label: I'm running the latest version of Tube Archivist and have read the [release notes](https://github.com/tubearchivist/tubearchivist/releases/latest).
|
||||
required: true
|
||||
- label: I have read through the [wiki](https://github.com/tubearchivist/tubearchivist/wiki) and the [readme](https://github.com/tubearchivist/tubearchivist#installing-and-updating), particularly the [common errors](https://github.com/tubearchivist/tubearchivist#common-errors) section.
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: os
|
||||
|
||||
2
.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml
vendored
2
.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml
vendored
@@ -15,7 +15,7 @@ body:
|
||||
options:
|
||||
- label: I have read and understand the [installation instructions](https://github.com/tubearchivist/tubearchivist#installing-and-updating).
|
||||
required: true
|
||||
- label: My issue is not described in the [potential pitfalls](https://github.com/tubearchivist/tubearchivist#potential-pitfalls) section.
|
||||
- label: My issue is not described in the [common errors](https://github.com/tubearchivist/tubearchivist#common-errors) section.
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
* [Extended Universe](#extended-universe)
|
||||
* [Installing and updating](#installing-and-updating)
|
||||
* [Getting Started](#getting-started)
|
||||
* [Potential pitfalls](#potential-pitfalls)
|
||||
* [Common Errors](#common-errors)
|
||||
* [Roadmap](#roadmap)
|
||||
* [Known limitations](#known-limitations)
|
||||
* [Donate](#donate)
|
||||
@@ -61,11 +61,11 @@ Once your YouTube video collection grows, it becomes hard to search and find a s
|
||||
- [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.
|
||||
|
||||
## Installing and updating
|
||||
There's dedicated user-contributed install steps under [docs/Installation.md](./docs/Installation.md) for Unraid, Truenas and Synology which you can use instead of this section if you happen to be using one of those. Otherwise, continue on.
|
||||
There's dedicated user-contributed install steps under [docs/Installation.md](./docs/Installation.md) for podman, Unraid, Truenas and Synology which you can use instead of this section if you happen to be using one of those. Otherwise, continue on.
|
||||
|
||||
For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation.
|
||||
|
||||
Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson` (an unofficial rebuild for arm64) instead of [the official one](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L27).
|
||||
Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson`, an unofficial rebuild for arm64.
|
||||
|
||||
This project requires docker. Ensure it is installed and running on your system.
|
||||
|
||||
@@ -165,7 +165,7 @@ You will see the current version number of **Tube Archivist** in the footer of t
|
||||
|
||||
There is a Helm Chart available at https://github.com/insuusvenerati/helm-charts. Mostly self-explanatory but feel free to ask questions in the discord / subreddit.
|
||||
|
||||
## Potential pitfalls
|
||||
## Common Errors
|
||||
### vm.max_map_count
|
||||
**Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.
|
||||
|
||||
|
||||
@@ -1,12 +1,40 @@
|
||||
# Detailed Installation Instructions for Various Platforms
|
||||
|
||||
## Table of Contents
|
||||
- [Podman](#podman)
|
||||
- [Unraid](#unraid)
|
||||
- [Truenas Scale](#truenas-scale)
|
||||
- [Synology](#synology)
|
||||
|
||||
These are beginners guides installation instructions for additional platforms generously provided by users of these platforms. When in doubt, verify the details with the [Readme](https://github.com/tubearchivist/tubearchivist#installing-and-updating). If you see any issues here while using these instructions, please contribute.
|
||||
|
||||
## Podman
|
||||
Podman handles container hostname resolving slightly differently than docker, so you need to make a few changes to the `docker-compose.yml` to get up and running.
|
||||
|
||||
### Step 1: Follow the installation instructions from the [README](https://github.com/tubearchivist/tubearchivist#installing-and-updating), with a few additional changes to the `docker-compose.yml`.
|
||||
|
||||
Edit these additional changes to the `docker-compose.yml`:
|
||||
- under `tubearchivist`->`image`:
|
||||
- prefix the container name with `docker.io/` (or the url of your repo of choice).
|
||||
- under `tubearchivist`->`environment`:
|
||||
- `ES_URL`: change `archivist-es` to the internal IP of the computer that will be running the containers.
|
||||
- `REDIS_HOST`: change `archivist-redis` to the internal IP of the computer that will be running the containers (should be the same as above).
|
||||
- under `archivist-redis`->`image`:
|
||||
- prefix the container name with `docker.io/` again.
|
||||
- under `archivist-redis`->`expose`:
|
||||
- change the whole entry from `expose: ["<PORT>"]` into `ports: ["<PORT>:<PORT>"].
|
||||
- under `archivist-es`->`image`:
|
||||
- prefix the container name with `docker.io/` again.
|
||||
- under `archivist-es`->`expose`:
|
||||
- change the whole entry from `expose: ["<PORT>"]` into `ports: ["<PORT>:<PORT>"].
|
||||
|
||||
### Step 2: Create service files (optional)
|
||||
|
||||
Since podman doesn't run as a service, it can't start containers after reboots, at least not without some help.
|
||||
|
||||
If you want to enable this behavior, you can follow [this example](https://techblog.jeppson.org/2020/04/create-podman-services-with-podman-compose/) to have `systemd` start up the containers with `podman-compose` when the computer boots up.
|
||||
|
||||
|
||||
## Unraid
|
||||
|
||||
Tube Archivist, and all if it's dependencies are located in the [community applications](https://forums.unraid.net/topic/38582-plug-in-community-applications/) store. The three containers you will need are as follows:
|
||||
@@ -195,7 +223,9 @@ Once all of the folders have been created, it should have a folder structure wit
|
||||
- "GID=0"
|
||||
- "xpack.security.enabled=true"
|
||||
- "ELASTIC_PASSWORD=verysecret"
|
||||
- "path.repo=/usr/share/elasticsearch/data/snapshot"
|
||||
> Do not use the default password as it is very insecure.
|
||||
> Activating snapshots for backups should only be done *after* setting the `path.repo` setting.
|
||||

|
||||
12. Click on the **Apply** button.
|
||||
13. Back on the **Create Container** screen, click the **Next** button.
|
||||
@@ -252,4 +282,4 @@ Once all of the folders have been created, it should have a folder structure wit
|
||||
<!-- This section is a Work In Progress -->
|
||||
|
||||
### From there, you should be able to start up your containers and you're good to go!
|
||||
If you're still having trouble, join us on [discord](https://discord.gg/AFwz8nE7BK) and come to the #synology channel.
|
||||
If you're still having trouble, join us on [discord](https://discord.gg/AFwz8nE7BK) and come to the #synology channel.
|
||||
|
||||
@@ -32,6 +32,15 @@ Additional settings passed to yt-dlp.
|
||||
- **Source Settings**: User created subtitles are provided from the uploader and are usually the video script. Auto generated is from YouTube, quality varies, particularly for auto translated tracks.
|
||||
- **Index Settings**: Enabling subtitle indexing will add the lines to Elasticsearch and will make subtitles searchable. This will increase the index size and is not recommended on low-end hardware.
|
||||
|
||||
## Comments
|
||||
- **Download and index comments**: Set your configuration for downloading and indexing comments. This takes the same values as documented in the `max_comments` section for the youtube extractor of [yt-dlp](https://github.com/yt-dlp/yt-dlp#youtube). Add without space between the four different fields: *max-comments,max-parents,max-replies,max-replies-per-thread*. Example:
|
||||
- `all,100,all,30`: Get 100 max-parents and 30 max-replies-per-thread.
|
||||
- `1000,all,all,50`: Get a total of 1000 comments over all, 50 replies per thread.
|
||||
- **Comment sort method**: Change sort method between *top* or *new*. The default is *top*, as decided by YouTube.
|
||||
- The [Refresh Metadata](#refresh-metadata) background task will get comments from your already archived videos, spreading the requests out over time.
|
||||
|
||||
Archiving comments is slow as only very few comments get returned per request with yt-dlp. Choose your configuration above wisely. Tube Archivist will download comments after the download queue finishes, your videos will be already available while the comments are getting downloaded.
|
||||
|
||||
## Cookie
|
||||
Importing your YouTube Cookie into Tube Archivist allows yt-dlp to bypass age restrictions, gives access to private videos and your *watch later* or *liked videos*.
|
||||
|
||||
@@ -102,7 +111,7 @@ Examples:
|
||||
NOTE:
|
||||
- Changes in the scheduler settings require a container restart to take effect.
|
||||
- Cron format as *number*/*number* are none standard cron and are not supported by the scheduler, for example **0 0/12 \*** is invalid, use **0 \*/12 \*** instead.
|
||||
- Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason * or wildcards for minutes are not supported.
|
||||
- Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason, the scheduler doesn't support schedules that trigger more than once per hour.
|
||||
|
||||
## Rescan Subscriptions
|
||||
That's the equivalent task as run from the downloads page looking through your channel and playlist and add missing videos to the download queue.
|
||||
|
||||
@@ -1,11 +1,23 @@
|
||||
# Video Page
|
||||
|
||||
Every video downloaded gets a dedicated page accessible at `/video/<video-id>` of your Tube Archivist.
|
||||
Every video downloaded gets a dedicated page accessible at `/video/<video-id>/` of your Tube Archivist.
|
||||
|
||||
Clicking on the channel name or the channel icon will bring you to the dedicated channel detail [page](Channels#channel-detail).
|
||||
|
||||
The button **Delete Video** will delete that video including the media file.
|
||||
|
||||
If available, a tag cloud will show, representing the tags set by the uploader.
|
||||
|
||||
The video description is truncated to the first few lines, click on *show more* to expand the whole description.
|
||||
|
||||
## Playlist
|
||||
When available, a playlist navigation will show at the bottom. Clicking on the playlist name will bring you to the dedicated [Playlist Detail](Playlists#playlist-detail) page showing all videos downloaded from that playlist. The number in square brackets indicates the position of the current video in that playlist.
|
||||
|
||||
Clicking on the next or previous video name or thumbnail will bring you to that dedicated video page.
|
||||
Clicking on the next or previous video name or thumbnail will bring you to that dedicated video page.
|
||||
|
||||
## Similar Videos
|
||||
Tube Archivist will show up to six similar videos in a grid. Similarity is detected from the **video title** and the **video tags**. This naturally will show some videos from the same channel, but can also return videos about the same topic from other channels.
|
||||
|
||||
When playing a video from the similar section with the inline player, the current video will get replaced, refresh the page to reset that or click on the video title to avoid that behavior.
|
||||
|
||||
## Comments
|
||||
If activated on the settings page, this will show the indexed comments. Reveal the threads by clicking the *+ Replies* button. Comments with a heart symbol are favorited by the uploader, comments by the uploader are highlighted in a different color.
|
||||
|
||||
@@ -12,6 +12,7 @@ Note:
|
||||
**Video**
|
||||
- [Video List](#video-list-view)
|
||||
- [Video Single](#video-item-view)
|
||||
- [Video Comments](#video-comment-view)
|
||||
- [Video Single Progress](#video-progress-view)
|
||||
- [Video Single Sponsorblock](#sponsor-block-view) WIP
|
||||
|
||||
@@ -78,6 +79,9 @@ Pass page number as a query parameter: `page=2`. Defaults to *0*, `page=1` is re
|
||||
## Video Item View
|
||||
/api/video/\<video_id>/
|
||||
|
||||
## Video Comment View
|
||||
/api/video/\<video_id>/comment/
|
||||
|
||||
## Video Progress View
|
||||
/api/video/\<video_id>/progress
|
||||
|
||||
|
||||
@@ -48,6 +48,8 @@ class SearchProcess:
|
||||
processed = self._process_playlist(result["_source"])
|
||||
if index == "ta_download":
|
||||
processed = self._process_download(result["_source"])
|
||||
if index == "ta_comment":
|
||||
processed = self._process_comment(result["_source"])
|
||||
|
||||
return processed
|
||||
|
||||
@@ -123,3 +125,17 @@ class SearchProcess:
|
||||
}
|
||||
)
|
||||
return dict(sorted(download_dict.items()))
|
||||
|
||||
def _process_comment(self, comment_dict):
|
||||
"""run on all comments, create reply thread"""
|
||||
all_comments = comment_dict["comment_comments"]
|
||||
processed_comments = []
|
||||
|
||||
for comment in all_comments:
|
||||
if comment["comment_parent"] == "root":
|
||||
comment.update({"comment_replies": []})
|
||||
processed_comments.append(comment)
|
||||
else:
|
||||
processed_comments[-1]["comment_replies"].append(comment)
|
||||
|
||||
return processed_comments
|
||||
|
||||
@@ -18,7 +18,9 @@ from api.views import (
|
||||
TaskApiView,
|
||||
VideoApiListView,
|
||||
VideoApiView,
|
||||
VideoCommentView,
|
||||
VideoProgressView,
|
||||
VideoSimilarView,
|
||||
VideoSponsorView,
|
||||
)
|
||||
from django.urls import path
|
||||
@@ -41,6 +43,16 @@ urlpatterns = [
|
||||
VideoProgressView.as_view(),
|
||||
name="api-video-progress",
|
||||
),
|
||||
path(
|
||||
"video/<slug:video_id>/comment/",
|
||||
VideoCommentView.as_view(),
|
||||
name="api-video-comment",
|
||||
),
|
||||
path(
|
||||
"video/<slug:video_id>/similar/",
|
||||
VideoSimilarView.as_view(),
|
||||
name="api-video-similar",
|
||||
),
|
||||
path(
|
||||
"video/<slug:video_id>/sponsor/",
|
||||
VideoSponsorView.as_view(),
|
||||
|
||||
@@ -43,7 +43,6 @@ class ApiBaseView(APIView):
|
||||
def get_document(self, document_id):
|
||||
"""get single document from es"""
|
||||
path = f"{self.search_base}{document_id}"
|
||||
print(path)
|
||||
response, status_code = ElasticWrap(path).get()
|
||||
try:
|
||||
self.response["data"] = SearchProcess(response).process()
|
||||
@@ -62,10 +61,11 @@ class ApiBaseView(APIView):
|
||||
}
|
||||
)
|
||||
|
||||
def get_document_list(self, request):
|
||||
def get_document_list(self, request, pagination=True):
|
||||
"""get a list of results"""
|
||||
print(self.search_base)
|
||||
self.initiate_pagination(request)
|
||||
if pagination:
|
||||
self.initiate_pagination(request)
|
||||
|
||||
es_handler = ElasticWrap(self.search_base)
|
||||
response, status_code = es_handler.get(data=self.data)
|
||||
self.response["data"] = SearchProcess(response).process()
|
||||
@@ -74,8 +74,11 @@ class ApiBaseView(APIView):
|
||||
else:
|
||||
self.status_code = 404
|
||||
|
||||
self.pagination_handler.validate(response["hits"]["total"]["value"])
|
||||
self.response["paginate"] = self.pagination_handler.pagination
|
||||
if pagination:
|
||||
self.pagination_handler.validate(
|
||||
response["hits"]["total"]["value"]
|
||||
)
|
||||
self.response["paginate"] = self.pagination_handler.pagination
|
||||
|
||||
|
||||
class VideoApiView(ApiBaseView):
|
||||
@@ -145,6 +148,46 @@ class VideoProgressView(ApiBaseView):
|
||||
return Response(self.response)
|
||||
|
||||
|
||||
class VideoCommentView(ApiBaseView):
|
||||
"""resolves to /api/video/<video_id>/comment/
|
||||
handle video comments
|
||||
GET: return all comments from video with reply threads
|
||||
"""
|
||||
|
||||
search_base = "ta_comment/_doc/"
|
||||
|
||||
def get(self, request, video_id):
|
||||
"""get video comments"""
|
||||
# pylint: disable=unused-argument
|
||||
self.get_document(video_id)
|
||||
|
||||
return Response(self.response, status=self.status_code)
|
||||
|
||||
|
||||
class VideoSimilarView(ApiBaseView):
|
||||
"""resolves to /api/video/<video-id>/similar/
|
||||
GET: return max 3 videos similar to this
|
||||
"""
|
||||
|
||||
search_base = "ta_video/_search/"
|
||||
|
||||
def get(self, request, video_id):
|
||||
"""get similar videos"""
|
||||
self.data = {
|
||||
"size": 6,
|
||||
"query": {
|
||||
"more_like_this": {
|
||||
"fields": ["tags", "title"],
|
||||
"like": {"_id": video_id},
|
||||
"min_term_freq": 1,
|
||||
"max_query_terms": 25,
|
||||
}
|
||||
},
|
||||
}
|
||||
self.get_document_list(request, pagination=False)
|
||||
return Response(self.response, status=self.status_code)
|
||||
|
||||
|
||||
class VideoSponsorView(ApiBaseView):
|
||||
"""resolves to /api/video/<video_id>/sponsor/
|
||||
handle sponsor block integration
|
||||
@@ -391,7 +434,6 @@ class DownloadApiListView(ApiBaseView):
|
||||
@staticmethod
|
||||
def post(request):
|
||||
"""add list of videos to download queue"""
|
||||
print(f"request meta data: {request.META}")
|
||||
data = request.data
|
||||
try:
|
||||
to_add = data["data"]
|
||||
@@ -478,10 +520,7 @@ class TaskApiView(ApiBaseView):
|
||||
|
||||
def post(self, request):
|
||||
"""handle post request"""
|
||||
|
||||
data = request.data
|
||||
print(data)
|
||||
response = TaskHandler(data).run_task()
|
||||
response = TaskHandler(request.data).run_task()
|
||||
|
||||
return Response(response)
|
||||
|
||||
@@ -616,6 +655,6 @@ class SearchView(ApiBaseView):
|
||||
return Response(
|
||||
{"message": "no search query specified"}, status=400
|
||||
)
|
||||
print("searching for: " + search_query)
|
||||
|
||||
search_results = SearchForm().multi_search(search_query)
|
||||
return Response(search_results)
|
||||
|
||||
@@ -262,4 +262,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
|
||||
|
||||
# TA application settings
|
||||
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
|
||||
TA_VERSION = "v0.2.4"
|
||||
TA_VERSION = "v0.3.0"
|
||||
|
||||
@@ -5,9 +5,10 @@ import sys
|
||||
|
||||
from django.apps import AppConfig
|
||||
from home.src.es.connect import ElasticWrap
|
||||
from home.src.es.index_setup import index_check
|
||||
from home.src.es.index_setup import ElasitIndexWrap
|
||||
from home.src.es.snapshot import ElasticSnapshot
|
||||
from home.src.ta.config import AppConfig as ArchivistConfig
|
||||
from home.src.ta.helper import clear_dl_cache
|
||||
from home.src.ta.ta_redis import RedisArchivist
|
||||
|
||||
|
||||
@@ -27,10 +28,11 @@ class StartupCheck:
|
||||
print("run startup checks")
|
||||
self.es_version_check()
|
||||
self.release_lock()
|
||||
index_check()
|
||||
ElasitIndexWrap().setup()
|
||||
self.sync_redis_state()
|
||||
self.set_redis_conf()
|
||||
self.make_folders()
|
||||
clear_dl_cache(self.config_handler.config)
|
||||
self.snapshot_check()
|
||||
self.set_has_run()
|
||||
|
||||
|
||||
@@ -27,6 +27,8 @@
|
||||
"subtitle": false,
|
||||
"subtitle_source": false,
|
||||
"subtitle_index": false,
|
||||
"comment_max": false,
|
||||
"comment_sort": "top",
|
||||
"cookie_import": false,
|
||||
"throttledratelimit": false,
|
||||
"integrate_ryd": false,
|
||||
|
||||
@@ -15,6 +15,7 @@ from home.src.download.subscriptions import PlaylistSubscription
|
||||
from home.src.download.yt_dlp_base import CookieHandler, YtWrap
|
||||
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||
from home.src.index.channel import YoutubeChannel
|
||||
from home.src.index.comments import Comments
|
||||
from home.src.index.playlist import YoutubePlaylist
|
||||
from home.src.index.video import YoutubeVideo, index_new_video
|
||||
from home.src.ta.config import AppConfig
|
||||
@@ -39,6 +40,7 @@ class DownloadPostProcess:
|
||||
self.auto_delete_all()
|
||||
self.auto_delete_overwrites()
|
||||
self.validate_playlists()
|
||||
self.get_comments()
|
||||
|
||||
def auto_delete_all(self):
|
||||
"""handle auto delete"""
|
||||
@@ -139,6 +141,27 @@ class DownloadPostProcess:
|
||||
|
||||
RedisArchivist().set_message(key, mess_dict, expire=expire)
|
||||
|
||||
def get_comments(self):
|
||||
"""get comments from youtube"""
|
||||
if not self.download.config["downloads"]["comment_max"]:
|
||||
return
|
||||
|
||||
total_videos = len(self.download.videos)
|
||||
for idx, video_id in enumerate(self.download.videos):
|
||||
comment = Comments(video_id, config=self.download.config)
|
||||
comment.build_json(notify=(idx, total_videos))
|
||||
comment.upload_comments()
|
||||
|
||||
key = "message:download"
|
||||
message = {
|
||||
"status": key,
|
||||
"level": "info",
|
||||
"title": "Download and index comments finished",
|
||||
"message": f"added comments for {total_videos} videos",
|
||||
}
|
||||
|
||||
RedisArchivist().set_message(key, message, expire=4)
|
||||
|
||||
|
||||
class VideoDownloader:
|
||||
"""
|
||||
@@ -155,6 +178,7 @@ class VideoDownloader:
|
||||
self.config = AppConfig().config
|
||||
self._build_obs()
|
||||
self.channels = set()
|
||||
self.videos = set()
|
||||
|
||||
def run_queue(self):
|
||||
"""setup download queue in redis loop until no more items"""
|
||||
@@ -187,6 +211,7 @@ class VideoDownloader:
|
||||
youtube_id, video_overwrites=self.video_overwrites
|
||||
)
|
||||
self.channels.add(vid_dict["channel"]["channel_id"])
|
||||
self.videos.add(vid_dict["youtube_id"])
|
||||
mess_dict = {
|
||||
"status": self.MSG,
|
||||
"level": "info",
|
||||
|
||||
224
tubearchivist/home/src/es/backup.py
Normal file
224
tubearchivist/home/src/es/backup.py
Normal file
@@ -0,0 +1,224 @@
|
||||
"""
|
||||
Functionality:
|
||||
- Handle json zip file based backup
|
||||
- create backup
|
||||
- restore backup
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
|
||||
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||
from home.src.ta.config import AppConfig
|
||||
from home.src.ta.helper import get_mapping, ignore_filelist
|
||||
|
||||
|
||||
class ElasticBackup:
|
||||
"""dump index to nd-json files for later bulk import"""
|
||||
|
||||
def __init__(self, reason=False):
|
||||
self.config = AppConfig().config
|
||||
self.cache_dir = self.config["application"]["cache_dir"]
|
||||
self.timestamp = datetime.now().strftime("%Y%m%d")
|
||||
self.index_config = get_mapping()
|
||||
self.reason = reason
|
||||
|
||||
def backup_all_indexes(self):
|
||||
"""backup all indexes, add reason to init"""
|
||||
print("backup all indexes")
|
||||
if not self.reason:
|
||||
raise ValueError("missing backup reason in ElasticBackup")
|
||||
|
||||
for index in self.index_config:
|
||||
index_name = index["index_name"]
|
||||
print(f"backup: export in progress for {index_name}")
|
||||
if not self.index_exists(index_name):
|
||||
print(f"skip backup for not yet existing index {index_name}")
|
||||
continue
|
||||
|
||||
self.backup_index(index_name)
|
||||
|
||||
self.zip_it()
|
||||
if self.reason == "auto":
|
||||
self.rotate_backup()
|
||||
|
||||
@staticmethod
|
||||
def backup_index(index_name):
|
||||
"""export all documents of a single index"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"sort": [{"_doc": {"order": "desc"}}],
|
||||
}
|
||||
paginate = IndexPaginate(
|
||||
f"ta_{index_name}", data, keep_source=True, callback=BackupCallback
|
||||
)
|
||||
_ = paginate.get_results()
|
||||
|
||||
def zip_it(self):
|
||||
"""pack it up into single zip file"""
|
||||
file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
|
||||
folder = os.path.join(self.cache_dir, "backup")
|
||||
|
||||
to_backup = []
|
||||
for file in os.listdir(folder):
|
||||
if file.endswith(".json"):
|
||||
to_backup.append(os.path.join(folder, file))
|
||||
|
||||
backup_file = os.path.join(folder, file_name)
|
||||
|
||||
comp = zipfile.ZIP_DEFLATED
|
||||
with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
|
||||
for backup_file in to_backup:
|
||||
zip_f.write(backup_file, os.path.basename(backup_file))
|
||||
|
||||
# cleanup
|
||||
for backup_file in to_backup:
|
||||
os.remove(backup_file)
|
||||
|
||||
def post_bulk_restore(self, file_name):
|
||||
"""send bulk to es"""
|
||||
file_path = os.path.join(self.cache_dir, file_name)
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
data = f.read()
|
||||
|
||||
if not data.strip():
|
||||
return
|
||||
|
||||
_, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
|
||||
|
||||
def get_all_backup_files(self):
|
||||
"""build all available backup files for view"""
|
||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
||||
backup_files = os.listdir(backup_dir)
|
||||
all_backup_files = ignore_filelist(backup_files)
|
||||
all_available_backups = [
|
||||
i
|
||||
for i in all_backup_files
|
||||
if i.startswith("ta_") and i.endswith(".zip")
|
||||
]
|
||||
all_available_backups.sort(reverse=True)
|
||||
|
||||
backup_dicts = []
|
||||
for backup_file in all_available_backups:
|
||||
file_split = backup_file.split("-")
|
||||
if len(file_split) == 2:
|
||||
timestamp = file_split[1].strip(".zip")
|
||||
reason = False
|
||||
elif len(file_split) == 3:
|
||||
timestamp = file_split[1]
|
||||
reason = file_split[2].strip(".zip")
|
||||
|
||||
to_add = {
|
||||
"filename": backup_file,
|
||||
"timestamp": timestamp,
|
||||
"reason": reason,
|
||||
}
|
||||
backup_dicts.append(to_add)
|
||||
|
||||
return backup_dicts
|
||||
|
||||
def restore(self, filename):
|
||||
"""
|
||||
restore from backup zip file
|
||||
call reset from ElasitIndexWrap first to start blank
|
||||
"""
|
||||
zip_content = self._unpack_zip_backup(filename)
|
||||
self._restore_json_files(zip_content)
|
||||
|
||||
def _unpack_zip_backup(self, filename):
|
||||
"""extract backup zip and return filelist"""
|
||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
||||
file_path = os.path.join(backup_dir, filename)
|
||||
|
||||
with zipfile.ZipFile(file_path, "r") as z:
|
||||
zip_content = z.namelist()
|
||||
z.extractall(backup_dir)
|
||||
|
||||
return zip_content
|
||||
|
||||
def _restore_json_files(self, zip_content):
|
||||
"""go through the unpacked files and restore"""
|
||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
||||
|
||||
for json_f in zip_content:
|
||||
|
||||
file_name = os.path.join(backup_dir, json_f)
|
||||
|
||||
if not json_f.startswith("es_") or not json_f.endswith(".json"):
|
||||
os.remove(file_name)
|
||||
continue
|
||||
|
||||
print("restoring: " + json_f)
|
||||
self.post_bulk_restore(file_name)
|
||||
os.remove(file_name)
|
||||
|
||||
@staticmethod
|
||||
def index_exists(index_name):
|
||||
"""check if index already exists to skip"""
|
||||
_, status_code = ElasticWrap(f"ta_{index_name}").get()
|
||||
exists = status_code == 200
|
||||
|
||||
return exists
|
||||
|
||||
def rotate_backup(self):
|
||||
"""delete old backups if needed"""
|
||||
rotate = self.config["scheduler"]["run_backup_rotate"]
|
||||
if not rotate:
|
||||
return
|
||||
|
||||
all_backup_files = self.get_all_backup_files()
|
||||
auto = [i for i in all_backup_files if i["reason"] == "auto"]
|
||||
|
||||
if len(auto) <= rotate:
|
||||
print("no backup files to rotate")
|
||||
return
|
||||
|
||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
||||
|
||||
all_to_delete = auto[rotate:]
|
||||
for to_delete in all_to_delete:
|
||||
file_path = os.path.join(backup_dir, to_delete["filename"])
|
||||
print(f"remove old backup file: {file_path}")
|
||||
os.remove(file_path)
|
||||
|
||||
|
||||
class BackupCallback:
|
||||
"""handle backup ndjson writer as callback for IndexPaginate"""
|
||||
|
||||
def __init__(self, source, index_name):
|
||||
self.source = source
|
||||
self.index_name = index_name
|
||||
self.timestamp = datetime.now().strftime("%Y%m%d")
|
||||
|
||||
def run(self):
|
||||
"""run the junk task"""
|
||||
file_content = self._build_bulk()
|
||||
self._write_es_json(file_content)
|
||||
|
||||
def _build_bulk(self):
|
||||
"""build bulk query data from all_results"""
|
||||
bulk_list = []
|
||||
|
||||
for document in self.source:
|
||||
document_id = document["_id"]
|
||||
es_index = document["_index"]
|
||||
action = {"index": {"_index": es_index, "_id": document_id}}
|
||||
source = document["_source"]
|
||||
bulk_list.append(json.dumps(action))
|
||||
bulk_list.append(json.dumps(source))
|
||||
|
||||
# add last newline
|
||||
bulk_list.append("\n")
|
||||
file_content = "\n".join(bulk_list)
|
||||
|
||||
return file_content
|
||||
|
||||
def _write_es_json(self, file_content):
|
||||
"""write nd-json file for es _bulk API to disk"""
|
||||
cache_dir = AppConfig().config["application"]["cache_dir"]
|
||||
file_name = f"es_{self.index_name.lstrip('ta_')}-{self.timestamp}.json"
|
||||
file_path = os.path.join(cache_dir, "backup", file_name)
|
||||
with open(file_path, "a+", encoding="utf-8") as f:
|
||||
f.write(file_content)
|
||||
@@ -37,8 +37,7 @@
|
||||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
"type": "date"
|
||||
},
|
||||
"channel_overwrites": {
|
||||
"properties": {
|
||||
@@ -120,8 +119,7 @@
|
||||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
"type": "date"
|
||||
},
|
||||
"channel_overwrites": {
|
||||
"properties": {
|
||||
@@ -193,6 +191,9 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"comment_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"stats" : {
|
||||
"properties" : {
|
||||
"average_rating" : {
|
||||
@@ -460,6 +461,75 @@
|
||||
},
|
||||
"number_of_replicas": "0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"index_name": "comment",
|
||||
"expected_map": {
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_last_refresh": {
|
||||
"type": "date"
|
||||
},
|
||||
"comment_channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_comments": {
|
||||
"properties": {
|
||||
"comment_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_text": {
|
||||
"type" : "text"
|
||||
},
|
||||
"comment_timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"comment_time_text": {
|
||||
"type" : "text"
|
||||
},
|
||||
"comment_likecount": {
|
||||
"type": "long"
|
||||
},
|
||||
"comment_is_favorited": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_author": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"comment_author_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_author_thumbnail": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_author_is_uploader": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_parent": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
"analysis": {
|
||||
"normalizer": {
|
||||
"to_lower": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"number_of_replicas": "0"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -5,22 +5,17 @@ functionality:
|
||||
- backup and restore metadata
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
|
||||
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||
from home.src.es.backup import ElasticBackup
|
||||
from home.src.es.connect import ElasticWrap
|
||||
from home.src.es.snapshot import ElasticSnapshot
|
||||
from home.src.ta.config import AppConfig
|
||||
from home.src.ta.helper import ignore_filelist
|
||||
from home.src.ta.helper import get_mapping
|
||||
|
||||
|
||||
class ElasticIndex:
|
||||
"""
|
||||
handle mapping and settings on elastic search for a given index
|
||||
"""
|
||||
"""interact with a single index"""
|
||||
|
||||
def __init__(self, index_name, expected_map, expected_set):
|
||||
def __init__(self, index_name, expected_map=False, expected_set=False):
|
||||
self.index_name = index_name
|
||||
self.expected_map = expected_map
|
||||
self.expected_set = expected_set
|
||||
@@ -61,23 +56,23 @@ class ElasticIndex:
|
||||
if list(value.keys()) == ["properties"]:
|
||||
for key_n, value_n in value["properties"].items():
|
||||
if key not in now_map:
|
||||
print(key_n, value_n)
|
||||
print(f"detected mapping change: {key_n}, {value_n}")
|
||||
return True
|
||||
if key_n not in now_map[key]["properties"].keys():
|
||||
print(key_n, value_n)
|
||||
print(f"detected mapping change: {key_n}, {value_n}")
|
||||
return True
|
||||
if not value_n == now_map[key]["properties"][key_n]:
|
||||
print(key_n, value_n)
|
||||
print(f"detected mapping change: {key_n}, {value_n}")
|
||||
return True
|
||||
|
||||
continue
|
||||
|
||||
# not nested
|
||||
if key not in now_map.keys():
|
||||
print(key, value)
|
||||
print(f"detected mapping change: {key}, {value}")
|
||||
return True
|
||||
if not value == now_map[key]:
|
||||
print(key, value)
|
||||
print(f"detected mapping change: {key}, {value}")
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -100,6 +95,8 @@ class ElasticIndex:
|
||||
|
||||
def rebuild_index(self):
|
||||
"""rebuild with new mapping"""
|
||||
print(f"applying new mappings to index ta_{self.index_name}...")
|
||||
self.create_blank(for_backup=True)
|
||||
self.reindex("backup")
|
||||
self.delete_index(backup=False)
|
||||
self.create_blank()
|
||||
@@ -126,279 +123,88 @@ class ElasticIndex:
|
||||
|
||||
_, _ = ElasticWrap(path).delete()
|
||||
|
||||
def create_blank(self):
|
||||
def create_blank(self, for_backup=False):
|
||||
"""apply new mapping and settings for blank new index"""
|
||||
print(f"create new blank index with name ta_{self.index_name}...")
|
||||
path = f"ta_{self.index_name}"
|
||||
if for_backup:
|
||||
path = f"{path}_backup"
|
||||
|
||||
data = {}
|
||||
if self.expected_set:
|
||||
data.update({"settings": self.expected_set})
|
||||
if self.expected_map:
|
||||
data.update({"mappings": {"properties": self.expected_map}})
|
||||
|
||||
_, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
|
||||
_, _ = ElasticWrap(path).put(data)
|
||||
|
||||
|
||||
class BackupCallback:
|
||||
"""handle backup ndjson writer as callback for IndexPaginate"""
|
||||
class ElasitIndexWrap:
|
||||
"""interact with all index mapping and setup"""
|
||||
|
||||
def __init__(self, source, index_name):
|
||||
self.source = source
|
||||
self.index_name = index_name
|
||||
self.timestamp = datetime.now().strftime("%Y%m%d")
|
||||
def __init__(self):
|
||||
self.index_config = get_mapping()
|
||||
self.backup_run = False
|
||||
|
||||
def run(self):
|
||||
"""run the junk task"""
|
||||
file_content = self._build_bulk()
|
||||
self._write_es_json(file_content)
|
||||
|
||||
def _build_bulk(self):
|
||||
"""build bulk query data from all_results"""
|
||||
bulk_list = []
|
||||
|
||||
for document in self.source:
|
||||
document_id = document["_id"]
|
||||
es_index = document["_index"]
|
||||
action = {"index": {"_index": es_index, "_id": document_id}}
|
||||
source = document["_source"]
|
||||
bulk_list.append(json.dumps(action))
|
||||
bulk_list.append(json.dumps(source))
|
||||
|
||||
# add last newline
|
||||
bulk_list.append("\n")
|
||||
file_content = "\n".join(bulk_list)
|
||||
|
||||
return file_content
|
||||
|
||||
def _write_es_json(self, file_content):
|
||||
"""write nd-json file for es _bulk API to disk"""
|
||||
cache_dir = AppConfig().config["application"]["cache_dir"]
|
||||
file_name = f"es_{self.index_name.lstrip('ta_')}-{self.timestamp}.json"
|
||||
file_path = os.path.join(cache_dir, "backup", file_name)
|
||||
with open(file_path, "a+", encoding="utf-8") as f:
|
||||
f.write(file_content)
|
||||
|
||||
|
||||
class ElasticBackup:
|
||||
"""dump index to nd-json files for later bulk import"""
|
||||
|
||||
def __init__(self, index_config, reason):
|
||||
self.config = AppConfig().config
|
||||
self.cache_dir = self.config["application"]["cache_dir"]
|
||||
self.timestamp = datetime.now().strftime("%Y%m%d")
|
||||
self.index_config = index_config
|
||||
self.reason = reason
|
||||
|
||||
@staticmethod
|
||||
def backup_index(index_name):
|
||||
"""export all documents of a single index"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"sort": [{"_doc": {"order": "desc"}}],
|
||||
}
|
||||
paginate = IndexPaginate(
|
||||
f"ta_{index_name}", data, keep_source=True, callback=BackupCallback
|
||||
)
|
||||
_ = paginate.get_results()
|
||||
|
||||
def zip_it(self):
|
||||
"""pack it up into single zip file"""
|
||||
file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
|
||||
folder = os.path.join(self.cache_dir, "backup")
|
||||
|
||||
to_backup = []
|
||||
for file in os.listdir(folder):
|
||||
if file.endswith(".json"):
|
||||
to_backup.append(os.path.join(folder, file))
|
||||
|
||||
backup_file = os.path.join(folder, file_name)
|
||||
|
||||
comp = zipfile.ZIP_DEFLATED
|
||||
with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
|
||||
for backup_file in to_backup:
|
||||
zip_f.write(backup_file, os.path.basename(backup_file))
|
||||
|
||||
# cleanup
|
||||
for backup_file in to_backup:
|
||||
os.remove(backup_file)
|
||||
|
||||
def post_bulk_restore(self, file_name):
|
||||
"""send bulk to es"""
|
||||
file_path = os.path.join(self.cache_dir, file_name)
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
data = f.read()
|
||||
|
||||
if not data.strip():
|
||||
return
|
||||
|
||||
_, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
|
||||
|
||||
def get_all_backup_files(self):
|
||||
"""build all available backup files for view"""
|
||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
||||
backup_files = os.listdir(backup_dir)
|
||||
all_backup_files = ignore_filelist(backup_files)
|
||||
all_available_backups = [
|
||||
i
|
||||
for i in all_backup_files
|
||||
if i.startswith("ta_") and i.endswith(".zip")
|
||||
]
|
||||
all_available_backups.sort(reverse=True)
|
||||
|
||||
backup_dicts = []
|
||||
for backup_file in all_available_backups:
|
||||
file_split = backup_file.split("-")
|
||||
if len(file_split) == 2:
|
||||
timestamp = file_split[1].strip(".zip")
|
||||
reason = False
|
||||
elif len(file_split) == 3:
|
||||
timestamp = file_split[1]
|
||||
reason = file_split[2].strip(".zip")
|
||||
|
||||
to_add = {
|
||||
"filename": backup_file,
|
||||
"timestamp": timestamp,
|
||||
"reason": reason,
|
||||
}
|
||||
backup_dicts.append(to_add)
|
||||
|
||||
return backup_dicts
|
||||
|
||||
def unpack_zip_backup(self, filename):
|
||||
"""extract backup zip and return filelist"""
|
||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
||||
file_path = os.path.join(backup_dir, filename)
|
||||
|
||||
with zipfile.ZipFile(file_path, "r") as z:
|
||||
zip_content = z.namelist()
|
||||
z.extractall(backup_dir)
|
||||
|
||||
return zip_content
|
||||
|
||||
def restore_json_files(self, zip_content):
|
||||
"""go through the unpacked files and restore"""
|
||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
||||
|
||||
for json_f in zip_content:
|
||||
|
||||
file_name = os.path.join(backup_dir, json_f)
|
||||
|
||||
if not json_f.startswith("es_") or not json_f.endswith(".json"):
|
||||
os.remove(file_name)
|
||||
def setup(self):
|
||||
"""setup elastic index, run at startup"""
|
||||
for index in self.index_config:
|
||||
index_name, expected_map, expected_set = self._config_split(index)
|
||||
handler = ElasticIndex(index_name, expected_map, expected_set)
|
||||
if not handler.exists:
|
||||
handler.create_blank()
|
||||
continue
|
||||
|
||||
print("restoring: " + json_f)
|
||||
self.post_bulk_restore(file_name)
|
||||
os.remove(file_name)
|
||||
rebuild = handler.validate()
|
||||
if rebuild:
|
||||
self._check_backup()
|
||||
handler.rebuild_index()
|
||||
continue
|
||||
|
||||
# else all good
|
||||
print(f"ta_{index_name} index is created and up to date...")
|
||||
|
||||
def reset(self):
|
||||
"""reset all indexes to blank"""
|
||||
self.delete_all()
|
||||
self.create_all_blank()
|
||||
|
||||
def delete_all(self):
|
||||
"""delete all indexes"""
|
||||
print("reset elastic index")
|
||||
for index in self.index_config:
|
||||
index_name, _, _ = self._config_split(index)
|
||||
handler = ElasticIndex(index_name)
|
||||
handler.delete_index(backup=False)
|
||||
|
||||
def create_all_blank(self):
|
||||
"""create all blank indexes"""
|
||||
print("create all new indexes in elastic from template")
|
||||
for index in self.index_config:
|
||||
index_name, expected_map, expected_set = self._config_split(index)
|
||||
handler = ElasticIndex(index_name, expected_map, expected_set)
|
||||
handler.create_blank()
|
||||
|
||||
@staticmethod
|
||||
def index_exists(index_name):
|
||||
"""check if index already exists to skip"""
|
||||
_, status_code = ElasticWrap(f"ta_{index_name}").get()
|
||||
exists = status_code == 200
|
||||
|
||||
return exists
|
||||
|
||||
def rotate_backup(self):
|
||||
"""delete old backups if needed"""
|
||||
rotate = self.config["scheduler"]["run_backup_rotate"]
|
||||
if not rotate:
|
||||
return
|
||||
|
||||
all_backup_files = self.get_all_backup_files()
|
||||
auto = [i for i in all_backup_files if i["reason"] == "auto"]
|
||||
|
||||
if len(auto) <= rotate:
|
||||
print("no backup files to rotate")
|
||||
return
|
||||
|
||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
||||
|
||||
all_to_delete = auto[rotate:]
|
||||
for to_delete in all_to_delete:
|
||||
file_path = os.path.join(backup_dir, to_delete["filename"])
|
||||
print(f"remove old backup file: {file_path}")
|
||||
os.remove(file_path)
|
||||
|
||||
|
||||
def get_mapping():
|
||||
"""read index_mapping.json and get expected mapping and settings"""
|
||||
with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
|
||||
index_config = json.load(f).get("index_config")
|
||||
|
||||
return index_config
|
||||
|
||||
|
||||
def index_check(force_restore=False):
|
||||
"""check if all indexes are created and have correct mapping"""
|
||||
|
||||
backed_up = False
|
||||
index_config = get_mapping()
|
||||
|
||||
for index in index_config:
|
||||
def _config_split(index):
|
||||
"""split index config keys"""
|
||||
index_name = index["index_name"]
|
||||
expected_map = index["expected_map"]
|
||||
expected_set = index["expected_set"]
|
||||
handler = ElasticIndex(index_name, expected_map, expected_set)
|
||||
# force restore
|
||||
if force_restore:
|
||||
handler.delete_index(backup=False)
|
||||
handler.create_blank()
|
||||
continue
|
||||
|
||||
# create new
|
||||
if not handler.exists:
|
||||
print(f"create new blank index with name ta_{index_name}...")
|
||||
handler.create_blank()
|
||||
continue
|
||||
return index_name, expected_map, expected_set
|
||||
|
||||
# validate index
|
||||
rebuild = handler.validate()
|
||||
if rebuild:
|
||||
# make backup before rebuild
|
||||
if not backed_up:
|
||||
print("running backup first")
|
||||
backup_all_indexes(reason="update")
|
||||
backed_up = True
|
||||
def _check_backup(self):
|
||||
"""create backup if needed"""
|
||||
if self.backup_run:
|
||||
return
|
||||
|
||||
print(f"applying new mappings to index ta_{index_name}...")
|
||||
handler.rebuild_index()
|
||||
continue
|
||||
config = AppConfig().config
|
||||
if config["application"]["enable_snapshot"]:
|
||||
# take snapshot if enabled
|
||||
ElasticSnapshot().take_snapshot_now(wait=True)
|
||||
else:
|
||||
# fallback to json backup
|
||||
ElasticBackup(reason="update").backup_all_indexes()
|
||||
|
||||
# else all good
|
||||
print(f"ta_{index_name} index is created and up to date...")
|
||||
|
||||
|
||||
def get_available_backups():
|
||||
"""return dict of available backups for settings view"""
|
||||
index_config = get_mapping()
|
||||
backup_handler = ElasticBackup(index_config, reason=False)
|
||||
all_backup_files = backup_handler.get_all_backup_files()
|
||||
return all_backup_files
|
||||
|
||||
|
||||
def backup_all_indexes(reason):
|
||||
"""backup all es indexes to disk"""
|
||||
index_config = get_mapping()
|
||||
backup_handler = ElasticBackup(index_config, reason)
|
||||
|
||||
for index in backup_handler.index_config:
|
||||
index_name = index["index_name"]
|
||||
print(f"backup: export in progress for {index_name}")
|
||||
if not backup_handler.index_exists(index_name):
|
||||
continue
|
||||
backup_handler.backup_index(index_name)
|
||||
|
||||
backup_handler.zip_it()
|
||||
|
||||
if reason == "auto":
|
||||
backup_handler.rotate_backup()
|
||||
|
||||
|
||||
def restore_from_backup(filename):
|
||||
"""restore indexes from backup file"""
|
||||
# delete
|
||||
index_check(force_restore=True)
|
||||
# recreate
|
||||
index_config = get_mapping()
|
||||
backup_handler = ElasticBackup(index_config, reason=False)
|
||||
zip_content = backup_handler.unpack_zip_backup(filename)
|
||||
backup_handler.restore_json_files(zip_content)
|
||||
self.backup_run = True
|
||||
|
||||
@@ -5,10 +5,11 @@ functionality:
|
||||
|
||||
from datetime import datetime
|
||||
from os import environ
|
||||
from time import sleep
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from home.src.es.connect import ElasticWrap
|
||||
from home.src.es.index_setup import get_mapping
|
||||
from home.src.ta.helper import get_mapping
|
||||
|
||||
|
||||
class ElasticSnapshot:
|
||||
@@ -116,6 +117,7 @@ class ElasticSnapshot:
|
||||
"repository": self.REPO,
|
||||
"config": {
|
||||
"indices": self.all_indices,
|
||||
"ignore_unavailable": True,
|
||||
"include_global_state": True,
|
||||
},
|
||||
"retention": {
|
||||
@@ -141,15 +143,42 @@ class ElasticSnapshot:
|
||||
print("snapshot: last snapshot is up-to-date")
|
||||
return outdated
|
||||
|
||||
def take_snapshot_now(self):
|
||||
def take_snapshot_now(self, wait=False):
|
||||
"""execute daily snapshot now"""
|
||||
path = f"_slm/policy/{self.POLICY}/_execute"
|
||||
response, statuscode = ElasticWrap(path).post()
|
||||
if statuscode == 200:
|
||||
print(f"snapshot: executing now: {response}")
|
||||
|
||||
if wait:
|
||||
self._wait_for_snapshot(response["snapshot_name"])
|
||||
|
||||
return response
|
||||
|
||||
def _wait_for_snapshot(self, snapshot_name):
|
||||
"""return after snapshot_name completes"""
|
||||
path = f"_snapshot/{self.REPO}/{snapshot_name}"
|
||||
|
||||
while True:
|
||||
# wait for task to be created
|
||||
sleep(1)
|
||||
_, statuscode = ElasticWrap(path).get()
|
||||
if statuscode == 200:
|
||||
break
|
||||
|
||||
while True:
|
||||
# wait for snapshot success
|
||||
response, statuscode = ElasticWrap(path).get()
|
||||
snapshot_state = response["snapshots"][0]["state"]
|
||||
if snapshot_state == "SUCCESS":
|
||||
break
|
||||
|
||||
print(f"snapshot: {snapshot_name} in state {snapshot_state}")
|
||||
print("snapshot: wait to complete")
|
||||
sleep(5)
|
||||
|
||||
print(f"snapshot: completed - {response}")
|
||||
|
||||
def get_snapshot_stats(self):
|
||||
"""get snapshot info for frontend"""
|
||||
snapshot_info = self._build_policy_details()
|
||||
|
||||
@@ -92,6 +92,12 @@ class ApplicationSettingsForm(forms.Form):
|
||||
("1", "enable subtitle index"),
|
||||
]
|
||||
|
||||
COMMENT_SORT_CHOICES = [
|
||||
("", "-- change comments sort settings --"),
|
||||
("top", "sort comments by top"),
|
||||
("new", "sort comments by new"),
|
||||
]
|
||||
|
||||
COOKIE_IMPORT_CHOICES = [
|
||||
("", "-- change cookie settings"),
|
||||
("0", "disable cookie"),
|
||||
@@ -120,6 +126,10 @@ class ApplicationSettingsForm(forms.Form):
|
||||
downloads_subtitle_index = forms.ChoiceField(
|
||||
widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False
|
||||
)
|
||||
downloads_comment_max = forms.CharField(required=False)
|
||||
downloads_comment_sort = forms.ChoiceField(
|
||||
widget=forms.Select, choices=COMMENT_SORT_CHOICES, required=False
|
||||
)
|
||||
downloads_cookie_import = forms.ChoiceField(
|
||||
widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False
|
||||
)
|
||||
|
||||
@@ -120,7 +120,7 @@ class SearchHandler:
|
||||
if "vid_thumb_url" in hit_keys:
|
||||
youtube_id = hit["source"]["youtube_id"]
|
||||
thumb_path = ThumbManager(youtube_id).vid_thumb_path()
|
||||
hit["source"]["vid_thumb_url"] = thumb_path
|
||||
hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
|
||||
|
||||
if "channel_last_refresh" in hit_keys:
|
||||
refreshed = hit["source"]["channel_last_refresh"]
|
||||
|
||||
@@ -50,7 +50,7 @@ class ChannelScraper:
|
||||
url = f"https://www.youtube.com/channel/{self.channel_id}/about?hl=en"
|
||||
cookies = {"CONSENT": "YES+xxxxxxxxxxxxxxxxxxxxxxxxxxx"}
|
||||
response = requests.get(
|
||||
url, cookies=cookies, headers=requests_headers()
|
||||
url, cookies=cookies, headers=requests_headers(), timeout=10
|
||||
)
|
||||
if response.ok:
|
||||
channel_page = response.text
|
||||
@@ -193,6 +193,9 @@ class YoutubeChannel(YouTubeItem):
|
||||
if not self.json_data and fallback:
|
||||
self._video_fallback(fallback)
|
||||
|
||||
if not self.json_data:
|
||||
return
|
||||
|
||||
self.get_channel_art()
|
||||
|
||||
def _video_fallback(self, fallback):
|
||||
@@ -275,6 +278,15 @@ class YoutubeChannel(YouTubeItem):
|
||||
}
|
||||
_, _ = ElasticWrap("ta_video/_delete_by_query").post(data)
|
||||
|
||||
def delete_es_comments(self):
|
||||
"""delete all comments from this channel"""
|
||||
data = {
|
||||
"query": {
|
||||
"term": {"comment_channel_id": {"value": self.youtube_id}}
|
||||
}
|
||||
}
|
||||
_, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
|
||||
|
||||
def delete_playlists(self):
|
||||
"""delete all indexed playlist from es"""
|
||||
all_playlists = self.get_indexed_playlists()
|
||||
@@ -301,6 +313,7 @@ class YoutubeChannel(YouTubeItem):
|
||||
self.delete_playlists()
|
||||
print(f"{self.youtube_id}: delete indexed videos")
|
||||
self.delete_es_videos()
|
||||
self.delete_es_comments()
|
||||
self.del_in_es()
|
||||
|
||||
def index_channel_playlists(self):
|
||||
@@ -408,7 +421,8 @@ class YoutubeChannel(YouTubeItem):
|
||||
to_write[key] = False
|
||||
continue
|
||||
if value in [0, "0"]:
|
||||
del to_write[key]
|
||||
if key in to_write:
|
||||
del to_write[key]
|
||||
continue
|
||||
if value == "1":
|
||||
to_write[key] = True
|
||||
|
||||
179
tubearchivist/home/src/index/comments.py
Normal file
179
tubearchivist/home/src/index/comments.py
Normal file
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
Functionality:
|
||||
- Download comments
|
||||
- Index comments in ES
|
||||
- Retrieve comments from ES
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from home.src.download.yt_dlp_base import YtWrap
|
||||
from home.src.es.connect import ElasticWrap
|
||||
from home.src.ta.config import AppConfig
|
||||
from home.src.ta.ta_redis import RedisArchivist
|
||||
|
||||
|
||||
class Comments:
|
||||
"""hold all comments functionality"""
|
||||
|
||||
def __init__(self, youtube_id, config=False):
|
||||
self.youtube_id = youtube_id
|
||||
self.es_path = f"ta_comment/_doc/{youtube_id}"
|
||||
self.json_data = False
|
||||
self.config = config
|
||||
self.is_activated = False
|
||||
self.comments_format = False
|
||||
|
||||
def build_json(self, notify=False):
|
||||
"""build json document for es"""
|
||||
print(f"{self.youtube_id}: get comments")
|
||||
self.check_config()
|
||||
if not self.is_activated:
|
||||
return
|
||||
|
||||
self._send_notification(notify)
|
||||
comments_raw, channel_id = self.get_yt_comments()
|
||||
if comments_raw:
|
||||
self.format_comments(comments_raw)
|
||||
else:
|
||||
self.comments_format = []
|
||||
|
||||
self.json_data = {
|
||||
"youtube_id": self.youtube_id,
|
||||
"comment_last_refresh": int(datetime.now().strftime("%s")),
|
||||
"comment_channel_id": channel_id,
|
||||
"comment_comments": self.comments_format,
|
||||
}
|
||||
|
||||
def check_config(self):
|
||||
"""read config if not attached"""
|
||||
if not self.config:
|
||||
self.config = AppConfig().config
|
||||
|
||||
self.is_activated = bool(self.config["downloads"]["comment_max"])
|
||||
|
||||
@staticmethod
|
||||
def _send_notification(notify):
|
||||
"""send notification for download post process message"""
|
||||
if not notify:
|
||||
return
|
||||
|
||||
key = "message:download"
|
||||
idx, total_videos = notify
|
||||
message = {
|
||||
"status": key,
|
||||
"level": "info",
|
||||
"title": "Download and index comments",
|
||||
"message": f"Progress: {idx + 1}/{total_videos}",
|
||||
}
|
||||
|
||||
RedisArchivist().set_message(key, message)
|
||||
|
||||
def build_yt_obs(self):
|
||||
"""
|
||||
get extractor config
|
||||
max-comments,max-parents,max-replies,max-replies-per-thread
|
||||
"""
|
||||
max_comments = self.config["downloads"]["comment_max"]
|
||||
max_comments_list = [i.strip() for i in max_comments.split(",")]
|
||||
comment_sort = self.config["downloads"]["comment_sort"]
|
||||
|
||||
yt_obs = {
|
||||
"skip_download": True,
|
||||
"quiet": False,
|
||||
"getcomments": True,
|
||||
"extractor_args": {
|
||||
"youtube": {
|
||||
"max_comments": max_comments_list,
|
||||
"comment_sort": [comment_sort],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
return yt_obs
|
||||
|
||||
def get_yt_comments(self):
|
||||
"""get comments from youtube"""
|
||||
yt_obs = self.build_yt_obs()
|
||||
info_json = YtWrap(yt_obs).extract(self.youtube_id)
|
||||
comments_raw = info_json.get("comments")
|
||||
channel_id = info_json.get("channel_id")
|
||||
return comments_raw, channel_id
|
||||
|
||||
def format_comments(self, comments_raw):
|
||||
"""process comments to match format"""
|
||||
comments = []
|
||||
|
||||
for comment in comments_raw:
|
||||
cleaned_comment = self.clean_comment(comment)
|
||||
comments.append(cleaned_comment)
|
||||
|
||||
self.comments_format = comments
|
||||
|
||||
def clean_comment(self, comment):
|
||||
"""parse metadata from comment for indexing"""
|
||||
time_text_datetime = datetime.utcfromtimestamp(comment["timestamp"])
|
||||
|
||||
if time_text_datetime.hour == 0 and time_text_datetime.minute == 0:
|
||||
format_string = "%Y-%m-%d"
|
||||
else:
|
||||
format_string = "%Y-%m-%d %H:%M"
|
||||
|
||||
time_text = time_text_datetime.strftime(format_string)
|
||||
|
||||
cleaned_comment = {
|
||||
"comment_id": comment["id"],
|
||||
"comment_text": comment["text"].replace("\xa0", ""),
|
||||
"comment_timestamp": comment["timestamp"],
|
||||
"comment_time_text": time_text,
|
||||
"comment_likecount": comment["like_count"],
|
||||
"comment_is_favorited": comment["is_favorited"],
|
||||
"comment_author": comment["author"],
|
||||
"comment_author_id": comment["author_id"],
|
||||
"comment_author_thumbnail": comment["author_thumbnail"],
|
||||
"comment_author_is_uploader": comment["author_is_uploader"],
|
||||
"comment_parent": comment["parent"],
|
||||
}
|
||||
|
||||
return cleaned_comment
|
||||
|
||||
def upload_comments(self):
|
||||
"""upload comments to es"""
|
||||
if not self.is_activated:
|
||||
return
|
||||
|
||||
_, _ = ElasticWrap(self.es_path).put(self.json_data)
|
||||
|
||||
vid_path = f"ta_video/_update/{self.youtube_id}"
|
||||
data = {"doc": {"comment_count": len(self.comments_format)}}
|
||||
_, _ = ElasticWrap(vid_path).post(data=data)
|
||||
|
||||
def delete_comments(self):
|
||||
"""delete comments from es"""
|
||||
print(f"{self.youtube_id}: delete comments")
|
||||
_, _ = ElasticWrap(self.es_path).delete(refresh=True)
|
||||
|
||||
def get_es_comments(self):
|
||||
"""get comments from ES"""
|
||||
response, statuscode = ElasticWrap(self.es_path).get()
|
||||
if statuscode == 404:
|
||||
print(f"comments: not found {self.youtube_id}")
|
||||
return False
|
||||
|
||||
return response.get("_source")
|
||||
|
||||
def reindex_comments(self):
|
||||
"""update comments from youtube"""
|
||||
self.check_config()
|
||||
if not self.is_activated:
|
||||
return
|
||||
|
||||
self.build_json()
|
||||
es_comments = self.get_es_comments()
|
||||
|
||||
if not self.comments_format and es_comments["comment_comments"]:
|
||||
# don't overwrite comments in es
|
||||
return
|
||||
|
||||
self.delete_comments()
|
||||
self.upload_comments()
|
||||
@@ -56,11 +56,11 @@ class YouTubeItem:
|
||||
"ta_channel": "channel_active",
|
||||
"ta_playlist": "playlist_active",
|
||||
}
|
||||
update_path = f"{self.index_name}/_update/{self.youtube_id}"
|
||||
path = f"{self.index_name}/_update/{self.youtube_id}?refresh=true"
|
||||
data = {
|
||||
"script": f"ctx._source.{key_match.get(self.index_name)} = false"
|
||||
}
|
||||
_, _ = ElasticWrap(update_path).post(data)
|
||||
_, _ = ElasticWrap(path).post(data)
|
||||
|
||||
def del_in_es(self):
|
||||
"""delete item from elastic search"""
|
||||
|
||||
@@ -41,6 +41,10 @@ class YoutubePlaylist(YouTubeItem):
|
||||
|
||||
if scrape or not self.json_data:
|
||||
self.get_from_youtube()
|
||||
if not self.youtube_meta:
|
||||
self.json_data = False
|
||||
return
|
||||
|
||||
self.process_youtube_meta()
|
||||
self.get_entries()
|
||||
self.json_data["playlist_entries"] = self.all_members
|
||||
@@ -76,12 +80,12 @@ class YoutubePlaylist(YouTubeItem):
|
||||
downloaded = entry["id"] in self.all_youtube_ids
|
||||
else:
|
||||
downloaded = False
|
||||
if not entry["uploader"]:
|
||||
if not entry["channel"]:
|
||||
continue
|
||||
to_append = {
|
||||
"youtube_id": entry["id"],
|
||||
"title": entry["title"],
|
||||
"uploader": entry["uploader"],
|
||||
"uploader": entry["channel"],
|
||||
"idx": idx,
|
||||
"downloaded": downloaded,
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ from home.src.download.yt_dlp_base import CookieHandler
|
||||
from home.src.download.yt_dlp_handler import VideoDownloader
|
||||
from home.src.es.connect import ElasticWrap
|
||||
from home.src.index.channel import YoutubeChannel
|
||||
from home.src.index.comments import Comments
|
||||
from home.src.index.playlist import YoutubePlaylist
|
||||
from home.src.index.video import YoutubeVideo
|
||||
from home.src.ta.config import AppConfig
|
||||
@@ -147,8 +148,7 @@ class Reindex:
|
||||
if integrate_ryd:
|
||||
self._get_unrated_vids()
|
||||
|
||||
@staticmethod
|
||||
def _reindex_single_video(youtube_id):
|
||||
def _reindex_single_video(self, youtube_id):
|
||||
"""refresh data for single video"""
|
||||
video = YoutubeVideo(youtube_id)
|
||||
|
||||
@@ -182,6 +182,8 @@ class Reindex:
|
||||
thumb_handler.delete_video_thumb()
|
||||
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
|
||||
|
||||
Comments(youtube_id, config=self.config).reindex_comments()
|
||||
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
@@ -194,6 +196,8 @@ class Reindex:
|
||||
channel.get_from_youtube()
|
||||
if not channel.json_data:
|
||||
channel.deactivate()
|
||||
channel.get_from_es()
|
||||
channel.sync_to_videos()
|
||||
return
|
||||
|
||||
channel.json_data["channel_subscribed"] = subscribed
|
||||
|
||||
@@ -11,6 +11,7 @@ import requests
|
||||
from django.conf import settings
|
||||
from home.src.es.connect import ElasticWrap
|
||||
from home.src.index import channel as ta_channel
|
||||
from home.src.index import comments as ta_comments
|
||||
from home.src.index import playlist as ta_playlist
|
||||
from home.src.index.generic import YouTubeItem
|
||||
from home.src.index.subtitle import YoutubeSubtitle
|
||||
@@ -302,6 +303,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||
self.del_in_playlists()
|
||||
self.del_in_es()
|
||||
self.delete_subtitles()
|
||||
self.delete_comments()
|
||||
|
||||
def del_in_playlists(self):
|
||||
"""remove downloaded in playlist"""
|
||||
@@ -326,6 +328,13 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||
print(f"{self.youtube_id}: delete subtitles")
|
||||
YoutubeSubtitle(self).delete(subtitles=subtitles)
|
||||
|
||||
def delete_comments(self):
|
||||
"""delete comments from es"""
|
||||
comments = ta_comments.Comments(self.youtube_id, config=self.config)
|
||||
comments.check_config()
|
||||
if comments.is_activated:
|
||||
comments.delete_comments()
|
||||
|
||||
def _get_ryd_stats(self):
|
||||
"""get optional stats from returnyoutubedislikeapi.com"""
|
||||
# pylint: disable=broad-except
|
||||
|
||||
@@ -3,6 +3,8 @@ Loose collection of helper functions
|
||||
- don't import AppConfig class here to avoid circular imports
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
@@ -117,6 +119,23 @@ def time_parser(timestamp):
|
||||
return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
|
||||
|
||||
|
||||
def clear_dl_cache(config):
|
||||
"""clear leftover files from dl cache"""
|
||||
print("clear download cache")
|
||||
cache_dir = os.path.join(config["application"]["cache_dir"], "download")
|
||||
for cached in os.listdir(cache_dir):
|
||||
to_delete = os.path.join(cache_dir, cached)
|
||||
os.remove(to_delete)
|
||||
|
||||
|
||||
def get_mapping():
|
||||
"""read index_mapping.json and get expected mapping and settings"""
|
||||
with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
|
||||
index_config = json.load(f).get("index_config")
|
||||
|
||||
return index_config
|
||||
|
||||
|
||||
class UrlListParser:
|
||||
"""take a multi line string and detect valid youtube ids"""
|
||||
|
||||
|
||||
@@ -17,7 +17,8 @@ from home.src.download.subscriptions import (
|
||||
)
|
||||
from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator
|
||||
from home.src.download.yt_dlp_handler import VideoDownloader
|
||||
from home.src.es.index_setup import backup_all_indexes, restore_from_backup
|
||||
from home.src.es.backup import ElasticBackup
|
||||
from home.src.es.index_setup import ElasitIndexWrap
|
||||
from home.src.index.channel import YoutubeChannel
|
||||
from home.src.index.filesystem import (
|
||||
ImportFolderScanner,
|
||||
@@ -25,7 +26,7 @@ from home.src.index.filesystem import (
|
||||
scan_filesystem,
|
||||
)
|
||||
from home.src.ta.config import AppConfig, ScheduleBuilder
|
||||
from home.src.ta.helper import UrlListParser
|
||||
from home.src.ta.helper import UrlListParser, clear_dl_cache
|
||||
from home.src.ta.ta_redis import RedisArchivist, RedisQueue
|
||||
|
||||
CONFIG = AppConfig().config
|
||||
@@ -168,7 +169,7 @@ def run_backup(reason="auto"):
|
||||
try:
|
||||
have_lock = my_lock.acquire(blocking=False)
|
||||
if have_lock:
|
||||
backup_all_indexes(reason)
|
||||
ElasticBackup(reason=reason).backup_all_indexes()
|
||||
else:
|
||||
print("Did not acquire lock for backup task.")
|
||||
finally:
|
||||
@@ -180,7 +181,8 @@ def run_backup(reason="auto"):
|
||||
@shared_task
|
||||
def run_restore_backup(filename):
|
||||
"""called from settings page, dump backup to zip file"""
|
||||
restore_from_backup(filename)
|
||||
ElasitIndexWrap().reset()
|
||||
ElasticBackup().restore(filename)
|
||||
print("index restore finished")
|
||||
|
||||
|
||||
@@ -192,11 +194,7 @@ def kill_dl(task_id):
|
||||
_ = RedisArchivist().del_message("dl_queue_id")
|
||||
RedisQueue().clear()
|
||||
|
||||
# clear cache
|
||||
cache_dir = os.path.join(CONFIG["application"]["cache_dir"], "download")
|
||||
for cached in os.listdir(cache_dir):
|
||||
to_delete = os.path.join(cache_dir, cached)
|
||||
os.remove(to_delete)
|
||||
clear_dl_cache(CONFIG)
|
||||
|
||||
# notify
|
||||
mess_dict = {
|
||||
|
||||
@@ -100,7 +100,7 @@
|
||||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
{% if video.source.player.progress %}
|
||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
||||
{% else %}
|
||||
|
||||
@@ -72,7 +72,7 @@
|
||||
<div class="video-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video_thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video_thumb">
|
||||
{% if show_ignored_only %}
|
||||
<span>ignored</span>
|
||||
{% else %}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
{% if video.source.player.progress %}
|
||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
||||
{% else %}
|
||||
@@ -98,7 +98,7 @@
|
||||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
{% if video.source.player.progress %}
|
||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
||||
{% else %}
|
||||
|
||||
@@ -102,7 +102,7 @@
|
||||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
{% if video.source.player.progress %}
|
||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
||||
{% else %}
|
||||
|
||||
@@ -77,7 +77,7 @@
|
||||
<ul>
|
||||
<li><span class="settings-current">bestvideo[height<=720]+bestaudio/best[height<=720]</span>: best audio and max video height of 720p.</li>
|
||||
<li><span class="settings-current">bestvideo[height<=1080]+bestaudio/best[height<=1080]</span>: best audio and max video height of 1080p.</li>
|
||||
<li><span class="settings-current">bestvideo[height<=1080][VCODEC*=avc1]+bestaudio[ACODEC*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
|
||||
<li><span class="settings-current">bestvideo[height<=1080][vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
|
||||
<li><span class="settings-current">0</span>: deactivate and download the best quality possible as decided by yt-dlp.</li>
|
||||
</ul>
|
||||
<i>Make sure your custom format gets merged into a single file. Check out the <a href="https://github.com/yt-dlp/yt-dlp#format-selection" target="_blank">documentation</a> for valid configurations.</i><br>
|
||||
@@ -114,6 +114,24 @@
|
||||
{{ app_form.downloads_subtitle_index }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="settings-group">
|
||||
<h2 id="comments">Comments</h2>
|
||||
<div class="settings-item">
|
||||
<p>Download and index comments: <span class="settings-current">{{ config.downloads.comment_max }}</span><br>
|
||||
<i>Follow the yt-dlp max_comments documentation, <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">max-comments,max-parents,max-replies,max-replies-per-thread</a>:</i><br>
|
||||
<p>Example configurations:</p>
|
||||
<ul>
|
||||
<li><span class="settings-current">all,100,all,30</span>: Get 100 max-parents and 30 max-replies-per-thread.</li>
|
||||
<li><span class="settings-current">1000,all,all,50</span>: Get a total of 1000 comments over all, 50 replies per thread.</li>
|
||||
</ul>
|
||||
{{ app_form.downloads_comment_max }}</p>
|
||||
</div>
|
||||
<div class="settings-item">
|
||||
<p>Selected comment sort method: <span class="settings-current">{{ config.downloads.comment_sort }}</span><br>
|
||||
<i>Select how many comments and threads to download:</i><br>
|
||||
{{ app_form.downloads_comment_sort }}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="settings-group">
|
||||
<h2 id="format">Cookie</h2>
|
||||
<div class="settings-item">
|
||||
@@ -186,7 +204,7 @@
|
||||
<p>Note:</p>
|
||||
<ul>
|
||||
<li>Changes in the scheduler settings require a container restart to take effect.</li>
|
||||
<li>Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason <i>*</i> or <i>wildcard</i> for minutes is not supported.</li>
|
||||
<li>Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason, the scheduler doesn't support schedules that trigger more than once per hour.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
{% block content %}
|
||||
{% load static %}
|
||||
{% load humanize %}
|
||||
<div class="video-main">
|
||||
<div class="video-modal"><span class="video-modal-text"></span></div>
|
||||
<div id="player" class="player-wrapper">
|
||||
<div class="video-main">
|
||||
<div class="video-modal"><span class="video-modal-text"></span></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="notifications" id="notifications"></div>
|
||||
<div class="sponsorblock" id="sponsorblock">
|
||||
@@ -69,15 +71,24 @@
|
||||
<p class="thumb-icon"><img class="dislike" src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-down">: {{ video.stats.dislike_count|intcomma }}</p>
|
||||
{% endif %}
|
||||
{% if video.stats.average_rating %}
|
||||
<p class="rating-stars">Rating:
|
||||
<div class="rating-stars">
|
||||
{% for star in video.stats.average_rating %}
|
||||
<img src="/static/img/icon-star-{{ star }}.svg" alt="{{ star }}">
|
||||
{% endfor %}
|
||||
</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% if video.tags %}
|
||||
<div class="description-box">
|
||||
<div class="video-tag-box">
|
||||
{% for tag in video.tags %}
|
||||
<span class="video-tag">{{ tag }}</span>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if video.description %}
|
||||
<div class="description-box">
|
||||
<p id="text-expand" class="description-text">
|
||||
@@ -123,6 +134,20 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
<div class="description-box">
|
||||
<h3>Similar Videos</h3>
|
||||
<div class="video-list grid grid-3" id="similar-videos">
|
||||
</div>
|
||||
<script>getSimilarVideos('{{ video.youtube_id }}')</script>
|
||||
</div>
|
||||
{% if video.comment_count %}
|
||||
<div class="comments-section">
|
||||
<h3>Comments: {{video.comment_count}}</h3>
|
||||
<div id="comments-list" class="comments-list">
|
||||
</div>
|
||||
</div>
|
||||
<script>getComments('{{ video.youtube_id }}')</script>
|
||||
{% endif %}
|
||||
</div>
|
||||
<script>
|
||||
var videoData = getVideoData('{{ video.youtube_id }}');
|
||||
|
||||
@@ -16,8 +16,8 @@ from django.http import JsonResponse
|
||||
from django.shortcuts import redirect, render
|
||||
from django.views import View
|
||||
from home.src.download.yt_dlp_base import CookieHandler
|
||||
from home.src.es.backup import ElasticBackup
|
||||
from home.src.es.connect import ElasticWrap
|
||||
from home.src.es.index_setup import get_available_backups
|
||||
from home.src.es.snapshot import ElasticSnapshot
|
||||
from home.src.frontend.api_calls import PostData
|
||||
from home.src.frontend.forms import (
|
||||
@@ -939,7 +939,7 @@ class SettingsView(View):
|
||||
config_handler = AppConfig(request.user.id)
|
||||
colors = config_handler.colors
|
||||
|
||||
available_backups = get_available_backups()
|
||||
available_backups = ElasticBackup().get_all_backup_files()
|
||||
user_form = UserSettingsForm()
|
||||
app_form = ApplicationSettingsForm()
|
||||
scheduler_form = SchedulerSettingsForm()
|
||||
|
||||
@@ -5,9 +5,9 @@ django-auth-ldap==4.1.0
|
||||
django-cors-headers==3.13.0
|
||||
djangorestframework==3.14.0
|
||||
Pillow==9.3.0
|
||||
redis==4.3.4
|
||||
redis==4.3.5
|
||||
requests==2.28.1
|
||||
ryd-client==0.0.6
|
||||
uWSGI==2.0.21
|
||||
whitenoise==6.2.0
|
||||
yt_dlp==2022.10.4
|
||||
yt_dlp==2022.11.11
|
||||
|
||||
@@ -619,7 +619,8 @@ video:-webkit-full-screen {
|
||||
margin-top: 1rem;
|
||||
}
|
||||
|
||||
.description-box {
|
||||
.description-box,
|
||||
.comments-section {
|
||||
margin-top: 1rem;
|
||||
padding: 15px;
|
||||
background-color: var(--highlight-bg);
|
||||
@@ -778,10 +779,26 @@ video:-webkit-full-screen {
|
||||
margin-left: 5px;
|
||||
}
|
||||
|
||||
.thumb-icon {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.video-tag-box {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.video-tag {
|
||||
padding: 5px 10px;
|
||||
margin: 5px;
|
||||
border: 1px solid var(--accent-font-light);
|
||||
}
|
||||
|
||||
.thumb-icon img,
|
||||
.rating-stars img {
|
||||
width: 20px;
|
||||
margin: 0;
|
||||
margin: 0 5px;
|
||||
filter: var(--img-filter);
|
||||
}
|
||||
|
||||
@@ -819,6 +836,44 @@ video:-webkit-full-screen {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.comment-box {
|
||||
padding-bottom: 1rem;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.comment-box h3 {
|
||||
line-break: anywhere;
|
||||
}
|
||||
|
||||
.comments-replies {
|
||||
display: none;
|
||||
padding-left: 1rem;
|
||||
border-left: 1px solid var(--accent-font-light);
|
||||
margin-top: 1rem;
|
||||
}
|
||||
|
||||
.comment-highlight {
|
||||
background-color: var(--main-font);
|
||||
padding: 3px;
|
||||
color: var(--accent-font-dark);
|
||||
font-family: Sen-bold, sans-serif;
|
||||
width: fit-content;
|
||||
}
|
||||
|
||||
.comment-meta {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.space-carrot {
|
||||
margin: 0 5px;
|
||||
}
|
||||
|
||||
.comment-like img {
|
||||
width: 20px;
|
||||
margin-left: 5px;
|
||||
filter: var(--img-filter-error);
|
||||
}
|
||||
|
||||
/* multi search page */
|
||||
.multi-search-box {
|
||||
padding-right: 20px;
|
||||
|
||||
@@ -1,63 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||
|
||||
<svg
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
width="500"
|
||||
height="500"
|
||||
viewBox="0 0 132.29197 132.29167"
|
||||
version="1.1"
|
||||
id="svg1303"
|
||||
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"
|
||||
sodipodi:docname="Icons_seen.svg">
|
||||
<defs
|
||||
id="defs1297" />
|
||||
<sodipodi:namedview
|
||||
id="base"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:zoom="1.0105705"
|
||||
inkscape:cx="84.208758"
|
||||
inkscape:cy="136.94831"
|
||||
inkscape:document-units="mm"
|
||||
inkscape:current-layer="layer1"
|
||||
showgrid="false"
|
||||
units="px"
|
||||
inkscape:window-width="1920"
|
||||
inkscape:window-height="1017"
|
||||
inkscape:window-x="-8"
|
||||
inkscape:window-y="-8"
|
||||
inkscape:window-maximized="1" />
|
||||
<metadata
|
||||
id="metadata1300">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title />
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
inkscape:label="Ebene 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1"
|
||||
transform="translate(0,-164.70764)">
|
||||
<path
|
||||
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
|
||||
d="M 66.145984,191.3255 A 71.797122,73.404487 0 0 0 2.5025987,230.88314 71.797122,73.404487 0 0 0 66.145984,270.38145 71.797122,73.404487 0 0 0 129.78937,230.82387 71.797122,73.404487 0 0 0 66.145984,191.3255 Z m -4.921549,7.22394 a 32.724755,32.724755 0 0 0 -13.334395,5.17759 12.828107,12.828107 0 0 1 2.180958,-0.18652 12.828107,12.828107 0 0 1 12.828141,12.82816 12.828107,12.828107 0 0 1 -12.828141,12.82822 12.828107,12.828107 0 0 1 -12.828192,-12.82822 12.828107,12.828107 0 0 1 0.04509,-0.91548 32.724755,32.724755 0 0 0 -3.86581,15.39964 32.724755,32.724755 0 0 0 27.161922,32.24981 A 59.09757,60.420619 0 0 1 13.7604,230.8774 59.09757,60.420619 0 0 1 61.224664,198.54948 Z m 10.482345,0.0563 a 59.09757,60.420619 0 0 1 46.82502,32.22523 59.09757,60.420619 0 0 1 -47.393377,32.32497 32.724755,32.724755 0 0 0 27.73169,-32.30187 32.724755,32.724755 0 0 0 -27.163333,-32.24833 z"
|
||||
id="path1091"
|
||||
inkscape:connector-curvature="0" />
|
||||
</g>
|
||||
</svg>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 26.5.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1"
|
||||
id="svg1303" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" xmlns:svg="http://www.w3.org/2000/svg" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" inkscape:version="0.92.4 (5da689c313, 2019-01-14)" sodipodi:docname="Icons_seen.svg"
|
||||
xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 500 500"
|
||||
style="enable-background:new 0 0 500 500;" xml:space="preserve">
|
||||
<sodipodi:namedview bordercolor="#666666" borderopacity="1.0" id="base" inkscape:current-layer="layer1" inkscape:cx="84.208758" inkscape:cy="136.94831" inkscape:document-units="mm" inkscape:pageopacity="0.0" inkscape:pageshadow="2" inkscape:window-height="1017" inkscape:window-maximized="1" inkscape:window-width="1920" inkscape:window-x="-8" inkscape:window-y="-8" inkscape:zoom="1.0105705" pagecolor="#ffffff" showgrid="false" units="px">
|
||||
</sodipodi:namedview>
|
||||
<g id="layer1" transform="translate(0,-164.70764)" inkscape:groupmode="layer" inkscape:label="Ebene 1">
|
||||
<path id="path1091" inkscape:connector-curvature="0" d="M250,259.4c-105.2,0.1-201.5,60-250,155.4C48.5,510.1,144.9,569.9,250,570
|
||||
c105.2-0.1,201.5-60,250-155.4C451.5,319.3,355.1,259.5,250,259.4z M230.7,287.8c-18.7,2.8-36.6,9.8-52.4,20.3
|
||||
c2.8-0.5,5.7-0.7,8.6-0.7c27.8,0,50.4,22.6,50.4,50.4c0,0,0,0,0,0c0,27.8-22.6,50.4-50.4,50.4c0,0,0,0,0,0
|
||||
c-27.8,0-50.4-22.6-50.4-50.4c0,0,0,0,0,0c0-1.2,0.1-2.4,0.2-3.6c-10,18.6-15.2,39.4-15.2,60.5c0,62.6,45,116,106.7,126.7
|
||||
c-78.3-7.6-147.6-55.2-183.9-126.6C81,342.6,151.3,294.7,230.7,287.8L230.7,287.8z M271.8,288c78.3,7.6,147.6,55.2,183.9,126.6
|
||||
c-36.7,72.1-106.9,120-186.2,127c62.6-9.7,108.9-63.5,108.9-126.9C378.5,352.1,333.5,298.7,271.8,288L271.8,288z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 2.8 KiB After Width: | Height: | Size: 2.0 KiB |
8
tubearchivist/static/img/icon-heart.svg
Normal file
8
tubearchivist/static/img/icon-heart.svg
Normal file
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 26.5.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
viewBox="0 0 500 500" style="enable-background:new 0 0 500 500;" xml:space="preserve">
|
||||
<path d="M499.9,159L499.9,159c0.1-1.7,0.1-3.4,0.1-5.2c0-69.5-58.6-129.7-130.9-129.7c-52.9,0-98.4,34-119,77.4h0
|
||||
c-20.7-43.4-66.2-77.4-119-77.4C58.6,24.1,0,84.4,0,153.9c0,1.7,0.1,3.4,0.1,5.2h0c0,0-7.4,82.6,84.5,172.7
|
||||
c41.8,41.9,88.5,81.6,165.4,144.1c76.9-62.5,123.6-102.3,165.4-144.1C507.2,241.6,499.9,159,499.9,159z"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 684 B |
@@ -1,76 +1,18 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||
|
||||
<svg
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
width="500"
|
||||
height="500"
|
||||
viewBox="0 0 132.29197 132.29167"
|
||||
version="1.1"
|
||||
id="svg1303"
|
||||
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"
|
||||
sodipodi:docname="icon_thumb.svg">
|
||||
<defs
|
||||
id="defs1297" />
|
||||
<sodipodi:namedview
|
||||
id="base"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:zoom="1.5046797"
|
||||
inkscape:cx="35.718548"
|
||||
inkscape:cy="203.39339"
|
||||
inkscape:document-units="mm"
|
||||
inkscape:current-layer="layer1"
|
||||
showgrid="false"
|
||||
units="px"
|
||||
inkscape:window-width="1920"
|
||||
inkscape:window-height="1009"
|
||||
inkscape:window-x="-8"
|
||||
inkscape:window-y="-8"
|
||||
inkscape:window-maximized="1"
|
||||
showguides="true"
|
||||
inkscape:guide-bbox="true">
|
||||
<sodipodi:guide
|
||||
position="80.99058,65.965029"
|
||||
orientation="0,1"
|
||||
id="guide816"
|
||||
inkscape:locked="false" />
|
||||
<sodipodi:guide
|
||||
position="65.965178,49.107299"
|
||||
orientation="1,0"
|
||||
id="guide818"
|
||||
inkscape:locked="false" />
|
||||
</sodipodi:namedview>
|
||||
<metadata
|
||||
id="metadata1300">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title />
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
inkscape:label="Ebene 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1"
|
||||
transform="translate(0,-164.70764)">
|
||||
<path
|
||||
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
|
||||
d="m 72.35681,178.12175 c -0.545876,-0.0192 -1.121779,0.0192 -1.729242,0.12259 -12.959134,2.14891 4.226681,21.04642 -10.867262,31.83193 -12.617074,10.52505 -17.206737,7.22264 -17.332488,10.67172 v 33.88171 c 1.891308,5.00813 14.483922,10.12847 27.868234,11.21944 13.384309,1.09096 32.098988,-1.73801 32.305468,-7.56421 0.10989,-5.27572 -2.896223,-5.39662 -2.896223,-5.39662 0,0 6.589043,-0.40752 6.808883,-6.12278 0.10966,-5.60545 -6.38265,-6.40691 -6.38265,-6.40691 0,0 8.31589,-0.23607 8.31589,-6.01832 0,-5.91485 -6.39662,-6.38072 -8.37539,-6.48104 1.22776,-0.0747 6.07353,-1.75141 6.04505,-7.59582 -0.0287,-6.03384 -9.280896,-5.53597 -25.919173,-6.63279 1.919791,-11.19377 3.258461,-35.14851 -7.841097,-35.50894 z m -45.69253,38.36631 c -1.562226,0 -2.819733,1.04793 -2.819733,2.34979 v 38.25943 c 0,1.30191 1.257507,2.34983 2.819733,2.34983 h 10.526021 c 1.562231,0 2.819739,-1.04792 2.819739,-2.34983 v -38.25943 c 0,-1.30186 -1.257508,-2.34979 -2.819739,-2.34979 z"
|
||||
id="rect1278"
|
||||
inkscape:connector-curvature="0" />
|
||||
</g>
|
||||
</svg>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 26.5.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1"
|
||||
id="svg1303" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" xmlns:svg="http://www.w3.org/2000/svg" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" inkscape:version="0.92.4 (5da689c313, 2019-01-14)" sodipodi:docname="icon_thumb.svg"
|
||||
xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 500 500"
|
||||
style="enable-background:new 0 0 500 500;" xml:space="preserve">
|
||||
<sodipodi:namedview bordercolor="#666666" borderopacity="1.0" id="base" inkscape:current-layer="layer1" inkscape:cx="35.718548" inkscape:cy="203.39339" inkscape:document-units="mm" inkscape:guide-bbox="true" inkscape:pageopacity="0.0" inkscape:pageshadow="2" inkscape:window-height="1009" inkscape:window-maximized="1" inkscape:window-width="1920" inkscape:window-x="-8" inkscape:window-y="-8" inkscape:zoom="1.5046797" pagecolor="#ffffff" showgrid="false" showguides="true" units="px">
|
||||
<sodipodi:guide id="guide816" inkscape:locked="false" orientation="0,1" position="80.99058,65.965029"></sodipodi:guide>
|
||||
<sodipodi:guide id="guide818" inkscape:locked="false" orientation="1,0" position="65.965178,49.107299"></sodipodi:guide>
|
||||
</sodipodi:namedview>
|
||||
<g id="layer1" transform="translate(0,-164.70764)" inkscape:groupmode="layer" inkscape:label="Ebene 1">
|
||||
<path id="rect1278" inkscape:connector-curvature="0" d="M285.3,164.7c-3.1-0.1-6.4,0.1-9.8,0.7c-73.7,12.2,24,119.6-61.8,180.9
|
||||
c-71.7,59.8-97.8,41.1-98.5,60.7v192.6c10.8,28.5,82.3,57.6,158.4,63.8c76.1,6.2,182.5-9.9,183.6-43c0.6-30-16.5-30.7-16.5-30.7
|
||||
s37.5-2.3,38.7-34.8c0.6-31.9-36.3-36.4-36.3-36.4s47.3-1.3,47.3-34.2c0-33.6-36.4-36.3-47.6-36.8c7-0.4,34.5-10,34.4-43.2
|
||||
c-0.2-34.3-52.8-31.5-147.3-37.7C340.8,302.9,348.4,166.8,285.3,164.7L285.3,164.7z M25.6,382.8c-8.9,0-16,6-16,13.4v217.5
|
||||
c0,7.4,7.1,13.4,16,13.4h59.8c8.9,0,16-6,16-13.4V396.2c0-7.4-7.1-13.4-16-13.4H25.6z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 3.2 KiB After Width: | Height: | Size: 2.2 KiB |
@@ -287,7 +287,7 @@ function resetToken() {
|
||||
document.getElementById('text-reveal').replaceWith(message);
|
||||
}
|
||||
|
||||
// restore from snapshot
|
||||
// restore from snapshot
|
||||
function restoreSnapshot(snapshotId) {
|
||||
console.log('restore ' + snapshotId);
|
||||
let apiEndpoint = '/api/snapshot/' + snapshotId + '/';
|
||||
@@ -793,7 +793,11 @@ function apiRequest(apiEndpoint, method, data) {
|
||||
xhttp.setRequestHeader('Authorization', 'Token ' + sessionToken);
|
||||
xhttp.setRequestHeader('Content-Type', 'application/json');
|
||||
xhttp.send(JSON.stringify(data));
|
||||
return JSON.parse(xhttp.responseText);
|
||||
if (xhttp.status === 404) {
|
||||
return false;
|
||||
} else {
|
||||
return JSON.parse(xhttp.responseText);
|
||||
}
|
||||
}
|
||||
|
||||
// Gets origin URL
|
||||
@@ -951,7 +955,7 @@ function createVideo(video, viewStyle) {
|
||||
// create video item div from template
|
||||
const videoId = video.youtube_id;
|
||||
// const mediaUrl = video.media_url;
|
||||
const thumbUrl = '/cache/' + video.vid_thumb_url;
|
||||
// const thumbUrl = '/cache/' + video.vid_thumb_url;
|
||||
const videoTitle = video.title;
|
||||
const videoPublished = video.published;
|
||||
const videoDuration = video.player.duration_str;
|
||||
@@ -968,7 +972,7 @@ function createVideo(video, viewStyle) {
|
||||
<a href="#player" data-id="${videoId}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap ${viewStyle}">
|
||||
<div class="video-thumb">
|
||||
<img src="${thumbUrl}" alt="video-thumb">
|
||||
<img src="${video.vid_thumb_url}" alt="video-thumb">
|
||||
</div>
|
||||
<div class="video-play">
|
||||
<img src="/static/img/icon-play.svg" alt="play-icon">
|
||||
@@ -1104,6 +1108,138 @@ function createFulltext(fullText) {
|
||||
return fullTextDiv;
|
||||
}
|
||||
|
||||
function getComments(videoId) {
|
||||
let apiEndpoint = '/api/video/' + videoId + '/comment/';
|
||||
let response = apiRequest(apiEndpoint, 'GET');
|
||||
let allComments = response.data;
|
||||
|
||||
writeComments(allComments);
|
||||
}
|
||||
|
||||
function writeComments(allComments) {
|
||||
let commentsListBox = document.getElementById('comments-list');
|
||||
for (let i = 0; i < allComments.length; i++) {
|
||||
const rootComment = allComments[i];
|
||||
|
||||
let commentBox = createCommentBox(rootComment, true);
|
||||
|
||||
// add replies to commentBox
|
||||
if (rootComment.comment_replies) {
|
||||
let commentReplyBox = document.createElement('div');
|
||||
commentReplyBox.setAttribute('class', 'comments-replies');
|
||||
commentReplyBox.setAttribute('id', rootComment.comment_id + '-replies');
|
||||
let totalReplies = rootComment.comment_replies.length;
|
||||
if (totalReplies > 0) {
|
||||
let replyButton = createReplyButton(rootComment.comment_id + '-replies', totalReplies);
|
||||
commentBox.appendChild(replyButton);
|
||||
}
|
||||
for (let j = 0; j < totalReplies; j++) {
|
||||
const commentReply = rootComment.comment_replies[j];
|
||||
let commentReplyDiv = createCommentBox(commentReply, false);
|
||||
commentReplyBox.appendChild(commentReplyDiv);
|
||||
}
|
||||
if (totalReplies > 0) {
|
||||
commentBox.appendChild(commentReplyBox);
|
||||
}
|
||||
}
|
||||
commentsListBox.appendChild(commentBox);
|
||||
}
|
||||
}
|
||||
|
||||
function createReplyButton(replyId, totalReplies) {
|
||||
let replyButton = document.createElement('button');
|
||||
replyButton.innerHTML = `<span id="toggle-icon">+</span> ${totalReplies} replies`;
|
||||
replyButton.setAttribute('data-id', replyId);
|
||||
replyButton.setAttribute('onclick', 'toggleCommentReplies(this)');
|
||||
return replyButton;
|
||||
}
|
||||
|
||||
function toggleCommentReplies(button) {
|
||||
let commentReplyId = button.getAttribute('data-id');
|
||||
let state = document.getElementById(commentReplyId).style.display;
|
||||
|
||||
if (state === 'none' || state === '') {
|
||||
document.getElementById(commentReplyId).style.display = 'block';
|
||||
button.querySelector('#toggle-icon').innerHTML = '-';
|
||||
} else {
|
||||
document.getElementById(commentReplyId).style.display = 'none';
|
||||
button.querySelector('#toggle-icon').innerHTML = '+';
|
||||
}
|
||||
}
|
||||
|
||||
function createCommentBox(comment, isRoot) {
|
||||
let commentBox = document.createElement('div');
|
||||
commentBox.setAttribute('class', 'comment-box');
|
||||
|
||||
let commentClass;
|
||||
if (isRoot) {
|
||||
commentClass = 'root-comment';
|
||||
} else {
|
||||
commentClass = 'reply-comment';
|
||||
}
|
||||
|
||||
commentBox.classList.add = commentClass;
|
||||
|
||||
let commentAuthor = document.createElement('h3');
|
||||
commentAuthor.innerText = comment.comment_author;
|
||||
if (comment.comment_author_is_uploader) {
|
||||
commentAuthor.setAttribute('class', 'comment-highlight');
|
||||
}
|
||||
commentBox.appendChild(commentAuthor);
|
||||
|
||||
let commentText = document.createElement('p');
|
||||
commentText.innerText = comment.comment_text;
|
||||
commentBox.appendChild(commentText);
|
||||
|
||||
const spacer = '<span class="space-carrot">|</span>';
|
||||
let commentMeta = document.createElement('div');
|
||||
commentMeta.setAttribute('class', 'comment-meta');
|
||||
|
||||
commentMeta.innerHTML = `<span>${comment.comment_time_text}</span>`;
|
||||
|
||||
if (comment.comment_likecount > 0) {
|
||||
let numberFormatted = formatNumbers(comment.comment_likecount);
|
||||
commentMeta.innerHTML += `${spacer}<span class="thumb-icon"><img src="/static/img/icon-thumb.svg"> ${numberFormatted}</span>`;
|
||||
}
|
||||
|
||||
if (comment.comment_is_favorited) {
|
||||
commentMeta.innerHTML += `${spacer}<span class="comment-like"><img src="/static/img/icon-heart.svg"></span>`;
|
||||
}
|
||||
|
||||
commentBox.appendChild(commentMeta);
|
||||
|
||||
return commentBox;
|
||||
}
|
||||
|
||||
function getSimilarVideos(videoId) {
|
||||
let apiEndpoint = '/api/video/' + videoId + '/similar/';
|
||||
let response = apiRequest(apiEndpoint, 'GET');
|
||||
if (!response) {
|
||||
populateEmpty();
|
||||
return;
|
||||
}
|
||||
let allSimilar = response.data;
|
||||
if (allSimilar.length > 0) {
|
||||
populateSimilar(allSimilar);
|
||||
}
|
||||
}
|
||||
|
||||
function populateSimilar(allSimilar) {
|
||||
let similarBox = document.getElementById('similar-videos');
|
||||
for (let i = 0; i < allSimilar.length; i++) {
|
||||
const similarRaw = allSimilar[i];
|
||||
let similarDiv = createVideo(similarRaw, 'grid');
|
||||
similarBox.appendChild(similarDiv);
|
||||
}
|
||||
}
|
||||
|
||||
function populateEmpty() {
|
||||
let similarBox = document.getElementById('similar-videos');
|
||||
let emptyMessage = document.createElement('p');
|
||||
emptyMessage.innerText = 'No similar videos found.';
|
||||
similarBox.appendChild(emptyMessage);
|
||||
}
|
||||
|
||||
// generic
|
||||
|
||||
function sendPost(payload) {
|
||||
|
||||
Reference in New Issue
Block a user