Compare commits

...

61 Commits

Author SHA1 Message Date
simon
e98d2d3590 bump footer version 2022-11-30 08:52:45 +07:00
3ster
61ccb1c47a Fix example format specifier for iOS-compatible downloads (#370) 2022-11-30 08:50:56 +07:00
simon
b4b287bda4 fix spelling 2022-11-28 21:36:24 +07:00
simon
ca8f4c9fa4 update docs for comments and similar videos 2022-11-28 21:31:49 +07:00
simon
18b8a17df1 Use snapshots for mapping backup, #build
Changes:
- Use system snapshots at startup to backup before mapping changes
- Refactor startup, index setup and backup classes
- bump yt-dlp and redis
2022-11-27 18:26:45 +07:00
simon
f192c39be8 make snapshot before mapping changes 2022-11-27 18:01:57 +07:00
simon
f1b89345e2 refactor ElasticBackup and ElasticIndex for better startup handling 2022-11-27 15:41:59 +07:00
simon
f6c702fba7 refactor and run clear_dl_cache at startup 2022-11-25 18:49:36 +07:00
simon
f619a5f72d fi none existing unset channel overwrite, #362 2022-11-25 18:18:02 +07:00
simon
f7dd77ee3e bump redis 2022-11-25 15:58:43 +07:00
simon
0f41c54406 fix linebreak for comments header 2022-11-25 15:58:35 +07:00
simon
13d1927840 clean up API logging 2022-11-24 11:11:36 +07:00
simon
d0124c1a5c update yt-dlp, fix channel entries extractor 2022-11-22 19:11:58 +07:00
simon
5ed80d983d fix deactivated comment extractor 2022-11-22 18:01:27 +07:00
simon
24d66e33a7 video page improvements, #build
Changed:
- Added similar videos
- Added video tag cloud
- Added comment reply toggle
- Added comments progress
- Fixed channel and playlist deactivate
2022-11-22 11:43:44 +07:00
simon
786caf7bc1 Merge branch 'master' into testing 2022-11-22 11:36:52 +07:00
Gabe Dunn
e4b7258f7d Add podman installation instructions to docs (#367)
* add podman instructions to installation docs

* add mention of podman differences and docs page in readme installation section

* make mention of podman instructions more efficient
2022-11-22 11:36:09 +07:00
simon
90258e75e6 add notification for comment indexing 2022-11-22 11:23:34 +07:00
simon
b48b7c7e72 fix channel deactivation, take 2 2022-11-22 10:17:44 +07:00
simon
454d520394 standard video thumb path building 2022-11-21 15:50:01 +07:00
simon
ee5f83e89f add player-wrapper to video.html 2022-11-21 15:31:54 +07:00
simon
fa518c029c frontend similar videos 2022-11-21 15:13:37 +07:00
simon
aa1d0b759e [API] implement similar video endpoint 2022-11-21 15:13:15 +07:00
simon
abfd109203 hide comment box overflow 2022-11-21 15:05:28 +07:00
simon
ebf53f6486 fix channel deactivation 2022-11-21 14:55:47 +07:00
simon
2450a4ad1d future proof arm64 instructions 2022-11-21 13:46:19 +07:00
simon
41ea0c2c58 better padding for comment reply threads 2022-11-21 10:38:30 +07:00
simon
acb24e38a6 add tag cloud to video page, short rating 2022-11-21 10:27:28 +07:00
simon
a1fbc0b436 fix linter 2022-11-21 10:05:28 +07:00
simon
e052935e2d add comment reply toggle 2022-11-21 09:55:33 +07:00
simon
4d2bd51603 Merge branch 'master' into testing 2022-11-21 07:31:46 +07:00
simon
b3d3ebe9d0 fix channel_last_refresh mapping for date math 2022-11-20 20:38:10 +07:00
simon
d05c00aac6 fix playlist deactivate for refresh 2022-11-20 16:26:54 +07:00
simon
0c8fcbd724 use mapping to fix error for temp backup index 2022-11-20 16:13:30 +07:00
Simon
7e67ed02aa fix checkbox label 2022-11-20 15:45:55 +07:00
simon
e101c4abc7 ignore unavailable index for snapshots 2022-11-20 12:33:39 +07:00
simon
6a7a84792e better feedback when skipping not yet existing index backup 2022-11-20 12:21:25 +07:00
simon
2fa907c478 Index Comments, #build
Changed:
- added comment download and index
- [API] added comment api endpoints
2022-11-18 11:36:51 +07:00
simon
cc965ac473 update TA_VERSION to unstable 2022-11-18 11:35:09 +07:00
simon
8f5311ff04 delete comments when deleting channel 2022-11-18 11:33:06 +07:00
simon
e590d456a1 delete comments when deleting video 2022-11-18 10:47:48 +07:00
simon
a5be5a719b reindex comments 2022-11-18 10:31:09 +07:00
simon
b5ceb264ec format comment like number 2022-11-18 09:51:18 +07:00
simon
31c967cc11 fix linter 2022-11-18 09:46:33 +07:00
simon
2818bd5d52 add comments to video template if available 2022-11-18 09:44:20 +07:00
simon
939cd7fb62 index comments as a DownloadPostProcess 2022-11-18 09:22:58 +07:00
simon
172ced7129 configure comments extraction 2022-11-18 08:59:04 +07:00
simon
11c122cba3 finetune comments icon position 2022-11-18 08:58:25 +07:00
simon
456bc57df3 Merge branch 'master' into testing 2022-11-14 16:34:15 +07:00
crocs
091024553b Fixed "Unseen" and "Thumb" icons (#363)
* Update channel.html

Updated wording, removed "matching" as it's not necessary.

* Update downloads.html

* Delete icon-thumb.svg

* Delete icon-eye.svg

* Add files via upload
2022-11-14 16:33:26 +07:00
simon
ad016df105 implement comment frontend 2022-11-14 16:31:49 +07:00
simon
79d134cea8 clean format comments date string 2022-11-14 11:59:59 +07:00
simon
fb046bed5b [API] implement get comments API view 2022-11-12 12:42:08 +07:00
simon
f6b6185fb2 rename ta_comment index, implement get comments from es 2022-11-12 12:40:14 +07:00
simon
5b7e3e877b implement basic comment archive 2022-11-12 11:48:48 +07:00
simon
3c5302336f Merge branch 'master' into testing 2022-11-10 10:06:42 +07:00
simon
55c9721b51 rename to *common errors*, better issue template 2022-11-10 10:05:14 +07:00
simon
fc788c586d better wording for scheduler frequency, #358 2022-11-10 10:05:14 +07:00
simon
e912cf7fb6 rename to *common errors*, better issue template 2022-11-10 09:53:49 +07:00
lamusmaser
c111050c8f Update Synology documentation to include snapshot flag details. (#361) 2022-11-10 09:38:07 +07:00
simon
0e8d3b76c1 better wording for scheduler frequency, #358 2022-11-08 08:19:09 +07:00
41 changed files with 1137 additions and 480 deletions

View File

@@ -11,10 +11,12 @@ body:
- type: checkboxes
id: latest
attributes:
label: Latest and Greatest
label: "I've read the documentation"
options:
- label: I'm running the latest version of Tube Archivist and have read the [release notes](https://github.com/tubearchivist/tubearchivist/releases/latest).
required: true
- label: I have read through the [wiki](https://github.com/tubearchivist/tubearchivist/wiki) and the [readme](https://github.com/tubearchivist/tubearchivist#installing-and-updating), particularly the [common errors](https://github.com/tubearchivist/tubearchivist#common-errors) section.
required: true
- type: input
id: os

View File

@@ -15,7 +15,7 @@ body:
options:
- label: I have read and understand the [installation instructions](https://github.com/tubearchivist/tubearchivist#installing-and-updating).
required: true
- label: My issue is not described in the [potential pitfalls](https://github.com/tubearchivist/tubearchivist#potential-pitfalls) section.
- label: My issue is not described in the [common errors](https://github.com/tubearchivist/tubearchivist#common-errors) section.
required: true
- type: input

View File

@@ -16,7 +16,7 @@
* [Extended Universe](#extended-universe)
* [Installing and updating](#installing-and-updating)
* [Getting Started](#getting-started)
* [Potential pitfalls](#potential-pitfalls)
* [Common Errors](#common-errors)
* [Roadmap](#roadmap)
* [Known limitations](#known-limitations)
* [Donate](#donate)
@@ -61,11 +61,11 @@ Once your YouTube video collection grows, it becomes hard to search and find a s
- [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.
## Installing and updating
There's dedicated user-contributed install steps under [docs/Installation.md](./docs/Installation.md) for Unraid, Truenas and Synology which you can use instead of this section if you happen to be using one of those. Otherwise, continue on.
There's dedicated user-contributed install steps under [docs/Installation.md](./docs/Installation.md) for podman, Unraid, Truenas and Synology which you can use instead of this section if you happen to be using one of those. Otherwise, continue on.
For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation.
Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson` (an unofficial rebuild for arm64) instead of [the official one](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L27).
Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson`, an unofficial rebuild for arm64.
This project requires docker. Ensure it is installed and running on your system.
@@ -165,7 +165,7 @@ You will see the current version number of **Tube Archivist** in the footer of t
There is a Helm Chart available at https://github.com/insuusvenerati/helm-charts. Mostly self-explanatory but feel free to ask questions in the discord / subreddit.
## Potential pitfalls
## Common Errors
### vm.max_map_count
**Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.

View File

@@ -1,12 +1,40 @@
# Detailed Installation Instructions for Various Platforms
## Table of Contents
- [Podman](#podman)
- [Unraid](#unraid)
- [Truenas Scale](#truenas-scale)
- [Synology](#synology)
These are beginners guides installation instructions for additional platforms generously provided by users of these platforms. When in doubt, verify the details with the [Readme](https://github.com/tubearchivist/tubearchivist#installing-and-updating). If you see any issues here while using these instructions, please contribute.
## Podman
Podman handles container hostname resolving slightly differently than docker, so you need to make a few changes to the `docker-compose.yml` to get up and running.
### Step 1: Follow the installation instructions from the [README](https://github.com/tubearchivist/tubearchivist#installing-and-updating), with a few additional changes to the `docker-compose.yml`.
Edit these additional changes to the `docker-compose.yml`:
- under `tubearchivist`->`image`:
- prefix the container name with `docker.io/` (or the url of your repo of choice).
- under `tubearchivist`->`environment`:
- `ES_URL`: change `archivist-es` to the internal IP of the computer that will be running the containers.
- `REDIS_HOST`: change `archivist-redis` to the internal IP of the computer that will be running the containers (should be the same as above).
- under `archivist-redis`->`image`:
- prefix the container name with `docker.io/` again.
- under `archivist-redis`->`expose`:
- change the whole entry from `expose: ["<PORT>"]` into `ports: ["<PORT>:<PORT>"].
- under `archivist-es`->`image`:
- prefix the container name with `docker.io/` again.
- under `archivist-es`->`expose`:
- change the whole entry from `expose: ["<PORT>"]` into `ports: ["<PORT>:<PORT>"].
### Step 2: Create service files (optional)
Since podman doesn't run as a service, it can't start containers after reboots, at least not without some help.
If you want to enable this behavior, you can follow [this example](https://techblog.jeppson.org/2020/04/create-podman-services-with-podman-compose/) to have `systemd` start up the containers with `podman-compose` when the computer boots up.
## Unraid
Tube Archivist, and all if it's dependencies are located in the [community applications](https://forums.unraid.net/topic/38582-plug-in-community-applications/) store. The three containers you will need are as follows:
@@ -195,7 +223,9 @@ Once all of the folders have been created, it should have a folder structure wit
- "GID=0"
- "xpack.security.enabled=true"
- "ELASTIC_PASSWORD=verysecret"
- "path.repo=/usr/share/elasticsearch/data/snapshot"
> Do not use the default password as it is very insecure.
> Activating snapshots for backups should only be done *after* setting the `path.repo` setting.
![Synology - ElasticSearch Environment Configurations](assets/Synology_0.2.0_Docker-ES-Env-Conf.png)
12. Click on the **Apply** button.
13. Back on the **Create Container** screen, click the **Next** button.
@@ -252,4 +282,4 @@ Once all of the folders have been created, it should have a folder structure wit
<!-- This section is a Work In Progress -->
### From there, you should be able to start up your containers and you're good to go!
If you're still having trouble, join us on [discord](https://discord.gg/AFwz8nE7BK) and come to the #synology channel.
If you're still having trouble, join us on [discord](https://discord.gg/AFwz8nE7BK) and come to the #synology channel.

View File

@@ -32,6 +32,15 @@ Additional settings passed to yt-dlp.
- **Source Settings**: User created subtitles are provided from the uploader and are usually the video script. Auto generated is from YouTube, quality varies, particularly for auto translated tracks.
- **Index Settings**: Enabling subtitle indexing will add the lines to Elasticsearch and will make subtitles searchable. This will increase the index size and is not recommended on low-end hardware.
## Comments
- **Download and index comments**: Set your configuration for downloading and indexing comments. This takes the same values as documented in the `max_comments` section for the youtube extractor of [yt-dlp](https://github.com/yt-dlp/yt-dlp#youtube). Add without space between the four different fields: *max-comments,max-parents,max-replies,max-replies-per-thread*. Example:
- `all,100,all,30`: Get 100 max-parents and 30 max-replies-per-thread.
- `1000,all,all,50`: Get a total of 1000 comments over all, 50 replies per thread.
- **Comment sort method**: Change sort method between *top* or *new*. The default is *top*, as decided by YouTube.
- The [Refresh Metadata](#refresh-metadata) background task will get comments from your already archived videos, spreading the requests out over time.
Archiving comments is slow as only very few comments get returned per request with yt-dlp. Choose your configuration above wisely. Tube Archivist will download comments after the download queue finishes, your videos will be already available while the comments are getting downloaded.
## Cookie
Importing your YouTube Cookie into Tube Archivist allows yt-dlp to bypass age restrictions, gives access to private videos and your *watch later* or *liked videos*.
@@ -102,7 +111,7 @@ Examples:
NOTE:
- Changes in the scheduler settings require a container restart to take effect.
- Cron format as *number*/*number* are none standard cron and are not supported by the scheduler, for example **0 0/12 \*** is invalid, use **0 \*/12 \*** instead.
- Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason * or wildcards for minutes are not supported.
- Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason, the scheduler doesn't support schedules that trigger more than once per hour.
## Rescan Subscriptions
That's the equivalent task as run from the downloads page looking through your channel and playlist and add missing videos to the download queue.

View File

@@ -1,11 +1,23 @@
# Video Page
Every video downloaded gets a dedicated page accessible at `/video/<video-id>` of your Tube Archivist.
Every video downloaded gets a dedicated page accessible at `/video/<video-id>/` of your Tube Archivist.
Clicking on the channel name or the channel icon will bring you to the dedicated channel detail [page](Channels#channel-detail).
The button **Delete Video** will delete that video including the media file.
If available, a tag cloud will show, representing the tags set by the uploader.
The video description is truncated to the first few lines, click on *show more* to expand the whole description.
## Playlist
When available, a playlist navigation will show at the bottom. Clicking on the playlist name will bring you to the dedicated [Playlist Detail](Playlists#playlist-detail) page showing all videos downloaded from that playlist. The number in square brackets indicates the position of the current video in that playlist.
Clicking on the next or previous video name or thumbnail will bring you to that dedicated video page.
Clicking on the next or previous video name or thumbnail will bring you to that dedicated video page.
## Similar Videos
Tube Archivist will show up to six similar videos in a grid. Similarity is detected from the **video title** and the **video tags**. This naturally will show some videos from the same channel, but can also return videos about the same topic from other channels.
When playing a video from the similar section with the inline player, the current video will get replaced, refresh the page to reset that or click on the video title to avoid that behavior.
## Comments
If activated on the settings page, this will show the indexed comments. Reveal the threads by clicking the *+ Replies* button. Comments with a heart symbol are favorited by the uploader, comments by the uploader are highlighted in a different color.

View File

@@ -12,6 +12,7 @@ Note:
**Video**
- [Video List](#video-list-view)
- [Video Single](#video-item-view)
- [Video Comments](#video-comment-view)
- [Video Single Progress](#video-progress-view)
- [Video Single Sponsorblock](#sponsor-block-view) WIP
@@ -78,6 +79,9 @@ Pass page number as a query parameter: `page=2`. Defaults to *0*, `page=1` is re
## Video Item View
/api/video/\<video_id>/
## Video Comment View
/api/video/\<video_id>/comment/
## Video Progress View
/api/video/\<video_id>/progress

View File

@@ -48,6 +48,8 @@ class SearchProcess:
processed = self._process_playlist(result["_source"])
if index == "ta_download":
processed = self._process_download(result["_source"])
if index == "ta_comment":
processed = self._process_comment(result["_source"])
return processed
@@ -123,3 +125,17 @@ class SearchProcess:
}
)
return dict(sorted(download_dict.items()))
def _process_comment(self, comment_dict):
"""run on all comments, create reply thread"""
all_comments = comment_dict["comment_comments"]
processed_comments = []
for comment in all_comments:
if comment["comment_parent"] == "root":
comment.update({"comment_replies": []})
processed_comments.append(comment)
else:
processed_comments[-1]["comment_replies"].append(comment)
return processed_comments

View File

@@ -18,7 +18,9 @@ from api.views import (
TaskApiView,
VideoApiListView,
VideoApiView,
VideoCommentView,
VideoProgressView,
VideoSimilarView,
VideoSponsorView,
)
from django.urls import path
@@ -41,6 +43,16 @@ urlpatterns = [
VideoProgressView.as_view(),
name="api-video-progress",
),
path(
"video/<slug:video_id>/comment/",
VideoCommentView.as_view(),
name="api-video-comment",
),
path(
"video/<slug:video_id>/similar/",
VideoSimilarView.as_view(),
name="api-video-similar",
),
path(
"video/<slug:video_id>/sponsor/",
VideoSponsorView.as_view(),

View File

@@ -43,7 +43,6 @@ class ApiBaseView(APIView):
def get_document(self, document_id):
"""get single document from es"""
path = f"{self.search_base}{document_id}"
print(path)
response, status_code = ElasticWrap(path).get()
try:
self.response["data"] = SearchProcess(response).process()
@@ -62,10 +61,11 @@ class ApiBaseView(APIView):
}
)
def get_document_list(self, request):
def get_document_list(self, request, pagination=True):
"""get a list of results"""
print(self.search_base)
self.initiate_pagination(request)
if pagination:
self.initiate_pagination(request)
es_handler = ElasticWrap(self.search_base)
response, status_code = es_handler.get(data=self.data)
self.response["data"] = SearchProcess(response).process()
@@ -74,8 +74,11 @@ class ApiBaseView(APIView):
else:
self.status_code = 404
self.pagination_handler.validate(response["hits"]["total"]["value"])
self.response["paginate"] = self.pagination_handler.pagination
if pagination:
self.pagination_handler.validate(
response["hits"]["total"]["value"]
)
self.response["paginate"] = self.pagination_handler.pagination
class VideoApiView(ApiBaseView):
@@ -145,6 +148,46 @@ class VideoProgressView(ApiBaseView):
return Response(self.response)
class VideoCommentView(ApiBaseView):
"""resolves to /api/video/<video_id>/comment/
handle video comments
GET: return all comments from video with reply threads
"""
search_base = "ta_comment/_doc/"
def get(self, request, video_id):
"""get video comments"""
# pylint: disable=unused-argument
self.get_document(video_id)
return Response(self.response, status=self.status_code)
class VideoSimilarView(ApiBaseView):
"""resolves to /api/video/<video-id>/similar/
GET: return max 3 videos similar to this
"""
search_base = "ta_video/_search/"
def get(self, request, video_id):
"""get similar videos"""
self.data = {
"size": 6,
"query": {
"more_like_this": {
"fields": ["tags", "title"],
"like": {"_id": video_id},
"min_term_freq": 1,
"max_query_terms": 25,
}
},
}
self.get_document_list(request, pagination=False)
return Response(self.response, status=self.status_code)
class VideoSponsorView(ApiBaseView):
"""resolves to /api/video/<video_id>/sponsor/
handle sponsor block integration
@@ -391,7 +434,6 @@ class DownloadApiListView(ApiBaseView):
@staticmethod
def post(request):
"""add list of videos to download queue"""
print(f"request meta data: {request.META}")
data = request.data
try:
to_add = data["data"]
@@ -478,10 +520,7 @@ class TaskApiView(ApiBaseView):
def post(self, request):
"""handle post request"""
data = request.data
print(data)
response = TaskHandler(data).run_task()
response = TaskHandler(request.data).run_task()
return Response(response)
@@ -616,6 +655,6 @@ class SearchView(ApiBaseView):
return Response(
{"message": "no search query specified"}, status=400
)
print("searching for: " + search_query)
search_results = SearchForm().multi_search(search_query)
return Response(search_results)

View File

@@ -262,4 +262,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
# TA application settings
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
TA_VERSION = "v0.2.4"
TA_VERSION = "v0.3.0"

View File

@@ -5,9 +5,10 @@ import sys
from django.apps import AppConfig
from home.src.es.connect import ElasticWrap
from home.src.es.index_setup import index_check
from home.src.es.index_setup import ElasitIndexWrap
from home.src.es.snapshot import ElasticSnapshot
from home.src.ta.config import AppConfig as ArchivistConfig
from home.src.ta.helper import clear_dl_cache
from home.src.ta.ta_redis import RedisArchivist
@@ -27,10 +28,11 @@ class StartupCheck:
print("run startup checks")
self.es_version_check()
self.release_lock()
index_check()
ElasitIndexWrap().setup()
self.sync_redis_state()
self.set_redis_conf()
self.make_folders()
clear_dl_cache(self.config_handler.config)
self.snapshot_check()
self.set_has_run()

View File

@@ -27,6 +27,8 @@
"subtitle": false,
"subtitle_source": false,
"subtitle_index": false,
"comment_max": false,
"comment_sort": "top",
"cookie_import": false,
"throttledratelimit": false,
"integrate_ryd": false,

View File

@@ -15,6 +15,7 @@ from home.src.download.subscriptions import PlaylistSubscription
from home.src.download.yt_dlp_base import CookieHandler, YtWrap
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.index.channel import YoutubeChannel
from home.src.index.comments import Comments
from home.src.index.playlist import YoutubePlaylist
from home.src.index.video import YoutubeVideo, index_new_video
from home.src.ta.config import AppConfig
@@ -39,6 +40,7 @@ class DownloadPostProcess:
self.auto_delete_all()
self.auto_delete_overwrites()
self.validate_playlists()
self.get_comments()
def auto_delete_all(self):
"""handle auto delete"""
@@ -139,6 +141,27 @@ class DownloadPostProcess:
RedisArchivist().set_message(key, mess_dict, expire=expire)
def get_comments(self):
"""get comments from youtube"""
if not self.download.config["downloads"]["comment_max"]:
return
total_videos = len(self.download.videos)
for idx, video_id in enumerate(self.download.videos):
comment = Comments(video_id, config=self.download.config)
comment.build_json(notify=(idx, total_videos))
comment.upload_comments()
key = "message:download"
message = {
"status": key,
"level": "info",
"title": "Download and index comments finished",
"message": f"added comments for {total_videos} videos",
}
RedisArchivist().set_message(key, message, expire=4)
class VideoDownloader:
"""
@@ -155,6 +178,7 @@ class VideoDownloader:
self.config = AppConfig().config
self._build_obs()
self.channels = set()
self.videos = set()
def run_queue(self):
"""setup download queue in redis loop until no more items"""
@@ -187,6 +211,7 @@ class VideoDownloader:
youtube_id, video_overwrites=self.video_overwrites
)
self.channels.add(vid_dict["channel"]["channel_id"])
self.videos.add(vid_dict["youtube_id"])
mess_dict = {
"status": self.MSG,
"level": "info",

View File

@@ -0,0 +1,224 @@
"""
Functionality:
- Handle json zip file based backup
- create backup
- restore backup
"""
import json
import os
import zipfile
from datetime import datetime
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig
from home.src.ta.helper import get_mapping, ignore_filelist
class ElasticBackup:
"""dump index to nd-json files for later bulk import"""
def __init__(self, reason=False):
self.config = AppConfig().config
self.cache_dir = self.config["application"]["cache_dir"]
self.timestamp = datetime.now().strftime("%Y%m%d")
self.index_config = get_mapping()
self.reason = reason
def backup_all_indexes(self):
"""backup all indexes, add reason to init"""
print("backup all indexes")
if not self.reason:
raise ValueError("missing backup reason in ElasticBackup")
for index in self.index_config:
index_name = index["index_name"]
print(f"backup: export in progress for {index_name}")
if not self.index_exists(index_name):
print(f"skip backup for not yet existing index {index_name}")
continue
self.backup_index(index_name)
self.zip_it()
if self.reason == "auto":
self.rotate_backup()
@staticmethod
def backup_index(index_name):
"""export all documents of a single index"""
data = {
"query": {"match_all": {}},
"sort": [{"_doc": {"order": "desc"}}],
}
paginate = IndexPaginate(
f"ta_{index_name}", data, keep_source=True, callback=BackupCallback
)
_ = paginate.get_results()
def zip_it(self):
"""pack it up into single zip file"""
file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
folder = os.path.join(self.cache_dir, "backup")
to_backup = []
for file in os.listdir(folder):
if file.endswith(".json"):
to_backup.append(os.path.join(folder, file))
backup_file = os.path.join(folder, file_name)
comp = zipfile.ZIP_DEFLATED
with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
for backup_file in to_backup:
zip_f.write(backup_file, os.path.basename(backup_file))
# cleanup
for backup_file in to_backup:
os.remove(backup_file)
def post_bulk_restore(self, file_name):
"""send bulk to es"""
file_path = os.path.join(self.cache_dir, file_name)
with open(file_path, "r", encoding="utf-8") as f:
data = f.read()
if not data.strip():
return
_, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
def get_all_backup_files(self):
"""build all available backup files for view"""
backup_dir = os.path.join(self.cache_dir, "backup")
backup_files = os.listdir(backup_dir)
all_backup_files = ignore_filelist(backup_files)
all_available_backups = [
i
for i in all_backup_files
if i.startswith("ta_") and i.endswith(".zip")
]
all_available_backups.sort(reverse=True)
backup_dicts = []
for backup_file in all_available_backups:
file_split = backup_file.split("-")
if len(file_split) == 2:
timestamp = file_split[1].strip(".zip")
reason = False
elif len(file_split) == 3:
timestamp = file_split[1]
reason = file_split[2].strip(".zip")
to_add = {
"filename": backup_file,
"timestamp": timestamp,
"reason": reason,
}
backup_dicts.append(to_add)
return backup_dicts
def restore(self, filename):
"""
restore from backup zip file
call reset from ElasitIndexWrap first to start blank
"""
zip_content = self._unpack_zip_backup(filename)
self._restore_json_files(zip_content)
def _unpack_zip_backup(self, filename):
"""extract backup zip and return filelist"""
backup_dir = os.path.join(self.cache_dir, "backup")
file_path = os.path.join(backup_dir, filename)
with zipfile.ZipFile(file_path, "r") as z:
zip_content = z.namelist()
z.extractall(backup_dir)
return zip_content
def _restore_json_files(self, zip_content):
"""go through the unpacked files and restore"""
backup_dir = os.path.join(self.cache_dir, "backup")
for json_f in zip_content:
file_name = os.path.join(backup_dir, json_f)
if not json_f.startswith("es_") or not json_f.endswith(".json"):
os.remove(file_name)
continue
print("restoring: " + json_f)
self.post_bulk_restore(file_name)
os.remove(file_name)
@staticmethod
def index_exists(index_name):
"""check if index already exists to skip"""
_, status_code = ElasticWrap(f"ta_{index_name}").get()
exists = status_code == 200
return exists
def rotate_backup(self):
"""delete old backups if needed"""
rotate = self.config["scheduler"]["run_backup_rotate"]
if not rotate:
return
all_backup_files = self.get_all_backup_files()
auto = [i for i in all_backup_files if i["reason"] == "auto"]
if len(auto) <= rotate:
print("no backup files to rotate")
return
backup_dir = os.path.join(self.cache_dir, "backup")
all_to_delete = auto[rotate:]
for to_delete in all_to_delete:
file_path = os.path.join(backup_dir, to_delete["filename"])
print(f"remove old backup file: {file_path}")
os.remove(file_path)
class BackupCallback:
"""handle backup ndjson writer as callback for IndexPaginate"""
def __init__(self, source, index_name):
self.source = source
self.index_name = index_name
self.timestamp = datetime.now().strftime("%Y%m%d")
def run(self):
"""run the junk task"""
file_content = self._build_bulk()
self._write_es_json(file_content)
def _build_bulk(self):
"""build bulk query data from all_results"""
bulk_list = []
for document in self.source:
document_id = document["_id"]
es_index = document["_index"]
action = {"index": {"_index": es_index, "_id": document_id}}
source = document["_source"]
bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(source))
# add last newline
bulk_list.append("\n")
file_content = "\n".join(bulk_list)
return file_content
def _write_es_json(self, file_content):
"""write nd-json file for es _bulk API to disk"""
cache_dir = AppConfig().config["application"]["cache_dir"]
file_name = f"es_{self.index_name.lstrip('ta_')}-{self.timestamp}.json"
file_path = os.path.join(cache_dir, "backup", file_name)
with open(file_path, "a+", encoding="utf-8") as f:
f.write(file_content)

View File

@@ -37,8 +37,7 @@
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
"type": "date"
},
"channel_overwrites": {
"properties": {
@@ -120,8 +119,7 @@
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
"type": "date"
},
"channel_overwrites": {
"properties": {
@@ -193,6 +191,9 @@
}
}
},
"comment_count": {
"type": "long"
},
"stats" : {
"properties" : {
"average_rating" : {
@@ -460,6 +461,75 @@
},
"number_of_replicas": "0"
}
},
{
"index_name": "comment",
"expected_map": {
"youtube_id": {
"type": "keyword"
},
"comment_last_refresh": {
"type": "date"
},
"comment_channel_id": {
"type": "keyword"
},
"comment_comments": {
"properties": {
"comment_id": {
"type": "keyword"
},
"comment_text": {
"type" : "text"
},
"comment_timestamp": {
"type": "date"
},
"comment_time_text": {
"type" : "text"
},
"comment_likecount": {
"type": "long"
},
"comment_is_favorited": {
"type": "boolean"
},
"comment_author": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"comment_author_id": {
"type": "keyword"
},
"comment_author_thumbnail": {
"type": "keyword"
},
"comment_author_is_uploader": {
"type": "boolean"
},
"comment_parent": {
"type": "keyword"
}
}
}
},
"expected_set": {
"analysis": {
"normalizer": {
"to_lower": {
"type": "custom",
"filter": ["lowercase"]
}
}
},
"number_of_replicas": "0"
}
}
]
}

View File

@@ -5,22 +5,17 @@ functionality:
- backup and restore metadata
"""
import json
import os
import zipfile
from datetime import datetime
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.es.backup import ElasticBackup
from home.src.es.connect import ElasticWrap
from home.src.es.snapshot import ElasticSnapshot
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.helper import get_mapping
class ElasticIndex:
"""
handle mapping and settings on elastic search for a given index
"""
"""interact with a single index"""
def __init__(self, index_name, expected_map, expected_set):
def __init__(self, index_name, expected_map=False, expected_set=False):
self.index_name = index_name
self.expected_map = expected_map
self.expected_set = expected_set
@@ -61,23 +56,23 @@ class ElasticIndex:
if list(value.keys()) == ["properties"]:
for key_n, value_n in value["properties"].items():
if key not in now_map:
print(key_n, value_n)
print(f"detected mapping change: {key_n}, {value_n}")
return True
if key_n not in now_map[key]["properties"].keys():
print(key_n, value_n)
print(f"detected mapping change: {key_n}, {value_n}")
return True
if not value_n == now_map[key]["properties"][key_n]:
print(key_n, value_n)
print(f"detected mapping change: {key_n}, {value_n}")
return True
continue
# not nested
if key not in now_map.keys():
print(key, value)
print(f"detected mapping change: {key}, {value}")
return True
if not value == now_map[key]:
print(key, value)
print(f"detected mapping change: {key}, {value}")
return True
return False
@@ -100,6 +95,8 @@ class ElasticIndex:
def rebuild_index(self):
"""rebuild with new mapping"""
print(f"applying new mappings to index ta_{self.index_name}...")
self.create_blank(for_backup=True)
self.reindex("backup")
self.delete_index(backup=False)
self.create_blank()
@@ -126,279 +123,88 @@ class ElasticIndex:
_, _ = ElasticWrap(path).delete()
def create_blank(self):
def create_blank(self, for_backup=False):
"""apply new mapping and settings for blank new index"""
print(f"create new blank index with name ta_{self.index_name}...")
path = f"ta_{self.index_name}"
if for_backup:
path = f"{path}_backup"
data = {}
if self.expected_set:
data.update({"settings": self.expected_set})
if self.expected_map:
data.update({"mappings": {"properties": self.expected_map}})
_, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
_, _ = ElasticWrap(path).put(data)
class BackupCallback:
"""handle backup ndjson writer as callback for IndexPaginate"""
class ElasitIndexWrap:
"""interact with all index mapping and setup"""
def __init__(self, source, index_name):
self.source = source
self.index_name = index_name
self.timestamp = datetime.now().strftime("%Y%m%d")
def __init__(self):
self.index_config = get_mapping()
self.backup_run = False
def run(self):
"""run the junk task"""
file_content = self._build_bulk()
self._write_es_json(file_content)
def _build_bulk(self):
"""build bulk query data from all_results"""
bulk_list = []
for document in self.source:
document_id = document["_id"]
es_index = document["_index"]
action = {"index": {"_index": es_index, "_id": document_id}}
source = document["_source"]
bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(source))
# add last newline
bulk_list.append("\n")
file_content = "\n".join(bulk_list)
return file_content
def _write_es_json(self, file_content):
"""write nd-json file for es _bulk API to disk"""
cache_dir = AppConfig().config["application"]["cache_dir"]
file_name = f"es_{self.index_name.lstrip('ta_')}-{self.timestamp}.json"
file_path = os.path.join(cache_dir, "backup", file_name)
with open(file_path, "a+", encoding="utf-8") as f:
f.write(file_content)
class ElasticBackup:
"""dump index to nd-json files for later bulk import"""
def __init__(self, index_config, reason):
self.config = AppConfig().config
self.cache_dir = self.config["application"]["cache_dir"]
self.timestamp = datetime.now().strftime("%Y%m%d")
self.index_config = index_config
self.reason = reason
@staticmethod
def backup_index(index_name):
"""export all documents of a single index"""
data = {
"query": {"match_all": {}},
"sort": [{"_doc": {"order": "desc"}}],
}
paginate = IndexPaginate(
f"ta_{index_name}", data, keep_source=True, callback=BackupCallback
)
_ = paginate.get_results()
def zip_it(self):
"""pack it up into single zip file"""
file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
folder = os.path.join(self.cache_dir, "backup")
to_backup = []
for file in os.listdir(folder):
if file.endswith(".json"):
to_backup.append(os.path.join(folder, file))
backup_file = os.path.join(folder, file_name)
comp = zipfile.ZIP_DEFLATED
with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
for backup_file in to_backup:
zip_f.write(backup_file, os.path.basename(backup_file))
# cleanup
for backup_file in to_backup:
os.remove(backup_file)
def post_bulk_restore(self, file_name):
"""send bulk to es"""
file_path = os.path.join(self.cache_dir, file_name)
with open(file_path, "r", encoding="utf-8") as f:
data = f.read()
if not data.strip():
return
_, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
def get_all_backup_files(self):
"""build all available backup files for view"""
backup_dir = os.path.join(self.cache_dir, "backup")
backup_files = os.listdir(backup_dir)
all_backup_files = ignore_filelist(backup_files)
all_available_backups = [
i
for i in all_backup_files
if i.startswith("ta_") and i.endswith(".zip")
]
all_available_backups.sort(reverse=True)
backup_dicts = []
for backup_file in all_available_backups:
file_split = backup_file.split("-")
if len(file_split) == 2:
timestamp = file_split[1].strip(".zip")
reason = False
elif len(file_split) == 3:
timestamp = file_split[1]
reason = file_split[2].strip(".zip")
to_add = {
"filename": backup_file,
"timestamp": timestamp,
"reason": reason,
}
backup_dicts.append(to_add)
return backup_dicts
def unpack_zip_backup(self, filename):
"""extract backup zip and return filelist"""
backup_dir = os.path.join(self.cache_dir, "backup")
file_path = os.path.join(backup_dir, filename)
with zipfile.ZipFile(file_path, "r") as z:
zip_content = z.namelist()
z.extractall(backup_dir)
return zip_content
def restore_json_files(self, zip_content):
"""go through the unpacked files and restore"""
backup_dir = os.path.join(self.cache_dir, "backup")
for json_f in zip_content:
file_name = os.path.join(backup_dir, json_f)
if not json_f.startswith("es_") or not json_f.endswith(".json"):
os.remove(file_name)
def setup(self):
"""setup elastic index, run at startup"""
for index in self.index_config:
index_name, expected_map, expected_set = self._config_split(index)
handler = ElasticIndex(index_name, expected_map, expected_set)
if not handler.exists:
handler.create_blank()
continue
print("restoring: " + json_f)
self.post_bulk_restore(file_name)
os.remove(file_name)
rebuild = handler.validate()
if rebuild:
self._check_backup()
handler.rebuild_index()
continue
# else all good
print(f"ta_{index_name} index is created and up to date...")
def reset(self):
"""reset all indexes to blank"""
self.delete_all()
self.create_all_blank()
def delete_all(self):
"""delete all indexes"""
print("reset elastic index")
for index in self.index_config:
index_name, _, _ = self._config_split(index)
handler = ElasticIndex(index_name)
handler.delete_index(backup=False)
def create_all_blank(self):
"""create all blank indexes"""
print("create all new indexes in elastic from template")
for index in self.index_config:
index_name, expected_map, expected_set = self._config_split(index)
handler = ElasticIndex(index_name, expected_map, expected_set)
handler.create_blank()
@staticmethod
def index_exists(index_name):
"""check if index already exists to skip"""
_, status_code = ElasticWrap(f"ta_{index_name}").get()
exists = status_code == 200
return exists
def rotate_backup(self):
"""delete old backups if needed"""
rotate = self.config["scheduler"]["run_backup_rotate"]
if not rotate:
return
all_backup_files = self.get_all_backup_files()
auto = [i for i in all_backup_files if i["reason"] == "auto"]
if len(auto) <= rotate:
print("no backup files to rotate")
return
backup_dir = os.path.join(self.cache_dir, "backup")
all_to_delete = auto[rotate:]
for to_delete in all_to_delete:
file_path = os.path.join(backup_dir, to_delete["filename"])
print(f"remove old backup file: {file_path}")
os.remove(file_path)
def get_mapping():
"""read index_mapping.json and get expected mapping and settings"""
with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
index_config = json.load(f).get("index_config")
return index_config
def index_check(force_restore=False):
"""check if all indexes are created and have correct mapping"""
backed_up = False
index_config = get_mapping()
for index in index_config:
def _config_split(index):
"""split index config keys"""
index_name = index["index_name"]
expected_map = index["expected_map"]
expected_set = index["expected_set"]
handler = ElasticIndex(index_name, expected_map, expected_set)
# force restore
if force_restore:
handler.delete_index(backup=False)
handler.create_blank()
continue
# create new
if not handler.exists:
print(f"create new blank index with name ta_{index_name}...")
handler.create_blank()
continue
return index_name, expected_map, expected_set
# validate index
rebuild = handler.validate()
if rebuild:
# make backup before rebuild
if not backed_up:
print("running backup first")
backup_all_indexes(reason="update")
backed_up = True
def _check_backup(self):
"""create backup if needed"""
if self.backup_run:
return
print(f"applying new mappings to index ta_{index_name}...")
handler.rebuild_index()
continue
config = AppConfig().config
if config["application"]["enable_snapshot"]:
# take snapshot if enabled
ElasticSnapshot().take_snapshot_now(wait=True)
else:
# fallback to json backup
ElasticBackup(reason="update").backup_all_indexes()
# else all good
print(f"ta_{index_name} index is created and up to date...")
def get_available_backups():
"""return dict of available backups for settings view"""
index_config = get_mapping()
backup_handler = ElasticBackup(index_config, reason=False)
all_backup_files = backup_handler.get_all_backup_files()
return all_backup_files
def backup_all_indexes(reason):
"""backup all es indexes to disk"""
index_config = get_mapping()
backup_handler = ElasticBackup(index_config, reason)
for index in backup_handler.index_config:
index_name = index["index_name"]
print(f"backup: export in progress for {index_name}")
if not backup_handler.index_exists(index_name):
continue
backup_handler.backup_index(index_name)
backup_handler.zip_it()
if reason == "auto":
backup_handler.rotate_backup()
def restore_from_backup(filename):
"""restore indexes from backup file"""
# delete
index_check(force_restore=True)
# recreate
index_config = get_mapping()
backup_handler = ElasticBackup(index_config, reason=False)
zip_content = backup_handler.unpack_zip_backup(filename)
backup_handler.restore_json_files(zip_content)
self.backup_run = True

View File

@@ -5,10 +5,11 @@ functionality:
from datetime import datetime
from os import environ
from time import sleep
from zoneinfo import ZoneInfo
from home.src.es.connect import ElasticWrap
from home.src.es.index_setup import get_mapping
from home.src.ta.helper import get_mapping
class ElasticSnapshot:
@@ -116,6 +117,7 @@ class ElasticSnapshot:
"repository": self.REPO,
"config": {
"indices": self.all_indices,
"ignore_unavailable": True,
"include_global_state": True,
},
"retention": {
@@ -141,15 +143,42 @@ class ElasticSnapshot:
print("snapshot: last snapshot is up-to-date")
return outdated
def take_snapshot_now(self):
def take_snapshot_now(self, wait=False):
"""execute daily snapshot now"""
path = f"_slm/policy/{self.POLICY}/_execute"
response, statuscode = ElasticWrap(path).post()
if statuscode == 200:
print(f"snapshot: executing now: {response}")
if wait:
self._wait_for_snapshot(response["snapshot_name"])
return response
def _wait_for_snapshot(self, snapshot_name):
"""return after snapshot_name completes"""
path = f"_snapshot/{self.REPO}/{snapshot_name}"
while True:
# wait for task to be created
sleep(1)
_, statuscode = ElasticWrap(path).get()
if statuscode == 200:
break
while True:
# wait for snapshot success
response, statuscode = ElasticWrap(path).get()
snapshot_state = response["snapshots"][0]["state"]
if snapshot_state == "SUCCESS":
break
print(f"snapshot: {snapshot_name} in state {snapshot_state}")
print("snapshot: wait to complete")
sleep(5)
print(f"snapshot: completed - {response}")
def get_snapshot_stats(self):
"""get snapshot info for frontend"""
snapshot_info = self._build_policy_details()

View File

@@ -92,6 +92,12 @@ class ApplicationSettingsForm(forms.Form):
("1", "enable subtitle index"),
]
COMMENT_SORT_CHOICES = [
("", "-- change comments sort settings --"),
("top", "sort comments by top"),
("new", "sort comments by new"),
]
COOKIE_IMPORT_CHOICES = [
("", "-- change cookie settings"),
("0", "disable cookie"),
@@ -120,6 +126,10 @@ class ApplicationSettingsForm(forms.Form):
downloads_subtitle_index = forms.ChoiceField(
widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False
)
downloads_comment_max = forms.CharField(required=False)
downloads_comment_sort = forms.ChoiceField(
widget=forms.Select, choices=COMMENT_SORT_CHOICES, required=False
)
downloads_cookie_import = forms.ChoiceField(
widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False
)

View File

@@ -120,7 +120,7 @@ class SearchHandler:
if "vid_thumb_url" in hit_keys:
youtube_id = hit["source"]["youtube_id"]
thumb_path = ThumbManager(youtube_id).vid_thumb_path()
hit["source"]["vid_thumb_url"] = thumb_path
hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
if "channel_last_refresh" in hit_keys:
refreshed = hit["source"]["channel_last_refresh"]

View File

@@ -50,7 +50,7 @@ class ChannelScraper:
url = f"https://www.youtube.com/channel/{self.channel_id}/about?hl=en"
cookies = {"CONSENT": "YES+xxxxxxxxxxxxxxxxxxxxxxxxxxx"}
response = requests.get(
url, cookies=cookies, headers=requests_headers()
url, cookies=cookies, headers=requests_headers(), timeout=10
)
if response.ok:
channel_page = response.text
@@ -193,6 +193,9 @@ class YoutubeChannel(YouTubeItem):
if not self.json_data and fallback:
self._video_fallback(fallback)
if not self.json_data:
return
self.get_channel_art()
def _video_fallback(self, fallback):
@@ -275,6 +278,15 @@ class YoutubeChannel(YouTubeItem):
}
_, _ = ElasticWrap("ta_video/_delete_by_query").post(data)
def delete_es_comments(self):
"""delete all comments from this channel"""
data = {
"query": {
"term": {"comment_channel_id": {"value": self.youtube_id}}
}
}
_, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
def delete_playlists(self):
"""delete all indexed playlist from es"""
all_playlists = self.get_indexed_playlists()
@@ -301,6 +313,7 @@ class YoutubeChannel(YouTubeItem):
self.delete_playlists()
print(f"{self.youtube_id}: delete indexed videos")
self.delete_es_videos()
self.delete_es_comments()
self.del_in_es()
def index_channel_playlists(self):
@@ -408,7 +421,8 @@ class YoutubeChannel(YouTubeItem):
to_write[key] = False
continue
if value in [0, "0"]:
del to_write[key]
if key in to_write:
del to_write[key]
continue
if value == "1":
to_write[key] = True

View File

@@ -0,0 +1,179 @@
"""
Functionality:
- Download comments
- Index comments in ES
- Retrieve comments from ES
"""
from datetime import datetime
from home.src.download.yt_dlp_base import YtWrap
from home.src.es.connect import ElasticWrap
from home.src.ta.config import AppConfig
from home.src.ta.ta_redis import RedisArchivist
class Comments:
"""hold all comments functionality"""
def __init__(self, youtube_id, config=False):
self.youtube_id = youtube_id
self.es_path = f"ta_comment/_doc/{youtube_id}"
self.json_data = False
self.config = config
self.is_activated = False
self.comments_format = False
def build_json(self, notify=False):
"""build json document for es"""
print(f"{self.youtube_id}: get comments")
self.check_config()
if not self.is_activated:
return
self._send_notification(notify)
comments_raw, channel_id = self.get_yt_comments()
if comments_raw:
self.format_comments(comments_raw)
else:
self.comments_format = []
self.json_data = {
"youtube_id": self.youtube_id,
"comment_last_refresh": int(datetime.now().strftime("%s")),
"comment_channel_id": channel_id,
"comment_comments": self.comments_format,
}
def check_config(self):
"""read config if not attached"""
if not self.config:
self.config = AppConfig().config
self.is_activated = bool(self.config["downloads"]["comment_max"])
@staticmethod
def _send_notification(notify):
"""send notification for download post process message"""
if not notify:
return
key = "message:download"
idx, total_videos = notify
message = {
"status": key,
"level": "info",
"title": "Download and index comments",
"message": f"Progress: {idx + 1}/{total_videos}",
}
RedisArchivist().set_message(key, message)
def build_yt_obs(self):
"""
get extractor config
max-comments,max-parents,max-replies,max-replies-per-thread
"""
max_comments = self.config["downloads"]["comment_max"]
max_comments_list = [i.strip() for i in max_comments.split(",")]
comment_sort = self.config["downloads"]["comment_sort"]
yt_obs = {
"skip_download": True,
"quiet": False,
"getcomments": True,
"extractor_args": {
"youtube": {
"max_comments": max_comments_list,
"comment_sort": [comment_sort],
}
},
}
return yt_obs
def get_yt_comments(self):
"""get comments from youtube"""
yt_obs = self.build_yt_obs()
info_json = YtWrap(yt_obs).extract(self.youtube_id)
comments_raw = info_json.get("comments")
channel_id = info_json.get("channel_id")
return comments_raw, channel_id
def format_comments(self, comments_raw):
"""process comments to match format"""
comments = []
for comment in comments_raw:
cleaned_comment = self.clean_comment(comment)
comments.append(cleaned_comment)
self.comments_format = comments
def clean_comment(self, comment):
"""parse metadata from comment for indexing"""
time_text_datetime = datetime.utcfromtimestamp(comment["timestamp"])
if time_text_datetime.hour == 0 and time_text_datetime.minute == 0:
format_string = "%Y-%m-%d"
else:
format_string = "%Y-%m-%d %H:%M"
time_text = time_text_datetime.strftime(format_string)
cleaned_comment = {
"comment_id": comment["id"],
"comment_text": comment["text"].replace("\xa0", ""),
"comment_timestamp": comment["timestamp"],
"comment_time_text": time_text,
"comment_likecount": comment["like_count"],
"comment_is_favorited": comment["is_favorited"],
"comment_author": comment["author"],
"comment_author_id": comment["author_id"],
"comment_author_thumbnail": comment["author_thumbnail"],
"comment_author_is_uploader": comment["author_is_uploader"],
"comment_parent": comment["parent"],
}
return cleaned_comment
def upload_comments(self):
"""upload comments to es"""
if not self.is_activated:
return
_, _ = ElasticWrap(self.es_path).put(self.json_data)
vid_path = f"ta_video/_update/{self.youtube_id}"
data = {"doc": {"comment_count": len(self.comments_format)}}
_, _ = ElasticWrap(vid_path).post(data=data)
def delete_comments(self):
"""delete comments from es"""
print(f"{self.youtube_id}: delete comments")
_, _ = ElasticWrap(self.es_path).delete(refresh=True)
def get_es_comments(self):
"""get comments from ES"""
response, statuscode = ElasticWrap(self.es_path).get()
if statuscode == 404:
print(f"comments: not found {self.youtube_id}")
return False
return response.get("_source")
def reindex_comments(self):
"""update comments from youtube"""
self.check_config()
if not self.is_activated:
return
self.build_json()
es_comments = self.get_es_comments()
if not self.comments_format and es_comments["comment_comments"]:
# don't overwrite comments in es
return
self.delete_comments()
self.upload_comments()

View File

@@ -56,11 +56,11 @@ class YouTubeItem:
"ta_channel": "channel_active",
"ta_playlist": "playlist_active",
}
update_path = f"{self.index_name}/_update/{self.youtube_id}"
path = f"{self.index_name}/_update/{self.youtube_id}?refresh=true"
data = {
"script": f"ctx._source.{key_match.get(self.index_name)} = false"
}
_, _ = ElasticWrap(update_path).post(data)
_, _ = ElasticWrap(path).post(data)
def del_in_es(self):
"""delete item from elastic search"""

View File

@@ -41,6 +41,10 @@ class YoutubePlaylist(YouTubeItem):
if scrape or not self.json_data:
self.get_from_youtube()
if not self.youtube_meta:
self.json_data = False
return
self.process_youtube_meta()
self.get_entries()
self.json_data["playlist_entries"] = self.all_members
@@ -76,12 +80,12 @@ class YoutubePlaylist(YouTubeItem):
downloaded = entry["id"] in self.all_youtube_ids
else:
downloaded = False
if not entry["uploader"]:
if not entry["channel"]:
continue
to_append = {
"youtube_id": entry["id"],
"title": entry["title"],
"uploader": entry["uploader"],
"uploader": entry["channel"],
"idx": idx,
"downloaded": downloaded,
}

View File

@@ -16,6 +16,7 @@ from home.src.download.yt_dlp_base import CookieHandler
from home.src.download.yt_dlp_handler import VideoDownloader
from home.src.es.connect import ElasticWrap
from home.src.index.channel import YoutubeChannel
from home.src.index.comments import Comments
from home.src.index.playlist import YoutubePlaylist
from home.src.index.video import YoutubeVideo
from home.src.ta.config import AppConfig
@@ -147,8 +148,7 @@ class Reindex:
if integrate_ryd:
self._get_unrated_vids()
@staticmethod
def _reindex_single_video(youtube_id):
def _reindex_single_video(self, youtube_id):
"""refresh data for single video"""
video = YoutubeVideo(youtube_id)
@@ -182,6 +182,8 @@ class Reindex:
thumb_handler.delete_video_thumb()
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
Comments(youtube_id, config=self.config).reindex_comments()
return
@staticmethod
@@ -194,6 +196,8 @@ class Reindex:
channel.get_from_youtube()
if not channel.json_data:
channel.deactivate()
channel.get_from_es()
channel.sync_to_videos()
return
channel.json_data["channel_subscribed"] = subscribed

View File

@@ -11,6 +11,7 @@ import requests
from django.conf import settings
from home.src.es.connect import ElasticWrap
from home.src.index import channel as ta_channel
from home.src.index import comments as ta_comments
from home.src.index import playlist as ta_playlist
from home.src.index.generic import YouTubeItem
from home.src.index.subtitle import YoutubeSubtitle
@@ -302,6 +303,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
self.del_in_playlists()
self.del_in_es()
self.delete_subtitles()
self.delete_comments()
def del_in_playlists(self):
"""remove downloaded in playlist"""
@@ -326,6 +328,13 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
print(f"{self.youtube_id}: delete subtitles")
YoutubeSubtitle(self).delete(subtitles=subtitles)
def delete_comments(self):
"""delete comments from es"""
comments = ta_comments.Comments(self.youtube_id, config=self.config)
comments.check_config()
if comments.is_activated:
comments.delete_comments()
def _get_ryd_stats(self):
"""get optional stats from returnyoutubedislikeapi.com"""
# pylint: disable=broad-except

View File

@@ -3,6 +3,8 @@ Loose collection of helper functions
- don't import AppConfig class here to avoid circular imports
"""
import json
import os
import random
import re
import string
@@ -117,6 +119,23 @@ def time_parser(timestamp):
return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
def clear_dl_cache(config):
"""clear leftover files from dl cache"""
print("clear download cache")
cache_dir = os.path.join(config["application"]["cache_dir"], "download")
for cached in os.listdir(cache_dir):
to_delete = os.path.join(cache_dir, cached)
os.remove(to_delete)
def get_mapping():
"""read index_mapping.json and get expected mapping and settings"""
with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
index_config = json.load(f).get("index_config")
return index_config
class UrlListParser:
"""take a multi line string and detect valid youtube ids"""

View File

@@ -17,7 +17,8 @@ from home.src.download.subscriptions import (
)
from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator
from home.src.download.yt_dlp_handler import VideoDownloader
from home.src.es.index_setup import backup_all_indexes, restore_from_backup
from home.src.es.backup import ElasticBackup
from home.src.es.index_setup import ElasitIndexWrap
from home.src.index.channel import YoutubeChannel
from home.src.index.filesystem import (
ImportFolderScanner,
@@ -25,7 +26,7 @@ from home.src.index.filesystem import (
scan_filesystem,
)
from home.src.ta.config import AppConfig, ScheduleBuilder
from home.src.ta.helper import UrlListParser
from home.src.ta.helper import UrlListParser, clear_dl_cache
from home.src.ta.ta_redis import RedisArchivist, RedisQueue
CONFIG = AppConfig().config
@@ -168,7 +169,7 @@ def run_backup(reason="auto"):
try:
have_lock = my_lock.acquire(blocking=False)
if have_lock:
backup_all_indexes(reason)
ElasticBackup(reason=reason).backup_all_indexes()
else:
print("Did not acquire lock for backup task.")
finally:
@@ -180,7 +181,8 @@ def run_backup(reason="auto"):
@shared_task
def run_restore_backup(filename):
"""called from settings page, dump backup to zip file"""
restore_from_backup(filename)
ElasitIndexWrap().reset()
ElasticBackup().restore(filename)
print("index restore finished")
@@ -192,11 +194,7 @@ def kill_dl(task_id):
_ = RedisArchivist().del_message("dl_queue_id")
RedisQueue().clear()
# clear cache
cache_dir = os.path.join(CONFIG["application"]["cache_dir"], "download")
for cached in os.listdir(cache_dir):
to_delete = os.path.join(cache_dir, cached)
os.remove(to_delete)
clear_dl_cache(CONFIG)
# notify
mess_dict = {

View File

@@ -100,7 +100,7 @@
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
{% if video.source.player.progress %}
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
{% else %}

View File

@@ -72,7 +72,7 @@
<div class="video-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video_thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video_thumb">
{% if show_ignored_only %}
<span>ignored</span>
{% else %}

View File

@@ -12,7 +12,7 @@
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
{% if video.source.player.progress %}
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
{% else %}
@@ -98,7 +98,7 @@
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
{% if video.source.player.progress %}
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
{% else %}

View File

@@ -102,7 +102,7 @@
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
{% if video.source.player.progress %}
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
{% else %}

View File

@@ -77,7 +77,7 @@
<ul>
<li><span class="settings-current">bestvideo[height<=720]+bestaudio/best[height<=720]</span>: best audio and max video height of 720p.</li>
<li><span class="settings-current">bestvideo[height<=1080]+bestaudio/best[height<=1080]</span>: best audio and max video height of 1080p.</li>
<li><span class="settings-current">bestvideo[height<=1080][VCODEC*=avc1]+bestaudio[ACODEC*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
<li><span class="settings-current">bestvideo[height<=1080][vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
<li><span class="settings-current">0</span>: deactivate and download the best quality possible as decided by yt-dlp.</li>
</ul>
<i>Make sure your custom format gets merged into a single file. Check out the <a href="https://github.com/yt-dlp/yt-dlp#format-selection" target="_blank">documentation</a> for valid configurations.</i><br>
@@ -114,6 +114,24 @@
{{ app_form.downloads_subtitle_index }}
</div>
</div>
<div class="settings-group">
<h2 id="comments">Comments</h2>
<div class="settings-item">
<p>Download and index comments: <span class="settings-current">{{ config.downloads.comment_max }}</span><br>
<i>Follow the yt-dlp max_comments documentation, <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">max-comments,max-parents,max-replies,max-replies-per-thread</a>:</i><br>
<p>Example configurations:</p>
<ul>
<li><span class="settings-current">all,100,all,30</span>: Get 100 max-parents and 30 max-replies-per-thread.</li>
<li><span class="settings-current">1000,all,all,50</span>: Get a total of 1000 comments over all, 50 replies per thread.</li>
</ul>
{{ app_form.downloads_comment_max }}</p>
</div>
<div class="settings-item">
<p>Selected comment sort method: <span class="settings-current">{{ config.downloads.comment_sort }}</span><br>
<i>Select how many comments and threads to download:</i><br>
{{ app_form.downloads_comment_sort }}</p>
</div>
</div>
<div class="settings-group">
<h2 id="format">Cookie</h2>
<div class="settings-item">
@@ -186,7 +204,7 @@
<p>Note:</p>
<ul>
<li>Changes in the scheduler settings require a container restart to take effect.</li>
<li>Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason <i>*</i> or <i>wildcard</i> for minutes is not supported.</li>
<li>Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason, the scheduler doesn't support schedules that trigger more than once per hour.</li>
</ul>
</div>
</div>

View File

@@ -2,8 +2,10 @@
{% block content %}
{% load static %}
{% load humanize %}
<div class="video-main">
<div class="video-modal"><span class="video-modal-text"></span></div>
<div id="player" class="player-wrapper">
<div class="video-main">
<div class="video-modal"><span class="video-modal-text"></span></div>
</div>
</div>
<div class="notifications" id="notifications"></div>
<div class="sponsorblock" id="sponsorblock">
@@ -69,15 +71,24 @@
<p class="thumb-icon"><img class="dislike" src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-down">: {{ video.stats.dislike_count|intcomma }}</p>
{% endif %}
{% if video.stats.average_rating %}
<p class="rating-stars">Rating:
<div class="rating-stars">
{% for star in video.stats.average_rating %}
<img src="/static/img/icon-star-{{ star }}.svg" alt="{{ star }}">
{% endfor %}
</p>
</div>
{% endif %}
</div>
</div>
</div>
{% if video.tags %}
<div class="description-box">
<div class="video-tag-box">
{% for tag in video.tags %}
<span class="video-tag">{{ tag }}</span>
{% endfor %}
</div>
</div>
{% endif %}
{% if video.description %}
<div class="description-box">
<p id="text-expand" class="description-text">
@@ -123,6 +134,20 @@
</div>
{% endfor %}
{% endif %}
<div class="description-box">
<h3>Similar Videos</h3>
<div class="video-list grid grid-3" id="similar-videos">
</div>
<script>getSimilarVideos('{{ video.youtube_id }}')</script>
</div>
{% if video.comment_count %}
<div class="comments-section">
<h3>Comments: {{video.comment_count}}</h3>
<div id="comments-list" class="comments-list">
</div>
</div>
<script>getComments('{{ video.youtube_id }}')</script>
{% endif %}
</div>
<script>
var videoData = getVideoData('{{ video.youtube_id }}');

View File

@@ -16,8 +16,8 @@ from django.http import JsonResponse
from django.shortcuts import redirect, render
from django.views import View
from home.src.download.yt_dlp_base import CookieHandler
from home.src.es.backup import ElasticBackup
from home.src.es.connect import ElasticWrap
from home.src.es.index_setup import get_available_backups
from home.src.es.snapshot import ElasticSnapshot
from home.src.frontend.api_calls import PostData
from home.src.frontend.forms import (
@@ -939,7 +939,7 @@ class SettingsView(View):
config_handler = AppConfig(request.user.id)
colors = config_handler.colors
available_backups = get_available_backups()
available_backups = ElasticBackup().get_all_backup_files()
user_form = UserSettingsForm()
app_form = ApplicationSettingsForm()
scheduler_form = SchedulerSettingsForm()

View File

@@ -5,9 +5,9 @@ django-auth-ldap==4.1.0
django-cors-headers==3.13.0
djangorestframework==3.14.0
Pillow==9.3.0
redis==4.3.4
redis==4.3.5
requests==2.28.1
ryd-client==0.0.6
uWSGI==2.0.21
whitenoise==6.2.0
yt_dlp==2022.10.4
yt_dlp==2022.11.11

View File

@@ -619,7 +619,8 @@ video:-webkit-full-screen {
margin-top: 1rem;
}
.description-box {
.description-box,
.comments-section {
margin-top: 1rem;
padding: 15px;
background-color: var(--highlight-bg);
@@ -778,10 +779,26 @@ video:-webkit-full-screen {
margin-left: 5px;
}
.thumb-icon {
display: flex;
}
.video-tag-box {
display: flex;
flex-wrap: wrap;
justify-content: center;
}
.video-tag {
padding: 5px 10px;
margin: 5px;
border: 1px solid var(--accent-font-light);
}
.thumb-icon img,
.rating-stars img {
width: 20px;
margin: 0;
margin: 0 5px;
filter: var(--img-filter);
}
@@ -819,6 +836,44 @@ video:-webkit-full-screen {
width: 100%;
}
.comment-box {
padding-bottom: 1rem;
overflow: hidden;
}
.comment-box h3 {
line-break: anywhere;
}
.comments-replies {
display: none;
padding-left: 1rem;
border-left: 1px solid var(--accent-font-light);
margin-top: 1rem;
}
.comment-highlight {
background-color: var(--main-font);
padding: 3px;
color: var(--accent-font-dark);
font-family: Sen-bold, sans-serif;
width: fit-content;
}
.comment-meta {
display: flex;
}
.space-carrot {
margin: 0 5px;
}
.comment-like img {
width: 20px;
margin-left: 5px;
filter: var(--img-filter-error);
}
/* multi search page */
.multi-search-box {
padding-right: 20px;

View File

@@ -1,63 +1,17 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="500"
height="500"
viewBox="0 0 132.29197 132.29167"
version="1.1"
id="svg1303"
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"
sodipodi:docname="Icons_seen.svg">
<defs
id="defs1297" />
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="1.0105705"
inkscape:cx="84.208758"
inkscape:cy="136.94831"
inkscape:document-units="mm"
inkscape:current-layer="layer1"
showgrid="false"
units="px"
inkscape:window-width="1920"
inkscape:window-height="1017"
inkscape:window-x="-8"
inkscape:window-y="-8"
inkscape:window-maximized="1" />
<metadata
id="metadata1300">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title />
</cc:Work>
</rdf:RDF>
</metadata>
<g
inkscape:label="Ebene 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(0,-164.70764)">
<path
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
d="M 66.145984,191.3255 A 71.797122,73.404487 0 0 0 2.5025987,230.88314 71.797122,73.404487 0 0 0 66.145984,270.38145 71.797122,73.404487 0 0 0 129.78937,230.82387 71.797122,73.404487 0 0 0 66.145984,191.3255 Z m -4.921549,7.22394 a 32.724755,32.724755 0 0 0 -13.334395,5.17759 12.828107,12.828107 0 0 1 2.180958,-0.18652 12.828107,12.828107 0 0 1 12.828141,12.82816 12.828107,12.828107 0 0 1 -12.828141,12.82822 12.828107,12.828107 0 0 1 -12.828192,-12.82822 12.828107,12.828107 0 0 1 0.04509,-0.91548 32.724755,32.724755 0 0 0 -3.86581,15.39964 32.724755,32.724755 0 0 0 27.161922,32.24981 A 59.09757,60.420619 0 0 1 13.7604,230.8774 59.09757,60.420619 0 0 1 61.224664,198.54948 Z m 10.482345,0.0563 a 59.09757,60.420619 0 0 1 46.82502,32.22523 59.09757,60.420619 0 0 1 -47.393377,32.32497 32.724755,32.724755 0 0 0 27.73169,-32.30187 32.724755,32.724755 0 0 0 -27.163333,-32.24833 z"
id="path1091"
inkscape:connector-curvature="0" />
</g>
</svg>
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 26.5.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1"
id="svg1303" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" xmlns:svg="http://www.w3.org/2000/svg" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" inkscape:version="0.92.4 (5da689c313, 2019-01-14)" sodipodi:docname="Icons_seen.svg"
xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 500 500"
style="enable-background:new 0 0 500 500;" xml:space="preserve">
<sodipodi:namedview bordercolor="#666666" borderopacity="1.0" id="base" inkscape:current-layer="layer1" inkscape:cx="84.208758" inkscape:cy="136.94831" inkscape:document-units="mm" inkscape:pageopacity="0.0" inkscape:pageshadow="2" inkscape:window-height="1017" inkscape:window-maximized="1" inkscape:window-width="1920" inkscape:window-x="-8" inkscape:window-y="-8" inkscape:zoom="1.0105705" pagecolor="#ffffff" showgrid="false" units="px">
</sodipodi:namedview>
<g id="layer1" transform="translate(0,-164.70764)" inkscape:groupmode="layer" inkscape:label="Ebene 1">
<path id="path1091" inkscape:connector-curvature="0" d="M250,259.4c-105.2,0.1-201.5,60-250,155.4C48.5,510.1,144.9,569.9,250,570
c105.2-0.1,201.5-60,250-155.4C451.5,319.3,355.1,259.5,250,259.4z M230.7,287.8c-18.7,2.8-36.6,9.8-52.4,20.3
c2.8-0.5,5.7-0.7,8.6-0.7c27.8,0,50.4,22.6,50.4,50.4c0,0,0,0,0,0c0,27.8-22.6,50.4-50.4,50.4c0,0,0,0,0,0
c-27.8,0-50.4-22.6-50.4-50.4c0,0,0,0,0,0c0-1.2,0.1-2.4,0.2-3.6c-10,18.6-15.2,39.4-15.2,60.5c0,62.6,45,116,106.7,126.7
c-78.3-7.6-147.6-55.2-183.9-126.6C81,342.6,151.3,294.7,230.7,287.8L230.7,287.8z M271.8,288c78.3,7.6,147.6,55.2,183.9,126.6
c-36.7,72.1-106.9,120-186.2,127c62.6-9.7,108.9-63.5,108.9-126.9C378.5,352.1,333.5,298.7,271.8,288L271.8,288z"/>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 2.8 KiB

After

Width:  |  Height:  |  Size: 2.0 KiB

View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 26.5.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 500 500" style="enable-background:new 0 0 500 500;" xml:space="preserve">
<path d="M499.9,159L499.9,159c0.1-1.7,0.1-3.4,0.1-5.2c0-69.5-58.6-129.7-130.9-129.7c-52.9,0-98.4,34-119,77.4h0
c-20.7-43.4-66.2-77.4-119-77.4C58.6,24.1,0,84.4,0,153.9c0,1.7,0.1,3.4,0.1,5.2h0c0,0-7.4,82.6,84.5,172.7
c41.8,41.9,88.5,81.6,165.4,144.1c76.9-62.5,123.6-102.3,165.4-144.1C507.2,241.6,499.9,159,499.9,159z"/>
</svg>

After

Width:  |  Height:  |  Size: 684 B

View File

@@ -1,76 +1,18 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="500"
height="500"
viewBox="0 0 132.29197 132.29167"
version="1.1"
id="svg1303"
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"
sodipodi:docname="icon_thumb.svg">
<defs
id="defs1297" />
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="1.5046797"
inkscape:cx="35.718548"
inkscape:cy="203.39339"
inkscape:document-units="mm"
inkscape:current-layer="layer1"
showgrid="false"
units="px"
inkscape:window-width="1920"
inkscape:window-height="1009"
inkscape:window-x="-8"
inkscape:window-y="-8"
inkscape:window-maximized="1"
showguides="true"
inkscape:guide-bbox="true">
<sodipodi:guide
position="80.99058,65.965029"
orientation="0,1"
id="guide816"
inkscape:locked="false" />
<sodipodi:guide
position="65.965178,49.107299"
orientation="1,0"
id="guide818"
inkscape:locked="false" />
</sodipodi:namedview>
<metadata
id="metadata1300">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title />
</cc:Work>
</rdf:RDF>
</metadata>
<g
inkscape:label="Ebene 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(0,-164.70764)">
<path
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
d="m 72.35681,178.12175 c -0.545876,-0.0192 -1.121779,0.0192 -1.729242,0.12259 -12.959134,2.14891 4.226681,21.04642 -10.867262,31.83193 -12.617074,10.52505 -17.206737,7.22264 -17.332488,10.67172 v 33.88171 c 1.891308,5.00813 14.483922,10.12847 27.868234,11.21944 13.384309,1.09096 32.098988,-1.73801 32.305468,-7.56421 0.10989,-5.27572 -2.896223,-5.39662 -2.896223,-5.39662 0,0 6.589043,-0.40752 6.808883,-6.12278 0.10966,-5.60545 -6.38265,-6.40691 -6.38265,-6.40691 0,0 8.31589,-0.23607 8.31589,-6.01832 0,-5.91485 -6.39662,-6.38072 -8.37539,-6.48104 1.22776,-0.0747 6.07353,-1.75141 6.04505,-7.59582 -0.0287,-6.03384 -9.280896,-5.53597 -25.919173,-6.63279 1.919791,-11.19377 3.258461,-35.14851 -7.841097,-35.50894 z m -45.69253,38.36631 c -1.562226,0 -2.819733,1.04793 -2.819733,2.34979 v 38.25943 c 0,1.30191 1.257507,2.34983 2.819733,2.34983 h 10.526021 c 1.562231,0 2.819739,-1.04792 2.819739,-2.34983 v -38.25943 c 0,-1.30186 -1.257508,-2.34979 -2.819739,-2.34979 z"
id="rect1278"
inkscape:connector-curvature="0" />
</g>
</svg>
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 26.5.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1"
id="svg1303" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" xmlns:svg="http://www.w3.org/2000/svg" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" inkscape:version="0.92.4 (5da689c313, 2019-01-14)" sodipodi:docname="icon_thumb.svg"
xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 500 500"
style="enable-background:new 0 0 500 500;" xml:space="preserve">
<sodipodi:namedview bordercolor="#666666" borderopacity="1.0" id="base" inkscape:current-layer="layer1" inkscape:cx="35.718548" inkscape:cy="203.39339" inkscape:document-units="mm" inkscape:guide-bbox="true" inkscape:pageopacity="0.0" inkscape:pageshadow="2" inkscape:window-height="1009" inkscape:window-maximized="1" inkscape:window-width="1920" inkscape:window-x="-8" inkscape:window-y="-8" inkscape:zoom="1.5046797" pagecolor="#ffffff" showgrid="false" showguides="true" units="px">
<sodipodi:guide id="guide816" inkscape:locked="false" orientation="0,1" position="80.99058,65.965029"></sodipodi:guide>
<sodipodi:guide id="guide818" inkscape:locked="false" orientation="1,0" position="65.965178,49.107299"></sodipodi:guide>
</sodipodi:namedview>
<g id="layer1" transform="translate(0,-164.70764)" inkscape:groupmode="layer" inkscape:label="Ebene 1">
<path id="rect1278" inkscape:connector-curvature="0" d="M285.3,164.7c-3.1-0.1-6.4,0.1-9.8,0.7c-73.7,12.2,24,119.6-61.8,180.9
c-71.7,59.8-97.8,41.1-98.5,60.7v192.6c10.8,28.5,82.3,57.6,158.4,63.8c76.1,6.2,182.5-9.9,183.6-43c0.6-30-16.5-30.7-16.5-30.7
s37.5-2.3,38.7-34.8c0.6-31.9-36.3-36.4-36.3-36.4s47.3-1.3,47.3-34.2c0-33.6-36.4-36.3-47.6-36.8c7-0.4,34.5-10,34.4-43.2
c-0.2-34.3-52.8-31.5-147.3-37.7C340.8,302.9,348.4,166.8,285.3,164.7L285.3,164.7z M25.6,382.8c-8.9,0-16,6-16,13.4v217.5
c0,7.4,7.1,13.4,16,13.4h59.8c8.9,0,16-6,16-13.4V396.2c0-7.4-7.1-13.4-16-13.4H25.6z"/>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 3.2 KiB

After

Width:  |  Height:  |  Size: 2.2 KiB

View File

@@ -287,7 +287,7 @@ function resetToken() {
document.getElementById('text-reveal').replaceWith(message);
}
// restore from snapshot
// restore from snapshot
function restoreSnapshot(snapshotId) {
console.log('restore ' + snapshotId);
let apiEndpoint = '/api/snapshot/' + snapshotId + '/';
@@ -793,7 +793,11 @@ function apiRequest(apiEndpoint, method, data) {
xhttp.setRequestHeader('Authorization', 'Token ' + sessionToken);
xhttp.setRequestHeader('Content-Type', 'application/json');
xhttp.send(JSON.stringify(data));
return JSON.parse(xhttp.responseText);
if (xhttp.status === 404) {
return false;
} else {
return JSON.parse(xhttp.responseText);
}
}
// Gets origin URL
@@ -951,7 +955,7 @@ function createVideo(video, viewStyle) {
// create video item div from template
const videoId = video.youtube_id;
// const mediaUrl = video.media_url;
const thumbUrl = '/cache/' + video.vid_thumb_url;
// const thumbUrl = '/cache/' + video.vid_thumb_url;
const videoTitle = video.title;
const videoPublished = video.published;
const videoDuration = video.player.duration_str;
@@ -968,7 +972,7 @@ function createVideo(video, viewStyle) {
<a href="#player" data-id="${videoId}" onclick="createPlayer(this)">
<div class="video-thumb-wrap ${viewStyle}">
<div class="video-thumb">
<img src="${thumbUrl}" alt="video-thumb">
<img src="${video.vid_thumb_url}" alt="video-thumb">
</div>
<div class="video-play">
<img src="/static/img/icon-play.svg" alt="play-icon">
@@ -1104,6 +1108,138 @@ function createFulltext(fullText) {
return fullTextDiv;
}
function getComments(videoId) {
let apiEndpoint = '/api/video/' + videoId + '/comment/';
let response = apiRequest(apiEndpoint, 'GET');
let allComments = response.data;
writeComments(allComments);
}
function writeComments(allComments) {
let commentsListBox = document.getElementById('comments-list');
for (let i = 0; i < allComments.length; i++) {
const rootComment = allComments[i];
let commentBox = createCommentBox(rootComment, true);
// add replies to commentBox
if (rootComment.comment_replies) {
let commentReplyBox = document.createElement('div');
commentReplyBox.setAttribute('class', 'comments-replies');
commentReplyBox.setAttribute('id', rootComment.comment_id + '-replies');
let totalReplies = rootComment.comment_replies.length;
if (totalReplies > 0) {
let replyButton = createReplyButton(rootComment.comment_id + '-replies', totalReplies);
commentBox.appendChild(replyButton);
}
for (let j = 0; j < totalReplies; j++) {
const commentReply = rootComment.comment_replies[j];
let commentReplyDiv = createCommentBox(commentReply, false);
commentReplyBox.appendChild(commentReplyDiv);
}
if (totalReplies > 0) {
commentBox.appendChild(commentReplyBox);
}
}
commentsListBox.appendChild(commentBox);
}
}
function createReplyButton(replyId, totalReplies) {
let replyButton = document.createElement('button');
replyButton.innerHTML = `<span id="toggle-icon">+</span> ${totalReplies} replies`;
replyButton.setAttribute('data-id', replyId);
replyButton.setAttribute('onclick', 'toggleCommentReplies(this)');
return replyButton;
}
function toggleCommentReplies(button) {
let commentReplyId = button.getAttribute('data-id');
let state = document.getElementById(commentReplyId).style.display;
if (state === 'none' || state === '') {
document.getElementById(commentReplyId).style.display = 'block';
button.querySelector('#toggle-icon').innerHTML = '-';
} else {
document.getElementById(commentReplyId).style.display = 'none';
button.querySelector('#toggle-icon').innerHTML = '+';
}
}
function createCommentBox(comment, isRoot) {
let commentBox = document.createElement('div');
commentBox.setAttribute('class', 'comment-box');
let commentClass;
if (isRoot) {
commentClass = 'root-comment';
} else {
commentClass = 'reply-comment';
}
commentBox.classList.add = commentClass;
let commentAuthor = document.createElement('h3');
commentAuthor.innerText = comment.comment_author;
if (comment.comment_author_is_uploader) {
commentAuthor.setAttribute('class', 'comment-highlight');
}
commentBox.appendChild(commentAuthor);
let commentText = document.createElement('p');
commentText.innerText = comment.comment_text;
commentBox.appendChild(commentText);
const spacer = '<span class="space-carrot">|</span>';
let commentMeta = document.createElement('div');
commentMeta.setAttribute('class', 'comment-meta');
commentMeta.innerHTML = `<span>${comment.comment_time_text}</span>`;
if (comment.comment_likecount > 0) {
let numberFormatted = formatNumbers(comment.comment_likecount);
commentMeta.innerHTML += `${spacer}<span class="thumb-icon"><img src="/static/img/icon-thumb.svg"> ${numberFormatted}</span>`;
}
if (comment.comment_is_favorited) {
commentMeta.innerHTML += `${spacer}<span class="comment-like"><img src="/static/img/icon-heart.svg"></span>`;
}
commentBox.appendChild(commentMeta);
return commentBox;
}
function getSimilarVideos(videoId) {
let apiEndpoint = '/api/video/' + videoId + '/similar/';
let response = apiRequest(apiEndpoint, 'GET');
if (!response) {
populateEmpty();
return;
}
let allSimilar = response.data;
if (allSimilar.length > 0) {
populateSimilar(allSimilar);
}
}
function populateSimilar(allSimilar) {
let similarBox = document.getElementById('similar-videos');
for (let i = 0; i < allSimilar.length; i++) {
const similarRaw = allSimilar[i];
let similarDiv = createVideo(similarRaw, 'grid');
similarBox.appendChild(similarDiv);
}
}
function populateEmpty() {
let similarBox = document.getElementById('similar-videos');
let emptyMessage = document.createElement('p');
emptyMessage.innerText = 'No similar videos found.';
similarBox.appendChild(emptyMessage);
}
// generic
function sendPost(payload) {