From 00379bfe26fea2b8a9cefda556157fc65bc9af55 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 31 Dec 2022 15:35:58 +0700 Subject: [PATCH 01/21] fix cookie validate bool logic --- tubearchivist/home/src/index/reindex.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 5b65afcc..261c7160 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -205,7 +205,7 @@ class Reindex(ReindexBase): def reindex_all(self): """reindex all in queue""" - if self.cookie_invalid(): + if not self.cookie_is_valid(): print("[reindex] cookie invalid, exiting...") return @@ -333,8 +333,8 @@ class Reindex(ReindexBase): handler.get_indexed() self.all_indexed_ids = [i["youtube_id"] for i in handler.all_videos] - def cookie_invalid(self): - """return true if cookie is enabled and invalid""" + def cookie_is_valid(self): + """return true if cookie is enabled and valid""" if not self.config["downloads"]["cookie_import"]: return False From 43a41973d9f4035e096836c58caf756b72052ee2 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 31 Dec 2022 16:06:33 +0700 Subject: [PATCH 02/21] fix missing run method call --- tubearchivist/home/src/index/reindex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 261c7160..dba7c89d 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -246,7 +246,7 @@ class Reindex(ReindexBase): try: self._reindex_single_video_call(youtube_id) except FileNotFoundError: - ChannelUrlFixer(youtube_id, self.config) + ChannelUrlFixer(youtube_id, self.config).run() self._reindex_single_video_call(youtube_id) def _reindex_single_video_call(self, youtube_id): From 3db6077adfe254df69b18ec94009db7dc6866a27 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 2 Jan 2023 13:28:07 +0700 Subject: [PATCH 03/21] bump pillow --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index a92e749e..ee87d042 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -4,7 +4,7 @@ Django==4.1.4 django-auth-ldap==4.1.0 django-cors-headers==3.13.0 djangorestframework==3.14.0 -Pillow==9.3.0 +Pillow==9.4.0 redis==4.4.0 requests==2.28.1 ryd-client==0.0.6 From 7941b0412bcf6ebfc12e598e56b37dc0c1daad75 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 2 Jan 2023 13:28:49 +0700 Subject: [PATCH 04/21] set vid_type for pending items in ta_download --- tubearchivist/home/apps.py | 8 ++++++-- tubearchivist/home/templates/home/downloads.html | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tubearchivist/home/apps.py b/tubearchivist/home/apps.py index 1356ac80..14b1d7c5 100644 --- a/tubearchivist/home/apps.py +++ b/tubearchivist/home/apps.py @@ -129,14 +129,18 @@ class StartupCheck: def es_set_vid_type(self): """update path 0.3.0 to 0.3.1, set default vid_type to video""" + index_list = ["ta_video", "ta_download"] data = { "query": { "bool": {"must_not": [{"exists": {"field": "vid_type"}}]} }, "script": {"source": "ctx._source['vid_type'] = 'video'"}, } - response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data) - print(f"ta_video vid_type index update ran: {response}") + + for index_name in index_list: + path = f"{index_name}/_update_by_query" + response, _ = ElasticWrap(path).post(data=data) + print(f"{index_name} vid_type index update ran: {response}") class HomeConfig(AppConfig): diff --git a/tubearchivist/home/templates/home/downloads.html b/tubearchivist/home/templates/home/downloads.html index c4e76349..7b515721 100644 --- a/tubearchivist/home/templates/home/downloads.html +++ b/tubearchivist/home/templates/home/downloads.html @@ -96,7 +96,7 @@ {% else %} - + {% endif %} From 46229db64ce0cfc2340cd66b685ebe6ad5bd0584 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 2 Jan 2023 16:50:53 +0700 Subject: [PATCH 05/21] [API] add download now endpoint, fix vid_type --- tubearchivist/api/README.md | 7 +++++++ tubearchivist/api/views.py | 16 ++++++++++++---- tubearchivist/home/src/download/queue.py | 6 ++++++ tubearchivist/home/src/frontend/api_calls.py | 12 ------------ tubearchivist/home/tasks.py | 11 ++++++++--- tubearchivist/static/script.js | 4 ++-- 6 files changed, 35 insertions(+), 21 deletions(-) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 1a95fb8c..2328f685 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -221,6 +221,13 @@ Add to queue previously ignored video: } ``` +Download existing video now: +```json +{ + "status": "priority" +} +``` + DELETE /api/download/\/ Forget or delete from download queue diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index c99871dc..12b2e196 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -15,7 +15,7 @@ from home.src.index.video import SponsorBlock, YoutubeVideo from home.src.ta.config import AppConfig from home.src.ta.helper import UrlListParser from home.src.ta.ta_redis import RedisArchivist, RedisQueue -from home.tasks import check_reindex, extrac_dl, subscribe_to +from home.tasks import check_reindex, download_single, extrac_dl, subscribe_to from rest_framework.authentication import ( SessionAuthentication, TokenAuthentication, @@ -395,7 +395,7 @@ class DownloadApiView(ApiBaseView): """ search_base = "ta_download/_doc/" - valid_status = ["pending", "ignore"] + valid_status = ["pending", "ignore", "priority"] def get(self, request, video_id): # pylint: disable=unused-argument @@ -411,9 +411,17 @@ class DownloadApiView(ApiBaseView): print(message) return Response({"message": message}, status=400) + pending_video, status_code = PendingInteract(video_id).get_item() + if status_code == 404: + message = f"{video_id}: item not found {status_code}" + return Response({"message": message}, status=404) + print(f"{video_id}: change status to {item_status}") - PendingInteract(video_id=video_id, status=item_status).update_status() - RedisQueue(queue_name="dl_queue").clear_item(video_id) + if item_status == "priority": + download_single.delay(pending_video) + else: + PendingInteract(video_id, item_status).update_status() + RedisQueue(queue_name="dl_queue").clear_item(video_id) return Response(request.data) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index a3338c9f..5233ffd2 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -117,6 +117,12 @@ class PendingInteract: path = f"ta_download/_update/{self.video_id}" _, _ = ElasticWrap(path).post(data=data) + def get_item(self): + """return pending item dict""" + path = f"ta_download/_doc/{self.video_id}" + response, status_code = ElasticWrap(path).get() + return response["_source"], status_code + class PendingList(PendingIndex): """manage the pending videos list""" diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index 96957921..524f55d6 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -13,7 +13,6 @@ from home.src.ta.helper import UrlListParser from home.src.ta.ta_redis import RedisArchivist, RedisQueue from home.tasks import ( download_pending, - download_single, index_channel_playlists, kill_dl, re_sync_thumbs, @@ -56,7 +55,6 @@ class PostData: "sort_order": self._sort_order, "hide_watched": self._hide_watched, "show_subed_only": self._show_subed_only, - "dlnow": self._dlnow, "show_ignored_only": self._show_ignored_only, "manual-import": self._manual_import, "re-embed": self._re_embed, @@ -178,16 +176,6 @@ class PostData: RedisArchivist().set_message(key, message) return {"success": True} - def _dlnow(self): - """start downloading single vid now""" - youtube_id = self.exec_val - print(f"{youtube_id}: downloading now") - running = download_single.delay(youtube_id=youtube_id) - task_id = running.id - print("set task id: " + task_id) - RedisArchivist().set_message("dl_queue_id", task_id) - return {"success": True} - def _show_ignored_only(self): """switch view on /downloads/ to show ignored only""" show_value = self.exec_val diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 034f805a..ec08e399 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -6,6 +6,7 @@ Functionality: because tasks are initiated at application start """ +import json import os from celery import Celery, shared_task @@ -107,11 +108,15 @@ def download_pending(): @shared_task -def download_single(youtube_id): +def download_single(pending_video): """start download single video now""" queue = RedisQueue(queue_name="dl_queue") - queue.add_priority(youtube_id) - print("Added to queue with priority: " + youtube_id) + to_add = { + "youtube_id": pending_video["youtube_id"], + "vid_type": pending_video["vid_type"], + } + queue.add_priority(json.dumps(to_add)) + print(f"Added to queue with priority: {to_add}") # start queue if needed have_lock = False my_lock = RedisArchivist().get_lock("downloading") diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 56174c4e..1028f84a 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -193,8 +193,8 @@ function toIgnore(button) { function downloadNow(button) { let youtube_id = button.getAttribute('data-id'); - let payload = JSON.stringify({ dlnow: youtube_id }); - sendPost(payload); + let apiEndpoint = '/api/download/' + youtube_id + '/'; + apiRequest(apiEndpoint, 'POST', { status: 'priority' }); document.getElementById(youtube_id).remove(); setTimeout(function () { checkMessages(); From 0c8830793cb2410ae3e174b1a025ac0a2ce216bd Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 2 Jan 2023 17:44:02 +0700 Subject: [PATCH 06/21] add vid_type tag to queue thumb --- tubearchivist/home/templates/home/downloads.html | 15 +++++++++------ tubearchivist/static/css/style.css | 12 ++++++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/tubearchivist/home/templates/home/downloads.html b/tubearchivist/home/templates/home/downloads.html index 7b515721..4d5bbf5f 100644 --- a/tubearchivist/home/templates/home/downloads.html +++ b/tubearchivist/home/templates/home/downloads.html @@ -73,11 +73,14 @@
video_thumb - {% if show_ignored_only %} - ignored - {% else %} - queued - {% endif %} +
+ {% if show_ignored_only %} + ignored + {% else %} + queued + {% endif %} + {{ video.source.vid_type }} +
@@ -96,7 +99,7 @@ {% else %} - + {% endif %}
diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index 4e747951..337c3bac 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -462,7 +462,7 @@ video:-webkit-full-screen { overflow: hidden; } -.video-item:hover .video-thumb span { +.video-item:hover .video-thumb div { opacity: 1; } @@ -486,16 +486,20 @@ video:-webkit-full-screen { position: relative; } -.video-thumb span { +.video-thumb div { position: absolute; top: 5px; - left: 5px; - background-color: var(--accent-font-light); + left: 0; padding: 5px; opacity: 0; transition: 300ms ease-in-out; } +.video-tags span { + background-color: var(--accent-font-light); + padding: 5px; +} + .video-play img { width: 40px; filter: var(--img-filter); From 14d5a7ee1ae864756ed1a6478a60b283640bf291 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 2 Jan 2023 21:27:49 +0700 Subject: [PATCH 07/21] fix progress bar and video-tags overlap --- tubearchivist/static/css/style.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index 337c3bac..f56b5c28 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -462,7 +462,7 @@ video:-webkit-full-screen { overflow: hidden; } -.video-item:hover .video-thumb div { +.video-item:hover .video-tags { opacity: 1; } @@ -486,7 +486,7 @@ video:-webkit-full-screen { position: relative; } -.video-thumb div { +.video-tags { position: absolute; top: 5px; left: 0; From ce4288ff66cd40dc185289f255e9c4f810ddb726 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 2 Jan 2023 21:36:08 +0700 Subject: [PATCH 08/21] add is_short standalone helper function --- tubearchivist/home/src/ta/helper.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index af17fc06..45e4190f 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -13,6 +13,7 @@ import unicodedata from datetime import datetime from urllib.parse import parse_qs, urlparse +import requests from home.src.download.yt_dlp_base import YtWrap @@ -136,6 +137,16 @@ def get_mapping(): return index_config +def is_short(youtube_id): + """check if youtube_id is a shorts video, bot not it it's not a shorts""" + shorts_url = f"https://www.youtube.com/shorts/{youtube_id}" + response = requests.head( + shorts_url, headers=requests_headers(), timeout=10 + ) + + return response.status_code == 200 + + class UrlListParser: """take a multi line string and detect valid youtube ids""" From eb812d2b0a994f4b003f915c5ae985a27c27a0c3 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 2 Jan 2023 22:16:20 +0700 Subject: [PATCH 09/21] add timeouts --- tubearchivist/home/src/es/connect.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index aac13dcc..ab73ea8e 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -34,9 +34,11 @@ class ElasticWrap: def get(self, data=False): """get data from es""" if data: - response = requests.get(self.url, json=data, auth=self.auth) + response = requests.get( + self.url, json=data, auth=self.auth, timeout=10 + ) else: - response = requests.get(self.url, auth=self.auth) + response = requests.get(self.url, auth=self.auth, timeout=10) if not response.ok: print(response.text) @@ -53,10 +55,16 @@ class ElasticWrap: if data: response = requests.post( - self.url, data=payload, headers=headers, auth=self.auth + self.url, + data=payload, + headers=headers, + auth=self.auth, + timeout=10, ) else: - response = requests.post(self.url, headers=headers, auth=self.auth) + response = requests.post( + self.url, headers=headers, auth=self.auth, timeout=10 + ) if not response.ok: print(response.text) @@ -67,7 +75,9 @@ class ElasticWrap: """put data to es""" if refresh: self.url = f"{self.url}/?refresh=true" - response = requests.put(f"{self.url}", json=data, auth=self.auth) + response = requests.put( + f"{self.url}", json=data, auth=self.auth, timeout=10 + ) if not response.ok: print(response.text) print(data) @@ -80,9 +90,11 @@ class ElasticWrap: if refresh: self.url = f"{self.url}/?refresh=true" if data: - response = requests.delete(self.url, json=data, auth=self.auth) + response = requests.delete( + self.url, json=data, auth=self.auth, timeout=10 + ) else: - response = requests.delete(self.url, auth=self.auth) + response = requests.delete(self.url, auth=self.auth, timeout=10) if not response.ok: print(response.text) From 72e9234b2b030765df673282479342f5d7394f23 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 4 Jan 2023 08:24:18 +0700 Subject: [PATCH 10/21] fix playlist validator only validating active playlists --- .../home/src/download/yt_dlp_handler.py | 3 ++- tubearchivist/home/src/index/channel.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 636f3fa5..7c8af776 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -104,7 +104,7 @@ class DownloadPostProcess: continue # validate from local - playlists = channel.get_indexed_playlists() + playlists = channel.get_indexed_playlists(active_only=True) all_channel_playlist = [i["playlist_id"] for i in playlists] self._validate_channel_playlist(all_channel_playlist, id_c) @@ -117,6 +117,7 @@ class DownloadPostProcess: playlist.build_json(scrape=True) if not playlist.json_data: playlist.deactivate() + continue playlist.add_vids_to_playlist() playlist.upload_to_es() diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 90e0c116..894ce322 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -392,14 +392,16 @@ class YoutubeChannel(YouTubeItem): all_entries = [(i["id"], i["title"]) for i in playlists["entries"]] self.all_playlists = all_entries - def get_indexed_playlists(self): + def get_indexed_playlists(self, active_only=False): """get all indexed playlists from channel""" - data = { - "query": { - "term": {"playlist_channel_id": {"value": self.youtube_id}} - }, - "sort": [{"playlist_channel.keyword": {"order": "desc"}}], - } + must_list = [ + {"term": {"playlist_channel_id": {"value": self.youtube_id}}} + ] + if active_only: + must_list.append({"term": {"playlist_active": {"value": True}}}) + + data = {"query": {"bool": {"must": must_list}}} + all_playlists = IndexPaginate("ta_playlist", data).get_results() return all_playlists From eccad8968bd120a02f934fa90656d3d322ab34b9 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 4 Jan 2023 11:53:46 +0700 Subject: [PATCH 11/21] add unknown field to VideoTypeEnum --- tubearchivist/home/src/index/video_constants.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubearchivist/home/src/index/video_constants.py b/tubearchivist/home/src/index/video_constants.py index d5df23a6..1b8dcd4e 100644 --- a/tubearchivist/home/src/index/video_constants.py +++ b/tubearchivist/home/src/index/video_constants.py @@ -1,7 +1,12 @@ +"""video constants""" + import enum class VideoTypeEnum(enum.Enum): + """all vid_type fields""" + VIDEO = "video" LIVE = "live" SHORT = "short" + UNKNOWN = "unknown" From e29930871122b163ec817387a90936314a4e24d2 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 6 Jan 2023 10:11:52 +0700 Subject: [PATCH 12/21] match VideoTypeEnum with yt channel paths --- tubearchivist/home/apps.py | 2 +- tubearchivist/home/src/download/queue.py | 10 +++++----- tubearchivist/home/src/download/subscriptions.py | 12 ++++++------ tubearchivist/home/src/download/yt_dlp_handler.py | 4 ++-- tubearchivist/home/src/index/video.py | 4 ++-- tubearchivist/home/src/index/video_constants.py | 6 +++--- tubearchivist/home/tasks.py | 2 +- tubearchivist/home/urls.py | 2 +- tubearchivist/home/views.py | 8 ++++---- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tubearchivist/home/apps.py b/tubearchivist/home/apps.py index 14b1d7c5..073afcc7 100644 --- a/tubearchivist/home/apps.py +++ b/tubearchivist/home/apps.py @@ -134,7 +134,7 @@ class StartupCheck: "query": { "bool": {"must_not": [{"exists": {"field": "vid_type"}}]} }, - "script": {"source": "ctx._source['vid_type'] = 'video'"}, + "script": {"source": "ctx._source['vid_type'] = 'videos'"}, } for index_name in index_list: diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index 5233ffd2..b571053d 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -163,7 +163,7 @@ class PendingList(PendingIndex): def _process_entry(self, entry): """process single entry from url list""" if entry["type"] == "video": - vid_type = entry.get("vid_type", VideoTypeEnum.VIDEO) + vid_type = entry.get("vid_type", VideoTypeEnum.VIDEOS) self._add_video(entry["url"], vid_type) elif entry["type"] == "channel": self._parse_channel(entry["url"]) @@ -173,7 +173,7 @@ class PendingList(PendingIndex): else: raise ValueError(f"invalid url_type: {entry}") - def _add_video(self, url, vid_type=VideoTypeEnum.VIDEO): + def _add_video(self, url, vid_type=VideoTypeEnum.VIDEOS): """add video to list""" if url not in self.missing_videos and url not in self.to_skip: self.missing_videos.append((url, vid_type)) @@ -197,7 +197,7 @@ class PendingList(PendingIndex): for video_id in youtube_ids: # FIXME: This will need to be adjusted to support Live/Shorts # from playlists - self._add_video(video_id, VideoTypeEnum.VIDEO) + self._add_video(video_id, VideoTypeEnum.VIDEOS) def add_to_pending(self, status="pending"): """add missing videos to pending list""" @@ -244,7 +244,7 @@ class PendingList(PendingIndex): if idx + 1 % 25 == 0: print("adding to queue progress: " + progress) - def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEO): + def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEOS): """get details from youtubedl for single pending video""" vid = YtWrap(self.yt_obs, self.config).extract(youtube_id) if not vid: @@ -260,7 +260,7 @@ class PendingList(PendingIndex): return self._parse_youtube_details(vid, vid_type) - def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEO): + def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS): """parse response""" vid_id = vid.get("id") duration_str = DurationConverter.get_str(vid["duration"]) diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index df77bdec..afc1490f 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -41,18 +41,18 @@ class ChannelSubscription: queries = [ ( - VideoTypeEnum.VIDEO, - "videos", + VideoTypeEnum.VIDEOS, + VideoTypeEnum.VIDEOS.value, self.config["subscriptions"]["channel_size"], ), ( - VideoTypeEnum.LIVE, - "streams", + VideoTypeEnum.STREAMS, + VideoTypeEnum.STREAMS.value, self.config["subscriptions"]["live_channel_size"], ), ( - VideoTypeEnum.SHORT, - "shorts", + VideoTypeEnum.SHORTS, + VideoTypeEnum.SHORTS.value, self.config["subscriptions"]["shorts_channel_size"], ), ] diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 7c8af776..34855484 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -189,7 +189,7 @@ class VideoDownloader: youtube_id = youtube_data.get("youtube_id") tmp_vid_type = youtube_data.get( - "vid_type", VideoTypeEnum.VIDEO.value + "vid_type", VideoTypeEnum.VIDEOS.value ) video_type = VideoTypeEnum(tmp_vid_type) print(f"Downloading type: {video_type}") @@ -269,7 +269,7 @@ class VideoDownloader: "youtube_id": i["youtube_id"], # Using .value in default val to match what would be # decoded when parsing json if not set - "vid_type": i.get("vid_type", VideoTypeEnum.VIDEO.value), + "vid_type": i.get("vid_type", VideoTypeEnum.VIDEOS.value), } ) for i in pending.all_pending diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 92cbfdc5..406ad66d 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -128,7 +128,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): self, youtube_id, video_overwrites=False, - video_type=VideoTypeEnum.VIDEO, + video_type=VideoTypeEnum.VIDEOS, ): super().__init__(youtube_id) self.channel_id = False @@ -406,7 +406,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def index_new_video( - youtube_id, video_overwrites=False, video_type=VideoTypeEnum.VIDEO + youtube_id, video_overwrites=False, video_type=VideoTypeEnum.VIDEOS ): """combined classes to create new video in index""" video = YoutubeVideo( diff --git a/tubearchivist/home/src/index/video_constants.py b/tubearchivist/home/src/index/video_constants.py index 1b8dcd4e..9ccd0464 100644 --- a/tubearchivist/home/src/index/video_constants.py +++ b/tubearchivist/home/src/index/video_constants.py @@ -6,7 +6,7 @@ import enum class VideoTypeEnum(enum.Enum): """all vid_type fields""" - VIDEO = "video" - LIVE = "live" - SHORT = "short" + VIDEOS = "videos" + STREAMS = "streams" + SHORTS = "shorts" UNKNOWN = "unknown" diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index ec08e399..dd940e0a 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -68,7 +68,7 @@ def update_subscribed(): playlist_videos = [ { "type": "video", - "vid_type": VideoTypeEnum.VIDEO, + "vid_type": VideoTypeEnum.VIDEOS, "url": i, } for i in missing_from_playlists diff --git a/tubearchivist/home/urls.py b/tubearchivist/home/urls.py index 6cb0c5ad..8dc8c9d2 100644 --- a/tubearchivist/home/urls.py +++ b/tubearchivist/home/urls.py @@ -47,7 +47,7 @@ urlpatterns = [ name="channel_id", ), path( - "channel//live/", + "channel//streams/", login_required(ChannelIdLiveView.as_view()), name="channel_id_live", ), diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index e5636c57..20647aa3 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -514,7 +514,7 @@ class ChannelIdView(ChannelIdBaseView): view_origin = "home" es_search = "ta_video/_search" - video_types = [VideoTypeEnum.VIDEO] + video_types = [VideoTypeEnum.VIDEOS] def get(self, request, channel_id): """get request""" @@ -584,11 +584,11 @@ class ChannelIdView(ChannelIdBaseView): class ChannelIdLiveView(ChannelIdView): - """resolves to /channel//live/ + """resolves to /channel//streams/ display single channel page from channel_id """ - video_types = [VideoTypeEnum.LIVE] + video_types = [VideoTypeEnum.STREAMS] class ChannelIdShortsView(ChannelIdView): @@ -596,7 +596,7 @@ class ChannelIdShortsView(ChannelIdView): display single channel page from channel_id """ - video_types = [VideoTypeEnum.SHORT] + video_types = [VideoTypeEnum.SHORTS] class ChannelIdAboutView(ChannelIdBaseView): From cd8b8aafc61f84d3299881503d3f1cf8da543212 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 6 Jan 2023 13:47:17 +0700 Subject: [PATCH 13/21] bump libraries --- tubearchivist/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index ee87d042..f2e56a7c 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4==4.11.1 celery==5.2.7 -Django==4.1.4 +Django==4.1.5 django-auth-ldap==4.1.0 django-cors-headers==3.13.0 djangorestframework==3.14.0 @@ -9,5 +9,5 @@ redis==4.4.0 requests==2.28.1 ryd-client==0.0.6 uWSGI==2.0.21 -whitenoise==6.2.0 -yt_dlp==2022.11.11 +whitenoise==6.3.0 +yt_dlp==2023.1.2 From d9f73622a5b928473a9a5d6013bb8f45309fcdf7 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 6 Jan 2023 15:37:55 +0700 Subject: [PATCH 14/21] refactor UrlParser into own module, rename enum fields --- tubearchivist/api/views.py | 4 +- tubearchivist/home/src/download/queue.py | 11 +- tubearchivist/home/src/frontend/api_calls.py | 4 +- tubearchivist/home/src/frontend/watched.py | 4 +- tubearchivist/home/src/ta/helper.py | 102 -------------- tubearchivist/home/src/ta/urlparser.py | 133 +++++++++++++++++++ tubearchivist/home/tasks.py | 11 +- tubearchivist/home/views.py | 5 +- 8 files changed, 157 insertions(+), 117 deletions(-) create mode 100644 tubearchivist/home/src/ta/urlparser.py diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 12b2e196..7146de72 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -13,8 +13,8 @@ from home.src.index.generic import Pagination from home.src.index.reindex import ReindexProgress from home.src.index.video import SponsorBlock, YoutubeVideo from home.src.ta.config import AppConfig -from home.src.ta.helper import UrlListParser from home.src.ta.ta_redis import RedisArchivist, RedisQueue +from home.src.ta.urlparser import Parser from home.tasks import check_reindex, download_single, extrac_dl, subscribe_to from rest_framework.authentication import ( SessionAuthentication, @@ -484,7 +484,7 @@ class DownloadApiListView(ApiBaseView): pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"] url_str = " ".join(pending) try: - youtube_ids = UrlListParser(url_str).process_list() + youtube_ids = Parser(url_str).parse() except ValueError: message = f"failed to parse: {url_str}" print(message) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index b571053d..c3571623 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -163,7 +163,7 @@ class PendingList(PendingIndex): def _process_entry(self, entry): """process single entry from url list""" if entry["type"] == "video": - vid_type = entry.get("vid_type", VideoTypeEnum.VIDEOS) + vid_type = self._get_vid_type(entry) self._add_video(entry["url"], vid_type) elif entry["type"] == "channel": self._parse_channel(entry["url"]) @@ -173,6 +173,15 @@ class PendingList(PendingIndex): else: raise ValueError(f"invalid url_type: {entry}") + @staticmethod + def _get_vid_type(entry): + """add vid type enum if available""" + vid_type_str = entry.get("vid_type") + if not vid_type_str: + return VideoTypeEnum.VIDEOS + + return VideoTypeEnum(vid_type_str) + def _add_video(self, url, vid_type=VideoTypeEnum.VIDEOS): """add video to list""" if url not in self.missing_videos and url not in self.to_skip: diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index 524f55d6..a7d38907 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -9,8 +9,8 @@ from home.src.download.subscriptions import ( PlaylistSubscription, ) from home.src.index.playlist import YoutubePlaylist -from home.src.ta.helper import UrlListParser from home.src.ta.ta_redis import RedisArchivist, RedisQueue +from home.src.ta.urlparser import Parser from home.tasks import ( download_pending, index_channel_playlists, @@ -123,7 +123,7 @@ class PostData: """unsubscribe from channels or playlists""" id_unsub = self.exec_val print(f"{id_unsub}: unsubscribe") - to_unsub_list = UrlListParser(id_unsub).process_list() + to_unsub_list = Parser(id_unsub).parse() for to_unsub in to_unsub_list: unsub_type = to_unsub["type"] unsub_id = to_unsub["url"] diff --git a/tubearchivist/home/src/frontend/watched.py b/tubearchivist/home/src/frontend/watched.py index 8978b961..2ce80180 100644 --- a/tubearchivist/home/src/frontend/watched.py +++ b/tubearchivist/home/src/frontend/watched.py @@ -6,7 +6,7 @@ functionality: from datetime import datetime from home.src.es.connect import ElasticWrap -from home.src.ta.helper import UrlListParser +from home.src.ta.urlparser import Parser class WatchState: @@ -34,7 +34,7 @@ class WatchState: def _dedect_type(self): """find youtube id type""" print(self.youtube_id) - url_process = UrlListParser(self.youtube_id).process_list() + url_process = Parser(self.youtube_id).parse() url_type = url_process[0]["type"] return url_type diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 45e4190f..51bbf428 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -11,10 +11,8 @@ import string import subprocess import unicodedata from datetime import datetime -from urllib.parse import parse_qs, urlparse import requests -from home.src.download.yt_dlp_base import YtWrap def clean_string(file_name): @@ -147,106 +145,6 @@ def is_short(youtube_id): return response.status_code == 200 -class UrlListParser: - """take a multi line string and detect valid youtube ids""" - - def __init__(self, url_str): - self.url_list = [i.strip() for i in url_str.split()] - - def process_list(self): - """loop through the list""" - youtube_ids = [] - for url in self.url_list: - parsed = urlparse(url) - print(f"processing: {url}") - print(parsed) - if not parsed.netloc: - # is not a url - id_type = self.find_valid_id(url) - youtube_id = url - elif "youtube.com" not in url and "youtu.be" not in url: - raise ValueError(f"{url} is not a youtube link") - elif parsed.path: - # is a url - youtube_id, id_type = self.detect_from_url(parsed) - else: - # not detected - raise ValueError(f"failed to detect {url}") - - youtube_ids.append({"url": youtube_id, "type": id_type}) - - return youtube_ids - - def detect_from_url(self, parsed): - """detect from parsed url""" - if parsed.netloc == "youtu.be": - # shortened - youtube_id = parsed.path.strip("/") - _ = self.find_valid_id(youtube_id) - return youtube_id, "video" - - if parsed.query: - # detect from query string - query_parsed = parse_qs(parsed.query) - if "v" in query_parsed: - youtube_id = query_parsed["v"][0] - _ = self.find_valid_id(youtube_id) - return youtube_id, "video" - - if "list" in query_parsed: - youtube_id = query_parsed["list"][0] - return youtube_id, "playlist" - - if parsed.path.startswith("/channel/"): - # channel id in url - youtube_id = parsed.path.split("/")[2] - _ = self.find_valid_id(youtube_id) - return youtube_id, "channel" - - # detect channel with yt_dlp - youtube_id = self.extract_channel_name(parsed.geturl()) - return youtube_id, "channel" - - @staticmethod - def find_valid_id(id_str): - """detect valid id from length of string""" - str_len = len(id_str) - if str_len == 11: - id_type = "video" - elif str_len == 24: - id_type = "channel" - elif str_len in [34, 18] or id_str in ["LL", "WL"]: - id_type = "playlist" - else: - # unable to parse - raise ValueError("not a valid id_str: " + id_str) - - return id_type - - @staticmethod - def extract_channel_name(url): - """find channel id from channel name with yt-dlp help""" - obs_request = { - "skip_download": True, - "extract_flat": True, - "playlistend": 0, - } - url_info = YtWrap(obs_request).extract(url) - channel_id = url_info.get("channel_id", False) - if channel_id: - return channel_id - - url = url_info.get("url", False) - if url: - # handle old channel name redirect with url path split - channel_id = urlparse(url).path.strip("/").split("/")[1] - - return channel_id - - print(f"failed to extract channel id from {url}") - raise ValueError - - class DurationConverter: """ using ffmpeg to get and parse duration from filepath diff --git a/tubearchivist/home/src/ta/urlparser.py b/tubearchivist/home/src/ta/urlparser.py new file mode 100644 index 00000000..32c6030a --- /dev/null +++ b/tubearchivist/home/src/ta/urlparser.py @@ -0,0 +1,133 @@ +""" +Functionality: +- detect valid youtube ids and links from multi line string +- identify vid_type if possible +""" + +from urllib.parse import parse_qs, urlparse + +from home.src.download.yt_dlp_base import YtWrap +from home.src.index.video_constants import VideoTypeEnum + + +class Parser: + """take a multi line string and detect valid youtube ids""" + + def __init__(self, url_str): + self.url_list = [i.strip() for i in url_str.split()] + + def parse(self): + """parse the list""" + ids = [] + for url in self.url_list: + parsed = urlparse(url) + if parsed.netloc: + # is url + identified = self.process_url(parsed) + else: + # is not url + identified = self._find_valid_id(url) + + if "vid_type" not in identified: + identified.update(self._detect_vid_type(parsed.path)) + + ids.append(identified) + + return ids + + def process_url(self, parsed): + """process as url""" + if parsed.netloc == "youtu.be": + # shortened + youtube_id = parsed.path.strip("/") + return self._validate_expected(youtube_id, "video") + + query_parsed = parse_qs(parsed.query) + if "v" in query_parsed: + # video from v query str + youtube_id = query_parsed["v"][0] + return self._validate_expected(youtube_id, "video") + + if "list" in query_parsed: + # playlist from list query str + youtube_id = query_parsed["list"][0] + return self._validate_expected(youtube_id, "playlist") + + all_paths = parsed.path.strip("/").split("/") + if all_paths[0] == "shorts": + # is shorts video + item = self._validate_expected(all_paths[1], "video") + item.update({"vid_type": VideoTypeEnum.SHORTS.value}) + return item + + if all_paths[0] == "channel": + return self._validate_expected(all_paths[1], "channel") + + # detect channel + channel_id = self._extract_channel_name(parsed.geturl()) + return {"type": "channel", "url": channel_id} + + def _validate_expected(self, youtube_id, expected_type): + """raise value error if not matching""" + matched = self._find_valid_id(youtube_id) + if matched["type"] != expected_type: + raise ValueError( + f"{youtube_id} not of expected type {expected_type}" + ) + + return {"type": expected_type, "url": youtube_id} + + def _find_valid_id(self, id_str): + """detect valid id from length of string""" + if id_str in ("LL", "WL"): + return {"type": "playlist", "url": id_str} + + if id_str.startswith("@"): + url = f"https://www.youtube.com/{id_str}" + channel_id = self._extract_channel_name(url) + return {"type": "channel", "url": channel_id} + + len_id_str = len(id_str) + if len_id_str == 11: + item_type = "video" + elif len_id_str == 24: + item_type = "channel" + elif len_id_str in (34, 18): + item_type = "playlist" + else: + raise ValueError(f"not a valid id_str: {id_str}") + + return {"type": item_type, "url": id_str} + + @staticmethod + def _extract_channel_name(url): + """find channel id from channel name with yt-dlp help""" + obs_request = { + "skip_download": True, + "extract_flat": True, + "playlistend": 0, + } + url_info = YtWrap(obs_request).extract(url) + channel_id = url_info.get("channel_id", False) + if channel_id: + return channel_id + + url = url_info.get("url", False) + if url: + # handle old channel name redirect with url path split + channel_id = urlparse(url).path.strip("/").split("/")[1] + + return channel_id + + print(f"failed to extract channel id from {url}") + raise ValueError + + def _detect_vid_type(self, path): + """try to match enum from path, needs to be serializable""" + last = path.strip("/").split("/")[-1] + try: + vid_type = VideoTypeEnum(last).value + except ValueError: + vid_type = VideoTypeEnum.UNKNOWN.value + + return {"vid_type": vid_type} diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index dd940e0a..d5138368 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -25,8 +25,9 @@ from home.src.index.filesystem import ImportFolderScanner, scan_filesystem from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder -from home.src.ta.helper import UrlListParser, clear_dl_cache +from home.src.ta.helper import clear_dl_cache from home.src.ta.ta_redis import RedisArchivist, RedisQueue +from home.src.ta.urlparser import Parser CONFIG = AppConfig().config REDIS_HOST = os.environ.get("REDIS_HOST") @@ -261,9 +262,8 @@ def re_sync_thumbs(): @shared_task def subscribe_to(url_str): """take a list of urls to subscribe to""" - to_subscribe_list = UrlListParser(url_str).process_list() - counter = 1 - for item in to_subscribe_list: + to_subscribe_list = Parser(url_str).parse() + for idx, item in enumerate(to_subscribe_list): to_sub_id = item["url"] if item["type"] == "playlist": PlaylistSubscription().process_url_str([item]) @@ -286,10 +286,9 @@ def subscribe_to(url_str): "status": key, "level": "info", "title": "Subscribing to Channels", - "message": f"Processing {counter} of {len(to_subscribe_list)}", + "message": f"Processing {idx + 1} of {len(to_subscribe_list)}", } RedisArchivist().set_message(key, message=message, expire=True) - counter = counter + 1 @shared_task diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 20647aa3..a4fc8fdc 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -38,8 +38,9 @@ from home.src.index.playlist import YoutubePlaylist from home.src.index.reindex import ReindexProgress from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder -from home.src.ta.helper import UrlListParser, time_parser +from home.src.ta.helper import time_parser from home.src.ta.ta_redis import RedisArchivist +from home.src.ta.urlparser import Parser from home.tasks import extrac_dl, index_channel_playlists, subscribe_to from rest_framework.authtoken.models import Token @@ -456,7 +457,7 @@ class DownloadView(ArchivistResultsView): url_str = request.POST.get("vid_url") print(url_str) try: - youtube_ids = UrlListParser(url_str).process_list() + youtube_ids = Parser(url_str).parse() except ValueError: # failed to process key = "message:add" From 646bc1b12e4f799df09cfd89863fc979e80a197b Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 7 Jan 2023 09:45:01 +0700 Subject: [PATCH 15/21] filter by vid_type for add to download queue --- tubearchivist/home/src/download/queue.py | 12 ++-- .../home/src/download/subscriptions.py | 58 ++++++++++++------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index c3571623..dd45fad2 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -162,11 +162,11 @@ class PendingList(PendingIndex): def _process_entry(self, entry): """process single entry from url list""" + vid_type = self._get_vid_type(entry) if entry["type"] == "video": - vid_type = self._get_vid_type(entry) self._add_video(entry["url"], vid_type) elif entry["type"] == "channel": - self._parse_channel(entry["url"]) + self._parse_channel(entry["url"], vid_type) elif entry["type"] == "playlist": self._parse_playlist(entry["url"]) PlaylistSubscription().process_url_str([entry], subscribed=False) @@ -178,21 +178,21 @@ class PendingList(PendingIndex): """add vid type enum if available""" vid_type_str = entry.get("vid_type") if not vid_type_str: - return VideoTypeEnum.VIDEOS + return VideoTypeEnum.UNKNOWN return VideoTypeEnum(vid_type_str) - def _add_video(self, url, vid_type=VideoTypeEnum.VIDEOS): + def _add_video(self, url, vid_type): """add video to list""" if url not in self.missing_videos and url not in self.to_skip: self.missing_videos.append((url, vid_type)) else: print(f"{url}: skipped adding already indexed video to download.") - def _parse_channel(self, url): + def _parse_channel(self, url, vid_type): """add all videos of channel to list""" video_results = ChannelSubscription().get_last_youtube_videos( - url, limit=False + url, limit=False, query_filter=vid_type ) for video_id, _, vid_type in video_results: self._add_video(video_id, vid_type) diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index afc1490f..f154b490 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -36,30 +36,15 @@ class ChannelSubscription: return all_channels - def get_last_youtube_videos(self, channel_id, limit=True): + def get_last_youtube_videos( + self, channel_id, limit=True, query_filter=VideoTypeEnum.UNKNOWN + ): """get a list of last videos from channel""" - - queries = [ - ( - VideoTypeEnum.VIDEOS, - VideoTypeEnum.VIDEOS.value, - self.config["subscriptions"]["channel_size"], - ), - ( - VideoTypeEnum.STREAMS, - VideoTypeEnum.STREAMS.value, - self.config["subscriptions"]["live_channel_size"], - ), - ( - VideoTypeEnum.SHORTS, - VideoTypeEnum.SHORTS.value, - self.config["subscriptions"]["shorts_channel_size"], - ), - ] + queries = self._build_queries(query_filter, limit) last_videos = [] - for vid_type, url, limit_amount in queries: + for vid_type, limit_amount in queries: obs = { "skip_download": True, "extract_flat": True, @@ -67,8 +52,9 @@ class ChannelSubscription: if limit: obs["playlistend"] = limit_amount + path = vid_type.value channel = YtWrap(obs, self.config).extract( - f"https://www.youtube.com/channel/{channel_id}/{url}" + f"https://www.youtube.com/channel/{channel_id}/{path}" ) if not channel: continue @@ -78,6 +64,36 @@ class ChannelSubscription: return last_videos + def _build_queries(self, query_filter, limit): + """build query list for vid_type""" + limit_map = { + "videos": self.config["subscriptions"]["channel_size"], + "streams": self.config["subscriptions"]["live_channel_size"], + "shorts": self.config["subscriptions"]["shorts_channel_size"], + } + + queries = [] + + if query_filter and query_filter.value != "unknown": + if limit: + query_limit = limit_map.get(query_filter.value) + else: + query_limit = False + + queries.append((query_filter, query_limit)) + + return queries + + for query_item, default_limit in limit_map.items(): + if limit: + query_limit = default_limit + else: + query_limit = False + + queries.append((VideoTypeEnum(query_item), query_limit)) + + return queries + def find_missing(self): """add missing videos from subscribed channels to pending""" all_channels = self.get_channels() From db4798754a17d1be1fcfe0402b8574928915728d Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 7 Jan 2023 09:58:53 +0700 Subject: [PATCH 16/21] match streams type with was_live --- tubearchivist/home/src/download/queue.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index dd45fad2..34afbbbf 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -267,6 +267,9 @@ class PendingList(PendingIndex): if vid["live_status"] in ["is_upcoming", "is_live"]: return False + if vid["live_status"] == "was_live": + vid_type = VideoTypeEnum.STREAMS + return self._parse_youtube_details(vid, vid_type) def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS): From d5885273ac39050fd360b78faa3d37e85fd3cf4e Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 7 Jan 2023 15:41:11 +0700 Subject: [PATCH 17/21] detect any shorts before adding to queue --- tubearchivist/home/src/download/queue.py | 23 +++++++++++++++++++---- tubearchivist/home/src/ta/helper.py | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index 34afbbbf..9e62841c 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -17,7 +17,7 @@ from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.playlist import YoutubePlaylist from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig -from home.src.ta.helper import DurationConverter +from home.src.ta.helper import DurationConverter, is_shorts from home.src.ta.ta_redis import RedisArchivist @@ -204,9 +204,8 @@ class PendingList(PendingIndex): video_results = playlist.json_data.get("playlist_entries") youtube_ids = [i["youtube_id"] for i in video_results] for video_id in youtube_ids: - # FIXME: This will need to be adjusted to support Live/Shorts - # from playlists - self._add_video(video_id, VideoTypeEnum.VIDEOS) + # match vid_type later + self._add_video(video_id, VideoTypeEnum.UNKNOWN) def add_to_pending(self, status="pending"): """add missing videos to pending list""" @@ -269,9 +268,25 @@ class PendingList(PendingIndex): if vid["live_status"] == "was_live": vid_type = VideoTypeEnum.STREAMS + else: + if self._check_shorts(vid): + vid_type = VideoTypeEnum.SHORTS return self._parse_youtube_details(vid, vid_type) + @staticmethod + def _check_shorts(vid): + """check if vid is shorts video""" + if vid["width"] > vid["height"]: + return False + + duration = vid.get("duration") + if duration and isinstance(duration, int): + if duration > 60: + return False + + return is_shorts(vid["id"]) + def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS): """parse response""" vid_id = vid.get("id") diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 51bbf428..f07dc805 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -135,7 +135,7 @@ def get_mapping(): return index_config -def is_short(youtube_id): +def is_shorts(youtube_id): """check if youtube_id is a shorts video, bot not it it's not a shorts""" shorts_url = f"https://www.youtube.com/shorts/{youtube_id}" response = requests.head( From 5888e5442a6220203e1be4289b7e118abb91ab54 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 7 Jan 2023 15:51:58 +0700 Subject: [PATCH 18/21] set default streams and shorts channel size to 50 --- tubearchivist/home/config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/config.json b/tubearchivist/home/config.json index de467fb5..aa446399 100644 --- a/tubearchivist/home/config.json +++ b/tubearchivist/home/config.json @@ -15,8 +15,8 @@ "auto_search": false, "auto_download": false, "channel_size": 50, - "live_channel_size": 0, - "shorts_channel_size": 0 + "live_channel_size": 50, + "shorts_channel_size": 50 }, "downloads": { "limit_count": false, From 9a526f93f4920b2d25a6ed95038a85d6e92c3708 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 7 Jan 2023 15:53:32 +0700 Subject: [PATCH 19/21] fix reindex active_key --- tubearchivist/home/src/index/reindex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index dba7c89d..b8058c66 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -100,7 +100,7 @@ class ReindexOutdated(ReindexBase): refresh_key = reindex_config["refresh_key"] now_lte = self.now - self.interval * 24 * 60 * 60 must_list = [ - {"match": {"active": True}}, + {"match": {reindex_config["active_key"]: True}}, {"range": {refresh_key: {"lte": now_lte}}}, ] data = { From 7d6a8799667db49ebb71c8fb4662ee646d5a8f22 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 7 Jan 2023 18:08:35 +0700 Subject: [PATCH 20/21] bump python base file --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 57a97819..db55bf2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # First stage to build python wheel -FROM python:3.10.8-slim-bullseye AS builder +FROM python:3.10.9-slim-bullseye AS builder ARG TARGETPLATFORM RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -14,7 +14,7 @@ COPY ./tubearchivist/requirements.txt /requirements.txt RUN pip install --user -r requirements.txt # build final image -FROM python:3.10.8-slim-bullseye as tubearchivist +FROM python:3.10.9-slim-bullseye as tubearchivist ARG TARGETPLATFORM ARG INSTALL_DEBUG From d033573bd6ff83b80614f54221eef1945167b3ea Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 7 Jan 2023 18:12:09 +0700 Subject: [PATCH 21/21] hide empty shorts and streams from page --- .../home/templates/home/channel_id.html | 12 +++- .../home/templates/home/channel_id_about.html | 12 +++- .../templates/home/channel_id_playlist.html | 12 +++- tubearchivist/home/views.py | 55 ++++++++++++++++++- tubearchivist/static/css/style.css | 1 + 5 files changed, 80 insertions(+), 12 deletions(-) diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index d23e54df..a4c273fc 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -8,9 +8,15 @@

Videos

 -

Live

 -

Shorts

 -

Playlists

 + {% if has_streams %} +

Streams

 + {% endif %} + {% if has_shorts %} +

Shorts

 + {% endif %} + {% if has_playlists %} +

Playlists

 + {% endif %}

About

 {% if has_pending %}

Downloads

 diff --git a/tubearchivist/home/templates/home/channel_id_about.html b/tubearchivist/home/templates/home/channel_id_about.html index 2500e600..1688148d 100644 --- a/tubearchivist/home/templates/home/channel_id_about.html +++ b/tubearchivist/home/templates/home/channel_id_about.html @@ -8,9 +8,15 @@

Videos

 -

Live

 -

Shorts

 -

Playlists

 + {% if has_streams %} +

Streams

 + {% endif %} + {% if has_shorts %} +

Shorts

 + {% endif %} + {% if has_playlists %} +

Playlists

 + {% endif %}

About

 {% if has_pending %}

Downloads

 diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index e43460ca..03b484ab 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -8,9 +8,15 @@

Videos

 -

Live

 -

Shorts

 -

Playlists

 + {% if has_streams %} +

Streams

 + {% endif %} + {% if has_shorts %} +

Shorts

 + {% endif %} + {% if has_playlists %} +

Playlists

 + {% endif %}

About

 {% if has_pending %}

Downloads

 diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index a4fc8fdc..4918aca4 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -489,6 +489,13 @@ class ChannelIdBaseView(ArchivistResultsView): return channel_info + def channel_pages(self, channel_id): + """get additional context for channel pages""" + self.channel_has_pending(channel_id) + self.channel_has_streams(channel_id) + self.channel_has_shorts(channel_id) + self.channel_has_playlist(channel_id) + def channel_has_pending(self, channel_id): """check if channel has pending videos in queue""" path = "ta_download/_search" @@ -502,11 +509,53 @@ class ChannelIdBaseView(ArchivistResultsView): ] } }, + "_source": False, } response, _ = ElasticWrap(path).get(data=data) self.context.update({"has_pending": bool(response["hits"]["hits"])}) + def channel_has_streams(self, channel_id): + """check if channel has streams videos""" + data = self.get_type_data("streams", channel_id) + response, _ = ElasticWrap("ta_video/_search").get(data=data) + + self.context.update({"has_streams": bool(response["hits"]["hits"])}) + + def channel_has_shorts(self, channel_id): + """check if channel has shorts videos""" + data = self.get_type_data("shorts", channel_id) + response, _ = ElasticWrap("ta_video/_search").get(data=data) + + self.context.update({"has_shorts": bool(response["hits"]["hits"])}) + + @staticmethod + def get_type_data(vid_type, channel): + """build data query for vid_type""" + return { + "size": 1, + "query": { + "bool": { + "must": [ + {"term": {"vid_type": {"value": vid_type}}}, + {"term": {"channel.channel_id": {"value": channel}}}, + ] + } + }, + "_source": False, + } + + def channel_has_playlist(self, channel_id): + """check if channel has any playlist indexed""" + path = "ta_playlist/_search" + data = { + "size": 1, + "query": {"term": {"playlist_channel_id": {"value": channel_id}}}, + "_source": False, + } + response, _ = ElasticWrap(path).get(data=data) + self.context.update({"has_playlists": bool(response["hits"]["hits"])}) + class ChannelIdView(ChannelIdBaseView): """resolves to /channel// @@ -523,7 +572,7 @@ class ChannelIdView(ChannelIdBaseView): self._update_view_data(channel_id) self.find_results() self.match_progress() - self.channel_has_pending(channel_id) + self.channel_pages(channel_id) if self.context["results"]: channel_info = self.context["results"][0]["source"]["channel"] @@ -610,7 +659,7 @@ class ChannelIdAboutView(ChannelIdBaseView): def get(self, request, channel_id): """handle get request""" self.initiate_vars(request) - self.channel_has_pending(channel_id) + self.channel_pages(channel_id) response, _ = ElasticWrap(f"ta_channel/_doc/{channel_id}").get() channel_info = SearchProcess(response).process() @@ -658,7 +707,7 @@ class ChannelIdPlaylistView(ChannelIdBaseView): self.initiate_vars(request) self._update_view_data(channel_id) self.find_results() - self.channel_has_pending(channel_id) + self.channel_pages(channel_id) channel_info = self.get_channel_meta(channel_id) channel_name = channel_info["channel_name"] diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index f56b5c28..56ab4e35 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -649,6 +649,7 @@ video:-webkit-full-screen { .info-box-item { display: flex; + flex-wrap: wrap; align-items: center; padding: 15px; background-color: var(--highlight-bg);