From 569d97e2f3f225ea0db64ee324d7c2c54ba010b0 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 10 Jul 2025 10:00:50 +0700 Subject: [PATCH] handle add to queue progress --- backend/common/src/search_processor.py | 14 ++-- backend/download/serializers.py | 2 +- backend/download/src/queue.py | 101 ++++++++++++++++++++----- backend/task/tasks.py | 2 +- backend/video/src/index.py | 4 + 5 files changed, 95 insertions(+), 28 deletions(-) diff --git a/backend/common/src/search_processor.py b/backend/common/src/search_processor.py index 714d33b0..e5c0cac4 100644 --- a/backend/common/src/search_processor.py +++ b/backend/common/src/search_processor.py @@ -165,15 +165,17 @@ class SearchProcess: def _process_download(self, download_dict): """run on single download item""" - video_id = download_dict["youtube_id"] - cache_root = EnvironmentSettings().get_cache_root() - vid_thumb_url = ThumbManager(video_id).vid_thumb_path() - published = date_parser(download_dict["published"]) + vid_thumb_url = None + if download_dict.get("vid_thumb_url"): + video_id = download_dict["youtube_id"] + cache_root = EnvironmentSettings().get_cache_root() + relative_path = ThumbManager(video_id).vid_thumb_path() + vid_thumb_url = f"{cache_root}/{relative_path}" download_dict.update( { - "vid_thumb_url": f"{cache_root}/{vid_thumb_url}", - "published": published, + "vid_thumb_url": vid_thumb_url, + "published": date_parser(download_dict["published"]), } ) return dict(sorted(download_dict.items())) diff --git a/backend/download/serializers.py b/backend/download/serializers.py index 631541df..c8afde33 100644 --- a/backend/download/serializers.py +++ b/backend/download/serializers.py @@ -19,7 +19,7 @@ class DownloadItemSerializer(serializers.Serializer): status = serializers.ChoiceField(choices=["pending", "ignore"]) timestamp = serializers.IntegerField(allow_null=True) title = serializers.CharField() - vid_thumb_url = serializers.CharField() + vid_thumb_url = serializers.CharField(allow_null=True) vid_type = serializers.ChoiceField(choices=VideoTypeEnum.values()) youtube_id = serializers.CharField() message = serializers.CharField(required=False) diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index fd3e0166..45311572 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -216,7 +216,7 @@ class PendingList(PendingIndex): self.get_channels() total = len(self.youtube_ids) for idx, entry in enumerate(self.youtube_ids): - self._process_entry(entry) + self._process_entry(entry, idx, total) rand_sleep(self.config) if not self.task: continue @@ -226,10 +226,12 @@ class PendingList(PendingIndex): progress=(idx + 1) / total, ) - def _process_entry(self, entry: dict): + def _process_entry(self, entry: dict, idx_url: int, total_url: int): """process single entry from url list""" if entry["type"] == "video": - self._add_video(entry["url"], entry["vid_type"]) + self._add_video( + entry["url"], entry["vid_type"], idx_url, total_url + ) elif entry["type"] == "channel": self._parse_channel(entry["url"], entry["vid_type"]) elif entry["type"] == "playlist": @@ -237,7 +239,7 @@ class PendingList(PendingIndex): else: raise ValueError(f"invalid url_type: {entry}") - def _add_video(self, url, vid_type): + def _add_video(self, url, vid_type, idx, total): """add video to list""" if self.auto_start and url in set( i["youtube_id"] for i in self.all_pending @@ -248,7 +250,13 @@ class PendingList(PendingIndex): if url in self.missing_videos or url in self.to_skip: print(f"{url}: skipped adding already indexed video to download.") else: - to_add = self._parse_video(url, vid_type) + to_add = self._parse_video( + url, + vid_type, + url_type="video", + idx=idx, + total=total, + ) if to_add: self.missing_videos.append(to_add) @@ -261,7 +269,8 @@ class PendingList(PendingIndex): channel_handler = YoutubeChannel(url) channel_handler.build_json(upload=False) - for video_data in video_results: + total = len(video_results) + for idx, video_data in enumerate(video_results): video_id = video_data["id"] if video_id in self.to_skip: continue @@ -276,10 +285,20 @@ class PendingList(PendingIndex): video_data["channel_id"] = channel_id to_add = self._parse_entry( - youtube_id=video_id, video_data=video_data + youtube_id=video_id, + video_data=video_data, + url_type="channel", + idx=idx, + total=total, ) else: - to_add = self._parse_video(video_id, vid_type) + to_add = self._parse_video( + video_id, + vid_type, + url_type="channel", + idx=idx, + total=total, + ) if to_add: self.missing_videos.append(to_add) @@ -290,7 +309,8 @@ class PendingList(PendingIndex): playlist.get_from_youtube() video_results = playlist.youtube_meta["entries"] - for video_data in video_results: + total = len(video_results) + for idx, video_data in enumerate(video_results): video_id = video_data["id"] if video_id in self.to_skip: continue @@ -303,26 +323,52 @@ class PendingList(PendingIndex): channel_id = playlist.youtube_meta["channel_id"] video_data["channel_id"] = channel_id - to_add = self._parse_entry(video_id, video_data) + to_add = self._parse_entry( + video_id, + video_data, + url_type="playlist", + idx=idx, + total=total, + ) else: - to_add = self._parse_video(video_id, vid_type=None) + to_add = self._parse_video( + video_id, + vid_type=None, + url_type="playlist", + idx=idx, + total=total, + ) if to_add: self.missing_videos.append(to_add) - def _parse_video(self, url, vid_type): + def _parse_video(self, url, vid_type, url_type, idx, total): """parse video""" video = YoutubeVideo(youtube_id=url) video.get_from_youtube() video_data = video.youtube_meta video_data["vid_type"] = vid_type - to_add = self._parse_entry(youtube_id=url, video_data=video_data) + to_add = self._parse_entry( + youtube_id=url, + video_data=video_data, + url_type=url_type, + idx=idx, + total=total, + ) + ThumbManager(item_id=url).download_video_thumb(to_add["vid_thumb_url"]) rand_sleep(self.config) return to_add - def _parse_entry(self, youtube_id: str, video_data: dict) -> dict | None: + def _parse_entry( + self, + youtube_id: str, + video_data: dict, + url_type: str, + idx: int, + total: int, + ) -> dict | None: """parse entry""" if video_data.get("id") != youtube_id: # skip premium videos with different id or redirects @@ -345,8 +391,8 @@ class PendingList(PendingIndex): "channel_id": video_data["channel_id"], "channel_indexed": video_data["channel_id"] in self.all_channels, } - thumb_url = to_add["vid_thumb_url"] - ThumbManager(to_add["youtube_id"]).download_video_thumb(thumb_url) + + self._notify_progress(url_type, video_data["title"], idx, total) return to_add @@ -355,9 +401,6 @@ class PendingList(PendingIndex): if "thumbnail" in video_data: return video_data["thumbnail"] - if "thumbnails" in video_data: - return video_data["thumbnails"][-1]["url"] - return None def __extract_published(self, video_data) -> str | int | None: @@ -430,6 +473,24 @@ class PendingList(PendingIndex): return len(self.missing_videos) + def _notify_progress(self, url_type, name, idx, total): + """notify extraction progress""" + if not self.task: + return + + if self.flat: + second_line = f"Bulk processing {total} items." + else: + second_line = f"Processing video {idx + 1}/{total}." + + self.task.send_progress( + message_lines=[ + f"Extracting '{name}' from {url_type.title()}.", + second_line, + ], + progress=(idx + 1) / total, + ) + def _notify_empty(self): """notify nothing to add""" if not self.task: @@ -437,7 +498,7 @@ class PendingList(PendingIndex): self.task.send_progress( message_lines=[ - "Extractinc videos completed.", + "Extracting videos completed.", "No new videos found to add.", ] ) diff --git a/backend/task/tasks.py b/backend/task/tasks.py index 8583e916..413c1683 100644 --- a/backend/task/tasks.py +++ b/backend/task/tasks.py @@ -261,7 +261,7 @@ def rescan_filesystem(self): handler = Scanner(task=self) handler.scan() handler.apply() - ThumbValidator(task=self).validate() + thumbnail_check.delay() @shared_task(bind=True, name="thumbnail_check", base=BaseTask) diff --git a/backend/video/src/index.py b/backend/video/src/index.py index 6f324eb3..636bc4bb 100644 --- a/backend/video/src/index.py +++ b/backend/video/src/index.py @@ -14,6 +14,7 @@ from common.src.es_connect import ElasticWrap from common.src.helper import get_duration_sec, get_duration_str, randomizor from common.src.index_generic import YouTubeItem from django.conf import settings +from download.src.thumbnails import ThumbManager from playlist.src import index as ta_playlist from ryd_client import ryd_client from user.src.user_config import UserConfig @@ -409,5 +410,8 @@ def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS): raise ValueError("failed to get metadata for " + youtube_id) video.check_subtitles() + url = video.json_data["vid_thumb_url"] + ThumbManager(item_id=video.youtube_id).download_video_thumb(url=url) video.upload_to_es() + return video.json_data