diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 1d48c109..c1a150c2 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -446,5 +446,9 @@ class DownloadPostProcess(DownloaderBase): def get_comments(self): """get comments from youtube""" - videos = RedisQueue(self.VIDEO_QUEUE).get_all() - CommentList(videos, task=self.task).index() + video_queue = RedisQueue(self.VIDEO_QUEUE) + comment_list = CommentList(task=self.task) + comment_list.add(video_ids=video_queue.get_all()) + + video_queue.clear() + comment_list.index() diff --git a/tubearchivist/home/src/index/comments.py b/tubearchivist/home/src/index/comments.py index 5dced73d..794cbc35 100644 --- a/tubearchivist/home/src/index/comments.py +++ b/tubearchivist/home/src/index/comments.py @@ -10,6 +10,7 @@ from datetime import datetime from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap from home.src.ta.config import AppConfig +from home.src.ta.ta_redis import RedisQueue class Comments: @@ -189,20 +190,30 @@ class Comments: class CommentList: """interact with comments in group""" - def __init__(self, video_ids, task=False): - self.video_ids = video_ids + COMMENT_QUEUE = "index:comment" + + def __init__(self, task=False): self.task = task self.config = AppConfig().config - def index(self): - """index comments for list, init with task object to notify""" + def add(self, video_ids: list[str]) -> None: + """add list of videos to get comments, if enabled in config""" if not self.config["downloads"].get("comment_max"): return - total_videos = len(self.video_ids) - for idx, youtube_id in enumerate(self.video_ids): + RedisQueue(self.COMMENT_QUEUE).add_list(video_ids) + + def index(self): + """run comment index""" + queue = RedisQueue(self.COMMENT_QUEUE) + while True: + total = queue.max_score() + youtube_id, idx = queue.get_next() + if not youtube_id or not idx or not total: + break + if self.task: - self.notify(idx, total_videos) + self.notify(idx, total) comment = Comments(youtube_id, config=self.config) comment.build_json() @@ -211,6 +222,6 @@ class CommentList: def notify(self, idx, total_videos): """send notification on task""" - message = [f"Add comments for new videos {idx + 1}/{total_videos}"] - progress = (idx + 1) / total_videos + message = [f"Add comments for new videos {idx}/{total_videos}"] + progress = idx / total_videos self.task.send_progress(message, progress=progress) diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index c650cedb..484271a8 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -89,7 +89,9 @@ class Scanner: ) index_new_video(youtube_id) - CommentList(self.to_index, task=self.task).index() + comment_list = CommentList(task=self.task) + comment_list.add(video_ids=list(self.to_index)) + comment_list.index() def url_fix(self) -> None: """ diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/home/src/index/manual.py index a25b0611..aa65ebdb 100644 --- a/tubearchivist/home/src/index/manual.py +++ b/tubearchivist/home/src/index/manual.py @@ -147,7 +147,9 @@ class ImportFolderScanner: ManualImport(current_video, self.CONFIG).run() video_ids = [i["video_id"] for i in self.to_import] - CommentList(video_ids, task=self.task).index() + comment_list = CommentList(task=self.task) + comment_list.add(video_ids=video_ids) + comment_list.index() def _notify(self, idx, current_video): """send notification back to task"""