diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index cb85dfb1..6dfa4fd6 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -96,7 +96,7 @@ class DownloadPostProcess: def validate_playlists(self): """look for playlist needing to update""" for id_c, channel_id in enumerate(self.download.channels): - channel = YoutubeChannel(channel_id) + channel = YoutubeChannel(channel_id, task=self.download.task) overwrites = self.pending.channel_overwrites.get(channel_id, False) if overwrites and overwrites.get("index_playlists"): # validate from remote diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index a7d38907..e3d658a0 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -13,7 +13,6 @@ from home.src.ta.ta_redis import RedisArchivist, RedisQueue from home.src.ta.urlparser import Parser from home.tasks import ( download_pending, - index_channel_playlists, kill_dl, re_sync_thumbs, rescan_filesystem, @@ -62,7 +61,6 @@ class PostData: "db-restore": self._db_restore, "fs-rescan": self._fs_rescan, "delete-playlist": self._delete_playlist, - "find-playlists": self._find_playlists, } return exec_map[self.to_exec] @@ -232,9 +230,3 @@ class PostData: YoutubePlaylist(playlist_id).delete_videos_playlist() return {"success": True} - - def _find_playlists(self): - """add all playlists of a channel""" - channel_id = self.exec_val - index_channel_playlists.delay(channel_id) - return {"success": True} diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 477d1782..d4e6b007 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -18,7 +18,6 @@ from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.generic import YouTubeItem from home.src.index.playlist import YoutubePlaylist from home.src.ta.helper import clean_string, requests_headers -from home.src.ta.ta_redis import RedisArchivist class ChannelScraper: @@ -167,12 +166,12 @@ class YoutubeChannel(YouTubeItem): es_path = False index_name = "ta_channel" yt_base = "https://www.youtube.com/channel/" - msg = "message:playlistscan" - def __init__(self, youtube_id): + def __init__(self, youtube_id, task=False): super().__init__(youtube_id) self.es_path = f"{self.index_name}/_doc/{youtube_id}" self.all_playlists = False + self.task = task def build_json(self, upload=False, fallback=False): """get from es or from youtube""" @@ -324,34 +323,27 @@ class YoutubeChannel(YouTubeItem): print(f"{self.youtube_id}: index all playlists") self.get_from_es() channel_name = self.json_data["channel_name"] - mess_dict = { - "status": self.msg, - "level": "info", - "title": "Looking for playlists", - "message": f"{channel_name}: Scanning channel in progress", - } - RedisArchivist().set_message(self.msg, mess_dict, expire=True) + self.task.send_progress([f"{channel_name}: Looking for Playlists"]) self.get_all_playlists() if not self.all_playlists: print(f"{self.youtube_id}: no playlists found.") return all_youtube_ids = self.get_all_video_ids() + total = len(self.all_playlists) for idx, playlist in enumerate(self.all_playlists): - self._notify_single_playlist(idx, playlist) + self._notify_single_playlist(idx, total) self._index_single_playlist(playlist, all_youtube_ids) + print("add playlist: " + playlist[1]) - def _notify_single_playlist(self, idx, playlist): + def _notify_single_playlist(self, idx, total): """send notification""" channel_name = self.json_data["channel_name"] - mess_dict = { - "status": self.msg, - "level": "info", - "title": f"{channel_name}: Scanning channel for playlists", - "message": f"Progress: {idx + 1}/{len(self.all_playlists)}", - } - RedisArchivist().set_message(self.msg, mess_dict, expire=True) - print("add playlist: " + playlist[1]) + message = [ + f"{channel_name}: Scanning channel for playlists", + f"Progress: {idx + 1}/{total}", + ] + self.task.send_progress(message, progress=(idx + 1) / total) @staticmethod def _index_single_playlist(playlist, all_youtube_ids): diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 8c99181d..66f9465c 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -315,19 +315,10 @@ def subscribe_to(url_str): SubscriptionHandler(url_str).subscribe() -@shared_task(name="index_playlists") -def index_channel_playlists(channel_id): +@shared_task(bind=True, name="index_playlists", base=BaseTask) +def index_channel_playlists(self, channel_id): """add all playlists of channel to index""" - channel = YoutubeChannel(channel_id) - # notify - key = "message:playlistscan" - mess_dict = { - "status": key, - "level": "info", - "title": "Looking for playlists", - "message": f"{channel_id}: Channel scan in progress", - } - RedisArchivist().set_message(key, mess_dict, expire=True) + channel = YoutubeChannel(channel_id, task=self) channel.index_channel_playlists() diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index db735d01..e2ccc6ab 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -619,21 +619,6 @@ class ChannelIdView(ChannelIdBaseView): to_append = {"term": {"player.watched": {"value": False}}} self.data["query"]["bool"]["must"].append(to_append) - @staticmethod - def post(request, channel_id): - """handle post request""" - print(f"handle post from {channel_id}") - channel_overwrite_form = ChannelOverwriteForm(request.POST) - if channel_overwrite_form.is_valid(): - overwrites = channel_overwrite_form.cleaned_data - print(f"{channel_id}: set overwrites {overwrites}") - channel_overwrites(channel_id, overwrites=overwrites) - if overwrites.get("index_playlists") == "1": - index_channel_playlists.delay(channel_id) - - sleep(1) - return redirect("channel_id", channel_id, permanent=True) - class ChannelIdLiveView(ChannelIdView): """resolves to /channel//streams/