diff --git a/backend/channel/src/index.py b/backend/channel/src/index.py index 65503ef7..ddb66d96 100644 --- a/backend/channel/src/index.py +++ b/backend/channel/src/index.py @@ -301,7 +301,7 @@ class YoutubeChannel(YouTubeItem): + "/playlists?view=1&sort=dd&shelf_id=0" ) obs = {"skip_download": True, "extract_flat": True} - playlists = YtWrap(obs, self.config).extract(url) + playlists, _ = YtWrap(obs, self.config).extract(url) if not playlists: self.all_playlists = [] return diff --git a/backend/common/src/index_generic.py b/backend/common/src/index_generic.py index 450c1f88..116d4965 100644 --- a/backend/common/src/index_generic.py +++ b/backend/common/src/index_generic.py @@ -26,6 +26,7 @@ class YouTubeItem: self.youtube_id = youtube_id self.es_path = f"{self.index_name}/_doc/{youtube_id}" self.config = AppConfig().config + self.error = None self.youtube_meta = False self.json_data = False @@ -43,7 +44,9 @@ class YouTubeItem: obs_request["extractor_args"] = {"youtube": {"lang": langs_list}} url = self.build_yt_url() - self.youtube_meta = YtWrap(obs_request, self.config).extract(url) + self.youtube_meta, self.error = YtWrap( + obs_request, self.config + ).extract(url) def get_from_es(self): """get indexed data from elastic search""" diff --git a/backend/common/src/urlparser.py b/backend/common/src/urlparser.py index 3db07ad6..3c75c82f 100644 --- a/backend/common/src/urlparser.py +++ b/backend/common/src/urlparser.py @@ -124,9 +124,9 @@ class Parser: "extract_flat": True, "playlistend": 0, } - url_info = YtWrap(obs_request).extract(url) + url_info, error = YtWrap(obs_request).extract(url) if not url_info: - raise ValueError(f"failed to retrieve content from URL: {url}") + raise ValueError(f"failed to retrieve URL: {error}") channel_id = url_info.get("channel_id", False) if channel_id: diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index 3a6f4100..da4bc646 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -369,6 +369,14 @@ class PendingList(PendingIndex): if not video.youtube_meta: print(f"{url}: video metadata extraction failed, skipping") + if self.task: + self.task.send_progress( + message_lines=[ + "Video extraction failed.", + f"{video.error}", + ], + level="error", + ) return None video.youtube_meta["vid_type"] = vid_type @@ -559,5 +567,6 @@ class PendingList(PendingIndex): message_lines=[ "Adding extracted videos failed.", f"Status code: {status_code}", - ] + ], + level="error", ) diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index 524202e5..65080dd7 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -61,7 +61,7 @@ class ChannelSubscription: obs["playlistend"] = limit_amount url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}" - channel_query = YtWrap(obs, self.config).extract(url) + channel_query, _ = YtWrap(obs, self.config).extract(url) if not channel_query: continue diff --git a/backend/download/src/yt_dlp_base.py b/backend/download/src/yt_dlp_base.py index c80f9d7d..68332da0 100644 --- a/backend/download/src/yt_dlp_base.py +++ b/backend/download/src/yt_dlp_base.py @@ -80,30 +80,33 @@ class YtWrap: return True, True - def extract(self, url): - """make extract request""" + def extract(self, url) -> tuple[dict | None, str | None]: + """ + make extract request + returns response, error + """ with yt_dlp.YoutubeDL(self.obs) as ydl: try: response = ydl.extract_info(url) except cookiejar.LoadError as err: print(f"cookie file is invalid: {err}") - return False + return None, str(err) except yt_dlp.utils.ExtractorError as err: print(f"{url}: failed to extract: {err}, continue...") - return False + return None, str(err) except yt_dlp.utils.DownloadError as err: if "This channel does not have a" in str(err): - return False + return None, None print(f"{url}: failed to get info from youtube: {err}") if "Temporary failure in name resolution" in str(err): raise ConnectionError("lost the internet, abort!") from err - return False + return None, str(err) self._validate_cookie() - return response + return response, None def _validate_cookie(self): """check cookie and write it back for next use""" @@ -146,7 +149,7 @@ class CookieHandler: AppConfig().update_config({"downloads": {"cookie_import": False}}) print("[cookie]: revoked") - def validate(self): + def validate(self) -> bool: """validate cookie using the liked videos playlist""" validation = RedisArchivist().get_message_dict("cookie:valid") if validation: @@ -159,8 +162,8 @@ class CookieHandler: "extract_flat": True, } validator = YtWrap(obs_request, self.config) - response = bool(validator.extract("LL")) - self.store_validation(response) + response, error = validator.extract("LL") + self.store_validation(bool(response)) # update in redis to avoid expiring modified = validator.obs["cookiefile"].getvalue().strip("\x00") @@ -173,15 +176,15 @@ class CookieHandler: "status": "message:download", "level": "error", "title": "Cookie validation failed, exiting...", - "message": "", + "message": error, } RedisArchivist().set_message( "message:download", mess_dict, expire=4 ) print("[cookie]: validation failed, exiting...") - print(f"[cookie]: validation success: {response}") - return response + print(f"[cookie]: validation success: {bool(response)}") + return bool(response) @staticmethod def store_validation(response): diff --git a/backend/task/tasks.py b/backend/task/tasks.py index 413c1683..cb76b8e8 100644 --- a/backend/task/tasks.py +++ b/backend/task/tasks.py @@ -39,10 +39,10 @@ class BaseTask(Task): RedisArchivist().set_message(key, message, expire=20) def on_success(self, retval, task_id, args, kwargs): - """callback task completed successfully""" + """callback task completed""" print(f"{task_id} success callback") message, key = self._build_message() - message.update({"messages": ["Task completed successfully"]}) + message.update({"messages": ["Task completed"]}) RedisArchivist().set_message(key, message, expire=5) def before_start(self, task_id, args, kwargs): @@ -58,9 +58,11 @@ class BaseTask(Task): task_title = TASK_CONFIG.get(self.name).get("title") Notifications(self.name).send(task_id, task_title) - def send_progress(self, message_lines, progress=False, title=False): + def send_progress( + self, message_lines, progress=False, title=False, level="info" + ): """send progress message""" - message, key = self._build_message() + message, key = self._build_message(level=level) message.update( { "messages": message_lines, diff --git a/backend/video/src/comments.py b/backend/video/src/comments.py index 2d13448c..4b145a75 100644 --- a/backend/video/src/comments.py +++ b/backend/video/src/comments.py @@ -79,7 +79,9 @@ class Comments: def get_yt_comments(self): """get comments from youtube""" yt_obs = self.build_yt_obs() - info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id) + info_json, _ = YtWrap(yt_obs, config=self.config).extract( + self.youtube_id + ) if not info_json: return False, False