refact yt-dlp info extract, show errors

backend/channel/src/index.py

@@ -301,7 +301,7 @@ class YoutubeChannel(YouTubeItem):
             + "/playlists?view=1&sort=dd&shelf_id=0"
         )
         obs = {"skip_download": True, "extract_flat": True}
-        playlists = YtWrap(obs, self.config).extract(url)
+        playlists, _ = YtWrap(obs, self.config).extract(url)
         if not playlists:
             self.all_playlists = []
             return

backend/common/src/index_generic.py

@@ -26,6 +26,7 @@ class YouTubeItem:
         self.youtube_id = youtube_id
         self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.config = AppConfig().config
+        self.error = None
         self.youtube_meta = False
         self.json_data = False
 
@@ -43,7 +44,9 @@ class YouTubeItem:
             obs_request["extractor_args"] = {"youtube": {"lang": langs_list}}
 
         url = self.build_yt_url()
-        self.youtube_meta = YtWrap(obs_request, self.config).extract(url)
+        self.youtube_meta, self.error = YtWrap(
+            obs_request, self.config
+        ).extract(url)
 
     def get_from_es(self):
         """get indexed data from elastic search"""

backend/common/src/urlparser.py

@@ -124,9 +124,9 @@ class Parser:
             "extract_flat": True,
             "playlistend": 0,
         }
-        url_info = YtWrap(obs_request).extract(url)
+        url_info, error = YtWrap(obs_request).extract(url)
         if not url_info:
-            raise ValueError(f"failed to retrieve content from URL: {url}")
+            raise ValueError(f"failed to retrieve URL: {error}")
 
         channel_id = url_info.get("channel_id", False)
         if channel_id:

backend/download/src/queue.py

@@ -369,6 +369,14 @@ class PendingList(PendingIndex):
 
         if not video.youtube_meta:
             print(f"{url}: video metadata extraction failed, skipping")
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[
+                        "Video extraction failed.",
+                        f"{video.error}",
+                    ],
+                    level="error",
+                )
             return None
 
         video.youtube_meta["vid_type"] = vid_type
@@ -559,5 +567,6 @@ class PendingList(PendingIndex):
             message_lines=[
                 "Adding extracted videos failed.",
                 f"Status code: {status_code}",
-            ]
+            ],
+            level="error",
         )

backend/download/src/subscriptions.py

@@ -61,7 +61,7 @@ class ChannelSubscription:
                 obs["playlistend"] = limit_amount
 
             url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
-            channel_query = YtWrap(obs, self.config).extract(url)
+            channel_query, _ = YtWrap(obs, self.config).extract(url)
             if not channel_query:
                 continue
 

backend/download/src/yt_dlp_base.py

@@ -80,30 +80,33 @@ class YtWrap:
 
         return True, True
 
-    def extract(self, url):
+    def extract(self, url) -> tuple[dict | None, str | None]:
-        """make extract request"""
+        """
+        make extract request
+        returns response, error
+        """
         with yt_dlp.YoutubeDL(self.obs) as ydl:
             try:
                 response = ydl.extract_info(url)
             except cookiejar.LoadError as err:
                 print(f"cookie file is invalid: {err}")
-                return False
+                return None, str(err)
             except yt_dlp.utils.ExtractorError as err:
                 print(f"{url}: failed to extract: {err}, continue...")
-                return False
+                return None, str(err)
             except yt_dlp.utils.DownloadError as err:
                 if "This channel does not have a" in str(err):
-                    return False
+                    return None, None
 
                 print(f"{url}: failed to get info from youtube: {err}")
                 if "Temporary failure in name resolution" in str(err):
                     raise ConnectionError("lost the internet, abort!") from err
 
-                return False
+                return None, str(err)
 
         self._validate_cookie()
 
-        return response
+        return response, None
 
     def _validate_cookie(self):
         """check cookie and write it back for next use"""
@@ -146,7 +149,7 @@ class CookieHandler:
         AppConfig().update_config({"downloads": {"cookie_import": False}})
         print("[cookie]: revoked")
 
-    def validate(self):
+    def validate(self) -> bool:
         """validate cookie using the liked videos playlist"""
         validation = RedisArchivist().get_message_dict("cookie:valid")
         if validation:
@@ -159,8 +162,8 @@ class CookieHandler:
             "extract_flat": True,
         }
         validator = YtWrap(obs_request, self.config)
-        response = bool(validator.extract("LL"))
+        response, error = validator.extract("LL")
-        self.store_validation(response)
+        self.store_validation(bool(response))
 
         # update in redis to avoid expiring
         modified = validator.obs["cookiefile"].getvalue().strip("\x00")
@@ -173,15 +176,15 @@ class CookieHandler:
                 "status": "message:download",
                 "level": "error",
                 "title": "Cookie validation failed, exiting...",
-                "message": "",
+                "message": error,
             }
             RedisArchivist().set_message(
                 "message:download", mess_dict, expire=4
             )
             print("[cookie]: validation failed, exiting...")
 
-        print(f"[cookie]: validation success: {response}")
+        print(f"[cookie]: validation success: {bool(response)}")
-        return response
+        return bool(response)
 
     @staticmethod
     def store_validation(response):

backend/task/tasks.py

@@ -39,10 +39,10 @@ class BaseTask(Task):
         RedisArchivist().set_message(key, message, expire=20)
 
     def on_success(self, retval, task_id, args, kwargs):
-        """callback task completed successfully"""
+        """callback task completed"""
         print(f"{task_id} success callback")
         message, key = self._build_message()
-        message.update({"messages": ["Task completed successfully"]})
+        message.update({"messages": ["Task completed"]})
         RedisArchivist().set_message(key, message, expire=5)
 
     def before_start(self, task_id, args, kwargs):
@@ -58,9 +58,11 @@ class BaseTask(Task):
         task_title = TASK_CONFIG.get(self.name).get("title")
         Notifications(self.name).send(task_id, task_title)
 
-    def send_progress(self, message_lines, progress=False, title=False):
+    def send_progress(
+        self, message_lines, progress=False, title=False, level="info"
+    ):
         """send progress message"""
-        message, key = self._build_message()
+        message, key = self._build_message(level=level)
         message.update(
             {
                 "messages": message_lines,

backend/video/src/comments.py

@@ -79,7 +79,9 @@ class Comments:
     def get_yt_comments(self):
         """get comments from youtube"""
         yt_obs = self.build_yt_obs()
-        info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id)
+        info_json, _ = YtWrap(yt_obs, config=self.config).extract(
+            self.youtube_id
+        )
         if not info_json:
             return False, False