untangle channel_overwrites in video index and queue

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -35,7 +35,7 @@ class VideoDownloader:
 
     def __init__(self, task=False):
         self.obs = False
-        self.video_overwrites = False
+        self.channel_overwrites = get_channel_overwrites()
         self.task = task
         self.config = AppConfig().config
         self._build_obs()
@@ -44,28 +44,24 @@ class VideoDownloader:
 
     def run_queue(self, auto_only=False):
         """setup download queue in redis loop until no more items"""
-        self._get_overwrites()
         while True:
             video_data = self._get_next(auto_only)
             if self.task.is_stopped() or not video_data:
                 self._reset_auto()
                 break
 
-            youtube_id = video_data.get("youtube_id")
+            youtube_id = video_data["youtube_id"]
+            channel_id = video_data["channel_id"]
             print(f"{youtube_id}: Downloading video")
             self._notify(video_data, "Validate download format")
 
-            success = self._dl_single_vid(youtube_id)
+            success = self._dl_single_vid(youtube_id, channel_id)
             if not success:
                 continue
 
             self._notify(video_data, "Add video metadata to index", progress=1)
-
+            video_type = VideoTypeEnum(video_data["vid_type"])
-            vid_dict = index_new_video(
+            vid_dict = index_new_video(youtube_id, video_type=video_type)
-                youtube_id,
-                video_overwrites=self.video_overwrites,
-                video_type=VideoTypeEnum(video_data["vid_type"]),
-            )
             self.channels.add(vid_dict["channel"]["channel_id"])
             self.videos.add(vid_dict["youtube_id"])
 
@@ -112,13 +108,6 @@ class VideoDownloader:
 
         return response["hits"]["hits"][0]["_source"]
 
-    def _get_overwrites(self):
-        """get channel overwrites"""
-        pending = PendingList()
-        pending.get_download()
-        pending.get_channels()
-        self.video_overwrites = pending.video_overwrites
-
     def _progress_hook(self, response):
         """process the progress_hooks from yt_dlp"""
         progress = False
@@ -209,21 +198,16 @@ class VideoDownloader:
 
         self.obs["postprocessors"] = postprocessors
 
-    def get_format_overwrites(self, youtube_id):
+    def _set_overwrites(self, obs: dict, channel_id: str) -> None:
-        """get overwrites from single video"""
+        """add overwrites to obs"""
-        overwrites = self.video_overwrites.get(youtube_id, False)
+        overwrites = self.channel_overwrites.get(channel_id)
-        if overwrites:
+        if overwrites and overwrites.get("download_format"):
-            return overwrites.get("download_format", False)
+            obs["format"] = overwrites.get("download_format")
 
-        return False
+    def _dl_single_vid(self, youtube_id: str, channel_id: str) -> bool:
-
-    def _dl_single_vid(self, youtube_id):
         """download single video"""
         obs = self.obs.copy()
-        format_overwrite = self.get_format_overwrites(youtube_id)
+        self._set_overwrites(obs, channel_id)
-        if format_overwrite:
-            obs["format"] = format_overwrite
-
         dl_cache = os.path.join(self.CACHE_DIR, "download")
 
         # check if already in cache to continue from there

tubearchivist/home/src/index/video.py

@@ -125,15 +125,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
     index_name = "ta_video"
     yt_base = "https://www.youtube.com/watch?v="
 
-    def __init__(
+    def __init__(self, youtube_id, video_type=VideoTypeEnum.VIDEOS):
-        self,
-        youtube_id,
-        video_overwrites=False,
-        video_type=VideoTypeEnum.VIDEOS,
-    ):
         super().__init__(youtube_id)
         self.channel_id = False
-        self.video_overwrites = video_overwrites
         self.video_type = video_type
         self.offline_import = False
 
@@ -165,13 +159,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         """check if need to run sponsor block"""
         integrate = self.config["downloads"]["integrate_sponsorblock"]
 
-        if self.video_overwrites:
+        if overwrite := self.json_data["channel"].get("channel_overwrites"):
-            single_overwrite = self.video_overwrites.get(self.youtube_id)
+            if not overwrite:
-            if not single_overwrite:
                 return integrate
 
-            if "integrate_sponsorblock" in single_overwrite:
+            if "integrate_sponsorblock" in overwrite:
-                return single_overwrite.get("integrate_sponsorblock")
+                return overwrite.get("integrate_sponsorblock")
 
         return integrate
 
@@ -399,13 +392,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         _, _ = ElasticWrap(path).post(data=data)
 
 
-def index_new_video(
+def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
-    youtube_id, video_overwrites=False, video_type=VideoTypeEnum.VIDEOS
-):
     """combined classes to create new video in index"""
-    video = YoutubeVideo(
+    video = YoutubeVideo(youtube_id, video_type=video_type)
-        youtube_id, video_overwrites=video_overwrites, video_type=video_type
-    )
     video.build_json()
     if not video.json_data:
         raise ValueError("failed to get metadata for " + youtube_id)