From 770990c568c741f4ee8681894dcec05b325b9139 Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sat, 11 May 2024 21:23:27 +0200
Subject: [PATCH] split playlist parsing in find_missing in playlist

---
 .../home/src/download/subscriptions.py        | 43 ++++++++-----------
 tubearchivist/home/src/index/playlist.py      |  1 +
 tubearchivist/home/src/ta/helper.py           | 19 ++++++++
 3 files changed, 37 insertions(+), 26 deletions(-)

diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py
index 06c4ad0e..143d6628 100644
--- a/tubearchivist/home/src/download/subscriptions.py
+++ b/tubearchivist/home/src/download/subscriptions.py
@@ -12,6 +12,7 @@ from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
+from home.src.ta.helper import is_missing
 from home.src.ta.urlparser import Parser
 
 
@@ -218,27 +219,15 @@ class PlaylistSubscription:
         playlist.json_data["playlist_subscribed"] = subscribe_status
         playlist.upload_to_es()
 
-    @staticmethod
-    def get_to_ignore():
-        """get all youtube_ids already downloaded or ignored"""
-        pending = queue.PendingList()
-        pending.get_download()
-        pending.get_indexed()
-
-        return pending.to_skip
-
     def find_missing(self):
         """find videos in subscribed playlists not downloaded yet"""
         all_playlists = [i["playlist_id"] for i in self.get_playlists()]
         if not all_playlists:
             return False
 
-        to_ignore = self.get_to_ignore()
-
         missing_videos = []
         total = len(all_playlists)
         for idx, playlist_id in enumerate(all_playlists):
-            size_limit = self.config["subscriptions"]["channel_size"]
             playlist = YoutubePlaylist(playlist_id)
             is_active = playlist.update_playlist()
             if not is_active:
@@ -246,27 +235,29 @@ class PlaylistSubscription:
                 continue
 
             playlist_entries = playlist.json_data["playlist_entries"]
+            size_limit = self.config["subscriptions"]["channel_size"]
             if size_limit:
                 del playlist_entries[size_limit:]
 
-            all_missing = [i for i in playlist_entries if not i["downloaded"]]
-
-            for video in all_missing:
-                youtube_id = video["youtube_id"]
-                if youtube_id not in to_ignore:
-                    missing_videos.append(youtube_id)
+            to_check = [
+                i["youtube_id"]
+                for i in playlist_entries
+                if i["downloaded"] is False
+            ]
+            needs_downloading = is_missing(to_check)
+            missing_videos.extend(needs_downloading)
 
             if not self.task:
                 continue
 
-            if self.task:
-                self.task.send_progress(
-                    message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
-                    progress=(idx + 1) / total,
-                )
-                if self.task.is_stopped():
-                    self.task.send_progress(["Received Stop signal."])
-                    break
+            if self.task.is_stopped():
+                self.task.send_progress(["Received Stop signal."])
+                break
+
+            self.task.send_progress(
+                message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
+                progress=(idx + 1) / total,
+            )
 
         return missing_videos
 
diff --git a/tubearchivist/home/src/index/playlist.py b/tubearchivist/home/src/index/playlist.py
index 0af5274f..38866a9f 100644
--- a/tubearchivist/home/src/index/playlist.py
+++ b/tubearchivist/home/src/index/playlist.py
@@ -142,6 +142,7 @@ class YoutubePlaylist(YouTubeItem):
             return False
 
         self.upload_to_es()
+        self.add_vids_to_playlist()
         return True
 
     def build_nav(self, youtube_id):
diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py
index e3089d85..4cecef4a 100644
--- a/tubearchivist/home/src/ta/helper.py
+++ b/tubearchivist/home/src/ta/helper.py
@@ -12,6 +12,7 @@ from datetime import datetime
 from urllib.parse import urlparse
 
 import requests
+from home.src.es.connect import IndexPaginate
 from home.src.ta.settings import EnvironmentSettings
 
 
@@ -222,3 +223,21 @@ def check_stylesheet(stylesheet: str):
         return stylesheet
 
     return "dark.css"
+
+
+def is_missing(
+    to_check: str | list[str], index_name: str = "ta_video,ta_download"
+) -> list[str]:
+    """id or list of ids that are missing from index_name"""
+    if isinstance(to_check, str):
+        to_check = [to_check]
+
+    data = {
+        "query": {"terms": {"youtube_id": to_check}},
+        "_source": ["youtube_id"],
+    }
+    result = IndexPaginate(index_name, data=data).get_results()
+    existing_ids = [i["youtube_id"] for i in result]
+    dl = [i for i in to_check if i not in existing_ids]
+
+    return dl