From d2649c29c3bd321dcc7e3bfea2195f2a5c1daa5e Mon Sep 17 00:00:00 2001
From: simon <simobilleter@gmail.com>
Date: Sat, 8 Apr 2023 16:00:16 +0700
Subject: [PATCH] add recent videos to reindex queue task

---
 tubearchivist/home/src/index/reindex.py | 26 +++++++++++++++++++++++--
 tubearchivist/home/tasks.py             |  7 +++++--
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py
index baeadfa8..b5ae83cd 100644
--- a/tubearchivist/home/src/index/reindex.py
+++ b/tubearchivist/home/src/index/reindex.py
@@ -49,6 +49,7 @@ class ReindexBase:
     }
 
     MULTIPLY = 1.2
+    DAYS3 = 60 * 60 * 24 * 3
 
     def __init__(self):
         self.config = AppConfig().config
@@ -62,13 +63,34 @@ class ReindexBase:
         RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
 
 
-class ReindexOutdated(ReindexBase):
-    """add outdated documents to reindex queue"""
+class ReindexPopulate(ReindexBase):
+    """add outdated and recent documents to reindex queue"""
 
     def __init__(self):
         super().__init__()
         self.interval = self.config["scheduler"]["check_reindex_days"]
 
+    def add_recent(self):
+        """add recent videos to refresh"""
+        gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
+        must_list = [
+            {"term": {"active": {"value": True}}},
+            {"range": {"published": {"gte": gte}}},
+        ]
+        data = {
+            "size": 10000,
+            "query": {"bool": {"must": must_list}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+        hits = response["hits"]["hits"]
+        if not hits:
+            return
+
+        all_ids = [i["_source"]["youtube_id"] for i in hits]
+        reindex_config = self.REINDEX_CONFIG.get("video")
+        self.populate(all_ids, reindex_config)
+
     def add_outdated(self):
         """add outdated documents"""
         for reindex_config in self.REINDEX_CONFIG.values():
diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py
index f2c5341e..a86c1dc1 100644
--- a/tubearchivist/home/tasks.py
+++ b/tubearchivist/home/tasks.py
@@ -21,7 +21,7 @@ from home.src.es.index_setup import ElasitIndexWrap
 from home.src.index.channel import YoutubeChannel
 from home.src.index.filesystem import Filesystem
 from home.src.index.manual import ImportFolderScanner
-from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated
+from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
 from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
 from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.task_manager import TaskManager
@@ -220,9 +220,12 @@ def check_reindex(self, data=False, extract_videos=False):
     manager.init(self)
     if not data:
         # started from scheduler
+        populate = ReindexPopulate()
         print(f"[task][{self.name}] reindex outdated documents")
+        self.send_progress("Add recent documents to the reindex Queue.")
+        populate.add_recent()
         self.send_progress("Add outdated documents to the reindex Queue.")
-        ReindexOutdated().add_outdated()
+        populate.add_outdated()
 
     Reindex(task=self).reindex_all()