update TA_VERSION v0.3.2

Changed:
- Fixed startup schedule builder for new installations
- Fixed shorts and streams page sizes set to 0

docs/Settings.md

@@ -10,7 +10,7 @@ Switch between the easy on the eyes dark theme and the burning bright theme.
 - **Page Size**: Defines how many results get displayed on a given page. Same value goes for all archive views.
 
 ## Subscriptions
-Settings related to the channel management.
+Settings related to the channel management. Disable shorts or streams by setting their page size to 0 (zero).
 - **Channel Page Size**: Defines how many pages will get analyzed by **Tube Archivist** each time you click on *Rescan Subscriptions*. The default page size used by yt-dlp is **50**, that's also the recommended value to set here. Any value higher will slow down the rescan process, for example if you set the value to 51, that means yt-dlp will have to go through 2 pages of results instead of 1 and by that doubling the time that process takes.
 - **Live Page Size**: Same as above, but for channel live streams.
 - **Shorts page Size**: Same as above, but for shorts videos.

tubearchivist/config/settings.py

@@ -262,4 +262,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.3.1"
+TA_VERSION = "v0.3.2"

tubearchivist/home/apps.py

@@ -27,6 +27,7 @@ class StartupCheck:
     def run(self):
         """run all startup checks"""
         print("run startup checks")
+        self.set_lock()
         self.es_version_check()
         self.release_lock()
         self.sync_redis_state()
@@ -37,14 +38,19 @@ class StartupCheck:
         self.snapshot_check()
         self.ta_version_check()
         self.es_set_vid_type()
-        self.set_has_run()
+        self.expire_lock()
 
     def get_has_run(self):
         """validate if check has already executed"""
         return self.redis_con.get_message("startup_check")
 
-    def set_has_run(self):
+    def set_lock(self):
+        """set lock to avoid executing once per thread"""
+        self.redis_con.set_message("startup_check", message={"status": True})
+
+    def expire_lock(self):
         """startup checks run"""
+        print("startup checks completed")
         message = {"status": True}
         self.redis_con.set_message("startup_check", message, expire=120)
 
@@ -75,7 +81,6 @@ class StartupCheck:
     def release_lock(self):
         """make sure there are no leftover locks set in redis"""
         all_locks = [
-            "startup_check",
             "manual_import",
             "downloading",
             "dl_queue",

tubearchivist/home/src/download/subscriptions.py

@@ -85,6 +85,10 @@ class ChannelSubscription:
             return queries
 
         for query_item, default_limit in limit_map.items():
+            if not default_limit:
+                # is deactivated in config
+                continue
+
             if limit:
                 query_limit = default_limit
             else:

tubearchivist/home/src/es/connect.py

@@ -3,6 +3,7 @@ functionality:
 - wrapper around requests to call elastic search
 - reusable search_after to extract total index
 """
+# pylint: disable=missing-timeout
 
 import json
 
@@ -55,16 +56,10 @@ class ElasticWrap:
 
         if data:
             response = requests.post(
-                self.url,
-                data=payload,
-                headers=headers,
-                auth=self.auth,
-                timeout=10,
+                self.url, data=payload, headers=headers, auth=self.auth
             )
         else:
-            response = requests.post(
-                self.url, headers=headers, auth=self.auth, timeout=10
-            )
+            response = requests.post(self.url, headers=headers, auth=self.auth)
 
         if not response.ok:
             print(response.text)
@@ -75,9 +70,7 @@ class ElasticWrap:
         """put data to es"""
         if refresh:
             self.url = f"{self.url}/?refresh=true"
-        response = requests.put(
-            f"{self.url}", json=data, auth=self.auth, timeout=10
-        )
+        response = requests.put(f"{self.url}", json=data, auth=self.auth)
         if not response.ok:
             print(response.text)
             print(data)
@@ -90,11 +83,9 @@ class ElasticWrap:
         if refresh:
             self.url = f"{self.url}/?refresh=true"
         if data:
-            response = requests.delete(
-                self.url, json=data, auth=self.auth, timeout=10
-            )
+            response = requests.delete(self.url, json=data, auth=self.auth)
         else:
-            response = requests.delete(self.url, auth=self.auth, timeout=10)
+            response = requests.delete(self.url, auth=self.auth)
 
         if not response.ok:
             print(response.text)

tubearchivist/home/src/ta/config.py

@@ -135,6 +135,7 @@ class AppConfig:
         # check for customizations
         if not redis_config:
             config = self.get_config()
+            config["scheduler"]["version_check"] = self._build_rand_daily()
             RedisArchivist().set_message("config", config)
             return
 

tubearchivist/home/tasks.py

@@ -301,7 +301,7 @@ def index_channel_playlists(channel_id):
         "status": key,
         "level": "info",
         "title": "Looking for playlists",
-        "message": f'Scanning channel "{channel.youtube_id}" in progress',
+        "message": f"{channel_id}: Channel scan in progress",
     }
     RedisArchivist().set_message(key, mess_dict, expire=True)
     channel.index_channel_playlists()
@@ -313,9 +313,6 @@ def version_check():
     ReleaseVersion().check()
 
 
-try:
-    app.conf.beat_schedule = ScheduleBuilder().build_schedule()
-except KeyError:
-    # update path to load new defaults
-    StartupCheck().sync_redis_state()
-    app.conf.beat_schedule = ScheduleBuilder().build_schedule()
+# load new defaults then start the schedule here
+StartupCheck().sync_redis_state()
+app.conf.beat_schedule = ScheduleBuilder().build_schedule()

tubearchivist/home/templates/home/settings.html

@@ -33,6 +33,7 @@
         {% csrf_token %}
         <div class="settings-group">
             <h2 id="subscriptions">Subscriptions</h2>
+            <p>Disable shorts or streams by setting their page size to 0 (zero).</p>
             <div class="settings-item">
                 <p>YouTube page size: <span class="settings-current">{{ config.subscriptions.channel_size }}</span></p>
                 <i>Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
@@ -40,12 +41,12 @@
             </div>
             <div class="settings-item">
                 <p>YouTube Live page size: <span class="settings-current">{{ config.subscriptions.live_channel_size }}</span></p>
-                <i>Live Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50, 0 to disable.</i><br>
+                <i>Live Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
                 {{ app_form.subscriptions_live_channel_size }}
             </div>
             <div class="settings-item">
                 <p>YouTube Shorts page size: <span class="settings-current">{{ config.subscriptions.shorts_channel_size }}</span></p>
-                <i>Shorts Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50, 0 to disable.</i><br>
+                <i>Shorts Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
                 {{ app_form.subscriptions_shorts_channel_size }}
             </div>
         </div>