Unkwnown type fix, page size null fix, #build

Changed:
- Fix for unknow vid_type when adding to the queue
- Fix for resetting page size
- Tests for VideoQueryBuilder

backend/appsettings/serializers.py

@@ -21,10 +21,14 @@ class AppConfigSubSerializer(
 ):
     """serialize app config subscriptions"""
 
-    channel_size = serializers.IntegerField(required=False)
+    channel_size = serializers.IntegerField(required=False, allow_null=True)
-    live_channel_size = serializers.IntegerField(required=False)
+    live_channel_size = serializers.IntegerField(
-    shorts_channel_size = serializers.IntegerField(required=False)
+        required=False, allow_null=True
-    playlist_size = serializers.IntegerField(required=False)
+    )
+    shorts_channel_size = serializers.IntegerField(
+        required=False, allow_null=True
+    )
+    playlist_size = serializers.IntegerField(required=False, allow_null=True)
     auto_start = serializers.BooleanField(required=False)
     extract_flat = serializers.BooleanField(required=False)
 

backend/channel/src/remote_query.py

@@ -84,7 +84,9 @@ class VideoQueryBuilder:
         limit: bool,
     ) -> tuple[VideoTypeEnum, int | None]:
         """Generic query for video page scraping."""
-        if not limit:
+        app_config_size = self.config["subscriptions"].get(config_key)
+        if not limit or app_config_size is None:
+            # treat None as unlimited
             return (video_type, None)
 
         if (
@@ -94,8 +96,8 @@ class VideoQueryBuilder:
             overwrite = self.channel_overwrites[overwrite_key]
             return (video_type, overwrite)
 
-        if overwrite := self.config["subscriptions"].get(config_key):
+        if app_config_size:
-            return (video_type, overwrite)
+            return (video_type, app_config_size)
 
         return (video_type, 0)
 

backend/channel/tests/test_src/test_remote_query.py

@@ -0,0 +1,117 @@
+"""test video query building"""
+
+# pylint: disable=redefined-outer-name
+
+from enum import Enum
+
+import pytest
+from channel.src.remote_query import VideoQueryBuilder
+from video.src.constants import VideoTypeEnum
+
+
+@pytest.fixture
+def default_config():
+    """from appsettings"""
+    return {
+        "subscriptions": {
+            "channel_size": 5,
+            "live_channel_size": 3,
+            "shorts_channel_size": 2,
+        }
+    }
+
+
+@pytest.fixture
+def empty_overwrites():
+    """from channel overwrites"""
+    return {}
+
+
+@pytest.fixture
+def overwrites():
+    """from channel overwrites"""
+    return {
+        "subscriptions_channel_size": 10,
+        "subscriptions_live_channel_size": 0,
+        "subscriptions_shorts_channel_size": None,
+    }
+
+
+def test_build_all_queries_with_limit(default_config, empty_overwrites):
+    """default, empty overwrite"""
+    builder = VideoQueryBuilder(default_config, empty_overwrites)
+    result = builder.build_queries(None, limit=True)
+    expected = [
+        (VideoTypeEnum.VIDEOS, 5),
+        (VideoTypeEnum.STREAMS, 3),
+        (VideoTypeEnum.SHORTS, 2),
+    ]
+    assert result == expected
+
+
+def test_build_all_queries_without_limit(default_config, empty_overwrites):
+    """limit disabled"""
+    builder = VideoQueryBuilder(default_config, empty_overwrites)
+    result = builder.build_queries(None, limit=False)
+    expected = [
+        (VideoTypeEnum.VIDEOS, None),
+        (VideoTypeEnum.STREAMS, None),
+        (VideoTypeEnum.SHORTS, None),
+    ]
+    assert result == expected
+
+
+def test_build_specific_query(default_config, empty_overwrites):
+    """single vid_type"""
+    builder = VideoQueryBuilder(default_config, empty_overwrites)
+    result = builder.build_queries(VideoTypeEnum.VIDEOS)
+    assert result == [(VideoTypeEnum.VIDEOS, 5)]
+
+
+def test_build_multiple_queries(default_config, empty_overwrites):
+    """vid_type list"""
+    builder = VideoQueryBuilder(default_config, empty_overwrites)
+    result = builder.build_queries(
+        [VideoTypeEnum.VIDEOS, VideoTypeEnum.SHORTS]
+    )
+    assert result == [(VideoTypeEnum.VIDEOS, 5), (VideoTypeEnum.SHORTS, 2)]
+
+
+def test_overwrite_applied(default_config, overwrites):
+    """with overwrite from channel config"""
+    builder = VideoQueryBuilder(default_config, overwrites)
+    result = builder.build_queries(None, limit=True)
+    expected = [
+        (VideoTypeEnum.VIDEOS, 10),  # Overwritten
+        # STREAMS is overwritten to 0, should be excluded
+        (VideoTypeEnum.SHORTS, 2),  # None in overwrite, fallback to config
+    ]
+    assert result == expected
+
+
+def test_no_limit_ignores_config_and_overwrites(default_config, overwrites):
+    """no limit single vid_type"""
+    builder = VideoQueryBuilder(default_config, overwrites)
+    result = builder.build_queries([VideoTypeEnum.STREAMS], limit=False)
+    assert result == [(VideoTypeEnum.STREAMS, None)]
+
+
+def test_zero_query_not_included(default_config):
+    """overwrite to zero to disable"""
+    overwrites = {"subscriptions_live_channel_size": 0}
+    builder = VideoQueryBuilder(default_config, overwrites)
+    result = builder.build_queries([VideoTypeEnum.STREAMS], limit=True)
+    assert not result  # Should be skipped due to 0
+
+
+def test_invalid_video_type_is_ignored(default_config):
+    """invalid enum"""
+    builder = VideoQueryBuilder(default_config)
+
+    class FakeEnum(Enum):
+        """invalid"""
+
+        INVALID = "invalid"
+
+    result = builder.build_queries([FakeEnum.INVALID], limit=True)
+    assert not result

backend/download/src/queue.py

@@ -381,7 +381,11 @@ class PendingList(PendingIndex):
 
     def __extract_vid_type(self, video_data) -> str:
         """build vid type"""
-        if "vid_type" in video_data:
+        if (
+            "vid_type" in video_data
+            and video_data["vid_type"]
+            and str(video_data["vid_type"]) in VideoTypeEnum.values_known()
+        ):
             return str(video_data["vid_type"])
 
         if video_data.get("live_status") == "was_live":

backend/download/src/subscriptions.py

@@ -34,28 +34,7 @@ class ChannelSubscription:
         if not all_channels:
             return 0
 
-        all_channel_urls: list[ParsedURLType] = []
+        all_channel_urls = self._process_channel_urls(all_channels)
-
-        for channel in all_channels:
-            channel_tabs = channel["channel_tabs"]
-            if not channel_tabs:
-                continue
-
-            enum = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs]
-            queries = VideoQueryBuilder(
-                config=self.config,
-                channel_overwrites=channel.get("channel_overwrites", {}),
-            ).build_queries(video_type=enum)
-
-            for query in queries:
-                all_channel_urls.append(
-                    ParsedURLType(
-                        type="channel",
-                        url=channel["channel_id"],
-                        vid_type=query[0],
-                        limit=query[1],
-                    )
-                )
 
         if self.task:
             self.task.send_progress([f"Scanning {len(all_channels)} channels"])
@@ -70,6 +49,34 @@ class ChannelSubscription:
 
         return added
 
+    def _process_channel_urls(self, all_channels: list[dict]):
+        """process channels, build queries"""
+
+        all_channel_urls: list[ParsedURLType] = []
+
+        for channel in all_channels:
+            channel_tabs = channel["channel_tabs"]
+            if not channel_tabs:
+                continue
+
+            enums = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs]
+            queries = VideoQueryBuilder(
+                config=self.config,
+                channel_overwrites=channel.get("channel_overwrites", {}),
+            ).build_queries(video_type=enums)
+
+            for query in queries:
+                all_channel_urls.append(
+                    ParsedURLType(
+                        type="channel",
+                        url=channel["channel_id"],
+                        vid_type=query[0],
+                        limit=query[1],
+                    )
+                )
+
+        return all_channel_urls
+
 
 class PlaylistSubscription:
     """scan subscribed playlists for videos to add to pending"""

frontend/src/pages/SettingsApplication.tsx

@@ -96,9 +96,9 @@ const SettingsApplication = () => {
     const { data: cookieStateResponseData } = cookieStateResponse ?? {};
 
     // Subscriptions
-    setVideoPageSize(appSettingsConfigData?.subscriptions.channel_size || null);
+    setVideoPageSize(appSettingsConfigData?.subscriptions.channel_size ?? null);
-    setLivePageSize(appSettingsConfigData?.subscriptions.live_channel_size || null);
+    setLivePageSize(appSettingsConfigData?.subscriptions.live_channel_size ?? null);
-    setShortPageSize(appSettingsConfigData?.subscriptions.shorts_channel_size || null);
+    setShortPageSize(appSettingsConfigData?.subscriptions.shorts_channel_size ?? null);
     setPlaylistPageSize(appSettingsConfigData?.subscriptions.playlist_size || null);
     setIsAutostart(appSettingsConfigData?.subscriptions.auto_start || false);
     setIsExtractFlat(appSettingsConfigData?.subscriptions.extract_flat || false);