Track channel tabs, #build

Changed:
- Added channel_tabs to channel index
- Limit subscription refresh to available tabs
- Fix custom playlists
- Fix empty add to queue response

backend/appsettings/index_mapping.json

@@ -62,6 +62,9 @@
                         }
                     }
                 },
+                "channel_tabs": {
+                    "type": "keyword"
+                },
                 "channel_overwrites": {
                     "properties": {
                         "download_format": {
@@ -107,10 +110,6 @@
                     "type": "text",
                     "index": false
                 },
-                "vid_thumb_base64": {
-                    "type": "text",
-                    "index": false
-                },
                 "date_downloaded": {
                     "type": "date",
                     "format": "epoch_second"
@@ -165,6 +164,9 @@
                                 }
                             }
                         },
+                        "channel_tabs": {
+                            "type": "keyword"
+                        },
                         "channel_overwrites": {
                             "properties": {
                                 "download_format": {

backend/channel/serializers.py

@@ -4,6 +4,7 @@
 
 from common.serializers import PaginationSerializer, ValidateUnknownFieldsMixin
 from rest_framework import serializers
+from video.src.constants import VideoTypeEnum
 
 
 class ChannelOverwriteSerializer(
@@ -45,6 +46,9 @@ class ChannelSerializer(serializers.Serializer):
     channel_tags = serializers.ListField(
         child=serializers.CharField(), required=False
     )
+    channel_tabs = serializers.ListField(
+        child=serializers.ChoiceField(VideoTypeEnum.values_known())
+    )
     channel_views = serializers.IntegerField()
     _index = serializers.CharField(required=False)
     _score = serializers.IntegerField(required=False)

backend/channel/src/index.py

@@ -7,12 +7,14 @@ functionality:
 import os
 from datetime import datetime
 
+from channel.src.remote_query import get_last_channel_videos
 from common.src.env_settings import EnvironmentSettings
 from common.src.es_connect import ElasticWrap, IndexPaginate
 from common.src.helper import rand_sleep
 from common.src.index_generic import YouTubeItem
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import YtWrap
+from video.src.constants import VideoTypeEnum
 
 
 class YoutubeChannel(YouTubeItem):
@@ -68,6 +70,7 @@ class YoutubeChannel(YouTubeItem):
             "channel_thumb_url": self._get_thumb_art(),
             "channel_tvart_url": self._get_tv_art(),
             "channel_views": self.youtube_meta.get("view_count") or 0,
+            "channel_tabs": self.get_channel_tabs(),
         }
 
     def _get_thumb_art(self):
@@ -103,6 +106,31 @@ class YoutubeChannel(YouTubeItem):
 
         return False
 
+    def get_channel_tabs(self) -> list[str]:
+        """get channel tabs"""
+        tabs = VideoTypeEnum.values_known()
+        config_cp = self.config.copy()
+        config_cp["subscriptions"] = {
+            "channel_size": 1,
+            "live_channel_size": 1,
+            "shorts_channel_size": 1,
+        }
+        tabs = []
+        for query_filter in VideoTypeEnum:
+            if query_filter == VideoTypeEnum.UNKNOWN:
+                continue
+
+            videos = get_last_channel_videos(
+                channel_id=self.youtube_id,
+                config=config_cp,
+                limit=True,
+                query_filter=query_filter,
+            )
+            if videos:
+                tabs.append(query_filter.value)
+
+        return tabs
+
     def _video_fallback(self, fallback):
         """use video metadata as fallback"""
         print(f"{self.youtube_id}: fallback to video metadata")

backend/channel/src/remote_query.py

@@ -12,7 +12,9 @@ class VideoQueryBuilder:
         self.channel_overwrites = channel_overwrites or {}
 
     def build_queries(
-        self, video_type: VideoTypeEnum | None, limit: bool = True
+        self,
+        video_type: VideoTypeEnum | list[VideoTypeEnum] | None,
+        limit: bool = True,
     ) -> list[tuple[VideoTypeEnum, int | None]]:
         """Build queries for all or specific video type."""
         query_methods = {
@@ -22,13 +24,21 @@ class VideoQueryBuilder:
         }
 
         if video_type:
-            # build query for specific type
+            # build query for specific type/s
-            query_method = query_methods.get(video_type)
+            if not isinstance(video_type, list):
-            if query_method:
+                video_type = [video_type]
+
+            queries = []
+            for video_type_item in video_type:
+                query_method = query_methods.get(video_type_item)
+                if not query_method:
+                    continue
+
                 query = query_method(limit)
                 if query[1] != 0:
-                    return [query]
+                    queries.append(query)
-                return []
+
+            return queries
 
         # Build and return queries for all video types
         queries = []

backend/common/src/search_processor.py

@@ -160,6 +160,7 @@ class SearchProcess:
                 "playlist_last_refresh": playlist_last_refresh,
             }
         )
+        print(playlist_dict)
 
         return dict(sorted(playlist_dict.items()))
 

backend/config/management/commands/ta_config_backup.py

@@ -1,76 +0,0 @@
-"""backup config for sqlite reset and restore"""
-
-import json
-from pathlib import Path
-
-from django.contrib.auth import get_user_model
-from django.core.management.base import BaseCommand
-from home.models import CustomPeriodicTask
-from home.src.ta.settings import EnvironmentSettings
-from rest_framework.authtoken.models import Token
-
-User = get_user_model()
-
-
-class Command(BaseCommand):
-    """export"""
-
-    help = "Exports all users and their auth tokens to a JSON file"
-    FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
-
-    def handle(self, *args, **kwargs):
-        """entry point"""
-
-        data = {
-            "user_data": self.get_users(),
-            "schedule_data": self.get_schedules(),
-        }
-
-        with open(self.FILE, "w", encoding="utf-8") as json_file:
-            json_file.write(json.dumps(data))
-
-    def get_users(self):
-        """get users"""
-
-        users = User.objects.all()
-
-        user_data = []
-
-        for user in users:
-            user_info = {
-                "username": user.name,
-                "is_staff": user.is_staff,
-                "is_superuser": user.is_superuser,
-                "password": user.password,
-                "tokens": [],
-            }
-
-            try:
-                token = Token.objects.get(user=user)
-                user_info["tokens"] = [token.key]
-            except Token.DoesNotExist:
-                user_info["tokens"] = []
-
-            user_data.append(user_info)
-
-        return user_data
-
-    def get_schedules(self):
-        """get schedules"""
-
-        all_schedules = CustomPeriodicTask.objects.all()
-        schedule_data = []
-
-        for schedule in all_schedules:
-            schedule_info = {
-                "name": schedule.name,
-                "crontab": {
-                    "minute": schedule.crontab.minute,
-                    "hour": schedule.crontab.hour,
-                    "day_of_week": schedule.crontab.day_of_week,
-                },
-            }
-
-            schedule_data.append(schedule_info)
-
-        return schedule_data

backend/config/management/commands/ta_config_restore.py

@@ -1,89 +0,0 @@
-"""restore config from backup"""
-
-import json
-from pathlib import Path
-
-from common.src.env_settings import EnvironmentSettings
-from django.core.management.base import BaseCommand
-from django_celery_beat.models import CrontabSchedule
-from rest_framework.authtoken.models import Token
-from task.models import CustomPeriodicTask
-from task.src.task_config import TASK_CONFIG
-from user.models import Account
-
-
-class Command(BaseCommand):
-    """export"""
-
-    help = "Exports all users and their auth tokens to a JSON file"
-    FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
-
-    def handle(self, *args, **options):
-        """handle"""
-        self.stdout.write("restore users and schedules")
-        data = self.get_config()
-        self.restore_users(data["user_data"])
-        self.restore_schedules(data["schedule_data"])
-        self.stdout.write(
-            self.style.SUCCESS(
-                "    ✓ restore completed. Please restart the container."
-            )
-        )
-
-    def get_config(self) -> dict:
-        """get config from backup"""
-        with open(self.FILE, "r", encoding="utf-8") as json_file:
-            data = json.loads(json_file.read())
-
-        self.stdout.write(
-            self.style.SUCCESS(f"    ✓ json file found: {self.FILE}")
-        )
-
-        return data
-
-    def restore_users(self, user_data: list[dict]) -> None:
-        """restore users from config"""
-        self.stdout.write("delete existing users")
-        Account.objects.all().delete()
-
-        self.stdout.write("recreate users")
-        for user_info in user_data:
-            user = Account.objects.create(
-                name=user_info["username"],
-                is_staff=user_info["is_staff"],
-                is_superuser=user_info["is_superuser"],
-                password=user_info["password"],
-            )
-            for token in user_info["tokens"]:
-                Token.objects.create(user=user, key=token)
-
-            self.stdout.write(
-                self.style.SUCCESS(
-                    f"    ✓ recreated user with name: {user_info['username']}"
-                )
-            )
-
-    def restore_schedules(self, schedule_data: list[dict]) -> None:
-        """restore schedules"""
-        self.stdout.write("delete existing schedules")
-        CustomPeriodicTask.objects.all().delete()
-
-        self.stdout.write("recreate schedules")
-        for schedule in schedule_data:
-            task_name = schedule["name"]
-            description = TASK_CONFIG[task_name].get("title")
-            crontab, _ = CrontabSchedule.objects.get_or_create(
-                minute=schedule["crontab"]["minute"],
-                hour=schedule["crontab"]["hour"],
-                day_of_week=schedule["crontab"]["day_of_week"],
-                timezone=EnvironmentSettings.TZ,
-            )
-            task = CustomPeriodicTask.objects.create(
-                name=task_name,
-                task=task_name,
-                description=description,
-                crontab=crontab,
-            )
-            self.stdout.write(
-                self.style.SUCCESS(f"    ✓ recreated schedule: {task}")
-            )

backend/config/management/commands/ta_index_channel_tags.py

@@ -0,0 +1,36 @@
+"""
+migration for 0.5.4 to 0.5.5
+index channel_tags for subscribed channels
+"""
+
+import time
+
+from channel.src.index import YoutubeChannel
+from common.src.helper import get_channels
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    """command"""
+
+    def handle(self, *args, **kwargs):
+        """handle task"""
+
+        self.stdout.write("channel tags initial index")
+
+        channels = get_channels(subscribed_only=True, source=["channel_id"])
+
+        for es_channel in channels:
+            channel = YoutubeChannel(es_channel["channel_id"])
+            channel.get_from_es()
+            channel_name = channel.json_data["channel_name"]
+            channel_tabs = channel.get_channel_tabs()
+            channel.json_data["channel_tabs"] = channel_tabs
+            channel.upload_to_es()
+            channel.sync_to_videos()
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"    ✓ updated '{channel_name}' tabs: {channel_tabs}"
+                )
+            )
+            time.sleep(5)

backend/config/management/commands/ta_startup.py

@@ -19,11 +19,11 @@ from common.src.ta_redis import RedisArchivist
 from django.core.management.base import BaseCommand, CommandError
 from django.utils import dateformat
 from django_celery_beat.models import CrontabSchedule, PeriodicTasks
-from redis.exceptions import ResponseError
 from task.models import CustomPeriodicTask
 from task.src.config_schedule import ScheduleBuilder
 from task.src.task_manager import TaskManager
 from task.tasks import version_check
+from video.src.constants import VideoTypeEnum
 
 TOPIC = """
 
@@ -49,14 +49,12 @@ class Command(BaseCommand):
         self._version_check()
         self._index_setup()
         self._snapshot_check()
-        self._mig_app_settings()
         self._create_default_schedules()
         self._update_schedule_tz()
         self._init_app_config()
-        self._mig_channel_tags()
-        self._mig_video_channel_tags()
         self._mig_fix_download_channel_indexed()
         self._mig_add_default_playlist_sort()
+        self._mig_set_channel_tabs()
 
     def _make_folders(self):
         """make expected cache folders"""
@@ -159,39 +157,6 @@ class Command(BaseCommand):
         self.stdout.write("[7] setup snapshots")
         ElasticSnapshot().setup()
 
-    def _mig_app_settings(self) -> None:
-        """update from v0.4.13 to v0.5.0, migrate application settings"""
-        self.stdout.write("[MIGRATION] move appconfig to ES")
-        try:
-            config = RedisArchivist().get_message("config")
-        except ResponseError:
-            self.stdout.write(
-                self.style.SUCCESS("    Redis does not support JSON decoding")
-            )
-            return
-
-        if not config or config == {"status": False}:
-            self.stdout.write(
-                self.style.SUCCESS("    no config values to migrate")
-            )
-            return
-
-        path = "ta_config/_doc/appsettings"
-        response, status_code = ElasticWrap(path).post(config)
-
-        if status_code in [200, 201]:
-            self.stdout.write(
-                self.style.SUCCESS("    ✓ migrated appconfig to ES")
-            )
-            RedisArchivist().del_message("config", save=True)
-            return
-
-        message = "    🗙 failed to migrate app config"
-        self.stdout.write(self.style.ERROR(message))
-        self.stdout.write(response)
-        sleep(60)
-        raise CommandError(message)
-
     def _create_default_schedules(self) -> None:
         """create default schedules for new installations"""
         self.stdout.write("[8] create initial schedules")
@@ -293,70 +258,6 @@ class Command(BaseCommand):
             self.style.SUCCESS(f"      Status code: {status_code}")
         )
 
-    def _mig_channel_tags(self) -> None:
-        """update from v0.4.13 to v0.5.0, migrate incorrect data types"""
-        self.stdout.write("[MIGRATION] fix incorrect channel tags types")
-        path = "ta_channel/_update_by_query"
-        data = {
-            "query": {"match": {"channel_tags": False}},
-            "script": {
-                "source": "ctx._source.channel_tags = []",
-                "lang": "painless",
-            },
-        }
-        response, status_code = ElasticWrap(path).post(data)
-        if status_code in [200, 201]:
-            updated = response.get("updated")
-            if updated:
-                self.stdout.write(
-                    self.style.SUCCESS(f"    ✓ fixed {updated} channel tags")
-                )
-            else:
-                self.stdout.write(
-                    self.style.SUCCESS("    no channel tags needed fixing")
-                )
-            return
-
-        message = "    🗙 failed to fix channel tags"
-        self.stdout.write(self.style.ERROR(message))
-        self.stdout.write(response)
-        sleep(60)
-        raise CommandError(message)
-
-    def _mig_video_channel_tags(self) -> None:
-        """update from v0.4.13 to v0.5.0, migrate incorrect data types"""
-        self.stdout.write("[MIGRATION] fix incorrect video channel tags types")
-        path = "ta_video/_update_by_query"
-        data = {
-            "query": {"match": {"channel.channel_tags": False}},
-            "script": {
-                "source": "ctx._source.channel.channel_tags = []",
-                "lang": "painless",
-            },
-        }
-        response, status_code = ElasticWrap(path).post(data)
-        if status_code in [200, 201]:
-            updated = response.get("updated")
-            if updated:
-                self.stdout.write(
-                    self.style.SUCCESS(
-                        f"    ✓ fixed {updated} video channel tags"
-                    )
-                )
-            else:
-                self.stdout.write(
-                    self.style.SUCCESS(
-                        "    no video channel tags needed fixing"
-                    )
-                )
-            return
-
-        message = "    🗙 failed to fix video channel tags"
-        self.stdout.write(self.style.ERROR(message))
-        self.stdout.write(response)
-        sleep(60)
-        raise CommandError(message)
-
     def _mig_fix_download_channel_indexed(self) -> None:
         """migrate from v0.5.2 to 0.5.3, fix missing channel_indexed"""
         self.stdout.write("[MIGRATION] fix incorrect video channel tags types")
@@ -424,3 +325,37 @@ class Command(BaseCommand):
         self.stdout.write(response)
         sleep(60)
         raise CommandError(message)
+
+    def _mig_set_channel_tabs(self) -> None:
+        """migrate from 0.5.4 to 0.5.5 set initial channel tabs"""
+        self.stdout.write("[MIGRATION] set default channel_tabs")
+
+        path = "ta_channel/_update_by_query"
+        tabs = VideoTypeEnum.values_known()
+        data = {
+            "query": {
+                "bool": {"must_not": [{"exists": {"field": "channel_tabs"}}]}
+            },
+            "script": {
+                "source": f"ctx._source.channel_tabs = {tabs}",
+                "lang": "painless",
+            },
+        }
+        response, status_code = ElasticWrap(path).post(data)
+        if status_code in [200, 201]:
+            updated = response.get("updated")
+            if updated:
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ updated {updated} channels")
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS("    no channels need updating")
+                )
+            return
+
+        message = "    🗙 failed to set default channel_tabs"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)

backend/download/src/queue.py

@@ -165,7 +165,7 @@ class PendingList(PendingIndex):
         else:
             raise ValueError(f"invalid url_type: {entry}")
 
-    def _add_video(self, url, vid_type):
+    def _add_video(self, url, vid_type) -> dict | None:
         """add video to list"""
         if self.auto_start and url in set(
             i["youtube_id"] for i in self.all_pending
@@ -175,10 +175,11 @@ class PendingList(PendingIndex):
 
         if url in self.missing_videos or url in self.to_skip:
             print(f"{url}: skipped adding already indexed video to download.")
-        else:
+            return None
-            to_add = self._parse_video(url, vid_type)
+
-            if to_add:
+        to_add = self._parse_video(url, vid_type)
-                self.missing_videos.append(to_add)
+        if to_add:
+            self.missing_videos.append(to_add)
 
         return to_add
 

backend/download/src/subscriptions.py

@@ -28,7 +28,8 @@ class ChannelSubscription:
             self.task.send_progress(["Looking up channels."])
 
         all_channels = get_channels(
-            subscribed_only=True, source=["channel_id", "channel_overwrites"]
+            subscribed_only=True,
+            source=["channel_id", "channel_overwrites", "channel_tabs"],
         )
         if not all_channels:
             return 0
@@ -36,10 +37,15 @@ class ChannelSubscription:
         all_channel_urls: list[ParsedURLType] = []
 
         for channel in all_channels:
+            channel_tabs = channel["channel_tabs"]
+            if not channel_tabs:
+                continue
+
+            enum = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs]
             queries = VideoQueryBuilder(
                 config=self.config,
                 channel_overwrites=channel.get("channel_overwrites", {}),
-            ).build_queries(video_type=None)
+            ).build_queries(video_type=enum)
 
             for query in queries:
                 all_channel_urls.append(

backend/playlist/src/index.py

@@ -325,6 +325,7 @@ class YoutubePlaylist(YouTubeItem):
         self.delete_metadata()
 
     def create(self, name):
+        """create custom playlist"""
         self.json_data = {
             "playlist_id": self.youtube_id,
             "playlist_active": False,
@@ -337,6 +338,7 @@ class YoutubePlaylist(YouTubeItem):
             "playlist_description": False,
             "playlist_thumbnail": False,
             "playlist_subscribed": False,
+            "playlist_sort_order": "top",
         }
         self.upload_to_es()
         self.get_playlist_art()

backend/playlist/src/query_building.py

@@ -45,7 +45,7 @@ class QueryBuilder:
 
         type_parsed = getattr(PlaylistTypesEnum, playlist_type.upper()).value
 
-        return {"match": {"playlist_type.keyword": type_parsed}}
+        return {"match": {"playlist_type": type_parsed}}
 
     def parse_sort(self) -> dict:
         """return sort"""

backend/playlist/tests/test_src/test_query_building.py

@@ -27,4 +27,4 @@ def test_parse_type():
         qb.parse_type("invalid")
 
     result = qb.parse_type("custom")
-    assert result == {"match": {"playlist_type.keyword": "custom"}}
+    assert result == {"match": {"playlist_type": "custom"}}

backend/playlist/views.py

@@ -239,6 +239,12 @@ class PlaylistApiView(ApiBaseView):
             error = ErrorResponseSerializer({"error": "playlist not found"})
             return Response(error.data, status=404)
 
+        if self.response["playlist_type"] == "custom":
+            error = ErrorResponseSerializer(
+                {"error": f"playlist with ID {playlist_id} is custom"}
+            )
+            return Response(error.data, status=400)
+
         subscribed = validated_data.get("playlist_subscribed")
         sort_order = validated_data.get("playlist_sort_order")
 

backend/video/src/index.py

@@ -174,15 +174,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         # extract
         self.channel_id = self.youtube_meta["channel_id"]
         last_refresh = int(datetime.now().timestamp())
-        # base64_blur = ThumbManager().get_base64_blur(self.youtube_id)
-        base64_blur = False
         # build json_data basics
         self.json_data = {
             "title": self.youtube_meta["title"],
             "description": self.youtube_meta.get("description", ""),
             "category": self.youtube_meta.get("categories", []),
             "vid_thumb_url": self.youtube_meta["thumbnail"],
-            "vid_thumb_base64": base64_blur,
             "tags": self.youtube_meta.get("tags", []),
             "published": self._build_published(),
             "vid_last_refresh": last_refresh,

frontend/src/pages/Home.tsx

@@ -74,7 +74,6 @@ export type VideoType = {
   tags: string[];
   title: string;
   vid_last_refresh: string;
-  vid_thumb_base64: boolean;
   vid_thumb_url: string;
   vid_type: string;
   youtube_id: string;

frontend/src/pages/Playlist.tsx

@@ -288,31 +288,33 @@ const Playlist = () => {
                   />
                 </div>
               )}
-              <div className="toggle">
+              {playlist.playlist_type !== 'custom' && (
-                <span>Switch sort order:</span>
+                <div className="toggle">
-                <div className="toggleBox">
+                  <span>Switch sort order:</span>
-                  <input
+                  <div className="toggleBox">
-                    id="playlist_sort_order"
+                    <input
-                    type="checkbox"
+                      id="playlist_sort_order"
-                    checked={playlist.playlist_sort_order === 'bottom'}
+                      type="checkbox"
-                    onChange={async () => {
+                      checked={playlist.playlist_sort_order === 'bottom'}
-                      const newSortOrder =
+                      onChange={async () => {
-                        playlist.playlist_sort_order === 'top' ? 'bottom' : 'top';
+                        const newSortOrder =
-                      await updatePlaylistSortOrder(playlist.playlist_id, newSortOrder);
+                          playlist.playlist_sort_order === 'top' ? 'bottom' : 'top';
-                      setRefresh(true);
+                        await updatePlaylistSortOrder(playlist.playlist_id, newSortOrder);
-                    }}
+                        setRefresh(true);
-                  />
+                      }}
-                  {playlist.playlist_sort_order === 'bottom' ? (
+                    />
-                    <label htmlFor="" className="onbtn">
+                    {playlist.playlist_sort_order === 'bottom' ? (
-                      On
+                      <label htmlFor="" className="onbtn">
-                    </label>
+                        On
-                  ) : (
+                      </label>
-                    <label htmlFor="" className="ofbtn">
+                    ) : (
-                      Off
+                      <label htmlFor="" className="ofbtn">
-                    </label>
+                        Off
-                  )}
+                      </label>
+                    )}
+                  </div>
                 </div>
-              </div>
+              )}
             </div>
           </div>
         </div>