renamed django app folder to backend

backend/playlist/src/constants.py

@@ -0,0 +1,10 @@
+"""playlist constants"""
+
+import enum
+
+
+class PlaylistTypesEnum(enum.Enum):
+    """all playlist_type options"""
+
+    REGULAR = "regular"
+    CUSTOM = "custom"

backend/playlist/src/index.py

@@ -0,0 +1,443 @@
+"""
+functionality:
+- get metadata from youtube for a playlist
+- index and update in es
+"""
+
+import json
+from datetime import datetime
+
+from channel.src import index as channel
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.index_generic import YouTubeItem
+from download.src.thumbnails import ThumbManager
+from video.src import index as ta_video
+
+
+class YoutubePlaylist(YouTubeItem):
+    """represents a single youtube playlist"""
+
+    es_path = False
+    index_name = "ta_playlist"
+    yt_obs = {
+        "extract_flat": True,
+        "allow_playlist_files": True,
+    }
+    yt_base = "https://www.youtube.com/playlist?list="
+
+    def __init__(self, youtube_id):
+        super().__init__(youtube_id)
+        self.all_members = False
+        self.nav = False
+
+    def build_json(self, scrape=False):
+        """collection to create json_data"""
+        self.get_from_es()
+        if self.json_data:
+            subscribed = self.json_data.get("playlist_subscribed")
+        else:
+            subscribed = False
+
+        if scrape or not self.json_data:
+            self.get_from_youtube()
+            if not self.youtube_meta:
+                self.json_data = False
+                return
+
+            self.process_youtube_meta()
+            self._ensure_channel()
+            ids_found = self.get_local_vids()
+            self.get_entries(ids_found)
+            self.json_data["playlist_entries"] = self.all_members
+            self.json_data["playlist_subscribed"] = subscribed
+
+    def process_youtube_meta(self):
+        """extract relevant fields from youtube"""
+        try:
+            playlist_thumbnail = self.youtube_meta["thumbnails"][-1]["url"]
+        except IndexError:
+            print(f"{self.youtube_id}: thumbnail extraction failed")
+            playlist_thumbnail = False
+
+        self.json_data = {
+            "playlist_id": self.youtube_id,
+            "playlist_active": True,
+            "playlist_name": self.youtube_meta["title"],
+            "playlist_channel": self.youtube_meta["channel"],
+            "playlist_channel_id": self.youtube_meta["channel_id"],
+            "playlist_thumbnail": playlist_thumbnail,
+            "playlist_description": self.youtube_meta["description"] or False,
+            "playlist_last_refresh": int(datetime.now().timestamp()),
+            "playlist_type": "regular",
+        }
+
+    def _ensure_channel(self):
+        """make sure channel is indexed"""
+        channel_id = self.json_data["playlist_channel_id"]
+        channel_handler = channel.YoutubeChannel(channel_id)
+        channel_handler.build_json(upload=True)
+
+    def get_local_vids(self) -> list[str]:
+        """get local video ids from youtube entries"""
+        entries = self.youtube_meta["entries"]
+        data = {
+            "query": {"terms": {"youtube_id": [i["id"] for i in entries]}},
+            "_source": ["youtube_id"],
+        }
+        indexed_vids = IndexPaginate("ta_video", data).get_results()
+        ids_found = [i["youtube_id"] for i in indexed_vids]
+
+        return ids_found
+
+    def get_entries(self, ids_found) -> None:
+        """get all videos in playlist, match downloaded with ids_found"""
+        all_members = []
+        for idx, entry in enumerate(self.youtube_meta["entries"]):
+            if not entry["channel"]:
+                continue
+
+            to_append = {
+                "youtube_id": entry["id"],
+                "title": entry["title"],
+                "uploader": entry["channel"],
+                "idx": idx,
+                "downloaded": entry["id"] in ids_found,
+            }
+            all_members.append(to_append)
+
+        self.all_members = all_members
+
+    def get_playlist_art(self):
+        """download artwork of playlist"""
+        url = self.json_data["playlist_thumbnail"]
+        ThumbManager(self.youtube_id, item_type="playlist").download(url)
+
+    def add_vids_to_playlist(self):
+        """sync the playlist id to videos"""
+        script = (
+            'if (!ctx._source.containsKey("playlist")) '
+            + "{ctx._source.playlist = [params.playlist]} "
+            + "else if (!ctx._source.playlist.contains(params.playlist)) "
+            + "{ctx._source.playlist.add(params.playlist)} "
+            + "else {ctx.op = 'none'}"
+        )
+
+        bulk_list = []
+        for entry in self.json_data["playlist_entries"]:
+            video_id = entry["youtube_id"]
+            action = {"update": {"_id": video_id, "_index": "ta_video"}}
+            source = {
+                "script": {
+                    "source": script,
+                    "lang": "painless",
+                    "params": {"playlist": self.youtube_id},
+                }
+            }
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(source))
+
+        # add last newline
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+
+        ElasticWrap("_bulk").post(query_str, ndjson=True)
+
+    def remove_vids_from_playlist(self):
+        """remove playlist ids from videos if needed"""
+        needed = [i["youtube_id"] for i in self.json_data["playlist_entries"]]
+        data = {
+            "query": {"match": {"playlist": self.youtube_id}},
+            "_source": ["youtube_id"],
+        }
+        result = IndexPaginate("ta_video", data).get_results()
+        to_remove = [
+            i["youtube_id"] for i in result if i["youtube_id"] not in needed
+        ]
+        s = "ctx._source.playlist.removeAll(Collections.singleton(params.rm))"
+        for video_id in to_remove:
+            query = {
+                "script": {
+                    "source": s,
+                    "lang": "painless",
+                    "params": {"rm": self.youtube_id},
+                },
+                "query": {"match": {"youtube_id": video_id}},
+            }
+            path = "ta_video/_update_by_query"
+            _, status_code = ElasticWrap(path).post(query)
+            if status_code == 200:
+                print(f"{self.youtube_id}: removed {video_id} from playlist")
+
+    def update_playlist(self, skip_on_empty=False):
+        """update metadata for playlist with data from YouTube"""
+        self.build_json(scrape=True)
+        if not self.json_data:
+            # return false to deactivate
+            return False
+
+        if skip_on_empty:
+            has_item_downloaded = any(
+                i["downloaded"] for i in self.json_data["playlist_entries"]
+            )
+            if not has_item_downloaded:
+                return True
+
+        self.upload_to_es()
+        self.add_vids_to_playlist()
+        self.remove_vids_from_playlist()
+        self.get_playlist_art()
+        return True
+
+    def build_nav(self, youtube_id):
+        """find next and previous in playlist of a given youtube_id"""
+        cache_root = EnvironmentSettings().get_cache_root()
+        all_entries_available = self.json_data["playlist_entries"]
+        all_entries = [i for i in all_entries_available if i["downloaded"]]
+        current = [i for i in all_entries if i["youtube_id"] == youtube_id]
+        # stop if not found or playlist of 1
+        if not current or not len(all_entries) > 1:
+            return
+
+        current_idx = all_entries.index(current[0])
+        if current_idx == 0:
+            previous_item = False
+        else:
+            previous_item = all_entries[current_idx - 1]
+            prev_id = previous_item["youtube_id"]
+            prev_thumb_path = ThumbManager(prev_id).vid_thumb_path()
+            previous_item["vid_thumb"] = f"{cache_root}/{prev_thumb_path}"
+
+        if current_idx == len(all_entries) - 1:
+            next_item = False
+        else:
+            next_item = all_entries[current_idx + 1]
+            next_id = next_item["youtube_id"]
+            next_thumb_path = ThumbManager(next_id).vid_thumb_path()
+            next_item["vid_thumb"] = f"{cache_root}/{next_thumb_path}"
+
+        self.nav = {
+            "playlist_meta": {
+                "current_idx": current[0]["idx"],
+                "playlist_id": self.youtube_id,
+                "playlist_name": self.json_data["playlist_name"],
+                "playlist_channel": self.json_data["playlist_channel"],
+            },
+            "playlist_previous": previous_item,
+            "playlist_next": next_item,
+        }
+        return
+
+    def delete_metadata(self):
+        """delete metadata for playlist"""
+        self.delete_videos_metadata()
+        script = (
+            "ctx._source.playlist.removeAll("
+            + "Collections.singleton(params.playlist)) "
+        )
+        data = {
+            "query": {
+                "term": {"playlist.keyword": {"value": self.youtube_id}}
+            },
+            "script": {
+                "source": script,
+                "lang": "painless",
+                "params": {"playlist": self.youtube_id},
+            },
+        }
+        _, _ = ElasticWrap("ta_video/_update_by_query").post(data)
+        self.del_in_es()
+
+    def is_custom_playlist(self):
+        self.get_from_es()
+        return self.json_data["playlist_type"] == "custom"
+
+    def delete_videos_metadata(self, channel_id=None):
+        """delete video metadata for a specific channel"""
+        self.get_from_es()
+        playlist = self.json_data["playlist_entries"]
+        i = 0
+        while i < len(playlist):
+            video_id = playlist[i]["youtube_id"]
+            video = ta_video.YoutubeVideo(video_id)
+            video.get_from_es()
+            if (
+                channel_id is None
+                or video.json_data["channel"]["channel_id"] == channel_id
+            ):
+                playlist.pop(i)
+                self.remove_playlist_from_video(video_id)
+                i -= 1
+            i += 1
+        self.set_playlist_thumbnail()
+        self.upload_to_es()
+
+    def delete_videos_playlist(self):
+        """delete playlist with all videos"""
+        print(f"{self.youtube_id}: delete playlist")
+        self.get_from_es()
+        all_youtube_id = [
+            i["youtube_id"]
+            for i in self.json_data["playlist_entries"]
+            if i["downloaded"]
+        ]
+        for youtube_id in all_youtube_id:
+            ta_video.YoutubeVideo(youtube_id).delete_media_file()
+
+        self.delete_metadata()
+
+    def create(self, name):
+        self.json_data = {
+            "playlist_id": self.youtube_id,
+            "playlist_active": False,
+            "playlist_name": name,
+            "playlist_last_refresh": int(datetime.now().timestamp()),
+            "playlist_entries": [],
+            "playlist_type": "custom",
+            "playlist_channel": None,
+            "playlist_channel_id": None,
+            "playlist_description": False,
+            "playlist_thumbnail": False,
+            "playlist_subscribed": False,
+        }
+        self.upload_to_es()
+        self.get_playlist_art()
+        return True
+
+    def add_video_to_playlist(self, video_id):
+        self.get_from_es()
+        video_metadata = self.get_video_metadata(video_id)
+        video_metadata["idx"] = len(self.json_data["playlist_entries"])
+
+        if not self.playlist_entries_contains(video_id):
+            self.json_data["playlist_entries"].append(video_metadata)
+            self.json_data["playlist_last_refresh"] = int(
+                datetime.now().timestamp()
+            )
+            self.set_playlist_thumbnail()
+            self.upload_to_es()
+            video = ta_video.YoutubeVideo(video_id)
+            video.get_from_es()
+            if "playlist" not in video.json_data:
+                video.json_data["playlist"] = []
+            video.json_data["playlist"].append(self.youtube_id)
+            video.upload_to_es()
+        return True
+
+    def remove_playlist_from_video(self, video_id):
+        video = ta_video.YoutubeVideo(video_id)
+        video.get_from_es()
+        if video.json_data is not None and "playlist" in video.json_data:
+            video.json_data["playlist"].remove(self.youtube_id)
+            video.upload_to_es()
+
+    def move_video(self, video_id, action, hide_watched=False):
+        self.get_from_es()
+        video_index = self.get_video_index(video_id)
+        playlist = self.json_data["playlist_entries"]
+        item = playlist[video_index]
+        playlist.pop(video_index)
+        if action == "remove":
+            self.remove_playlist_from_video(item["youtube_id"])
+        else:
+            if action == "up":
+                while True:
+                    video_index = max(0, video_index - 1)
+                    if (
+                        not hide_watched
+                        or video_index == 0
+                        or (
+                            not self.get_video_is_watched(
+                                playlist[video_index]["youtube_id"]
+                            )
+                        )
+                    ):
+                        break
+            elif action == "down":
+                while True:
+                    video_index = min(len(playlist), video_index + 1)
+                    if (
+                        not hide_watched
+                        or video_index == len(playlist)
+                        or (
+                            not self.get_video_is_watched(
+                                playlist[video_index - 1]["youtube_id"]
+                            )
+                        )
+                    ):
+                        break
+            elif action == "top":
+                video_index = 0
+            else:
+                video_index = len(playlist)
+            playlist.insert(video_index, item)
+        self.json_data["playlist_last_refresh"] = int(
+            datetime.now().timestamp()
+        )
+
+        for i, item in enumerate(playlist):
+            item["idx"] = i
+
+        self.set_playlist_thumbnail()
+        self.upload_to_es()
+
+        return True
+
+    def del_video(self, video_id):
+        playlist = self.json_data["playlist_entries"]
+
+        i = 0
+        while i < len(playlist):
+            if video_id == playlist[i]["youtube_id"]:
+                playlist.pop(i)
+                self.set_playlist_thumbnail()
+                i -= 1
+            i += 1
+
+    def get_video_index(self, video_id):
+        for i, child in enumerate(self.json_data["playlist_entries"]):
+            if child["youtube_id"] == video_id:
+                return i
+        return -1
+
+    def playlist_entries_contains(self, video_id):
+        return (
+            len(
+                list(
+                    filter(
+                        lambda x: x["youtube_id"] == video_id,
+                        self.json_data["playlist_entries"],
+                    )
+                )
+            )
+            > 0
+        )
+
+    def get_video_is_watched(self, video_id):
+        video = ta_video.YoutubeVideo(video_id)
+        video.get_from_es()
+        return video.json_data["player"]["watched"]
+
+    def set_playlist_thumbnail(self):
+        playlist = self.json_data["playlist_entries"]
+        self.json_data["playlist_thumbnail"] = False
+
+        for video in playlist:
+            url = ThumbManager(video["youtube_id"]).vid_thumb_path()
+            if url is not None:
+                self.json_data["playlist_thumbnail"] = url
+                break
+        self.get_playlist_art()
+
+    def get_video_metadata(self, video_id):
+        video = ta_video.YoutubeVideo(video_id)
+        video.get_from_es()
+        video_json_data = {
+            "youtube_id": video.json_data["youtube_id"],
+            "title": video.json_data["title"],
+            "uploader": video.json_data["channel"]["channel_name"],
+            "idx": 0,
+            "downloaded": "date_downloaded" in video.json_data
+            and video.json_data["date_downloaded"] > 0,
+        }
+        return video_json_data

backend/playlist/src/query_building.py

@@ -0,0 +1,53 @@
+"""build query for playlists"""
+
+from playlist.src.constants import PlaylistTypesEnum
+
+
+class QueryBuilder:
+    """contain functionality"""
+
+    def __init__(self, **kwargs):
+        self.request_params = kwargs
+
+    def build_data(self) -> dict:
+        """build data dict"""
+        data = {}
+        data["query"] = self.build_query()
+        if sort := self.parse_sort():
+            data.update(sort)
+
+        return data
+
+    def build_query(self) -> dict:
+        """build query key"""
+        must_list = []
+        channel = self.request_params.get("channel")
+        if channel:
+            must_list.append({"match": {"playlist_channel_id": channel[0]}})
+
+        subscribed = self.request_params.get("subscribed")
+        if subscribed:
+            subed_bool = subscribed[0] == "true"
+            must_list.append({"match": {"playlist_subscribed": subed_bool}})
+
+        playlist_type = self.request_params.get("type")
+        if playlist_type:
+            type_list = self.parse_type(playlist_type[0])
+            must_list.append(type_list)
+
+        query = {"bool": {"must": must_list}}
+
+        return query
+
+    def parse_type(self, playlist_type: str) -> dict:
+        """parse playlist type"""
+        if not hasattr(PlaylistTypesEnum, playlist_type.upper()):
+            raise ValueError(f"'{playlist_type}' not in PlaylistTypesEnum")
+
+        type_parsed = getattr(PlaylistTypesEnum, playlist_type.upper()).value
+
+        return {"match": {"playlist_type.keyword": type_parsed}}
+
+    def parse_sort(self) -> dict:
+        """return sort"""
+        return {"sort": [{"playlist_name.keyword": {"order": "asc"}}]}

backend/playlist/tests/test_src/test_query_building.py

@@ -0,0 +1,30 @@
+"""test playlist query building"""
+
+import pytest
+from playlist.src.query_building import QueryBuilder
+
+
+def test_build_data():
+    """test for correct key building"""
+    qb = QueryBuilder(
+        channel=["test_channel"],
+        subscribed=["true"],
+        type=["regular"],
+    )
+    result = qb.build_data()
+    must_list = result["query"]["bool"]["must"]
+    assert "query" in result
+    assert "sort" in result
+    assert result["sort"] == [{"playlist_name.keyword": {"order": "asc"}}]
+    assert {"match": {"playlist_channel_id": "test_channel"}} in must_list
+    assert {"match": {"playlist_subscribed": True}} in must_list
+
+
+def test_parse_type():
+    """validate type"""
+    qb = QueryBuilder(type=["regular"])
+    with pytest.raises(ValueError):
+        qb.parse_type("invalid")
+
+    result = qb.parse_type("custom")
+    assert result == {"match": {"playlist_type.keyword": "custom"}}

backend/playlist/urls.py

@@ -0,0 +1,17 @@
+"""all playlist API urls"""
+
+from django.urls import path
+from playlist import views
+
+urlpatterns = [
+    path(
+        "",
+        views.PlaylistApiListView.as_view(),
+        name="api-playlist-list",
+    ),
+    path(
+        "<slug:playlist_id>/",
+        views.PlaylistApiView.as_view(),
+        name="api-playlist",
+    ),
+]

backend/playlist/views.py

@@ -0,0 +1,116 @@
+"""all playlist API views"""
+
+from common.views_base import AdminWriteOnly, ApiBaseView
+from download.src.subscriptions import PlaylistSubscription
+from playlist.src.index import YoutubePlaylist
+from playlist.src.query_building import QueryBuilder
+from rest_framework import status
+from rest_framework.response import Response
+from task.tasks import subscribe_to
+from user.src.user_config import UserConfig
+
+
+class PlaylistApiListView(ApiBaseView):
+    """resolves to /api/playlist/
+    GET: returns list of indexed playlists
+    params:
+    - channel:str=<channel-id>
+    - subscribed: bool
+    - type:enum=regular|custom
+    POST: change subscribe state
+    """
+
+    search_base = "ta_playlist/_search/"
+    permission_classes = [AdminWriteOnly]
+
+    def get(self, request):
+        """get request"""
+        try:
+            data = QueryBuilder(**request.GET).build_data()
+        except ValueError as err:
+            return Response({"error": str(err)}, status=400)
+
+        self.data = data
+        self.get_document_list(request)
+
+        return Response(self.response)
+
+    def post(self, request):
+        """subscribe/unsubscribe to list of playlists"""
+        data = request.data
+        try:
+            to_add = data["data"]
+        except KeyError:
+            message = "missing expected data key"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        pending = []
+        for playlist_item in to_add:
+            playlist_id = playlist_item["playlist_id"]
+            if playlist_item["playlist_subscribed"]:
+                pending.append(playlist_id)
+            else:
+                self._unsubscribe(playlist_id)
+
+        if pending:
+            url_str = " ".join(pending)
+            subscribe_to.delay(url_str, expected_type="playlist")
+
+        return Response(data)
+
+    @staticmethod
+    def _unsubscribe(playlist_id: str):
+        """unsubscribe"""
+        print(f"[{playlist_id}] unsubscribe from playlist")
+        PlaylistSubscription().change_subscribe(
+            playlist_id, subscribe_status=False
+        )
+
+
+class PlaylistApiView(ApiBaseView):
+    """resolves to /api/playlist/<playlist_id>/
+    GET: returns metadata dict of playlist
+    """
+
+    search_base = "ta_playlist/_doc/"
+    permission_classes = [AdminWriteOnly]
+    valid_custom_actions = ["create", "remove", "up", "down", "top", "bottom"]
+
+    def get(self, request, playlist_id):
+        # pylint: disable=unused-argument
+        """get request"""
+        self.get_document(playlist_id)
+        return Response(self.response, status=self.status_code)
+
+    def post(self, request, playlist_id):
+        """post to custom playlist to add a video to list"""
+        playlist = YoutubePlaylist(playlist_id)
+        if not playlist.is_custom_playlist():
+            message = f"playlist with ID {playlist_id} is not custom"
+            return Response({"message": message}, status=400)
+
+        action = request.data.get("action")
+        if action not in self.valid_custom_actions:
+            message = f"invalid action: {action}"
+            return Response({"message": message}, status=400)
+
+        video_id = request.data.get("video_id")
+        if action == "create":
+            playlist.add_video_to_playlist(video_id)
+        else:
+            hide = UserConfig(request.user.id).get_value("hide_watched")
+            playlist.move_video(video_id, action, hide_watched=hide)
+
+        return Response({"success": True}, status=status.HTTP_201_CREATED)
+
+    def delete(self, request, playlist_id):
+        """delete playlist"""
+        print(f"{playlist_id}: delete playlist")
+        delete_videos = request.GET.get("delete-videos", False)
+        if delete_videos:
+            YoutubePlaylist(playlist_id).delete_videos_playlist()
+        else:
+            YoutubePlaylist(playlist_id).delete_metadata()
+
+        return Response({"success": True})