renamed django app folder to backend

backend/video/src/comments.py

@@ -0,0 +1,227 @@
+"""
+Functionality:
+- Download comments
+- Index comments in ES
+- Retrieve comments from ES
+"""
+
+from datetime import datetime
+
+from appsettings.src.config import AppConfig
+from common.src.es_connect import ElasticWrap
+from common.src.ta_redis import RedisQueue
+from download.src.yt_dlp_base import YtWrap
+
+
+class Comments:
+    """interact with comments per video"""
+
+    def __init__(self, youtube_id, config=False):
+        self.youtube_id = youtube_id
+        self.es_path = f"ta_comment/_doc/{youtube_id}"
+        self.json_data = False
+        self.config = config
+        self.is_activated = False
+        self.comments_format = False
+
+    def build_json(self):
+        """build json document for es"""
+        print(f"{self.youtube_id}: get comments")
+        self.check_config()
+        if not self.is_activated:
+            return
+
+        comments_raw, channel_id = self.get_yt_comments()
+        if not comments_raw and not channel_id:
+            return
+
+        self.format_comments(comments_raw)
+
+        self.json_data = {
+            "youtube_id": self.youtube_id,
+            "comment_last_refresh": int(datetime.now().timestamp()),
+            "comment_channel_id": channel_id,
+            "comment_comments": self.comments_format,
+        }
+
+    def check_config(self):
+        """read config if not attached"""
+        if not self.config:
+            self.config = AppConfig().config
+
+        self.is_activated = bool(self.config["downloads"]["comment_max"])
+
+    def build_yt_obs(self):
+        """
+        get extractor config
+        max-comments,max-parents,max-replies,max-replies-per-thread
+        """
+        max_comments = self.config["downloads"]["comment_max"]
+        max_comments_list = [i.strip() for i in max_comments.split(",")]
+        comment_sort = self.config["downloads"]["comment_sort"]
+
+        yt_obs = {
+            "check_formats": None,
+            "skip_download": True,
+            "getcomments": True,
+            "ignoreerrors": True,
+            "extractor_args": {
+                "youtube": {
+                    "max_comments": max_comments_list,
+                    "comment_sort": [comment_sort],
+                    "player_client": ["ios", "web"],  # workaround yt-dlp #9554
+                }
+            },
+        }
+
+        return yt_obs
+
+    def get_yt_comments(self):
+        """get comments from youtube"""
+        yt_obs = self.build_yt_obs()
+        info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id)
+        if not info_json:
+            return False, False
+
+        comments_raw = info_json.get("comments")
+        channel_id = info_json.get("channel_id")
+        return comments_raw, channel_id
+
+    def format_comments(self, comments_raw):
+        """process comments to match format"""
+        comments = []
+
+        if comments_raw:
+            for comment in comments_raw:
+                cleaned_comment = self.clean_comment(comment)
+                if not cleaned_comment:
+                    continue
+
+                comments.append(cleaned_comment)
+
+        self.comments_format = comments
+
+    def clean_comment(self, comment):
+        """parse metadata from comment for indexing"""
+        if not comment.get("text"):
+            # comment text can be empty
+            print(f"{self.youtube_id}: Failed to extract text, {comment}")
+            return False
+
+        time_text_datetime = datetime.utcfromtimestamp(comment["timestamp"])
+
+        if time_text_datetime.hour == 0 and time_text_datetime.minute == 0:
+            format_string = "%Y-%m-%d"
+        else:
+            format_string = "%Y-%m-%d %H:%M"
+
+        time_text = time_text_datetime.strftime(format_string)
+
+        if not comment.get("author"):
+            comment["author"] = comment.get("author_id", "Unknown")
+
+        cleaned_comment = {
+            "comment_id": comment["id"],
+            "comment_text": comment["text"].replace("\xa0", ""),
+            "comment_timestamp": comment["timestamp"],
+            "comment_time_text": time_text,
+            "comment_likecount": comment.get("like_count", None),
+            "comment_is_favorited": comment.get("is_favorited", False),
+            "comment_author": comment["author"],
+            "comment_author_id": comment["author_id"],
+            "comment_author_thumbnail": comment["author_thumbnail"],
+            "comment_author_is_uploader": comment.get(
+                "author_is_uploader", False
+            ),
+            "comment_parent": comment["parent"],
+        }
+
+        return cleaned_comment
+
+    def upload_comments(self):
+        """upload comments to es"""
+        if not self.is_activated:
+            return
+
+        print(f"{self.youtube_id}: upload comments")
+        _, _ = ElasticWrap(self.es_path).put(self.json_data)
+
+        vid_path = f"ta_video/_update/{self.youtube_id}"
+        data = {"doc": {"comment_count": len(self.comments_format)}}
+        _, _ = ElasticWrap(vid_path).post(data=data)
+
+    def delete_comments(self):
+        """delete comments from es"""
+        print(f"{self.youtube_id}: delete comments")
+        _, _ = ElasticWrap(self.es_path).delete(refresh=True)
+
+    def get_es_comments(self):
+        """get comments from ES"""
+        response, statuscode = ElasticWrap(self.es_path).get()
+        if statuscode == 404:
+            print(f"comments: not found {self.youtube_id}")
+            return False
+
+        return response.get("_source")
+
+    def reindex_comments(self):
+        """update comments from youtube"""
+        self.check_config()
+        if not self.is_activated:
+            return
+
+        self.build_json()
+        if not self.json_data:
+            return
+
+        es_comments = self.get_es_comments()
+
+        if not self.comments_format:
+            return
+
+        if not self.comments_format and es_comments["comment_comments"]:
+            # don't overwrite comments in es
+            return
+
+        self.delete_comments()
+        self.upload_comments()
+
+
+class CommentList:
+    """interact with comments in group"""
+
+    COMMENT_QUEUE = "index:comment"
+
+    def __init__(self, task=False):
+        self.task = task
+        self.config = AppConfig().config
+
+    def add(self, video_ids: list[str]) -> None:
+        """add list of videos to get comments, if enabled in config"""
+        if not self.config["downloads"].get("comment_max"):
+            return
+
+        RedisQueue(self.COMMENT_QUEUE).add_list(video_ids)
+
+    def index(self):
+        """run comment index"""
+        queue = RedisQueue(self.COMMENT_QUEUE)
+        while True:
+            total = queue.max_score()
+            youtube_id, idx = queue.get_next()
+            if not youtube_id or not idx or not total:
+                break
+
+            if self.task:
+                self.notify(idx, total)
+
+            comment = Comments(youtube_id, config=self.config)
+            comment.build_json()
+            if comment.json_data:
+                comment.upload_comments()
+
+    def notify(self, idx, total_videos):
+        """send notification on task"""
+        message = [f"Add comments for new videos {idx}/{total_videos}"]
+        progress = idx / total_videos
+        self.task.send_progress(message, progress=progress)

backend/video/src/constants.py

@@ -0,0 +1,30 @@
+"""video constants"""
+
+import enum
+
+
+class VideoTypeEnum(enum.Enum):
+    """all vid_type fields"""
+
+    VIDEOS = "videos"
+    STREAMS = "streams"
+    SHORTS = "shorts"
+    UNKNOWN = "unknown"
+
+
+class SortEnum(enum.Enum):
+    """all sort by options"""
+
+    PUBLISHED = "published"
+    DOWNLOADED = "date_downloaded"
+    VIEWS = "stats.view_count"
+    LIKES = "stats.like_count"
+    DURATION = "player.duration"
+    MEDIASIZE = "media_size"
+
+
+class OrderEnum(enum.Enum):
+    """all order by options"""
+
+    ASC = "asc"
+    DESC = "desc"

backend/video/src/index.py

@@ -0,0 +1,404 @@
+"""
+functionality:
+- get metadata from youtube for a video
+- index and update in es
+"""
+
+import os
+from datetime import datetime
+
+import requests
+from channel.src import index as ta_channel
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import get_duration_sec, get_duration_str, randomizor
+from common.src.index_generic import YouTubeItem
+from django.conf import settings
+from playlist.src import index as ta_playlist
+from ryd_client import ryd_client
+from user.src.user_config import UserConfig
+from video.src.comments import Comments
+from video.src.constants import VideoTypeEnum
+from video.src.media_streams import MediaStreamExtractor
+from video.src.subtitle import YoutubeSubtitle
+
+
+class SponsorBlock:
+    """handle sponsor block integration"""
+
+    API = "https://sponsor.ajay.app/api"
+
+    def __init__(self, user_id=False):
+        self.user_id = user_id
+        self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
+        self.last_refresh = int(datetime.now().timestamp())
+
+    def get_sb_id(self) -> str:
+        """get sponsorblock for the userid or generate if needed"""
+        if not self.user_id:
+            raise ValueError("missing request user id")
+
+        user = UserConfig(self.user_id)
+        sb_id = user.get_value("sponsorblock_id")
+        if not sb_id:
+            sb_id = randomizor(32)
+            user.set_value("sponsorblock_id", sb_id)
+
+        return sb_id
+
+    def get_timestamps(self, youtube_id):
+        """get timestamps from the API"""
+        url = f"{self.API}/skipSegments?videoID={youtube_id}"
+        headers = {"User-Agent": self.user_agent}
+        print(f"{youtube_id}: get sponsorblock timestamps")
+        try:
+            response = requests.get(url, headers=headers, timeout=10)
+        except requests.ReadTimeout:
+            print(f"{youtube_id}: sponsorblock API timeout")
+            return False
+
+        if not response.ok:
+            print(f"{youtube_id}: sponsorblock failed: {response.status_code}")
+            if response.status_code == 503:
+                return False
+
+            sponsor_dict = {
+                "last_refresh": self.last_refresh,
+                "is_enabled": True,
+                "segments": [],
+            }
+        else:
+            all_segments = response.json()
+            sponsor_dict = self._get_sponsor_dict(all_segments)
+
+        return sponsor_dict
+
+    def _get_sponsor_dict(self, all_segments):
+        """format and process response"""
+        _ = [i.pop("description", None) for i in all_segments]
+        has_unlocked = not any(i.get("locked") for i in all_segments)
+
+        sponsor_dict = {
+            "last_refresh": self.last_refresh,
+            "has_unlocked": has_unlocked,
+            "is_enabled": True,
+            "segments": all_segments,
+        }
+        return sponsor_dict
+
+    def post_timestamps(self, youtube_id, start_time, end_time):
+        """post timestamps to api"""
+        user_id = self.get_sb_id()
+        data = {
+            "videoID": youtube_id,
+            "startTime": start_time,
+            "endTime": end_time,
+            "category": "sponsor",
+            "userID": user_id,
+            "userAgent": self.user_agent,
+        }
+        url = f"{self.API}/skipSegments?videoID={youtube_id}"
+        print(f"post: {data}")
+        print(f"to: {url}")
+
+        return {"success": True}, 200
+
+    def vote_on_segment(self, uuid, vote):
+        """send vote on existing segment"""
+        user_id = self.get_sb_id()
+        data = {
+            "UUID": uuid,
+            "userID": user_id,
+            "type": vote,
+        }
+        url = f"{self.API}/api/voteOnSponsorTime"
+        print(f"post: {data}")
+        print(f"to: {url}")
+
+        return {"success": True}, 200
+
+
+class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
+    """represents a single youtube video"""
+
+    es_path = False
+    index_name = "ta_video"
+    yt_base = "https://www.youtube.com/watch?v="
+
+    def __init__(self, youtube_id, video_type=VideoTypeEnum.VIDEOS):
+        super().__init__(youtube_id)
+        self.channel_id = False
+        self.video_type = video_type
+        self.offline_import = False
+
+    def build_json(self, youtube_meta_overwrite=False, media_path=False):
+        """build json dict of video"""
+        self.get_from_youtube()
+        if not self.youtube_meta and not youtube_meta_overwrite:
+            return
+
+        if not self.youtube_meta:
+            self.youtube_meta = youtube_meta_overwrite
+            self.offline_import = True
+
+        self.process_youtube_meta()
+        self._add_channel()
+        self._add_stats()
+        self.add_file_path()
+        self.add_player(media_path)
+        self.add_streams(media_path)
+        if self.config["downloads"]["integrate_ryd"]:
+            self._get_ryd_stats()
+
+        if self._check_get_sb():
+            self._get_sponsorblock()
+
+        return
+
+    def _check_get_sb(self):
+        """check if need to run sponsor block"""
+        integrate = self.config["downloads"]["integrate_sponsorblock"]
+
+        if overwrite := self.json_data["channel"].get("channel_overwrites"):
+            if not overwrite:
+                return integrate
+
+            if "integrate_sponsorblock" in overwrite:
+                return overwrite.get("integrate_sponsorblock")
+
+        return integrate
+
+    def process_youtube_meta(self):
+        """extract relevant fields from youtube"""
+        self._validate_id()
+        # extract
+        self.channel_id = self.youtube_meta["channel_id"]
+        upload_date = self.youtube_meta["upload_date"]
+        upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
+        published = upload_date_time.strftime("%Y-%m-%d")
+        last_refresh = int(datetime.now().timestamp())
+        # base64_blur = ThumbManager().get_base64_blur(self.youtube_id)
+        base64_blur = False
+        # build json_data basics
+        self.json_data = {
+            "title": self.youtube_meta["title"],
+            "description": self.youtube_meta.get("description", ""),
+            "category": self.youtube_meta.get("categories", []),
+            "vid_thumb_url": self.youtube_meta["thumbnail"],
+            "vid_thumb_base64": base64_blur,
+            "tags": self.youtube_meta.get("tags", []),
+            "published": published,
+            "vid_last_refresh": last_refresh,
+            "date_downloaded": last_refresh,
+            "youtube_id": self.youtube_id,
+            # Using .value to make json encodable
+            "vid_type": self.video_type.value,
+            "active": True,
+        }
+
+    def _validate_id(self):
+        """validate expected video ID, raise value error on mismatch"""
+        remote_id = self.youtube_meta["id"]
+
+        if not self.youtube_id == remote_id:
+            # unexpected redirect
+            message = (
+                f"[reindex][{self.youtube_id}] got an unexpected redirect "
+                + f"to {remote_id}, you are probably getting blocked by YT. "
+                "See FAQ for more details."
+            )
+            raise ValueError(message)
+
+    def _add_channel(self):
+        """add channel dict to video json_data"""
+        channel = ta_channel.YoutubeChannel(self.channel_id)
+        channel.build_json(upload=True, fallback=self.youtube_meta)
+        self.json_data.update({"channel": channel.json_data})
+
+    def _add_stats(self):
+        """add stats dicst to json_data"""
+        stats = {
+            "view_count": self.youtube_meta.get("view_count", 0),
+            "like_count": self.youtube_meta.get("like_count", 0),
+            "dislike_count": self.youtube_meta.get("dislike_count", 0),
+            "average_rating": self.youtube_meta.get("average_rating", 0),
+        }
+        self.json_data.update({"stats": stats})
+
+    def build_dl_cache_path(self):
+        """find video path in dl cache"""
+        cache_dir = EnvironmentSettings.CACHE_DIR
+        video_id = self.json_data["youtube_id"]
+        cache_path = f"{cache_dir}/download/{video_id}.mp4"
+        if os.path.exists(cache_path):
+            return cache_path
+
+        channel_path = os.path.join(
+            EnvironmentSettings.MEDIA_DIR,
+            self.json_data["channel"]["channel_id"],
+            f"{video_id}.mp4",
+        )
+        if os.path.exists(channel_path):
+            return channel_path
+
+        raise FileNotFoundError
+
+    def add_player(self, media_path=False):
+        """add player information for new videos"""
+        vid_path = media_path or self.build_dl_cache_path()
+        duration = get_duration_sec(vid_path)
+
+        self.json_data.update(
+            {
+                "player": {
+                    "watched": False,
+                    "duration": duration,
+                    "duration_str": get_duration_str(duration),
+                }
+            }
+        )
+
+    def add_streams(self, media_path=False):
+        """add stream metadata"""
+        vid_path = media_path or self.build_dl_cache_path()
+        media = MediaStreamExtractor(vid_path)
+        self.json_data.update(
+            {
+                "streams": media.extract_metadata(),
+                "media_size": media.get_file_size(),
+            }
+        )
+
+    def add_file_path(self):
+        """build media_url for where file will be located"""
+        self.json_data["media_url"] = os.path.join(
+            self.json_data["channel"]["channel_id"],
+            self.json_data["youtube_id"] + ".mp4",
+        )
+
+    def delete_media_file(self):
+        """delete video file, meta data"""
+        print(f"{self.youtube_id}: delete video")
+        self.get_from_es()
+        if not self.json_data:
+            raise FileNotFoundError
+
+        video_base = EnvironmentSettings.MEDIA_DIR
+        media_url = self.json_data.get("media_url")
+        file_path = os.path.join(video_base, media_url)
+        try:
+            os.remove(file_path)
+        except FileNotFoundError:
+            print(f"{self.youtube_id}: failed {media_url}, continue.")
+
+        self.del_in_playlists()
+        self.del_in_es()
+        self.delete_subtitles()
+        self.delete_comments()
+
+    def del_in_playlists(self):
+        """remove downloaded in playlist"""
+        all_playlists = self.json_data.get("playlist")
+        if not all_playlists:
+            return
+
+        for playlist_id in all_playlists:
+            print(f"{playlist_id}: delete video {self.youtube_id}")
+            playlist = ta_playlist.YoutubePlaylist(playlist_id)
+            playlist.get_from_es()
+            entries = playlist.json_data["playlist_entries"]
+            for idx, entry in enumerate(entries):
+                if entry["youtube_id"] == self.youtube_id:
+                    playlist.json_data["playlist_entries"][idx].update(
+                        {"downloaded": False}
+                    )
+                    if playlist.json_data["playlist_type"] == "custom":
+                        playlist.del_video(self.youtube_id)
+            playlist.upload_to_es()
+
+    def delete_subtitles(self, subtitles=False):
+        """delete indexed subtitles"""
+        print(f"{self.youtube_id}: delete subtitles")
+        YoutubeSubtitle(self).delete(subtitles=subtitles)
+
+    def delete_comments(self):
+        """delete comments from es"""
+        comments = Comments(self.youtube_id, config=self.config)
+        comments.check_config()
+        if comments.is_activated:
+            comments.delete_comments()
+
+    def _get_ryd_stats(self):
+        """get optional stats from returnyoutubedislikeapi.com"""
+        # pylint: disable=broad-except
+        try:
+            print(f"{self.youtube_id}: get ryd stats")
+            result = ryd_client.get(self.youtube_id)
+        except Exception as err:
+            print(f"{self.youtube_id}: failed to query ryd api {err}")
+            return
+
+        if result["status"] == 404:
+            return
+
+        dislikes = {
+            "dislike_count": result.get("dislikes", 0),
+            "average_rating": result.get("rating", 0),
+        }
+        self.json_data["stats"].update(dislikes)
+
+    def _get_sponsorblock(self):
+        """get optional sponsorblock timestamps from sponsor.ajay.app"""
+        sponsorblock = SponsorBlock().get_timestamps(self.youtube_id)
+        if sponsorblock:
+            self.json_data["sponsorblock"] = sponsorblock
+
+    def check_subtitles(self, subtitle_files=False):
+        """optionally add subtitles"""
+        if self.offline_import and subtitle_files:
+            indexed = self._offline_subtitles(subtitle_files)
+            self.json_data["subtitles"] = indexed
+            return
+
+        handler = YoutubeSubtitle(self)
+        subtitles = handler.get_subtitles()
+        if subtitles:
+            indexed = handler.download_subtitles(relevant_subtitles=subtitles)
+            self.json_data["subtitles"] = indexed
+
+    def _offline_subtitles(self, subtitle_files):
+        """import offline subtitles"""
+        base_name, _ = os.path.splitext(self.json_data["media_url"])
+        subtitles = []
+        for subtitle in subtitle_files:
+            lang = subtitle.split(".")[-2]
+            subtitle_media_url = f"{base_name}.{lang}.vtt"
+            to_add = {
+                "ext": "vtt",
+                "url": False,
+                "name": lang,
+                "lang": lang,
+                "source": "file",
+                "media_url": subtitle_media_url,
+            }
+            subtitles.append(to_add)
+
+        return subtitles
+
+    def update_media_url(self):
+        """update only media_url in es for reindex channel rename"""
+        data = {"doc": {"media_url": self.json_data["media_url"]}}
+        path = f"{self.index_name}/_update/{self.youtube_id}"
+        _, _ = ElasticWrap(path).post(data=data)
+
+
+def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
+    """combined classes to create new video in index"""
+    video = YoutubeVideo(youtube_id, video_type=video_type)
+    video.build_json()
+    if not video.json_data:
+        raise ValueError("failed to get metadata for " + youtube_id)
+
+    video.check_subtitles()
+    video.upload_to_es()
+    return video.json_data

backend/video/src/media_streams.py

@@ -0,0 +1,81 @@
+"""extract metadata from video streams"""
+
+import json
+import subprocess
+from os import stat
+
+
+class MediaStreamExtractor:
+    """extract stream metadata"""
+
+    def __init__(self, media_path):
+        self.media_path = media_path
+        self.metadata = []
+
+    def extract_metadata(self):
+        """entry point to extract metadata"""
+
+        cmd = [
+            "ffprobe",
+            "-v",
+            "quiet",
+            "-print_format",
+            "json",
+            "-show_streams",
+            "-show_format",
+            self.media_path,
+        ]
+        result = subprocess.run(
+            cmd, capture_output=True, text=True, check=False
+        )
+
+        if result.returncode != 0:
+            return self.metadata
+
+        streams = json.loads(result.stdout).get("streams")
+        for stream in streams:
+            self.process_stream(stream)
+
+        return self.metadata
+
+    def process_stream(self, stream):
+        """parse stream to metadata"""
+        codec_type = stream.get("codec_type")
+        if codec_type == "video":
+            self._extract_video_metadata(stream)
+        elif codec_type == "audio":
+            self._extract_audio_metadata(stream)
+        else:
+            return
+
+    def _extract_video_metadata(self, stream):
+        """parse video metadata"""
+        if "bit_rate" not in stream:
+            # is probably thumbnail
+            return
+
+        self.metadata.append(
+            {
+                "type": "video",
+                "index": stream["index"],
+                "codec": stream["codec_name"],
+                "width": stream["width"],
+                "height": stream["height"],
+                "bitrate": int(stream["bit_rate"]),
+            }
+        )
+
+    def _extract_audio_metadata(self, stream):
+        """extract audio metadata"""
+        self.metadata.append(
+            {
+                "type": "audio",
+                "index": stream["index"],
+                "codec": stream.get("codec_name", "undefined"),
+                "bitrate": int(stream.get("bit_rate", 0)),
+            }
+        )
+
+    def get_file_size(self):
+        """get filesize in bytes"""
+        return stat(self.media_path).st_size

backend/video/src/query_building.py

@@ -0,0 +1,99 @@
+"""build query for video fetching"""
+
+from common.src.ta_redis import RedisArchivist
+from video.src.constants import OrderEnum, SortEnum, VideoTypeEnum
+
+
+class QueryBuilder:
+    """contain functionality"""
+
+    WATCH_OPTIONS = ["watched", "unwatched", "continue"]
+
+    def __init__(self, user_id: int, **kwargs):
+        self.user_id = user_id
+        self.request_params = kwargs
+
+    def build_data(self) -> dict:
+        """build data dict"""
+        data = {}
+        data["query"] = self.build_query()
+        if sort := self.parse_sort():
+            data.update(sort)
+
+        return data
+
+    def build_query(self) -> dict:
+        """build query key"""
+        must_list = []
+        channel = self.request_params.get("channel")
+        if channel:
+            must_list.append({"match": {"channel.channel_id": channel[0]}})
+
+        playlist = self.request_params.get("playlist")
+        if playlist:
+            must_list.append({"match": {"playlist.keyword": playlist[0]}})
+
+        watch = self.request_params.get("watch")
+        if watch:
+            watch_must_list = self.parse_watch(watch[0])
+            must_list.append(watch_must_list)
+
+        video_type = self.request_params.get("type")
+        if video_type:
+            type_list_list = self.parse_type(video_type[0])
+            must_list.append(type_list_list)
+
+        query = {"bool": {"must": must_list}}
+
+        return query
+
+    def parse_watch(self, watch: str) -> dict:
+        """build query"""
+        if watch not in self.WATCH_OPTIONS:
+            raise ValueError(f"'{watch}' not in {self.WATCH_OPTIONS}")
+
+        if watch == "continue":
+            continue_must = self._build_continue_must()
+            return continue_must
+
+        return {"match": {"player.watched": watch == "watched"}}
+
+    def _build_continue_must(self):
+        results = RedisArchivist().list_items(f"{self.user_id}:progress:")
+        if not results:
+            return None
+
+        ids = [{"match": {"youtube_id": i.get("youtube_id")}} for i in results]
+        continue_ids = {"bool": {"should": ids}}
+
+        return continue_ids
+
+    def parse_type(self, video_type: str):
+        """parse video type"""
+        if not hasattr(VideoTypeEnum, video_type.upper()):
+            raise ValueError(f"'{video_type}' not in VideoTypeEnum")
+
+        vid_type = getattr(VideoTypeEnum, video_type.upper()).value
+
+        return {"match": {"vid_type": vid_type}}
+
+    def parse_sort(self) -> dict | None:
+        """build sort key"""
+        sort = self.request_params.get("sort")
+        if not sort:
+            return None
+
+        sort = sort[0]
+        if not hasattr(SortEnum, sort.upper()):
+            raise ValueError(f"'{sort}' not in SortEnum")
+
+        sort_field = getattr(SortEnum, sort.upper()).value
+
+        order = self.request_params.get("order", ["desc"])
+        order = order[0]
+        if not hasattr(OrderEnum, order.upper()):
+            raise ValueError(f"'{order}' not in OrderEnum")
+
+        order_by = getattr(OrderEnum, order.upper()).value
+
+        return {"sort": [{sort_field: {"order": order_by}}]}

backend/video/src/subtitle.py

@@ -0,0 +1,335 @@
+"""
+functionality:
+- download subtitles
+- parse subtitles into it's cues
+- index dubtitles
+"""
+
+import json
+import os
+from datetime import datetime
+
+import requests
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import requests_headers
+
+
+class YoutubeSubtitle:
+    """handle video subtitle functionality"""
+
+    def __init__(self, video):
+        self.video = video
+        self.languages = False
+
+    def _sub_conf_parse(self):
+        """add additional conf values to self"""
+        languages_raw = self.video.config["downloads"]["subtitle"]
+        if languages_raw:
+            self.languages = [i.strip() for i in languages_raw.split(",")]
+
+    def get_subtitles(self):
+        """check what to do"""
+        self._sub_conf_parse()
+        if not self.languages:
+            # no subtitles
+            return False
+
+        relevant_subtitles = []
+        for lang in self.languages:
+            user_sub = self._get_user_subtitles(lang)
+            if user_sub:
+                relevant_subtitles.append(user_sub)
+                continue
+
+            if self.video.config["downloads"]["subtitle_source"] == "auto":
+                auto_cap = self._get_auto_caption(lang)
+                if auto_cap:
+                    relevant_subtitles.append(auto_cap)
+
+        return relevant_subtitles
+
+    def _get_auto_caption(self, lang):
+        """get auto_caption subtitles"""
+        print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
+        all_subtitles = self.video.youtube_meta.get("automatic_captions")
+
+        if not all_subtitles:
+            return False
+
+        video_media_url = self.video.json_data["media_url"]
+        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
+        all_formats = all_subtitles.get(lang)
+        if not all_formats:
+            return False
+
+        subtitle_json3 = [i for i in all_formats if i["ext"] == "json3"]
+        if not subtitle_json3:
+            print(f"{self.video.youtube_id}-{lang}: json3 not processed")
+            return False
+
+        subtitle = subtitle_json3[0]
+        subtitle.update(
+            {"lang": lang, "source": "auto", "media_url": media_url}
+        )
+
+        return subtitle
+
+    def _normalize_lang(self):
+        """normalize country specific language keys"""
+        all_subtitles = self.video.youtube_meta.get("subtitles")
+        if not all_subtitles:
+            return False
+
+        all_keys = list(all_subtitles.keys())
+        for key in all_keys:
+            lang = key.split("-")[0]
+            old = all_subtitles.pop(key)
+            if lang == "live_chat":
+                continue
+            all_subtitles[lang] = old
+
+        return all_subtitles
+
+    def _get_user_subtitles(self, lang):
+        """get subtitles uploaded from channel owner"""
+        print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
+        all_subtitles = self._normalize_lang()
+        if not all_subtitles:
+            return False
+
+        video_media_url = self.video.json_data["media_url"]
+        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
+        all_formats = all_subtitles.get(lang)
+        if not all_formats:
+            # no user subtitles found
+            return False
+
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
+        subtitle.update(
+            {"lang": lang, "source": "user", "media_url": media_url}
+        )
+
+        return subtitle
+
+    def download_subtitles(self, relevant_subtitles):
+        """download subtitle files to archive"""
+        videos_base = EnvironmentSettings.MEDIA_DIR
+        indexed = []
+        for subtitle in relevant_subtitles:
+            dest_path = os.path.join(videos_base, subtitle["media_url"])
+            source = subtitle["source"]
+            lang = subtitle.get("lang")
+            response = requests.get(
+                subtitle["url"], headers=requests_headers(), timeout=30
+            )
+            if not response.ok:
+                print(f"{self.video.youtube_id}: failed to download subtitle")
+                print(response.text)
+                continue
+
+            if not response.text:
+                print(f"{self.video.youtube_id}: skip empty subtitle")
+                continue
+
+            parser = SubtitleParser(response.text, lang, source)
+            parser.process()
+            if not parser.all_cues:
+                continue
+
+            subtitle_str = parser.get_subtitle_str()
+            self._write_subtitle_file(dest_path, subtitle_str)
+            if self.video.config["downloads"]["subtitle_index"]:
+                query_str = parser.create_bulk_import(self.video, source)
+                self._index_subtitle(query_str)
+
+            indexed.append(subtitle)
+
+        return indexed
+
+    def _write_subtitle_file(self, dest_path, subtitle_str):
+        """write subtitle file to disk"""
+        # create folder here for first video of channel
+        os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
+        with open(dest_path, "w", encoding="utf-8") as subfile:
+            subfile.write(subtitle_str)
+
+        host_uid = EnvironmentSettings.HOST_UID
+        host_gid = EnvironmentSettings.HOST_GID
+        if host_uid and host_gid:
+            os.chown(dest_path, host_uid, host_gid)
+
+    @staticmethod
+    def _index_subtitle(query_str):
+        """send subtitle to es for indexing"""
+        _, _ = ElasticWrap("_bulk").post(data=query_str, ndjson=True)
+
+    def delete(self, subtitles=False):
+        """delete subtitles from index and filesystem"""
+        youtube_id = self.video.youtube_id
+        videos_base = EnvironmentSettings.MEDIA_DIR
+        # delete files
+        if subtitles:
+            files = [i["media_url"] for i in subtitles]
+        else:
+            if not self.video.json_data.get("subtitles"):
+                return
+
+            files = [i["media_url"] for i in self.video.json_data["subtitles"]]
+
+        for file_name in files:
+            file_path = os.path.join(videos_base, file_name)
+            try:
+                os.remove(file_path)
+            except FileNotFoundError:
+                print(f"{youtube_id}: {file_path} failed to delete")
+        # delete from index
+        path = "ta_subtitle/_delete_by_query?refresh=true"
+        data = {"query": {"term": {"youtube_id": {"value": youtube_id}}}}
+        _, _ = ElasticWrap(path).post(data=data)
+
+
+class SubtitleParser:
+    """parse subtitle str from youtube"""
+
+    def __init__(self, subtitle_str, lang, source):
+        self.subtitle_raw = json.loads(subtitle_str)
+        self.lang = lang
+        self.source = source
+        self.all_cues = False
+
+    def process(self):
+        """extract relevant que data"""
+        self.all_cues = []
+        all_events = self.subtitle_raw.get("events")
+
+        if not all_events:
+            return
+
+        if self.source == "auto":
+            all_events = self._flat_auto_caption(all_events)
+
+        for idx, event in enumerate(all_events):
+            if "dDurationMs" not in event or "segs" not in event:
+                # some events won't have a duration or segs
+                print(f"skipping subtitle event without content: {event}")
+                continue
+
+            cue = {
+                "start": self._ms_conv(event["tStartMs"]),
+                "end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
+                "text": "".join([i.get("utf8") for i in event["segs"]]),
+                "idx": idx + 1,
+            }
+            self.all_cues.append(cue)
+
+    @staticmethod
+    def _flat_auto_caption(all_events):
+        """flatten autocaption segments"""
+        flatten = []
+        for event in all_events:
+            if "segs" not in event.keys():
+                continue
+            text = "".join([i.get("utf8") for i in event.get("segs")])
+            if not text.strip():
+                continue
+
+            if flatten:
+                # fix overlapping retiming issue
+                last = flatten[-1]
+                if "dDurationMs" not in last or "segs" not in last:
+                    # some events won't have a duration or segs
+                    print(f"skipping subtitle event without content: {event}")
+                    continue
+
+                last_end = last["tStartMs"] + last["dDurationMs"]
+                if event["tStartMs"] < last_end:
+                    joined = last["segs"][0]["utf8"] + "\n" + text
+                    last["segs"][0]["utf8"] = joined
+                    continue
+
+            event.update({"segs": [{"utf8": text}]})
+            flatten.append(event)
+
+        return flatten
+
+    @staticmethod
+    def _ms_conv(ms):
+        """convert ms to timestamp"""
+        hours = str((ms // (1000 * 60 * 60)) % 24).zfill(2)
+        minutes = str((ms // (1000 * 60)) % 60).zfill(2)
+        secs = str((ms // 1000) % 60).zfill(2)
+        millis = str(ms % 1000).zfill(3)
+
+        return f"{hours}:{minutes}:{secs}.{millis}"
+
+    def get_subtitle_str(self):
+        """create vtt text str from cues"""
+        subtitle_str = f"WEBVTT\nKind: captions\nLanguage: {self.lang}"
+
+        for cue in self.all_cues:
+            stamp = f"{cue.get('start')} --> {cue.get('end')}"
+            cue_text = f"\n\n{cue.get('idx')}\n{stamp}\n{cue.get('text')}"
+            subtitle_str = subtitle_str + cue_text
+
+        return subtitle_str
+
+    def create_bulk_import(self, video, source):
+        """subtitle lines for es import"""
+        documents = self._create_documents(video, source)
+        bulk_list = []
+
+        for document in documents:
+            document_id = document.get("subtitle_fragment_id")
+            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(document))
+
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+
+        return query_str
+
+    def _create_documents(self, video, source):
+        """process documents"""
+        documents = self._chunk_list(video.youtube_id)
+        channel = video.json_data.get("channel")
+        meta_dict = {
+            "youtube_id": video.youtube_id,
+            "title": video.json_data.get("title"),
+            "subtitle_channel": channel.get("channel_name"),
+            "subtitle_channel_id": channel.get("channel_id"),
+            "subtitle_last_refresh": int(datetime.now().timestamp()),
+            "subtitle_lang": self.lang,
+            "subtitle_source": source,
+        }
+
+        _ = [i.update(meta_dict) for i in documents]
+
+        return documents
+
+    def _chunk_list(self, youtube_id):
+        """join cues for bulk import"""
+        chunk_list = []
+
+        chunk = {}
+        for cue in self.all_cues:
+            if chunk:
+                text = f"{chunk.get('subtitle_line')} {cue.get('text')}\n"
+                chunk["subtitle_line"] = text
+            else:
+                idx = len(chunk_list) + 1
+                chunk = {
+                    "subtitle_index": idx,
+                    "subtitle_line": cue.get("text"),
+                    "subtitle_start": cue.get("start"),
+                }
+
+            chunk["subtitle_fragment_id"] = f"{youtube_id}-{self.lang}-{idx}"
+
+            if cue["idx"] % 5 == 0:
+                chunk["subtitle_end"] = cue.get("end")
+                chunk_list.append(chunk)
+                chunk = {}
+
+        return chunk_list

backend/video/tests/test_src/test_query_building.py

@@ -0,0 +1,68 @@
+"""test video query building"""
+
+import pytest
+from video.src.query_building import QueryBuilder
+
+
+def test_initialization():
+    """init constructor"""
+    qb = QueryBuilder(user_id=1)
+    assert qb.user_id == 1
+    assert not qb.request_params
+
+
+def test_build_data():
+    """test for correct key building"""
+    qb = QueryBuilder(
+        user_id=1,
+        channel=["test_channel"],
+        playlist=["test_playlist"],
+        watch=["watched"],
+        type=["videos"],
+        sort=["published"],
+        order=["desc"],
+    )
+    result = qb.build_data()
+    assert "query" in result
+    assert "sort" in result
+    assert result["sort"] == [{"published": {"order": "desc"}}]
+
+
+def test_parse_watch():
+    """watched query building"""
+    qb = QueryBuilder(user_id=1, watch=["watched"])
+    result = qb.parse_watch("watched")
+    assert result == {"match": {"player.watched": True}}
+
+    result = qb.parse_watch("unwatched")
+    assert result == {"match": {"player.watched": False}}
+
+    with pytest.raises(ValueError):
+        qb.parse_watch("invalid")
+
+
+def test_parse_type():
+    """test type is parsed"""
+    qb = QueryBuilder(user_id=1, type=["videos"])
+    with pytest.raises(ValueError):
+        qb.parse_type("invalid")
+
+    result = qb.parse_type("videos")
+    assert result == {"match": {"vid_type": "videos"}}
+
+
+def test_parse_sort():
+    """test sort and order"""
+    qb = QueryBuilder(user_id=1, sort=["views"], order=["desc"])
+    result = qb.parse_sort()
+    assert result == {"sort": [{"stats.view_count": {"order": "desc"}}]}
+
+    with pytest.raises(ValueError):
+        qb = QueryBuilder(user_id=1, sort=["invalid"])
+        qb.parse_sort()
+
+    with pytest.raises(ValueError):
+        qb = QueryBuilder(
+            user_id=1, sort=["stats.view_count"], order=["invalid"]
+        )
+        qb.parse_sort()

backend/video/urls.py

@@ -0,0 +1,33 @@
+"""all video API urls"""
+
+from django.urls import path
+from video import views
+
+urlpatterns = [
+    path("", views.VideoApiListView.as_view(), name="api-video-list"),
+    path(
+        "<slug:video_id>/",
+        views.VideoApiView.as_view(),
+        name="api-video",
+    ),
+    path(
+        "<slug:video_id>/nav/",
+        views.VideoApiNavView.as_view(),
+        name="api-video-nav",
+    ),
+    path(
+        "<slug:video_id>/progress/",
+        views.VideoProgressView.as_view(),
+        name="api-video-progress",
+    ),
+    path(
+        "<slug:video_id>/comment/",
+        views.VideoCommentView.as_view(),
+        name="api-video-comment",
+    ),
+    path(
+        "<slug:video_id>/similar/",
+        views.VideoSimilarView.as_view(),
+        name="api-video-similar",
+    ),
+]

backend/video/views.py

@@ -0,0 +1,172 @@
+"""all API views for video endpoints"""
+
+from common.src.ta_redis import RedisArchivist
+from common.views_base import AdminWriteOnly, ApiBaseView
+from playlist.src.index import YoutubePlaylist
+from rest_framework.response import Response
+from video.src.index import YoutubeVideo
+from video.src.query_building import QueryBuilder
+
+
+class VideoApiListView(ApiBaseView):
+    """resolves to /api/video/
+    GET: returns list of videos
+    params:
+    - playlist:str=<playlist-id>
+    - channel:str=<channel-id>
+    - watch:enum=watched|unwatched|continue
+    - sort:enum=published|downloaded|views|likes|duration|filesize
+    - order:enum=asc|desc
+    - type:enum=videos|streams|shorts
+    """
+
+    search_base = "ta_video/_search/"
+
+    def get(self, request):
+        """get request"""
+        try:
+            data = QueryBuilder(request.user.id, **request.GET).build_data()
+        except ValueError as err:
+            return Response({"error": str(err)}, status=400)
+
+        self.data = data
+        self.get_document_list(request)
+
+        return Response(self.response)
+
+
+class VideoApiView(ApiBaseView):
+    """resolves to /api/video/<video_id>/
+    GET: returns metadata dict of video
+    """
+
+    search_base = "ta_video/_doc/"
+    permission_classes = [AdminWriteOnly]
+
+    def get(self, request, video_id):
+        # pylint: disable=unused-argument
+        """get request"""
+        self.get_document(video_id)
+        return Response(self.response, status=self.status_code)
+
+    def delete(self, request, video_id):
+        # pylint: disable=unused-argument
+        """delete single video"""
+        message = {"video": video_id}
+        try:
+            YoutubeVideo(video_id).delete_media_file()
+            status_code = 200
+            message.update({"state": "delete"})
+        except FileNotFoundError:
+            status_code = 404
+            message.update({"state": "not found"})
+
+        return Response(message, status=status_code)
+
+
+class VideoApiNavView(ApiBaseView):
+    """resolves to /api/video/<video-id>/nav/
+    GET: returns playlist nav
+    """
+
+    search_base = "ta_video/_doc/"
+
+    def get(self, request, video_id):
+        # pylint: disable=unused-argument
+        """get request"""
+        self.get_document(video_id)
+        if self.status_code != 200:
+            return Response(status=self.status_code)
+
+        print(self.response)
+
+        playlist_nav = []
+
+        if not self.response["data"].get("playlist"):
+            return Response(playlist_nav)
+
+        for playlist_id in self.response["data"]["playlist"]:
+            playlist = YoutubePlaylist(playlist_id)
+            playlist.get_from_es()
+            playlist.build_nav(video_id)
+            if playlist.nav:
+                playlist_nav.append(playlist.nav)
+
+        return Response(playlist_nav, status=self.status_code)
+
+
+class VideoProgressView(ApiBaseView):
+    """resolves to /api/video/<video_id>/progress/
+    handle progress status for video
+    """
+
+    def get(self, request, video_id):
+        """get progress for a single video"""
+        user_id = request.user.id
+        key = f"{user_id}:progress:{video_id}"
+        video_progress = RedisArchivist().get_message(key)
+        position = video_progress.get("position", 0)
+
+        self.response = {
+            "youtube_id": video_id,
+            "user_id": user_id,
+            "position": position,
+        }
+        return Response(self.response)
+
+    def post(self, request, video_id):
+        """set progress position in redis"""
+        position = request.data.get("position", 0)
+        key = f"{request.user.id}:progress:{video_id}"
+        message = {"position": position, "youtube_id": video_id}
+        RedisArchivist().set_message(key, message)
+        self.response = request.data
+        return Response(self.response)
+
+    def delete(self, request, video_id):
+        """delete progress position"""
+        key = f"{request.user.id}:progress:{video_id}"
+        RedisArchivist().del_message(key)
+        self.response = {"progress-reset": video_id}
+
+        return Response(self.response)
+
+
+class VideoCommentView(ApiBaseView):
+    """resolves to /api/video/<video_id>/comment/
+    handle video comments
+    GET: return all comments from video with reply threads
+    """
+
+    search_base = "ta_comment/_doc/"
+
+    def get(self, request, video_id):
+        """get video comments"""
+        # pylint: disable=unused-argument
+        self.get_document(video_id)
+
+        return Response(self.response, status=self.status_code)
+
+
+class VideoSimilarView(ApiBaseView):
+    """resolves to /api/video/<video-id>/similar/
+    GET: return max 6 videos similar to this
+    """
+
+    search_base = "ta_video/_search/"
+
+    def get(self, request, video_id):
+        """get similar videos"""
+        self.data = {
+            "size": 6,
+            "query": {
+                "more_like_this": {
+                    "fields": ["tags", "title"],
+                    "like": {"_id": video_id},
+                    "min_term_freq": 1,
+                    "max_query_terms": 25,
+                }
+            },
+        }
+        self.get_document_list(request, pagination=False)
+        return Response(self.response, status=self.status_code)