From 4c5f56e191bde5a8a32144d4944bef5e675bee89 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 18 Jul 2024 12:34:48 +0200 Subject: [PATCH] split video app --- tubearchivist/api/urls.py | 30 --- tubearchivist/api/views.py | 175 ------------------ tubearchivist/config/settings.py | 1 + tubearchivist/config/urls.py | 1 + tubearchivist/home/src/download/queue.py | 2 +- .../home/src/download/subscriptions.py | 4 +- .../home/src/download/yt_dlp_handler.py | 6 +- tubearchivist/home/src/index/filesystem.py | 4 +- tubearchivist/home/src/index/manual.py | 4 +- tubearchivist/home/src/index/playlist.py | 2 +- tubearchivist/home/src/index/reindex.py | 4 +- tubearchivist/home/src/ta/urlparser.py | 2 +- tubearchivist/home/views.py | 2 +- tubearchivist/video/__init__.py | 0 tubearchivist/video/migrations/__init__.py | 0 tubearchivist/video/src/__init__.py | 0 .../{home/src/index => video/src}/comments.py | 0 .../src/constants.py} | 0 .../src/index/video.py => video/src/index.py} | 10 +- .../src/media_streams.py} | 0 .../{home/src/index => video/src}/subtitle.py | 0 tubearchivist/video/urls.py | 28 +++ tubearchivist/video/views.py | 127 +++++++++++++ 23 files changed, 177 insertions(+), 225 deletions(-) create mode 100644 tubearchivist/video/__init__.py create mode 100644 tubearchivist/video/migrations/__init__.py create mode 100644 tubearchivist/video/src/__init__.py rename tubearchivist/{home/src/index => video/src}/comments.py (100%) rename tubearchivist/{home/src/index/video_constants.py => video/src/constants.py} (100%) rename tubearchivist/{home/src/index/video.py => video/src/index.py} (97%) rename tubearchivist/{home/src/index/video_streams.py => video/src/media_streams.py} (100%) rename tubearchivist/{home/src/index => video/src}/subtitle.py (100%) create mode 100644 tubearchivist/video/urls.py create mode 100644 tubearchivist/video/views.py diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 4ad57a73..4ef05e61 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -6,36 +6,6 @@ from django.urls import path urlpatterns = [ path("ping/", views.PingView.as_view(), name="ping"), path("login/", views.LoginApiView.as_view(), name="api-login"), - path( - "video/", - views.VideoApiListView.as_view(), - name="api-video-list", - ), - path( - "video//", - views.VideoApiView.as_view(), - name="api-video", - ), - path( - "video//progress/", - views.VideoProgressView.as_view(), - name="api-video-progress", - ), - path( - "video//comment/", - views.VideoCommentView.as_view(), - name="api-video-comment", - ), - path( - "video//similar/", - views.VideoSimilarView.as_view(), - name="api-video-similar", - ), - path( - "video//sponsor/", - views.VideoSponsorView.as_view(), - name="api-video-sponsor", - ), path( "channel/", views.ChannelApiListView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 5d231da5..2f205fe5 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -26,7 +26,6 @@ from home.src.index.channel import YoutubeChannel from home.src.index.generic import Pagination from home.src.index.playlist import YoutubePlaylist from home.src.index.reindex import ReindexProgress -from home.src.index.video import SponsorBlock, YoutubeVideo from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.notify import Notifications, get_all_notifications from home.src.ta.settings import EnvironmentSettings @@ -138,180 +137,6 @@ class ApiBaseView(APIView): self.response["paginate"] = self.pagination_handler.pagination -class VideoApiView(ApiBaseView): - """resolves to /api/video// - GET: returns metadata dict of video - """ - - search_base = "ta_video/_doc/" - permission_classes = [AdminWriteOnly] - - def get(self, request, video_id): - # pylint: disable=unused-argument - """get request""" - self.get_document(video_id) - return Response(self.response, status=self.status_code) - - def delete(self, request, video_id): - # pylint: disable=unused-argument - """delete single video""" - message = {"video": video_id} - try: - YoutubeVideo(video_id).delete_media_file() - status_code = 200 - message.update({"state": "delete"}) - except FileNotFoundError: - status_code = 404 - message.update({"state": "not found"}) - - return Response(message, status=status_code) - - -class VideoApiListView(ApiBaseView): - """resolves to /api/video/ - GET: returns list of videos - """ - - search_base = "ta_video/_search/" - - def get(self, request): - """get request""" - self.data.update({"sort": [{"published": {"order": "desc"}}]}) - self.get_document_list(request) - - return Response(self.response) - - -class VideoProgressView(ApiBaseView): - """resolves to /api/video//progress/ - handle progress status for video - """ - - def get(self, request, video_id): - """get progress for a single video""" - user_id = request.user.id - key = f"{user_id}:progress:{video_id}" - video_progress = RedisArchivist().get_message(key) - position = video_progress.get("position", 0) - - self.response = { - "youtube_id": video_id, - "user_id": user_id, - "position": position, - } - return Response(self.response) - - def post(self, request, video_id): - """set progress position in redis""" - position = request.data.get("position", 0) - key = f"{request.user.id}:progress:{video_id}" - message = {"position": position, "youtube_id": video_id} - RedisArchivist().set_message(key, message) - self.response = request.data - return Response(self.response) - - def delete(self, request, video_id): - """delete progress position""" - key = f"{request.user.id}:progress:{video_id}" - RedisArchivist().del_message(key) - self.response = {"progress-reset": video_id} - - return Response(self.response) - - -class VideoCommentView(ApiBaseView): - """resolves to /api/video//comment/ - handle video comments - GET: return all comments from video with reply threads - """ - - search_base = "ta_comment/_doc/" - - def get(self, request, video_id): - """get video comments""" - # pylint: disable=unused-argument - self.get_document(video_id) - - return Response(self.response, status=self.status_code) - - -class VideoSimilarView(ApiBaseView): - """resolves to /api/video//similar/ - GET: return max 6 videos similar to this - """ - - search_base = "ta_video/_search/" - - def get(self, request, video_id): - """get similar videos""" - self.data = { - "size": 6, - "query": { - "more_like_this": { - "fields": ["tags", "title"], - "like": {"_id": video_id}, - "min_term_freq": 1, - "max_query_terms": 25, - } - }, - } - self.get_document_list(request, pagination=False) - return Response(self.response, status=self.status_code) - - -class VideoSponsorView(ApiBaseView): - """resolves to /api/video//sponsor/ - handle sponsor block integration - """ - - search_base = "ta_video/_doc/" - - def get(self, request, video_id): - """get sponsor info""" - # pylint: disable=unused-argument - - self.get_document(video_id) - if not self.response.get("data"): - message = {"message": "video not found"} - return Response(message, status=404) - - sponsorblock = self.response["data"].get("sponsorblock") - - return Response(sponsorblock) - - def post(self, request, video_id): - """post verification and timestamps""" - if "segment" in request.data: - response, status_code = self._create_segment(request, video_id) - elif "vote" in request.data: - response, status_code = self._vote_on_segment(request) - - return Response(response, status=status_code) - - @staticmethod - def _create_segment(request, video_id): - """create segment in API""" - start_time = request.data["segment"]["startTime"] - end_time = request.data["segment"]["endTime"] - response, status_code = SponsorBlock(request.user.id).post_timestamps( - video_id, start_time, end_time - ) - - return response, status_code - - @staticmethod - def _vote_on_segment(request): - """validate on existing segment""" - user_id = request.user.id - uuid = request.data["vote"]["uuid"] - vote = request.data["vote"]["yourVote"] - response, status_code = SponsorBlock(user_id).vote_on_segment( - uuid, vote - ) - - return response, status_code - - class ChannelApiView(ApiBaseView): """resolves to /api/channel// GET: returns metadata dict of channel diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 39742ab9..2a7c8edb 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -62,6 +62,7 @@ INSTALLED_APPS = [ "rest_framework", "rest_framework.authtoken", "api", + "video", "config", ] diff --git a/tubearchivist/config/urls.py b/tubearchivist/config/urls.py index 6b373657..146330fa 100644 --- a/tubearchivist/config/urls.py +++ b/tubearchivist/config/urls.py @@ -20,5 +20,6 @@ from django.urls import include, path urlpatterns = [ path("", include("home.urls")), path("api/", include("api.urls")), + path("api/video/", include("video.urls")), path("admin/", admin.site.urls), ] diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index 5f0a86bb..bf303305 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -12,9 +12,9 @@ from home.src.download.thumbnails import ThumbManager from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.playlist import YoutubePlaylist -from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig from home.src.ta.helper import get_duration_str, is_shorts +from video.src.constants import VideoTypeEnum class PendingIndex: diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index 34cd87e2..078a9e49 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -9,11 +9,11 @@ from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import IndexPaginate from home.src.index.channel import YoutubeChannel from home.src.index.playlist import YoutubePlaylist -from home.src.index.video import YoutubeVideo -from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig from home.src.ta.helper import is_missing from home.src.ta.urlparser import Parser +from video.src.constants import VideoTypeEnum +from video.src.index import YoutubeVideo class ChannelSubscription: diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 5c4ad48d..9d2dc0a6 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -15,14 +15,14 @@ from home.src.download.subscriptions import PlaylistSubscription from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.channel import YoutubeChannel -from home.src.index.comments import CommentList from home.src.index.playlist import YoutubePlaylist -from home.src.index.video import YoutubeVideo, index_new_video -from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig from home.src.ta.helper import get_channel_overwrites, ignore_filelist from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisQueue +from video.src.comments import CommentList +from video.src.constants import VideoTypeEnum +from video.src.index import YoutubeVideo, index_new_video class DownloaderBase: diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index 484271a8..d31fdbc5 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -6,10 +6,10 @@ Functionality: import os from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.index.comments import CommentList -from home.src.index.video import YoutubeVideo, index_new_video from home.src.ta.helper import ignore_filelist from home.src.ta.settings import EnvironmentSettings +from video.src.comments import CommentList +from video.src.index import YoutubeVideo, index_new_video class Scanner: diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/home/src/index/manual.py index aa65ebdb..8a4a316e 100644 --- a/tubearchivist/home/src/index/manual.py +++ b/tubearchivist/home/src/index/manual.py @@ -12,12 +12,12 @@ import shutil import subprocess from home.src.download.thumbnails import ThumbManager -from home.src.index.comments import CommentList -from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist from home.src.ta.settings import EnvironmentSettings from PIL import Image +from video.src.comments import CommentList +from video.src.index import YoutubeVideo from yt_dlp.utils import ISO639Utils diff --git a/tubearchivist/home/src/index/playlist.py b/tubearchivist/home/src/index/playlist.py index a7cef0f0..65206f33 100644 --- a/tubearchivist/home/src/index/playlist.py +++ b/tubearchivist/home/src/index/playlist.py @@ -11,7 +11,7 @@ from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index import channel from home.src.index.generic import YouTubeItem -from home.src.index.video import YoutubeVideo +from video.src.index import YoutubeVideo class YoutubePlaylist(YouTubeItem): diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 51563998..527d44cc 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -16,12 +16,12 @@ from home.src.download.thumbnails import ThumbManager from home.src.download.yt_dlp_base import CookieHandler from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.channel import YoutubeChannel -from home.src.index.comments import Comments from home.src.index.playlist import YoutubePlaylist -from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisQueue +from video.src.comments import Comments +from video.src.index import YoutubeVideo class ReindexConfigType(TypedDict): diff --git a/tubearchivist/home/src/ta/urlparser.py b/tubearchivist/home/src/ta/urlparser.py index 924961ac..662f51d4 100644 --- a/tubearchivist/home/src/ta/urlparser.py +++ b/tubearchivist/home/src/ta/urlparser.py @@ -7,7 +7,7 @@ Functionality: from urllib.parse import parse_qs, urlparse from home.src.download.yt_dlp_base import YtWrap -from home.src.index.video_constants import VideoTypeEnum +from video.src.constants import VideoTypeEnum class Parser: diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index ec7858cf..f9cbc4c8 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -44,7 +44,6 @@ from home.src.index.channel import channel_overwrites from home.src.index.generic import Pagination from home.src.index.playlist import YoutubePlaylist from home.src.index.reindex import ReindexProgress -from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.config_schedule import ScheduleBuilder from home.src.ta.helper import check_stylesheet, time_parser @@ -54,6 +53,7 @@ from home.src.ta.ta_redis import RedisArchivist from home.src.ta.users import UserConfig from home.tasks import index_channel_playlists, subscribe_to from rest_framework.authtoken.models import Token +from video.src.constants import VideoTypeEnum class ArchivistViewConfig(View): diff --git a/tubearchivist/video/__init__.py b/tubearchivist/video/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/video/migrations/__init__.py b/tubearchivist/video/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/video/src/__init__.py b/tubearchivist/video/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/home/src/index/comments.py b/tubearchivist/video/src/comments.py similarity index 100% rename from tubearchivist/home/src/index/comments.py rename to tubearchivist/video/src/comments.py diff --git a/tubearchivist/home/src/index/video_constants.py b/tubearchivist/video/src/constants.py similarity index 100% rename from tubearchivist/home/src/index/video_constants.py rename to tubearchivist/video/src/constants.py diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/video/src/index.py similarity index 97% rename from tubearchivist/home/src/index/video.py rename to tubearchivist/video/src/index.py index 22efc088..a02bc7f6 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/video/src/index.py @@ -11,16 +11,16 @@ import requests from django.conf import settings from home.src.es.connect import ElasticWrap from home.src.index import channel as ta_channel -from home.src.index import comments as ta_comments from home.src.index import playlist as ta_playlist from home.src.index.generic import YouTubeItem -from home.src.index.subtitle import YoutubeSubtitle -from home.src.index.video_constants import VideoTypeEnum -from home.src.index.video_streams import MediaStreamExtractor from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor from home.src.ta.settings import EnvironmentSettings from home.src.ta.users import UserConfig from ryd_client import ryd_client +from video.src.comments import Comments +from video.src.constants import VideoTypeEnum +from video.src.media_streams import MediaStreamExtractor +from video.src.subtitle import YoutubeSubtitle class SponsorBlock: @@ -323,7 +323,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def delete_comments(self): """delete comments from es""" - comments = ta_comments.Comments(self.youtube_id, config=self.config) + comments = Comments(self.youtube_id, config=self.config) comments.check_config() if comments.is_activated: comments.delete_comments() diff --git a/tubearchivist/home/src/index/video_streams.py b/tubearchivist/video/src/media_streams.py similarity index 100% rename from tubearchivist/home/src/index/video_streams.py rename to tubearchivist/video/src/media_streams.py diff --git a/tubearchivist/home/src/index/subtitle.py b/tubearchivist/video/src/subtitle.py similarity index 100% rename from tubearchivist/home/src/index/subtitle.py rename to tubearchivist/video/src/subtitle.py diff --git a/tubearchivist/video/urls.py b/tubearchivist/video/urls.py new file mode 100644 index 00000000..80b8dbc8 --- /dev/null +++ b/tubearchivist/video/urls.py @@ -0,0 +1,28 @@ +"""all video API urls""" + +from django.urls import path +from video import views + +urlpatterns = [ + path("", views.VideoApiListView.as_view(), name="api-video-list"), + path( + "/", + views.VideoApiView.as_view(), + name="api-video", + ), + path( + "/progress/", + views.VideoProgressView.as_view(), + name="api-video-progress", + ), + path( + "/comment/", + views.VideoCommentView.as_view(), + name="api-video-comment", + ), + path( + "/similar/", + views.VideoSimilarView.as_view(), + name="api-video-similar", + ), +] diff --git a/tubearchivist/video/views.py b/tubearchivist/video/views.py new file mode 100644 index 00000000..68725290 --- /dev/null +++ b/tubearchivist/video/views.py @@ -0,0 +1,127 @@ +"""all API views for video endpoints""" + +from api.views import AdminWriteOnly, ApiBaseView +from home.src.ta.ta_redis import RedisArchivist +from rest_framework.response import Response +from video.src.index import YoutubeVideo + + +class VideoApiView(ApiBaseView): + """resolves to /api/video// + GET: returns metadata dict of video + """ + + search_base = "ta_video/_doc/" + permission_classes = [AdminWriteOnly] + + def get(self, request, video_id): + # pylint: disable=unused-argument + """get request""" + self.get_document(video_id) + return Response(self.response, status=self.status_code) + + def delete(self, request, video_id): + # pylint: disable=unused-argument + """delete single video""" + message = {"video": video_id} + try: + YoutubeVideo(video_id).delete_media_file() + status_code = 200 + message.update({"state": "delete"}) + except FileNotFoundError: + status_code = 404 + message.update({"state": "not found"}) + + return Response(message, status=status_code) + + +class VideoApiListView(ApiBaseView): + """resolves to /api/video/ + GET: returns list of videos + """ + + search_base = "ta_video/_search/" + + def get(self, request): + """get request""" + self.data.update({"sort": [{"published": {"order": "desc"}}]}) + self.get_document_list(request) + + return Response(self.response) + + +class VideoProgressView(ApiBaseView): + """resolves to /api/video//progress/ + handle progress status for video + """ + + def get(self, request, video_id): + """get progress for a single video""" + user_id = request.user.id + key = f"{user_id}:progress:{video_id}" + video_progress = RedisArchivist().get_message(key) + position = video_progress.get("position", 0) + + self.response = { + "youtube_id": video_id, + "user_id": user_id, + "position": position, + } + return Response(self.response) + + def post(self, request, video_id): + """set progress position in redis""" + position = request.data.get("position", 0) + key = f"{request.user.id}:progress:{video_id}" + message = {"position": position, "youtube_id": video_id} + RedisArchivist().set_message(key, message) + self.response = request.data + return Response(self.response) + + def delete(self, request, video_id): + """delete progress position""" + key = f"{request.user.id}:progress:{video_id}" + RedisArchivist().del_message(key) + self.response = {"progress-reset": video_id} + + return Response(self.response) + + +class VideoCommentView(ApiBaseView): + """resolves to /api/video//comment/ + handle video comments + GET: return all comments from video with reply threads + """ + + search_base = "ta_comment/_doc/" + + def get(self, request, video_id): + """get video comments""" + # pylint: disable=unused-argument + self.get_document(video_id) + + return Response(self.response, status=self.status_code) + + +class VideoSimilarView(ApiBaseView): + """resolves to /api/video//similar/ + GET: return max 6 videos similar to this + """ + + search_base = "ta_video/_search/" + + def get(self, request, video_id): + """get similar videos""" + self.data = { + "size": 6, + "query": { + "more_like_this": { + "fields": ["tags", "title"], + "like": {"_id": video_id}, + "min_term_freq": 1, + "max_query_terms": 25, + } + }, + } + self.get_document_list(request, pagination=False) + return Response(self.response, status=self.status_code)