From 457f6a0b8727999ade726d95f17f2e9b849e5f9f Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 11 Jan 2025 16:39:50 +0700 Subject: [PATCH] implement po_token extractor args --- backend/appsettings/src/config.py | 2 + backend/appsettings/urls.py | 5 ++ backend/appsettings/views.py | 32 ++++++++- backend/download/src/queue.py | 1 + backend/download/src/yt_dlp_base.py | 44 ++++++++++++ frontend/src/api/actions/deletePoToken.ts | 9 +++ frontend/src/api/actions/updatePoToken.ts | 10 +++ .../src/api/loader/loadAppsettingsConfig.ts | 1 + frontend/src/pages/SettingsApplication.tsx | 71 +++++++++++++++++-- 9 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 frontend/src/api/actions/deletePoToken.ts create mode 100644 frontend/src/api/actions/updatePoToken.ts diff --git a/backend/appsettings/src/config.py b/backend/appsettings/src/config.py index b135c355..8778f038 100644 --- a/backend/appsettings/src/config.py +++ b/backend/appsettings/src/config.py @@ -40,6 +40,7 @@ class DownloadsConfigType(TypedDict): comment_max: str | bool comment_sort: Literal["top", "new"] cookie_import: bool + potoken: bool throttledratelimit: int extractor_lang: str | bool integrate_ryd: bool @@ -85,6 +86,7 @@ class AppConfig: "comment_max": False, "comment_sort": "top", "cookie_import": False, + "potoken": False, "throttledratelimit": False, "extractor_lang": False, "integrate_ryd": False, diff --git a/backend/appsettings/urls.py b/backend/appsettings/urls.py index a93a2198..14d1c416 100644 --- a/backend/appsettings/urls.py +++ b/backend/appsettings/urls.py @@ -34,6 +34,11 @@ urlpatterns = [ views.CookieView.as_view(), name="api-cookie", ), + path( + "potoken/", + views.POTokenView.as_view(), + name="api-potoken", + ), path( "token/", views.TokenView.as_view(), diff --git a/backend/appsettings/views.py b/backend/appsettings/views.py index e847ebe4..d1ccd972 100644 --- a/backend/appsettings/views.py +++ b/backend/appsettings/views.py @@ -5,7 +5,7 @@ from appsettings.src.config import AppConfig from appsettings.src.snapshot import ElasticSnapshot from common.src.ta_redis import RedisArchivist from common.views_base import AdminOnly, ApiBaseView -from download.src.yt_dlp_base import CookieHandler +from download.src.yt_dlp_base import CookieHandler, POTokenHandler from rest_framework.authtoken.models import Token from rest_framework.response import Response from task.src.task_manager import TaskCommand @@ -252,6 +252,36 @@ class CookieView(ApiBaseView): return validation +class POTokenView(ApiBaseView): + """handle PO token""" + + permission_classes = [AdminOnly] + + def get(self, request): + """get token""" + config = AppConfig().config + potoken = POTokenHandler(config).get() + return Response({"potoken": potoken}) + + def post(self, request): + """post token""" + config = AppConfig().config + new_token = request.data.get("potoken") + if not new_token: + message = "missing potoken key in request data" + print(message) + return Response({"message": message}, status=400) + + POTokenHandler(config).set_token(new_token) + return Response({"potoken": new_token}) + + def delete(self, request): + """delete token""" + config = AppConfig().config + POTokenHandler(config).revoke_token() + return Response({"potoken": None}) + + class TokenView(ApiBaseView): """resolves to /api/appsettings/token/ DELETE: revoke the token diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index cf0e40d9..d2dc4fe3 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -251,6 +251,7 @@ class PendingList(PendingIndex): self._notify_add(idx, total) video_details = self.get_youtube_details(youtube_id, vid_type) if not video_details: + rand_sleep(self.config) continue video_details.update( diff --git a/backend/download/src/yt_dlp_base.py b/backend/download/src/yt_dlp_base.py index 55311a5f..2e59c5ac 100644 --- a/backend/download/src/yt_dlp_base.py +++ b/backend/download/src/yt_dlp_base.py @@ -11,6 +11,7 @@ from io import StringIO import yt_dlp from appsettings.src.config import AppConfig from common.src.ta_redis import RedisArchivist +from django.conf import settings class YtWrap: @@ -36,6 +37,10 @@ class YtWrap: self.obs.update(self.obs_request) if self.config: self.add_cookie() + self.add_potoken() + + if getattr(settings, "DEBUG", False): + print(self.obs) def add_cookie(self): """add cookie if enabled""" @@ -43,6 +48,21 @@ class YtWrap: cookie_io = CookieHandler(self.config).get() self.obs["cookiefile"] = cookie_io + def add_potoken(self): + """add potoken if enabled""" + if self.config["downloads"].get("potoken"): + potoken = POTokenHandler(self.config).get() + self.obs.update( + { + "extractor_args": { + "youtube": { + "po_token": [potoken], + "player-client": ["web", "default"], + }, + } + } + ) + def download(self, url): """make download request""" with yt_dlp.YoutubeDL(self.obs) as ydl: @@ -148,3 +168,27 @@ class CookieHandler: "validated_str": now.strftime("%Y-%m-%d %H:%M"), } RedisArchivist().set_message("cookie:valid", message) + + +class POTokenHandler: + """handle po token""" + + REDIS_KEY = "potoken" + + def __init__(self, config): + self.config = config + + def get(self) -> str | None: + """get PO token""" + potoken = RedisArchivist().get_message_str(self.REDIS_KEY) + return potoken + + def set_token(self, new_token: str) -> None: + """set new PO token""" + RedisArchivist().set_message(self.REDIS_KEY, new_token) + AppConfig().update_config({"downloads.potoken": True}) + + def revoke_token(self) -> None: + """revoke token""" + RedisArchivist().del_message(self.REDIS_KEY) + AppConfig().update_config({"downloads.potoken": False}) diff --git a/frontend/src/api/actions/deletePoToken.ts b/frontend/src/api/actions/deletePoToken.ts new file mode 100644 index 00000000..b8ffc8c5 --- /dev/null +++ b/frontend/src/api/actions/deletePoToken.ts @@ -0,0 +1,9 @@ +import APIClient from '../../functions/APIClient'; + +const deletePoToken = async () => { + return APIClient('/api/appsettings/potoken/', { + method: 'DELETE', + }); +}; + +export default deletePoToken; diff --git a/frontend/src/api/actions/updatePoToken.ts b/frontend/src/api/actions/updatePoToken.ts new file mode 100644 index 00000000..1c6355ef --- /dev/null +++ b/frontend/src/api/actions/updatePoToken.ts @@ -0,0 +1,10 @@ +import APIClient from '../../functions/APIClient'; + +const updatePoToken = async (potoken: string) => { + return APIClient('/api/appsettings/potoken/', { + method: 'POST', + body: { potoken }, + }); +}; + +export default updatePoToken; diff --git a/frontend/src/api/loader/loadAppsettingsConfig.ts b/frontend/src/api/loader/loadAppsettingsConfig.ts index f83cbc19..34ba3e38 100644 --- a/frontend/src/api/loader/loadAppsettingsConfig.ts +++ b/frontend/src/api/loader/loadAppsettingsConfig.ts @@ -21,6 +21,7 @@ export type AppSettingsConfigType = { comment_max: string | null; comment_sort: string; cookie_import: boolean; + potoken: boolean; throttledratelimit: number | null; extractor_lang: string | null; integrate_ryd: boolean; diff --git a/frontend/src/pages/SettingsApplication.tsx b/frontend/src/pages/SettingsApplication.tsx index c5c131cc..33f9e2d7 100644 --- a/frontend/src/pages/SettingsApplication.tsx +++ b/frontend/src/pages/SettingsApplication.tsx @@ -17,6 +17,8 @@ import updateCookie from '../api/actions/updateCookie'; import loadCookie, { CookieStateType } from '../api/loader/loadCookie'; import deleteCookie from '../api/actions/deleteCookie'; import validateCookie from '../api/actions/validateCookie'; +import deletePoToken from '../api/actions/deletePoToken'; +import updatePoToken from '../api/actions/updatePoToken'; type SnapshotType = { id: string; @@ -81,9 +83,8 @@ const SettingsApplication = () => { // Cookie const [cookieFormData, setCookieFormData] = useState(''); const [showCookieForm, setShowCookieForm] = useState(false); - // const [cookieImport, setCookieImport] = useState(false); - // const [validatingCookie, setValidatingCookie] = useState(false); - // const [cookieResponse, setCookieResponse] = useState(); + const [poTokenFormData, setPoTokenFormData] = useState('web+'); + const [showPoTokenForm, setShowPoTokenForm] = useState(false); // Integrations const [showApiToken, setShowApiToken] = useState(false); @@ -129,9 +130,6 @@ const SettingsApplication = () => { setCommentsMax(appSettingsConfig.downloads.comment_max); setCommentsSort(appSettingsConfig.downloads.comment_sort); - // Cookie - // setCookieImport(appSettingsConfig?.downloads.cookie_import); - // Integrations setDownloadDislikes(appSettingsConfig.downloads.integrate_ryd); setEnableSponsorBlock(appSettingsConfig.downloads.integrate_sponsorblock); @@ -172,6 +170,18 @@ const SettingsApplication = () => { setRefresh(true); }; + const handlePoTokenRevoke = async () => { + await deletePoToken(); + setRefresh(true); + }; + + const handlePoTokenUpdate = async () => { + await updatePoToken(poTokenFormData); + setPoTokenFormData('web+'); + setShowPoTokenForm(false); + setRefresh(true); + }; + useEffect(() => { fetchData(); }, []); @@ -509,6 +519,55 @@ const SettingsApplication = () => { )} +
+
+

Add PO Token

+
+
+ {response?.appSettingsConfig?.downloads.potoken ? ( + <> +

PO Token enabled.

+ + + ) : ( +

PO Token disabled

+ )} + {showPoTokenForm ? ( +
+ { + setPoTokenFormData(e.target.value); + }} + /> + {poTokenFormData !== 'web+' && ( + + )} + +
+ ) : ( +
+ +
+ )} +
+

Integrations