diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 51732e8f..ff4c2fce 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -11,36 +11,11 @@ urlpatterns = [ views.RefreshView.as_view(), name="api-refresh", ), - path( - "snapshot/", - views.SnapshotApiListView.as_view(), - name="api-snapshot-list", - ), - path( - "snapshot//", - views.SnapshotApiView.as_view(), - name="api-snapshot", - ), - path( - "backup/", - views.BackupApiListView.as_view(), - name="api-backup-list", - ), - path( - "backup//", - views.BackupApiView.as_view(), - name="api-backup", - ), path( "config/user/", views.UserConfigView.as_view(), name="api-config-user", ), - path( - "cookie/", - views.CookieView.as_view(), - name="api-cookie", - ), path( "watched/", views.WatchedView.as_view(), @@ -51,11 +26,6 @@ urlpatterns = [ views.SearchView.as_view(), name="api-search", ), - path( - "token/", - views.TokenView.as_view(), - name="api-token", - ), path( "notification/", views.NotificationView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index a78b8a1c..4afc9975 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -10,14 +10,11 @@ from api.src.aggs import ( WatchProgress, ) from api.src.search_processor import SearchProcess -from download.src.yt_dlp_base import CookieHandler -from home.src.es.backup import ElasticBackup +from appsettings.src.reindex import ReindexProgress from home.src.es.connect import ElasticWrap -from home.src.es.snapshot import ElasticSnapshot from home.src.frontend.searching import SearchForm from home.src.frontend.watched import WatchState from home.src.index.generic import Pagination -from home.src.index.reindex import ReindexProgress from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist @@ -31,8 +28,7 @@ from rest_framework.authtoken.models import Token from rest_framework.authtoken.views import ObtainAuthToken from rest_framework.response import Response from rest_framework.views import APIView -from task.src.task_manager import TaskCommand -from task.tasks import check_reindex, run_restore_backup +from task.tasks import check_reindex def check_admin(user): @@ -164,148 +160,6 @@ class LoginApiView(ObtainAuthToken): ) -class SnapshotApiListView(ApiBaseView): - """resolves to /api/snapshot/ - GET: returns snapshot config plus list of existing snapshots - POST: take snapshot now - """ - - permission_classes = [AdminOnly] - - @staticmethod - def get(request): - """handle get request""" - # pylint: disable=unused-argument - snapshots = ElasticSnapshot().get_snapshot_stats() - - return Response(snapshots) - - @staticmethod - def post(request): - """take snapshot now with post request""" - # pylint: disable=unused-argument - response = ElasticSnapshot().take_snapshot_now() - - return Response(response) - - -class SnapshotApiView(ApiBaseView): - """resolves to /api/snapshot// - GET: return a single snapshot - POST: restore snapshot - DELETE: delete a snapshot - """ - - permission_classes = [AdminOnly] - - @staticmethod - def get(request, snapshot_id): - """handle get request""" - # pylint: disable=unused-argument - snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id) - - if not snapshot: - return Response({"message": "snapshot not found"}, status=404) - - return Response(snapshot) - - @staticmethod - def post(request, snapshot_id): - """restore snapshot with post request""" - # pylint: disable=unused-argument - response = ElasticSnapshot().restore_all(snapshot_id) - if not response: - message = {"message": "failed to restore snapshot"} - return Response(message, status=400) - - return Response(response) - - @staticmethod - def delete(request, snapshot_id): - """delete snapshot from index""" - # pylint: disable=unused-argument - response = ElasticSnapshot().delete_single_snapshot(snapshot_id) - if not response: - message = {"message": "failed to delete snapshot"} - return Response(message, status=400) - - return Response(response) - - -class BackupApiListView(ApiBaseView): - """resolves to /api/backup/ - GET: returns list of available zip backups - POST: take zip backup now - """ - - permission_classes = [AdminOnly] - task_name = "run_backup" - - @staticmethod - def get(request): - """handle get request""" - # pylint: disable=unused-argument - backup_files = ElasticBackup().get_all_backup_files() - return Response(backup_files) - - def post(self, request): - """handle post request""" - # pylint: disable=unused-argument - response = TaskCommand().start(self.task_name) - message = { - "message": "backup task started", - "task_id": response["task_id"], - } - - return Response(message) - - -class BackupApiView(ApiBaseView): - """resolves to /api/backup// - GET: return a single backup - POST: restore backup - DELETE: delete backup - """ - - permission_classes = [AdminOnly] - task_name = "restore_backup" - - @staticmethod - def get(request, filename): - """get single backup""" - # pylint: disable=unused-argument - backup_file = ElasticBackup().build_backup_file_data(filename) - if not backup_file: - message = {"message": "file not found"} - return Response(message, status=404) - - return Response(backup_file) - - def post(self, request, filename): - """restore backup file""" - # pylint: disable=unused-argument - task = run_restore_backup.delay(filename) - message = { - "message": "backup restore task started", - "filename": filename, - "task_id": task.id, - } - return Response(message) - - @staticmethod - def delete(request, filename): - """delete backup file""" - # pylint: disable=unused-argument - - backup_file = ElasticBackup().delete_file(filename) - if not backup_file: - message = {"message": "file not found"} - return Response(message, status=404) - - message = {"message": f"file {filename} deleted"} - return Response(message) - - class RefreshView(ApiBaseView): """resolves to /api/refresh/ GET: get refresh progress @@ -376,60 +230,6 @@ class UserConfigView(ApiBaseView): return Response(response) -class CookieView(ApiBaseView): - """resolves to /api/cookie/ - GET: check if cookie is enabled - POST: verify validity of cookie - PUT: import cookie - """ - - permission_classes = [AdminOnly] - - @staticmethod - def get(request): - """handle get request""" - # pylint: disable=unused-argument - config = AppConfig().config - valid = RedisArchivist().get_message("cookie:valid") - response = {"cookie_enabled": config["downloads"]["cookie_import"]} - response.update(valid) - - return Response(response) - - @staticmethod - def post(request): - """handle post request""" - # pylint: disable=unused-argument - config = AppConfig().config - validated = CookieHandler(config).validate() - - return Response({"cookie_validated": validated}) - - @staticmethod - def put(request): - """handle put request""" - # pylint: disable=unused-argument - config = AppConfig().config - cookie = request.data.get("cookie") - if not cookie: - message = "missing cookie key in request data" - print(message) - return Response({"message": message}, status=400) - - print(f"cookie preview:\n\n{cookie[:300]}") - handler = CookieHandler(config) - handler.set_cookie(cookie) - validated = handler.validate() - if not validated: - handler.revoke() - message = {"cookie_import": "fail", "cookie_validated": validated} - print(f"cookie: {message}") - return Response({"message": message}, status=400) - - message = {"cookie_import": "done", "cookie_validated": validated} - return Response(message) - - class WatchedView(ApiBaseView): """resolves to /api/watched/ POST: change watched state of video, channel or playlist @@ -467,20 +267,6 @@ class SearchView(ApiBaseView): return Response(search_results) -class TokenView(ApiBaseView): - """resolves to /api/token/ - DELETE: revoke the token - """ - - permission_classes = [AdminOnly] - - @staticmethod - def delete(request): - print("revoke API token") - request.user.auth_token.delete() - return Response({"success": True}) - - class NotificationView(ApiBaseView): """resolves to /api/notification/ GET: returns a list of notifications diff --git a/tubearchivist/appsettings/__init__.py b/tubearchivist/appsettings/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/appsettings/index_mapping.json similarity index 100% rename from tubearchivist/home/src/es/index_mapping.json rename to tubearchivist/appsettings/index_mapping.json diff --git a/tubearchivist/appsettings/migrations/__init__.py b/tubearchivist/appsettings/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/appsettings/src/__init__.py b/tubearchivist/appsettings/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/home/src/es/backup.py b/tubearchivist/appsettings/src/backup.py similarity index 100% rename from tubearchivist/home/src/es/backup.py rename to tubearchivist/appsettings/src/backup.py diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/appsettings/src/filesystem.py similarity index 100% rename from tubearchivist/home/src/index/filesystem.py rename to tubearchivist/appsettings/src/filesystem.py diff --git a/tubearchivist/home/src/es/index_setup.py b/tubearchivist/appsettings/src/index_setup.py similarity index 97% rename from tubearchivist/home/src/es/index_setup.py rename to tubearchivist/appsettings/src/index_setup.py index c913da75..84484021 100644 --- a/tubearchivist/home/src/es/index_setup.py +++ b/tubearchivist/appsettings/src/index_setup.py @@ -5,9 +5,9 @@ functionality: - backup and restore metadata """ -from home.src.es.backup import ElasticBackup +from appsettings.src.backup import ElasticBackup +from appsettings.src.snapshot import ElasticSnapshot from home.src.es.connect import ElasticWrap -from home.src.es.snapshot import ElasticSnapshot from home.src.ta.config import AppConfig from home.src.ta.helper import get_mapping @@ -111,6 +111,8 @@ class ElasticIndex: elif method == "restore": source = f"ta_{self.index_name}_backup" destination = f"ta_{self.index_name}" + else: + raise ValueError("invalid method, expected 'backup' or 'restore'") data = {"source": {"index": source}, "dest": {"index": destination}} _, _ = ElasticWrap("_reindex?refresh=true").post(data=data) diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/appsettings/src/manual.py similarity index 100% rename from tubearchivist/home/src/index/manual.py rename to tubearchivist/appsettings/src/manual.py diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/appsettings/src/reindex.py similarity index 100% rename from tubearchivist/home/src/index/reindex.py rename to tubearchivist/appsettings/src/reindex.py diff --git a/tubearchivist/home/src/es/snapshot.py b/tubearchivist/appsettings/src/snapshot.py similarity index 100% rename from tubearchivist/home/src/es/snapshot.py rename to tubearchivist/appsettings/src/snapshot.py diff --git a/tubearchivist/appsettings/urls.py b/tubearchivist/appsettings/urls.py new file mode 100644 index 00000000..ff5edd0b --- /dev/null +++ b/tubearchivist/appsettings/urls.py @@ -0,0 +1,37 @@ +"""all app settings API urls""" + +from appsettings import views +from django.urls import path + +urlpatterns = [ + path( + "snapshot/", + views.SnapshotApiListView.as_view(), + name="api-snapshot-list", + ), + path( + "snapshot//", + views.SnapshotApiView.as_view(), + name="api-snapshot", + ), + path( + "backup/", + views.BackupApiListView.as_view(), + name="api-backup-list", + ), + path( + "backup//", + views.BackupApiView.as_view(), + name="api-backup", + ), + path( + "cookie/", + views.CookieView.as_view(), + name="api-cookie", + ), + path( + "token/", + views.TokenView.as_view(), + name="api-token", + ), +] diff --git a/tubearchivist/appsettings/views.py b/tubearchivist/appsettings/views.py new file mode 100644 index 00000000..c5480b61 --- /dev/null +++ b/tubearchivist/appsettings/views.py @@ -0,0 +1,222 @@ +"""all app settings API views""" + +from api.views import AdminOnly, ApiBaseView +from appsettings.src.backup import ElasticBackup +from appsettings.src.snapshot import ElasticSnapshot +from download.src.yt_dlp_base import CookieHandler +from home.src.ta.config import AppConfig +from home.src.ta.ta_redis import RedisArchivist +from rest_framework.response import Response +from task.src.task_manager import TaskCommand +from task.tasks import run_restore_backup + + +class SnapshotApiListView(ApiBaseView): + """resolves to /api/appsettings/snapshot/ + GET: returns snapshot config plus list of existing snapshots + POST: take snapshot now + """ + + permission_classes = [AdminOnly] + + @staticmethod + def get(request): + """handle get request""" + # pylint: disable=unused-argument + snapshots = ElasticSnapshot().get_snapshot_stats() + + return Response(snapshots) + + @staticmethod + def post(request): + """take snapshot now with post request""" + # pylint: disable=unused-argument + response = ElasticSnapshot().take_snapshot_now() + + return Response(response) + + +class SnapshotApiView(ApiBaseView): + """resolves to /api/appsettings/snapshot// + GET: return a single snapshot + POST: restore snapshot + DELETE: delete a snapshot + """ + + permission_classes = [AdminOnly] + + @staticmethod + def get(request, snapshot_id): + """handle get request""" + # pylint: disable=unused-argument + snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id) + + if not snapshot: + return Response({"message": "snapshot not found"}, status=404) + + return Response(snapshot) + + @staticmethod + def post(request, snapshot_id): + """restore snapshot with post request""" + # pylint: disable=unused-argument + response = ElasticSnapshot().restore_all(snapshot_id) + if not response: + message = {"message": "failed to restore snapshot"} + return Response(message, status=400) + + return Response(response) + + @staticmethod + def delete(request, snapshot_id): + """delete snapshot from index""" + # pylint: disable=unused-argument + response = ElasticSnapshot().delete_single_snapshot(snapshot_id) + if not response: + message = {"message": "failed to delete snapshot"} + return Response(message, status=400) + + return Response(response) + + +class BackupApiListView(ApiBaseView): + """resolves to /api/appsettings/backup/ + GET: returns list of available zip backups + POST: take zip backup now + """ + + permission_classes = [AdminOnly] + task_name = "run_backup" + + @staticmethod + def get(request): + """handle get request""" + # pylint: disable=unused-argument + backup_files = ElasticBackup().get_all_backup_files() + return Response(backup_files) + + def post(self, request): + """handle post request""" + # pylint: disable=unused-argument + response = TaskCommand().start(self.task_name) + message = { + "message": "backup task started", + "task_id": response["task_id"], + } + + return Response(message) + + +class BackupApiView(ApiBaseView): + """resolves to /api/appsettings/backup// + GET: return a single backup + POST: restore backup + DELETE: delete backup + """ + + permission_classes = [AdminOnly] + task_name = "restore_backup" + + @staticmethod + def get(request, filename): + """get single backup""" + # pylint: disable=unused-argument + backup_file = ElasticBackup().build_backup_file_data(filename) + if not backup_file: + message = {"message": "file not found"} + return Response(message, status=404) + + return Response(backup_file) + + def post(self, request, filename): + """restore backup file""" + # pylint: disable=unused-argument + task = run_restore_backup.delay(filename) + message = { + "message": "backup restore task started", + "filename": filename, + "task_id": task.id, + } + return Response(message) + + @staticmethod + def delete(request, filename): + """delete backup file""" + # pylint: disable=unused-argument + + backup_file = ElasticBackup().delete_file(filename) + if not backup_file: + message = {"message": "file not found"} + return Response(message, status=404) + + message = {"message": f"file {filename} deleted"} + return Response(message) + + +class CookieView(ApiBaseView): + """resolves to /api/appsettings/cookie/ + GET: check if cookie is enabled + POST: verify validity of cookie + PUT: import cookie + """ + + permission_classes = [AdminOnly] + + @staticmethod + def get(request): + """handle get request""" + # pylint: disable=unused-argument + config = AppConfig().config + valid = RedisArchivist().get_message("cookie:valid") + response = {"cookie_enabled": config["downloads"]["cookie_import"]} + response.update(valid) + + return Response(response) + + @staticmethod + def post(request): + """handle post request""" + # pylint: disable=unused-argument + config = AppConfig().config + validated = CookieHandler(config).validate() + + return Response({"cookie_validated": validated}) + + @staticmethod + def put(request): + """handle put request""" + # pylint: disable=unused-argument + config = AppConfig().config + cookie = request.data.get("cookie") + if not cookie: + message = "missing cookie key in request data" + print(message) + return Response({"message": message}, status=400) + + print(f"cookie preview:\n\n{cookie[:300]}") + handler = CookieHandler(config) + handler.set_cookie(cookie) + validated = handler.validate() + if not validated: + handler.revoke() + message = {"cookie_import": "fail", "cookie_validated": validated} + print(f"cookie: {message}") + return Response({"message": message}, status=400) + + message = {"cookie_import": "done", "cookie_validated": validated} + return Response(message) + + +class TokenView(ApiBaseView): + """resolves to /api/appsettings/token/ + DELETE: revoke the token + """ + + permission_classes = [AdminOnly] + + @staticmethod + def delete(request): + """delete the token, new will get created automatically""" + print("revoke API token") + request.user.auth_token.delete() + return Response({"success": True}) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 80ca118b..b9c1b0d2 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -67,6 +67,7 @@ INSTALLED_APPS = [ "playlist", "download", "task", + "appsettings", "config", ] diff --git a/tubearchivist/config/urls.py b/tubearchivist/config/urls.py index b229f0a7..f0324718 100644 --- a/tubearchivist/config/urls.py +++ b/tubearchivist/config/urls.py @@ -25,5 +25,6 @@ urlpatterns = [ path("api/playlist/", include("playlist.urls")), path("api/download/", include("download.urls")), path("api/task/", include("task.urls")), + path("api/appsettings/", include("appsettings.urls")), path("admin/", admin.site.urls), ] diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 5e069804..c89246f4 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -131,7 +131,7 @@ def clear_dl_cache(cache_dir: str) -> int: def get_mapping() -> dict: """read index_mapping.json and get expected mapping and settings""" - with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f: + with open("appsettings/index_mapping.json", "r", encoding="utf-8") as f: index_config: dict = json.load(f).get("index_config") return index_config diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 60c2c2b0..78fcb67a 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -11,6 +11,9 @@ from time import sleep from api.src.search_processor import SearchProcess, process_aggs from api.views import check_admin +from appsettings.src.backup import ElasticBackup +from appsettings.src.reindex import ReindexProgress +from appsettings.src.snapshot import ElasticSnapshot from channel.src.index import channel_overwrites from django.conf import settings from django.contrib.auth import login @@ -22,9 +25,7 @@ from django.utils.decorators import method_decorator from django.views import View from download.src.queue import PendingInteract from download.src.yt_dlp_base import CookieHandler -from home.src.es.backup import ElasticBackup from home.src.es.connect import ElasticWrap -from home.src.es.snapshot import ElasticSnapshot from home.src.frontend.forms import ( AddToQueueForm, ApplicationSettingsForm, @@ -41,7 +42,6 @@ from home.src.frontend.forms_schedule import ( SchedulerSettingsForm, ) from home.src.index.generic import Pagination -from home.src.index.reindex import ReindexProgress from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.helper import check_stylesheet, time_parser from home.src.ta.notify import Notifications, get_all_notifications diff --git a/tubearchivist/task/tasks.py b/tubearchivist/task/tasks.py index 2759b13b..f0685e45 100644 --- a/tubearchivist/task/tasks.py +++ b/tubearchivist/task/tasks.py @@ -6,6 +6,11 @@ Functionality: - handle task locking """ +from appsettings.src.backup import ElasticBackup +from appsettings.src.filesystem import Scanner +from appsettings.src.index_setup import ElasitIndexWrap +from appsettings.src.manual import ImportFolderScanner +from appsettings.src.reindex import Reindex, ReindexManual, ReindexPopulate from celery import Task, shared_task from celery.exceptions import Retry from channel.src.index import YoutubeChannel @@ -13,11 +18,6 @@ from download.src.queue import PendingList from download.src.subscriptions import SubscriptionHandler, SubscriptionScanner from download.src.thumbnails import ThumbFilesystem, ThumbValidator from download.src.yt_dlp_handler import VideoDownloader -from home.src.es.backup import ElasticBackup -from home.src.es.index_setup import ElasitIndexWrap -from home.src.index.filesystem import Scanner -from home.src.index.manual import ImportFolderScanner -from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate from home.src.ta.config import ReleaseVersion from home.src.ta.notify import Notifications from home.src.ta.ta_redis import RedisArchivist