diff --git a/tubearchivist/api/src/aggs.py b/tubearchivist/api/src/aggs.py index 1e2f54c6..ac6c2e69 100644 --- a/tubearchivist/api/src/aggs.py +++ b/tubearchivist/api/src/aggs.py @@ -117,7 +117,7 @@ class WatchProgress(AggBase): all_duration = int(aggregations["total_duration"].get("value")) response.update( { - "all": { + "total": { "duration": all_duration, "duration_str": get_duration_str(all_duration), "items": aggregations["total_vids"].get("value"), diff --git a/tubearchivist/api/src/search_processor.py b/tubearchivist/api/src/search_processor.py index 232474d4..9f77b05e 100644 --- a/tubearchivist/api/src/search_processor.py +++ b/tubearchivist/api/src/search_processor.py @@ -7,15 +7,14 @@ Functionality: import urllib.parse from home.src.download.thumbnails import ThumbManager -from home.src.ta.config import AppConfig from home.src.ta.helper import date_praser, get_duration_str +from home.src.ta.settings import EnvironmentSettings class SearchProcess: """process search results""" - CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR def __init__(self, response): self.response = response diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index a01a8749..bb72322b 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -18,6 +18,7 @@ from home.src.index.playlist import YoutubePlaylist from home.src.index.reindex import ReindexProgress from home.src.index.video import SponsorBlock, YoutubeVideo from home.src.ta.config import AppConfig, ReleaseVersion +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskCommand, TaskManager from home.src.ta.urlparser import Parser @@ -28,28 +29,56 @@ from home.tasks import ( extrac_dl, subscribe_to, ) +from rest_framework import permissions from rest_framework.authentication import ( SessionAuthentication, TokenAuthentication, ) from rest_framework.authtoken.models import Token from rest_framework.authtoken.views import ObtainAuthToken -from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView +def check_admin(user): + """check for admin permission for restricted views""" + return user.is_staff or user.groups.filter(name="admin").exists() + + +class AdminOnly(permissions.BasePermission): + """allow only admin""" + + def has_permission(self, request, view): + return check_admin(request.user) + + +class AdminWriteOnly(permissions.BasePermission): + """allow only admin writes""" + + def has_permission(self, request, view): + if request.method in permissions.SAFE_METHODS: + return permissions.IsAuthenticated().has_permission(request, view) + + return check_admin(request.user) + + class ApiBaseView(APIView): """base view to inherit from""" authentication_classes = [SessionAuthentication, TokenAuthentication] - permission_classes = [IsAuthenticated] + permission_classes = [permissions.IsAuthenticated] search_base = "" data = "" def __init__(self): super().__init__() - self.response = {"data": False, "config": AppConfig().config} + self.response = { + "data": False, + "config": { + "enable_cast": EnvironmentSettings.ENABLE_CAST, + "downloads": AppConfig().config["downloads"], + }, + } self.data = {"query": {"match_all": {}}} self.status_code = False self.context = False @@ -102,6 +131,7 @@ class VideoApiView(ApiBaseView): """ search_base = "ta_video/_doc/" + permission_classes = [AdminWriteOnly] def get(self, request, video_id): # pylint: disable=unused-argument @@ -165,7 +195,6 @@ class VideoProgressView(ApiBaseView): message = {"position": position, "youtube_id": video_id} RedisArchivist().set_message(key, message) self.response = request.data - return Response(self.response) def delete(self, request, video_id): @@ -276,6 +305,7 @@ class ChannelApiView(ApiBaseView): """ search_base = "ta_channel/_doc/" + permission_classes = [AdminWriteOnly] def get(self, request, channel_id): # pylint: disable=unused-argument @@ -306,6 +336,7 @@ class ChannelApiListView(ApiBaseView): search_base = "ta_channel/_search/" valid_filter = ["subscribed"] + permission_classes = [AdminWriteOnly] def get(self, request): """get request""" @@ -419,6 +450,7 @@ class PlaylistApiListView(ApiBaseView): """ search_base = "ta_playlist/_search/" + permission_classes = [AdminWriteOnly] def get(self, request): """handle get request""" @@ -467,6 +499,7 @@ class PlaylistApiView(ApiBaseView): """ search_base = "ta_playlist/_doc/" + permission_classes = [AdminWriteOnly] def get(self, request, playlist_id): # pylint: disable=unused-argument @@ -513,6 +546,7 @@ class DownloadApiView(ApiBaseView): search_base = "ta_download/_doc/" valid_status = ["pending", "ignore", "priority"] + permission_classes = [AdminOnly] def get(self, request, video_id): # pylint: disable=unused-argument @@ -559,6 +593,7 @@ class DownloadApiListView(ApiBaseView): search_base = "ta_download/_search/" valid_filter = ["pending", "ignore"] + permission_classes = [AdminOnly] def get(self, request): """get request""" @@ -667,6 +702,8 @@ class SnapshotApiListView(ApiBaseView): POST: take snapshot now """ + permission_classes = [AdminOnly] + @staticmethod def get(request): """handle get request""" @@ -691,6 +728,8 @@ class SnapshotApiView(ApiBaseView): DELETE: delete a snapshot """ + permission_classes = [AdminOnly] + @staticmethod def get(request, snapshot_id): """handle get request""" @@ -730,6 +769,8 @@ class TaskListView(ApiBaseView): GET: return a list of all stored task results """ + permission_classes = [AdminOnly] + def get(self, request): """handle get request""" # pylint: disable=unused-argument @@ -744,6 +785,8 @@ class TaskNameListView(ApiBaseView): POST: start new background process """ + permission_classes = [AdminOnly] + def get(self, request, task_name): """handle get request""" # pylint: disable=unused-argument @@ -782,6 +825,7 @@ class TaskIDView(ApiBaseView): """ valid_commands = ["stop", "kill"] + permission_classes = [AdminOnly] def get(self, request, task_id): """handle get request""" @@ -833,6 +877,8 @@ class RefreshView(ApiBaseView): POST: start a manual refresh task """ + permission_classes = [AdminOnly] + def get(self, request): """handle get request""" request_type = request.GET.get("type") @@ -866,6 +912,8 @@ class CookieView(ApiBaseView): PUT: import cookie """ + permission_classes = [AdminOnly] + @staticmethod def get(request): """handle get request""" @@ -953,6 +1001,8 @@ class TokenView(ApiBaseView): DELETE: revoke the token """ + permission_classes = [AdminOnly] + @staticmethod def delete(request): print("revoke API token") diff --git a/tubearchivist/config/management/commands/ta_connection.py b/tubearchivist/config/management/commands/ta_connection.py index 4e0f59aa..94b5e0dc 100644 --- a/tubearchivist/config/management/commands/ta_connection.py +++ b/tubearchivist/config/management/commands/ta_connection.py @@ -3,12 +3,12 @@ Functionality: - check that all connections are working """ -from os import environ from time import sleep import requests from django.core.management.base import BaseCommand, CommandError from home.src.es.connect import ElasticWrap +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist TOPIC = """ @@ -156,10 +156,8 @@ class Command(BaseCommand): " 🗙 path.repo env var not found. " + "set the following env var to the ES container:\n" + " path.repo=" - + environ.get( - "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot" - ), + + EnvironmentSettings.ES_SNAPSHOT_DIR ) - self.stdout.write(self.style.ERROR(f"{message}")) + self.stdout.write(self.style.ERROR(message)) sleep(60) raise CommandError(message) diff --git a/tubearchivist/config/management/commands/ta_envcheck.py b/tubearchivist/config/management/commands/ta_envcheck.py index 446962d9..76c5ba10 100644 --- a/tubearchivist/config/management/commands/ta_envcheck.py +++ b/tubearchivist/config/management/commands/ta_envcheck.py @@ -11,6 +11,7 @@ import re from django.core.management.base import BaseCommand, CommandError from home.models import Account +from home.src.ta.settings import EnvironmentSettings LOGO = """ @@ -96,18 +97,14 @@ class Command(BaseCommand): def _elastic_user_overwrite(self): """check for ELASTIC_USER overwrite""" - self.stdout.write("[2] set default ES user") - if not os.environ.get("ELASTIC_USER"): - os.environ.setdefault("ELASTIC_USER", "elastic") - - env = os.environ.get("ELASTIC_USER") - + self.stdout.write("[2] check ES user overwrite") + env = EnvironmentSettings.ES_USER self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}")) def _ta_port_overwrite(self): """set TA_PORT overwrite for nginx""" self.stdout.write("[3] check TA_PORT overwrite") - overwrite = os.environ.get("TA_PORT") + overwrite = EnvironmentSettings.TA_PORT if not overwrite: self.stdout.write(self.style.SUCCESS(" TA_PORT is not set")) return @@ -125,7 +122,7 @@ class Command(BaseCommand): def _ta_uwsgi_overwrite(self): """set TA_UWSGI_PORT overwrite""" self.stdout.write("[4] check TA_UWSGI_PORT overwrite") - overwrite = os.environ.get("TA_UWSGI_PORT") + overwrite = EnvironmentSettings.TA_UWSGI_PORT if not overwrite: message = " TA_UWSGI_PORT is not set" self.stdout.write(self.style.SUCCESS(message)) @@ -151,7 +148,7 @@ class Command(BaseCommand): def _enable_cast_overwrite(self): """cast workaround, remove auth for static files in nginx""" self.stdout.write("[5] check ENABLE_CAST overwrite") - overwrite = os.environ.get("ENABLE_CAST") + overwrite = EnvironmentSettings.ENABLE_CAST if not overwrite: self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set")) return @@ -174,8 +171,8 @@ class Command(BaseCommand): self.stdout.write(self.style.SUCCESS(message)) return - name = os.environ.get("TA_USERNAME") - password = os.environ.get("TA_PASSWORD") + name = EnvironmentSettings.TA_USERNAME + password = EnvironmentSettings.TA_PASSWORD Account.objects.create_superuser(name, password) message = f" ✓ new superuser with name {name} created" self.stdout.write(self.style.SUCCESS(message)) diff --git a/tubearchivist/config/management/commands/ta_migpath.py b/tubearchivist/config/management/commands/ta_migpath.py index 607c58ee..05f0b76f 100644 --- a/tubearchivist/config/management/commands/ta_migpath.py +++ b/tubearchivist/config/management/commands/ta_migpath.py @@ -6,8 +6,8 @@ import shutil from django.core.management.base import BaseCommand from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings TOPIC = """ @@ -58,8 +58,7 @@ class FolderMigration: """migrate video archive folder""" def __init__(self): - self.config = AppConfig().config - self.videos = self.config["application"]["videos"] + self.videos = EnvironmentSettings.MEDIA_DIR self.bulk_list = [] def get_to_migrate(self): @@ -84,8 +83,8 @@ class FolderMigration: def create_folders(self, to_migrate): """create required channel folders""" - host_uid = self.config["application"]["HOST_UID"] - host_gid = self.config["application"]["HOST_GID"] + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate} for channel_id in all_channel_ids: diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index 602731f8..b10fc997 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -8,12 +8,11 @@ import os from time import sleep from django.core.management.base import BaseCommand, CommandError -from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.es.index_setup import ElasitIndexWrap from home.src.es.snapshot import ElasticSnapshot -from home.src.index.video_streams import MediaStreamExtractor from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.helper import clear_dl_cache +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskManager from home.src.ta.users import UserConfig @@ -43,8 +42,6 @@ class Command(BaseCommand): self._version_check() self._mig_index_setup() self._mig_snapshot_check() - self._mig_set_streams() - self._mig_set_autostart() self._mig_move_users_to_es() def _sync_redis_state(self): @@ -69,7 +66,7 @@ class Command(BaseCommand): "playlists", "videos", ] - cache_dir = AppConfig().config["application"]["cache_dir"] + cache_dir = EnvironmentSettings.CACHE_DIR for folder in folders: folder_path = os.path.join(cache_dir, folder) os.makedirs(folder_path, exist_ok=True) @@ -119,8 +116,7 @@ class Command(BaseCommand): def _clear_dl_cache(self): """clear leftover files from dl cache""" self.stdout.write("[5] clear leftover files from dl cache") - config = AppConfig().config - leftover_files = clear_dl_cache(config) + leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR) if leftover_files: self.stdout.write( self.style.SUCCESS(f" ✓ cleared {leftover_files} files") @@ -149,79 +145,6 @@ class Command(BaseCommand): self.stdout.write("[MIGRATION] setup snapshots") ElasticSnapshot().setup() - def _mig_set_streams(self): - """migration: update from 0.3.5 to 0.3.6, set streams and media_size""" - self.stdout.write("[MIGRATION] index streams and media size") - videos = AppConfig().config["application"]["videos"] - data = { - "query": { - "bool": {"must_not": [{"exists": {"field": "streams"}}]} - }, - "_source": ["media_url", "youtube_id"], - } - all_missing = IndexPaginate("ta_video", data).get_results() - if not all_missing: - self.stdout.write(" no videos need updating") - return - - total = len(all_missing) - for idx, missing in enumerate(all_missing): - media_url = missing["media_url"] - youtube_id = missing["youtube_id"] - media_path = os.path.join(videos, media_url) - if not os.path.exists(media_path): - self.stdout.write(f" file not found: {media_path}") - self.stdout.write(" run file system rescan to fix") - continue - - media = MediaStreamExtractor(media_path) - vid_data = { - "doc": { - "streams": media.extract_metadata(), - "media_size": media.get_file_size(), - } - } - path = f"ta_video/_update/{youtube_id}" - response, status_code = ElasticWrap(path).post(data=vid_data) - if not status_code == 200: - self.stdout.errors( - f" update failed: {path}, {response}, {status_code}" - ) - - if idx % 100 == 0: - self.stdout.write(f" progress {idx}/{total}") - - def _mig_set_autostart(self): - """migration: update from 0.3.5 to 0.3.6 set auto_start to false""" - self.stdout.write("[MIGRATION] set default download auto_start") - data = { - "query": { - "bool": {"must_not": [{"exists": {"field": "auto_start"}}]} - }, - "script": {"source": "ctx._source['auto_start'] = false"}, - } - path = "ta_download/_update_by_query" - response, status_code = ElasticWrap(path).post(data=data) - if status_code == 200: - updated = response.get("updated", 0) - if updated: - self.stdout.write( - self.style.SUCCESS( - f" ✓ {updated} videos updated in ta_download" - ) - ) - else: - self.stdout.write( - " no videos needed updating in ta_download" - ) - return - - message = " 🗙 ta_download auto_start update failed" - self.stdout.write(self.style.ERROR(message)) - self.stdout.write(response) - sleep(60) - raise CommandError(message) - def _mig_move_users_to_es(self): # noqa: C901 """migration: update from 0.4.1 to 0.4.2 move user config to ES""" self.stdout.write("[MIGRATION] move user configuration to ES") @@ -299,12 +222,12 @@ class Command(BaseCommand): f" ✓ Settings for user '{user}' migrated to ES" ) ) - except Exception as e: + except Exception as err: message = " 🗙 user migration to ES failed" self.stdout.write(self.style.ERROR(message)) - self.stdout.write(self.style.ERROR(e)) + self.stdout.write(self.style.ERROR(err)) sleep(60) - raise CommandError(message) + raise CommandError(message) from err else: self.stdout.write( self.style.SUCCESS( diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 01759857..765a6f2b 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -17,8 +17,8 @@ from pathlib import Path import ldap from corsheaders.defaults import default_headers from django_auth_ldap.config import LDAPSearch -from home.src.ta.config import AppConfig from home.src.ta.helper import ta_host_parser +from home.src.ta.settings import EnvironmentSettings # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -27,7 +27,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ -PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode()) +PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode()) SECRET_KEY = PW_HASH.hexdigest() # SECURITY WARNING: don't run with debug turned on in production! @@ -180,7 +180,7 @@ if bool(environ.get("TA_LDAP")): # Database # https://docs.djangoproject.com/en/3.2/ref/settings/#databases -CACHE_DIR = AppConfig().config["application"]["cache_dir"] +CACHE_DIR = EnvironmentSettings.CACHE_DIR DB_PATH = path.join(CACHE_DIR, "db.sqlite3") DATABASES = { "default": { @@ -228,7 +228,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")): # https://docs.djangoproject.com/en/3.2/topics/i18n/ LANGUAGE_CODE = "en-us" -TIME_ZONE = environ.get("TZ") or "UTC" +TIME_ZONE = EnvironmentSettings.TZ USE_I18N = True USE_L10N = True USE_TZ = True @@ -269,4 +269,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" -TA_VERSION = "v0.4.2" +TA_VERSION = "v0.4.3-unstable" diff --git a/tubearchivist/home/config.json b/tubearchivist/home/config.json index 26d3bf90..1b655f47 100644 --- a/tubearchivist/home/config.json +++ b/tubearchivist/home/config.json @@ -25,10 +25,6 @@ "integrate_sponsorblock": false }, "application": { - "app_root": "/app", - "cache_dir": "/cache", - "videos": "/youtube", - "enable_cast": false, "enable_snapshot": true }, "scheduler": { diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index ca498c58..896f6031 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -11,7 +11,7 @@ from time import sleep import requests from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig +from home.src.ta.settings import EnvironmentSettings from mutagen.mp4 import MP4, MP4Cover from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError @@ -21,8 +21,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True class ThumbManagerBase: """base class for thumbnail management""" - CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR VIDEO_DIR = os.path.join(CACHE_DIR, "videos") CHANNEL_DIR = os.path.join(CACHE_DIR, "channels") PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists") @@ -70,7 +69,7 @@ class ThumbManagerBase: img_raw = Image.open(self.fallback) return img_raw - app_root = self.CONFIG["application"]["app_root"] + app_root = EnvironmentSettings.APP_DIR default_map = { "video": os.path.join( app_root, "static/img/default-video-thumb.jpg" @@ -380,9 +379,8 @@ class ThumbFilesystem: class EmbedCallback: """callback class to embed thumbnails""" - CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] - MEDIA_DIR = CONFIG["application"]["videos"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR + MEDIA_DIR = EnvironmentSettings.MEDIA_DIR FORMAT = MP4Cover.FORMAT_JPEG def __init__(self, source, index_name, counter=0): diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index b39601cd..b1c8fa75 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -10,6 +10,7 @@ from http import cookiejar from io import StringIO import yt_dlp +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist @@ -86,6 +87,7 @@ class CookieHandler: def __init__(self, config): self.cookie_io = False self.config = config + self.cache_dir = EnvironmentSettings.CACHE_DIR def get(self): """get cookie io stream""" @@ -95,8 +97,9 @@ class CookieHandler: def import_cookie(self): """import cookie from file""" - cache_path = self.config["application"]["cache_dir"] - import_path = os.path.join(cache_path, "import", "cookies.google.txt") + import_path = os.path.join( + self.cache_dir, "import", "cookies.google.txt" + ) try: with open(import_path, encoding="utf-8") as cookie_file: diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 9a865aba..efcdb9b4 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -21,6 +21,7 @@ from home.src.index.video import YoutubeVideo, index_new_video from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings class DownloadPostProcess: @@ -153,6 +154,8 @@ class VideoDownloader: self.youtube_id_list = youtube_id_list self.task = task self.config = AppConfig().config + self.cache_dir = EnvironmentSettings.CACHE_DIR + self.media_dir = EnvironmentSettings.MEDIA_DIR self._build_obs() self.channels = set() self.videos = set() @@ -262,10 +265,7 @@ class VideoDownloader: """initial obs""" self.obs = { "merge_output_format": "mp4", - "outtmpl": ( - self.config["application"]["cache_dir"] - + "/download/%(id)s.mp4" - ), + "outtmpl": (self.cache_dir + "/download/%(id)s.mp4"), "progress_hooks": [self._progress_hook], "noprogress": True, "continuedl": True, @@ -340,7 +340,7 @@ class VideoDownloader: if format_overwrite: obs["format"] = format_overwrite - dl_cache = self.config["application"]["cache_dir"] + "/download/" + dl_cache = self.cache_dir + "/download/" # check if already in cache to continue from there all_cached = ignore_filelist(os.listdir(dl_cache)) @@ -370,20 +370,20 @@ class VideoDownloader: def move_to_archive(self, vid_dict): """move downloaded video from cache to archive""" - videos = self.config["application"]["videos"] - host_uid = self.config["application"]["HOST_UID"] - host_gid = self.config["application"]["HOST_GID"] + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID # make folder - folder = os.path.join(videos, vid_dict["channel"]["channel_id"]) + folder = os.path.join( + self.media_dir, vid_dict["channel"]["channel_id"] + ) if not os.path.exists(folder): os.makedirs(folder) if host_uid and host_gid: os.chown(folder, host_uid, host_gid) # move media file media_file = vid_dict["youtube_id"] + ".mp4" - cache_dir = self.config["application"]["cache_dir"] - old_path = os.path.join(cache_dir, "download", media_file) - new_path = os.path.join(videos, vid_dict["media_url"]) + old_path = os.path.join(self.cache_dir, "download", media_file) + new_path = os.path.join(self.media_dir, vid_dict["media_url"]) # move media file and fix permission shutil.move(old_path, new_path, copy_function=shutil.copyfile) if host_uid and host_gid: diff --git a/tubearchivist/home/src/es/backup.py b/tubearchivist/home/src/es/backup.py index 3dc1cf5a..b23592a4 100644 --- a/tubearchivist/home/src/es/backup.py +++ b/tubearchivist/home/src/es/backup.py @@ -13,6 +13,7 @@ from datetime import datetime from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.ta.config import AppConfig from home.src.ta.helper import get_mapping, ignore_filelist +from home.src.ta.settings import EnvironmentSettings class ElasticBackup: @@ -22,7 +23,7 @@ class ElasticBackup: def __init__(self, reason=False, task=False): self.config = AppConfig().config - self.cache_dir = self.config["application"]["cache_dir"] + self.cache_dir = EnvironmentSettings.CACHE_DIR self.timestamp = datetime.now().strftime("%Y%m%d") self.index_config = get_mapping() self.reason = reason @@ -217,6 +218,7 @@ class BackupCallback: self.index_name = index_name self.counter = counter self.timestamp = datetime.now().strftime("%Y%m%d") + self.cache_dir = EnvironmentSettings.CACHE_DIR def run(self): """run the junk task""" @@ -243,9 +245,8 @@ class BackupCallback: def _write_es_json(self, file_content): """write nd-json file for es _bulk API to disk""" - cache_dir = AppConfig().config["application"]["cache_dir"] index = self.index_name.lstrip("ta_") file_name = f"es_{index}-{self.timestamp}-{self.counter}.json" - file_path = os.path.join(cache_dir, "backup", file_name) + file_path = os.path.join(self.cache_dir, "backup", file_name) with open(file_path, "a+", encoding="utf-8") as f: f.write(file_content) diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index a7c3ff5f..f343b8e4 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -6,11 +6,11 @@ functionality: # pylint: disable=missing-timeout import json -import os from typing import Any import requests import urllib3 +from home.src.ta.settings import EnvironmentSettings class ElasticWrap: @@ -18,16 +18,14 @@ class ElasticWrap: returns response json and status code tuple """ - ES_URL: str = str(os.environ.get("ES_URL")) - ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD")) - ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic") - ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL")) - def __init__(self, path: str): - self.url: str = f"{self.ES_URL}/{path}" - self.auth: tuple[str, str] = (self.ES_USER, self.ES_PASS) + self.url: str = f"{EnvironmentSettings.ES_URL}/{path}" + self.auth: tuple[str, str] = ( + EnvironmentSettings.ES_USER, + EnvironmentSettings.ES_PASS, + ) - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) def get( @@ -43,7 +41,7 @@ class ElasticWrap: "timeout": timeout, } - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False if data: @@ -78,7 +76,7 @@ class ElasticWrap: } ) - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False response = requests.post(self.url, **kwargs) @@ -103,7 +101,7 @@ class ElasticWrap: "auth": self.auth, } - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False response = requests.put(self.url, **kwargs) @@ -130,7 +128,7 @@ class ElasticWrap: if data: kwargs["json"] = data - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False response = requests.delete(self.url, **kwargs) diff --git a/tubearchivist/home/src/es/snapshot.py b/tubearchivist/home/src/es/snapshot.py index 15fc82c6..0cff51e6 100644 --- a/tubearchivist/home/src/es/snapshot.py +++ b/tubearchivist/home/src/es/snapshot.py @@ -4,12 +4,12 @@ functionality: """ from datetime import datetime -from os import environ from time import sleep from zoneinfo import ZoneInfo from home.src.es.connect import ElasticWrap from home.src.ta.helper import get_mapping +from home.src.ta.settings import EnvironmentSettings class ElasticSnapshot: @@ -19,9 +19,7 @@ class ElasticSnapshot: REPO_SETTINGS = { "compress": "true", "chunk_size": "1g", - "location": environ.get( - "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot" - ), + "location": EnvironmentSettings.ES_SNAPSHOT_DIR, } POLICY = "ta_daily" @@ -256,7 +254,7 @@ class ElasticSnapshot: expected_format = "%Y-%m-%dT%H:%M:%S.%fZ" date = datetime.strptime(date_utc, expected_format) local_datetime = date.replace(tzinfo=ZoneInfo("localtime")) - converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ"))) + converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ)) converted_str = converted.strftime("%Y-%m-%d %H:%M") return converted_str diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 19070626..02e57266 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -14,6 +14,7 @@ from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.generic import YouTubeItem from home.src.index.playlist import YoutubePlaylist +from home.src.ta.settings import EnvironmentSettings class YoutubeChannel(YouTubeItem): @@ -134,7 +135,7 @@ class YoutubeChannel(YouTubeItem): def _info_json_fallback(self): """read channel info.json for additional metadata""" info_json = os.path.join( - self.config["application"]["cache_dir"], + EnvironmentSettings.CACHE_DIR, "import", f"{self.youtube_id}.info.json", ) @@ -178,7 +179,7 @@ class YoutubeChannel(YouTubeItem): def get_folder_path(self): """get folder where media files get stored""" folder_path = os.path.join( - self.app_conf["videos"], + EnvironmentSettings.MEDIA_DIR, self.json_data["channel_id"], ) return folder_path diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index ecd35092..ab208c22 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -8,14 +8,14 @@ import os from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.comments import CommentList from home.src.index.video import YoutubeVideo, index_new_video -from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings class Scanner: """scan index and filesystem""" - VIDEOS: str = AppConfig().config["application"]["videos"] + VIDEOS: str = EnvironmentSettings.MEDIA_DIR def __init__(self, task=False) -> None: self.task = task diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index a5f624de..e18623ee 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -26,7 +26,6 @@ class YouTubeItem: self.youtube_id = youtube_id self.es_path = f"{self.index_name}/_doc/{youtube_id}" self.config = AppConfig().config - self.app_conf = self.config["application"] self.youtube_meta = False self.json_data = False diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/home/src/index/manual.py index 3a19588f..a25b0611 100644 --- a/tubearchivist/home/src/index/manual.py +++ b/tubearchivist/home/src/index/manual.py @@ -16,6 +16,7 @@ from home.src.index.comments import CommentList from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings from PIL import Image from yt_dlp.utils import ISO639Utils @@ -28,7 +29,7 @@ class ImportFolderScanner: """ CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR IMPORT_DIR = os.path.join(CACHE_DIR, "import") """All extensions should be in lowercase until better handling is in place. @@ -433,9 +434,9 @@ class ManualImport: def _move_to_archive(self, json_data): """move identified media file to archive""" - videos = self.config["application"]["videos"] - host_uid = self.config["application"]["HOST_UID"] - host_gid = self.config["application"]["HOST_GID"] + videos = EnvironmentSettings.MEDIA_DIR + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID channel, file = os.path.split(json_data["media_url"]) channel_folder = os.path.join(videos, channel) @@ -472,7 +473,7 @@ class ManualImport: os.remove(subtitle_file) channel_info = os.path.join( - self.config["application"]["cache_dir"], + EnvironmentSettings.CACHE_DIR, "import", f"{json_data['channel']['channel_id']}.info.json", ) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index b89c00d9..e87b30aa 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -19,6 +19,7 @@ from home.src.index.comments import Comments from home.src.index.playlist import YoutubePlaylist from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisQueue @@ -293,7 +294,7 @@ class Reindex(ReindexBase): # get new media_url = os.path.join( - self.config["application"]["videos"], es_meta["media_url"] + EnvironmentSettings.MEDIA_DIR, es_meta["media_url"] ) video.build_json(media_path=media_url) if not video.youtube_meta: @@ -311,10 +312,6 @@ class Reindex(ReindexBase): video.json_data["playlist"] = es_meta.get("playlist") video.upload_to_es() - if es_meta.get("media_url") != video.json_data["media_url"]: - self._rename_media_file( - es_meta.get("media_url"), video.json_data["media_url"] - ) thumb_handler = ThumbManager(youtube_id) thumb_handler.delete_video_thumb() @@ -325,14 +322,6 @@ class Reindex(ReindexBase): return - def _rename_media_file(self, media_url_is, media_url_should): - """handle title change""" - print(f"[reindex] fix media_url {media_url_is} to {media_url_should}") - videos = self.config["application"]["videos"] - old_path = os.path.join(videos, media_url_is) - new_path = os.path.join(videos, media_url_should) - os.rename(old_path, new_path) - def _reindex_single_channel(self, channel_id): """refresh channel data and sync to videos""" # read current state diff --git a/tubearchivist/home/src/index/subtitle.py b/tubearchivist/home/src/index/subtitle.py index 18af56ac..b57ad55b 100644 --- a/tubearchivist/home/src/index/subtitle.py +++ b/tubearchivist/home/src/index/subtitle.py @@ -12,6 +12,7 @@ from datetime import datetime import requests from home.src.es.connect import ElasticWrap from home.src.ta.helper import requests_headers +from home.src.ta.settings import EnvironmentSettings class YoutubeSubtitle: @@ -113,7 +114,7 @@ class YoutubeSubtitle: def download_subtitles(self, relevant_subtitles): """download subtitle files to archive""" - videos_base = self.video.config["application"]["videos"] + videos_base = EnvironmentSettings.MEDIA_DIR indexed = [] for subtitle in relevant_subtitles: dest_path = os.path.join(videos_base, subtitle["media_url"]) @@ -149,8 +150,8 @@ class YoutubeSubtitle: with open(dest_path, "w", encoding="utf-8") as subfile: subfile.write(subtitle_str) - host_uid = self.video.config["application"]["HOST_UID"] - host_gid = self.video.config["application"]["HOST_GID"] + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID if host_uid and host_gid: os.chown(dest_path, host_uid, host_gid) @@ -162,7 +163,7 @@ class YoutubeSubtitle: def delete(self, subtitles=False): """delete subtitles from index and filesystem""" youtube_id = self.video.youtube_id - videos_base = self.video.config["application"]["videos"] + videos_base = EnvironmentSettings.MEDIA_DIR # delete files if subtitles: files = [i["media_url"] for i in subtitles] diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 606e32f3..b4143506 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -18,6 +18,7 @@ from home.src.index.subtitle import YoutubeSubtitle from home.src.index.video_constants import VideoTypeEnum from home.src.index.video_streams import MediaStreamExtractor from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor +from home.src.ta.settings import EnvironmentSettings from home.src.ta.users import UserConfig from ryd_client import ryd_client @@ -226,14 +227,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def build_dl_cache_path(self): """find video path in dl cache""" - cache_dir = self.app_conf["cache_dir"] + cache_dir = EnvironmentSettings.CACHE_DIR video_id = self.json_data["youtube_id"] cache_path = f"{cache_dir}/download/{video_id}.mp4" if os.path.exists(cache_path): return cache_path channel_path = os.path.join( - self.app_conf["videos"], + EnvironmentSettings.MEDIA_DIR, self.json_data["channel"]["channel_id"], f"{video_id}.mp4", ) @@ -282,7 +283,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): if not self.json_data: raise FileNotFoundError - video_base = self.app_conf["videos"] + video_base = EnvironmentSettings.MEDIA_DIR media_url = self.json_data.get("media_url") file_path = os.path.join(video_base, media_url) try: diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index a32d0830..c6e9e68f 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -5,7 +5,6 @@ Functionality: """ import json -import os import re from random import randint from time import sleep @@ -28,7 +27,6 @@ class AppConfig: if not config: config = self.get_config_file() - config["application"].update(self.get_config_env()) return config def get_config_file(self): @@ -36,25 +34,8 @@ class AppConfig: with open("home/config.json", "r", encoding="utf-8") as f: config_file = json.load(f) - config_file["application"].update(self.get_config_env()) - return config_file - @staticmethod - def get_config_env(): - """read environment application variables. - - Connection to ES is managed in ElasticWrap and the - connection to Redis is managed in RedisArchivist.""" - - application = { - "HOST_UID": int(os.environ.get("HOST_UID", False)), - "HOST_GID": int(os.environ.get("HOST_GID", False)), - "enable_cast": bool(os.environ.get("ENABLE_CAST")), - } - - return application - @staticmethod def get_config_redis(): """read config json set from redis to overwrite defaults""" diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index db6e4b6c..8bce8098 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -112,13 +112,13 @@ def time_parser(timestamp: str) -> float: return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds) -def clear_dl_cache(config: dict) -> int: +def clear_dl_cache(cache_dir: str) -> int: """clear leftover files from dl cache""" print("clear download cache") - cache_dir = os.path.join(config["application"]["cache_dir"], "download") - leftover_files = ignore_filelist(os.listdir(cache_dir)) + download_cache_dir = os.path.join(cache_dir, "download") + leftover_files = ignore_filelist(os.listdir(download_cache_dir)) for cached in leftover_files: - to_delete = os.path.join(cache_dir, cached) + to_delete = os.path.join(download_cache_dir, cached) os.remove(to_delete) return len(leftover_files) @@ -178,7 +178,7 @@ def get_duration_str(seconds: int) -> str: for unit_label, unit_seconds in units: if seconds >= unit_seconds: unit_count, seconds = divmod(seconds, unit_seconds) - duration_parts.append(f"{unit_count}{unit_label}") + duration_parts.append(f"{unit_count:02}{unit_label}") return " ".join(duration_parts) diff --git a/tubearchivist/home/src/ta/settings.py b/tubearchivist/home/src/ta/settings.py new file mode 100644 index 00000000..16f36994 --- /dev/null +++ b/tubearchivist/home/src/ta/settings.py @@ -0,0 +1,95 @@ +""" +Functionality: +- read and write application config backed by ES +- encapsulate persistence of application properties +""" + +from os import environ + + +class EnvironmentSettings: + """ + Handle settings for the application that are driven from the environment. + These will not change when the user is using the application. + These settings are only provided only on startup. + """ + + HOST_UID: int = int(environ.get("HOST_UID", False)) + HOST_GID: int = int(environ.get("HOST_GID", False)) + ENABLE_CAST: bool = bool(environ.get("ENABLE_CAST")) + TZ: str = str(environ.get("TZ", "UTC")) + TA_PORT: int = int(environ.get("TA_PORT", False)) + TA_UWSGI_PORT: int = int(environ.get("TA_UWSGI_PORT", False)) + TA_USERNAME: str = str(environ.get("TA_USERNAME")) + TA_PASSWORD: str = str(environ.get("TA_PASSWORD")) + + # Application Paths + MEDIA_DIR: str = str(environ.get("TA_MEDIA_DIR", "/youtube")) + APP_DIR: str = str(environ.get("TA_APP_DIR", "/app")) + CACHE_DIR: str = str(environ.get("TA_CACHE_DIR", "/cache")) + + # Redis + REDIS_HOST: str = str(environ.get("REDIS_HOST")) + REDIS_PORT: int = int(environ.get("REDIS_PORT", 6379)) + REDIS_NAME_SPACE: str = str(environ.get("REDIS_NAME_SPACE", "ta:")) + + # ElasticSearch + ES_URL: str = str(environ.get("ES_URL")) + ES_PASS: str = str(environ.get("ELASTIC_PASSWORD")) + ES_USER: str = str(environ.get("ELASTIC_USER", "elastic")) + ES_SNAPSHOT_DIR: str = str( + environ.get( + "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot" + ) + ) + ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL")) + + def print_generic(self): + """print generic env vars""" + print( + f""" + HOST_UID: {self.HOST_UID} + HOST_GID: {self.HOST_GID} + TZ: {self.TZ} + ENABLE_CAST: {self.ENABLE_CAST} + TA_PORT: {self.TA_PORT} + TA_UWSGI_PORT: {self.TA_UWSGI_PORT} + TA_USERNAME: {self.TA_USERNAME} + TA_PASSWORD: *****""" + ) + + def print_paths(self): + """debug paths set""" + print( + f""" + MEDIA_DIR: {self.MEDIA_DIR} + APP_DIR: {self.APP_DIR} + CACHE_DIR: {self.CACHE_DIR}""" + ) + + def print_redis_conf(self): + """debug redis conf paths""" + print( + f""" + REDIS_HOST: {self.REDIS_HOST} + REDIS_PORT: {self.REDIS_PORT} + REDIS_NAME_SPACE: {self.REDIS_NAME_SPACE}""" + ) + + def print_es_paths(self): + """debug es conf""" + print( + f""" + ES_URL: {self.ES_URL} + ES_PASS: ***** + ES_USER: {self.ES_USER} + ES_SNAPSHOT_DIR: {self.ES_SNAPSHOT_DIR} + ES_DISABLE_VERIFY_SSL: {self.ES_DISABLE_VERIFY_SSL}""" + ) + + def print_all(self): + """print all""" + self.print_generic() + self.print_paths() + self.print_redis_conf() + self.print_es_paths() diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index 77de5287..de821fbc 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -6,20 +6,21 @@ functionality: """ import json -import os import redis +from home.src.ta.settings import EnvironmentSettings class RedisBase: """connection base for redis""" - REDIS_HOST: str = str(os.environ.get("REDIS_HOST")) - REDIS_PORT: int = int(os.environ.get("REDIS_PORT") or 6379) - NAME_SPACE: str = "ta:" + NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE def __init__(self): - self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT) + self.conn = redis.Redis( + host=EnvironmentSettings.REDIS_HOST, + port=EnvironmentSettings.REDIS_PORT, + ) class RedisArchivist(RedisBase): diff --git a/tubearchivist/home/src/ta/users.py b/tubearchivist/home/src/ta/users.py index c337381f..57b342c3 100644 --- a/tubearchivist/home/src/ta/users.py +++ b/tubearchivist/home/src/ta/users.py @@ -28,12 +28,7 @@ class UserConfigType(TypedDict, total=False): class UserConfig: - """Handle settings for an individual user - - Create getters and setters for usage in the application. - Although tedious it helps prevents everything caring about how properties - are persisted. Plus it allows us to save anytime any value is set. - """ + """Handle settings for an individual user""" _DEFAULT_USER_SETTINGS = UserConfigType( colors="dark", diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index a2c4cdbf..88b09695 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -24,13 +24,14 @@ from home.src.index.manual import ImportFolderScanner from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.notify import Notifications +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskManager from home.src.ta.urlparser import Parser CONFIG = AppConfig().config -REDIS_HOST = os.environ.get("REDIS_HOST") -REDIS_PORT = os.environ.get("REDIS_PORT") or 6379 +REDIS_HOST = EnvironmentSettings.REDIS_HOST +REDIS_PORT = EnvironmentSettings.REDIS_PORT os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") app = Celery( @@ -39,9 +40,11 @@ app = Celery( backend=f"redis://{REDIS_HOST}:{REDIS_PORT}", result_extended=True, ) -app.config_from_object("django.conf:settings", namespace="ta:") +app.config_from_object( + "django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE +) app.autodiscover_tasks() -app.conf.timezone = os.environ.get("TZ") or "UTC" +app.conf.timezone = EnvironmentSettings.TZ class BaseTask(Task): diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index 2adce0d3..b7b998ec 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -1,4 +1,5 @@ {% load static %} +{% load auth_extras %} @@ -57,9 +58,11 @@
playlists
 + {% if request.user|has_group:"admin" or request.user.is_staff %}
downloads
 + {% endif %}
diff --git a/tubearchivist/home/templates/home/base_settings.html b/tubearchivist/home/templates/home/base_settings.html index 3eb6c13b..31787ef5 100644 --- a/tubearchivist/home/templates/home/base_settings.html +++ b/tubearchivist/home/templates/home/base_settings.html @@ -1,14 +1,17 @@ {# Base file for all of the settings pages to ensure a common menu #} {% extends "home/base.html" %} {% load static %} +{% load auth_extras %} {% block content %}

Dashboard

User

 -

Application

 -

Scheduling

 -

Actions

 + {% if request.user|has_group:"admin" or request.user.is_staff %} +

Application

 +

Scheduling

 +

Actions

 + {% endif %}
{% block settings_content %}{% endblock %} diff --git a/tubearchivist/home/templates/home/channel.html b/tubearchivist/home/templates/home/channel.html index 00913452..83a163d3 100644 --- a/tubearchivist/home/templates/home/channel.html +++ b/tubearchivist/home/templates/home/channel.html @@ -2,11 +2,13 @@ {% load static %} {% load humanize %} {% block content %} +{% load auth_extras %}

Channels

+ {% if request.user|has_group:"admin" or request.user.is_staff %}
add-icon
@@ -17,6 +19,7 @@
+ {% endif %}
diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 838615b6..0d45bd87 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -2,6 +2,8 @@ {% block content %} {% load static %} {% load humanize %} +{% load auth_extras %} +
channel_banner @@ -19,7 +21,9 @@ {% endif %}

About

 {% if has_pending %} -

Downloads

 + {% if request.user|has_group:"admin" or request.user.is_staff %} +

Downloads

 + {% endif %} {% endif %}
@@ -38,7 +42,9 @@

Subscribers: {{ channel_info.channel_subs|intcomma }}

{% endif %} {% if channel_info.channel_subscribed %} + {% if request.user|has_group:"admin" or request.user.is_staff %} + {% endif %} {% else %} {% endif %} diff --git a/tubearchivist/home/templates/home/channel_id_about.html b/tubearchivist/home/templates/home/channel_id_about.html index d4afcbc7..cd965f31 100644 --- a/tubearchivist/home/templates/home/channel_id_about.html +++ b/tubearchivist/home/templates/home/channel_id_about.html @@ -2,6 +2,7 @@ {% block content %} {% load static %} {% load humanize %} +{% load auth_extras %}
channel_banner @@ -19,7 +20,9 @@ {% endif %}

About

 {% if has_pending %} -

Downloads

 + {% if request.user|has_group:"admin" or request.user.is_staff %} +

Downloads

 + {% endif %} {% endif %}
@@ -56,19 +59,21 @@ {% elif channel_info.channel_views > 0 %}

Channel views: {{ channel_info.channel_views|intcomma }}

{% endif %} -
- -
- Delete {{ channel_info.channel_name }} including all videos? -
-
- {% if reindex %} -

Reindex scheduled

- {% else %} -
- - + {% if request.user|has_group:"admin" or request.user.is_staff %} +
+ +
+ Delete {{ channel_info.channel_name }} including all videos? +
+ {% if reindex %} +

Reindex scheduled

+ {% else %} +
+ + +
+ {% endif %} {% endif %}
@@ -90,53 +95,55 @@
{% endif %} -
-
-

Customize {{ channel_info.channel_name }}

-
- {% csrf_token %} -
-

Download format: - {% if channel_info.channel_overwrites.download_format %} - {{ channel_info.channel_overwrites.download_format }} - {% else %} - False - {% endif %}

- {{ channel_overwrite_form.download_format }}
-
-
-

Auto delete watched videos after x days: - {% if channel_info.channel_overwrites.autodelete_days %} - {{ channel_info.channel_overwrites.autodelete_days }} - {% else %} - False - {% endif %}

- {{ channel_overwrite_form.autodelete_days }}
-
-
-

Index playlists: - {% if channel_info.channel_overwrites.index_playlists %} - {{ channel_info.channel_overwrites.index_playlists }} - {% else %} - False - {% endif %}

- {{ channel_overwrite_form.index_playlists }}
-
-
-

Enable SponsorBlock: - {% if channel_info.channel_overwrites.integrate_sponsorblock %} - {{ channel_info.channel_overwrites.integrate_sponsorblock }} - {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %} - Disabled - {% else %} - False - {% endif %}

- {{ channel_overwrite_form.integrate_sponsorblock }}
-
- -
+ {% if request.user|has_group:"admin" or request.user.is_staff %} +
+
+

Customize {{ channel_info.channel_name }}

+
+ {% csrf_token %} +
+

Download format: + {% if channel_info.channel_overwrites.download_format %} + {{ channel_info.channel_overwrites.download_format }} + {% else %} + False + {% endif %}

+ {{ channel_overwrite_form.download_format }}
+
+
+

Auto delete watched videos after x days: + {% if channel_info.channel_overwrites.autodelete_days %} + {{ channel_info.channel_overwrites.autodelete_days }} + {% else %} + False + {% endif %}

+ {{ channel_overwrite_form.autodelete_days }}
+
+
+

Index playlists: + {% if channel_info.channel_overwrites.index_playlists %} + {{ channel_info.channel_overwrites.index_playlists }} + {% else %} + False + {% endif %}

+ {{ channel_overwrite_form.index_playlists }}
+
+
+

Enable SponsorBlock: + {% if channel_info.channel_overwrites.integrate_sponsorblock %} + {{ channel_info.channel_overwrites.integrate_sponsorblock }} + {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %} + Disabled + {% else %} + False + {% endif %}

+ {{ channel_overwrite_form.integrate_sponsorblock }}
+
+ +
+
-
+ {% endif %}
{% endblock content %} \ No newline at end of file diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index 2a199d00..539d4388 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -2,6 +2,7 @@ {% block content %} {% load static %} {% load humanize %} +{% load auth_extras %}
channel_banner @@ -19,7 +20,9 @@ {% endif %}

About

 {% if has_pending %} -

Downloads

 + {% if request.user|has_group:"admin" or request.user.is_staff %} +

Downloads

 + {% endif %} {% endif %}
@@ -53,7 +56,7 @@

{{ playlist.playlist_channel }}

{{ playlist.playlist_name }}

Last refreshed: {{ playlist.playlist_last_refresh }}

- {% if playlist.playlist_subscribed %} + {% if playlist.playlist_subscribed and request.user|has_group:"admin" or request.user.is_staff %} {% else %} diff --git a/tubearchivist/home/templates/home/playlist.html b/tubearchivist/home/templates/home/playlist.html index c6e02031..ab2d788c 100644 --- a/tubearchivist/home/templates/home/playlist.html +++ b/tubearchivist/home/templates/home/playlist.html @@ -1,11 +1,15 @@ {% extends "home/base.html" %} {% load static %} {% block content %} +{% load auth_extras %} +

Playlists

+ {% if request.user|has_group:"admin" or request.user.is_staff %} +
add-icon
@@ -16,6 +20,8 @@
+ {% endif %} +
diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index 3faf449b..5fbe1946 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -2,6 +2,8 @@ {% load static %} {% load humanize %} {% block content %} +{% load auth_extras %} +

{{ playlist_info.playlist_name }}

@@ -27,7 +29,9 @@

Last refreshed: {{ playlist_info.playlist_last_refresh }}

Playlist: {% if playlist_info.playlist_subscribed %} + {% if request.user|has_group:"admin" or request.user.is_staff %} + {% endif %} {% else %} {% endif %} diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html index 698d2a8d..b9ad04a1 100644 --- a/tubearchivist/home/templates/home/video.html +++ b/tubearchivist/home/templates/home/video.html @@ -2,6 +2,7 @@ {% block content %} {% load static %} {% load humanize %} +{% load auth_extras %}

@@ -81,15 +82,19 @@ {% if reindex %}

Reindex scheduled

{% else %} + {% if request.user|has_group:"admin" or request.user.is_staff %}
+ {% endif %} {% endif %} - + + {% if request.user|has_group:"admin" or request.user.is_staff %}
Are you sure?
+ {% endif %}
diff --git a/tubearchivist/home/templatetags/__init__.py b/tubearchivist/home/templatetags/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/home/templatetags/auth_extras.py b/tubearchivist/home/templatetags/auth_extras.py new file mode 100644 index 00000000..a41b4b1e --- /dev/null +++ b/tubearchivist/home/templatetags/auth_extras.py @@ -0,0 +1,8 @@ +from django import template + +register = template.Library() + + +@register.filter(name="has_group") +def has_group(user, group_name): + return user.groups.filter(name=group_name).exists() diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 488bef05..dae3d175 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -9,11 +9,14 @@ import urllib.parse from time import sleep from api.src.search_processor import SearchProcess, process_aggs +from api.views import check_admin from django.conf import settings from django.contrib.auth import login +from django.contrib.auth.decorators import user_passes_test from django.contrib.auth.forms import AuthenticationForm from django.http import Http404, JsonResponse from django.shortcuts import redirect, render +from django.utils.decorators import method_decorator from django.views import View from home.src.download.queue import PendingInteract from home.src.download.yt_dlp_base import CookieHandler @@ -39,6 +42,7 @@ from home.src.index.reindex import ReindexProgress from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.helper import time_parser +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.users import UserConfig from home.tasks import index_channel_playlists, subscribe_to @@ -53,7 +57,6 @@ class ArchivistViewConfig(View): self.view_origin = view_origin self.user_id = False self.user_conf: UserConfig = False - self.default_conf = False self.context = False def get_all_view_styles(self): @@ -70,11 +73,10 @@ class ArchivistViewConfig(View): """build default context for every view""" self.user_id = user_id self.user_conf = UserConfig(self.user_id) - self.default_conf = AppConfig().config self.context = { "colors": self.user_conf.get_value("colors"), - "cast": self.default_conf["application"]["enable_cast"], + "cast": EnvironmentSettings.ENABLE_CAST, "sort_by": self.user_conf.get_value("sort_by"), "sort_order": self.user_conf.get_value("sort_order"), "view_style": self.user_conf.get_value( @@ -317,6 +319,7 @@ class AboutView(MinView): return render(request, "home/about.html", context) +@method_decorator(user_passes_test(check_admin), name="dispatch") class DownloadView(ArchivistResultsView): """resolves to /download/ handle the download queue @@ -597,6 +600,7 @@ class ChannelIdAboutView(ChannelIdBaseView): return render(request, "home/channel_id_about.html", self.context) + @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def post(request, channel_id): """handle post request""" @@ -681,6 +685,7 @@ class ChannelView(ArchivistResultsView): "term": {"channel_subscribed": {"value": True}} } + @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def post(request): """handle http post requests""" @@ -824,6 +829,7 @@ class PlaylistView(ArchivistResultsView): } } + @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def post(request): """handle post from search form""" @@ -870,7 +876,7 @@ class VideoView(MinView): "video": video_data, "playlist_nav": playlist_nav, "title": video_data.get("title"), - "cast": config_handler.config["application"]["enable_cast"], + "cast": EnvironmentSettings.ENABLE_CAST, "config": config_handler.config, "position": time_parser(request.GET.get("t")), "reindex": reindex.get("state"), @@ -986,6 +992,7 @@ class SettingsUserView(MinView): return redirect("settings_user", permanent=True) +@method_decorator(user_passes_test(check_admin), name="dispatch") class SettingsApplicationView(MinView): """resolves to /settings/application/ handle the settings sub-page for application configuration, @@ -1075,6 +1082,7 @@ class SettingsApplicationView(MinView): RedisArchivist().set_message(key, message=message, expire=True) +@method_decorator(user_passes_test(check_admin), name="dispatch") class SettingsSchedulingView(MinView): """resolves to /settings/scheduling/ handle the settings sub-page for scheduling settings, @@ -1108,6 +1116,7 @@ class SettingsSchedulingView(MinView): return redirect("settings_scheduling", permanent=True) +@method_decorator(user_passes_test(check_admin), name="dispatch") class SettingsActionsView(MinView): """resolves to /settings/actions/ handle the settings actions sub-page diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 11ab42d9..e5a5daa2 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,13 +1,13 @@ -apprise==1.5.0 +apprise==1.6.0 celery==5.3.4 Django==4.2.6 django-auth-ldap==4.6.0 -django-cors-headers==4.2.0 +django-cors-headers==4.3.0 djangorestframework==3.14.0 -Pillow==10.0.1 +Pillow==10.1.0 redis==5.0.1 requests==2.31.0 ryd-client==0.0.6 uWSGI==2.0.22 -whitenoise==6.5.0 -yt-dlp==2023.10.7 +whitenoise==6.6.0 +yt-dlp==2023.10.13 diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index e6d3b862..cb40b268 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -470,7 +470,7 @@ function createPlayer(button) { // If cast integration is enabled create cast button let castButton = ''; - if (videoData.config.application.enable_cast) { + if (videoData.config.enable_cast) { castButton = ``; } diff --git a/tubearchivist/static/stats.js b/tubearchivist/static/stats.js index 23732667..3336753d 100644 --- a/tubearchivist/static/stats.js +++ b/tubearchivist/static/stats.js @@ -8,13 +8,18 @@ function primaryStats() { let apiEndpoint = '/api/stats/primary/'; let responseData = apiRequest(apiEndpoint, 'GET'); let primaryBox = document.getElementById('primaryBox'); + clearLoading(primaryBox); + let videoTile = buildVideoTile(responseData); primaryBox.appendChild(videoTile); + let channelTile = buildChannelTile(responseData); primaryBox.appendChild(channelTile); + let playlistTile = buildPlaylistTile(responseData); primaryBox.appendChild(playlistTile); + let downloadTile = buildDownloadTile(responseData); primaryBox.appendChild(downloadTile); } @@ -26,51 +31,133 @@ function clearLoading(dashBox) { function buildTile(titleText) { let tile = document.createElement('div'); tile.classList.add('info-box-item'); + let title = document.createElement('h3'); + title.innerText = titleText; tile.appendChild(title); + return tile; } +function buildTileContenTable(content, rowsWanted) { + let contentEntries = Object.entries(content); + + const nbsp = '\u00A0'; // No-Break Space https://www.compart.com/en/unicode/U+00A0 + + // Do not add spacing rows when on mobile device + const isMobile = window.matchMedia('(max-width: 600px)'); + if (!isMobile.matches) { + if (contentEntries.length < rowsWanted) { + const rowsToAdd = rowsWanted - contentEntries.length; + + for (let i = 0; i < rowsToAdd; i++) { + contentEntries.push([nbsp, nbsp]); + } + } + } + + const table = document.createElement('table'); + table.classList.add('agg-channel-table'); + const tableBody = document.createElement('tbody'); + + for (const [key, value] of contentEntries) { + const row = document.createElement('tr'); + + const leftCell = document.createElement('td'); + leftCell.classList.add('agg-channel-name'); + + // Do not add ":" when its a spacing entry + const keyText = key === nbsp ? key : `${key}: `; + const leftText = document.createTextNode(keyText); + leftCell.appendChild(leftText); + + const rightCell = document.createElement('td'); + rightCell.classList.add('agg-channel-right-align'); + + const rightText = document.createTextNode(value); + rightCell.appendChild(rightText); + + row.appendChild(leftCell); + row.appendChild(rightCell); + + tableBody.appendChild(row); + } + + table.appendChild(tableBody); + + return table; +} + function buildVideoTile(responseData) { - let tile = buildTile(`Total Videos: ${responseData.videos.total || 0}`); - let message = document.createElement('p'); - message.innerHTML = ` - videos: ${responseData.videos.videos || 0}
- shorts: ${responseData.videos.shorts || 0}
- streams: ${responseData.videos.streams || 0}
- `; - tile.appendChild(message); + let tile = buildTile(`Video types: `); + + const total = responseData.videos.total || 0; + const videos = responseData.videos.videos || 0; + const shorts = responseData.videos.shorts || 0; + const streams = responseData.videos.streams || 0; + + const content = { + Videos: `${videos}/${total}`, + Shorts: `${shorts}/${total}`, + Streams: `${streams}/${total}`, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); return tile; } function buildChannelTile(responseData) { - let tile = buildTile(`Total Channels: ${responseData.channels.total || 0}`); - let message = document.createElement('p'); - message.innerHTML = `subscribed: ${responseData.channels.sub_true || 0}`; - tile.appendChild(message); + let tile = buildTile(`Channels: `); + + const total = responseData.channels.total || 0; + const subscribed = responseData.channels.sub_true || 0; + + const content = { + Subscribed: `${subscribed}/${total}`, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); return tile; } function buildPlaylistTile(responseData) { - let tile = buildTile(`Total Playlists: ${responseData.playlists.total || 0}`); - let message = document.createElement('p'); - message.innerHTML = `subscribed: ${responseData.playlists.sub_true || 0}`; - tile.appendChild(message); + let tile = buildTile(`Playlists:`); + + const total = responseData.playlists.total || 0; + const subscribed = responseData.playlists.sub_true || 0; + + const content = { + Subscribed: `${subscribed}/${total}`, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); return tile; } function buildDownloadTile(responseData) { let tile = buildTile('Downloads'); - let message = document.createElement('p'); - message.innerHTML = ` - pending: ${responseData.downloads.pending || 0}
- ignored: ${responseData.downloads.ignore || 0}
- `; - tile.appendChild(message); + + const pending = responseData.downloads.pending || 0; + const ignored = responseData.downloads.ignore || 0; + + const content = { + Pending: pending, + Ignored: ignored, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); return tile; } @@ -81,24 +168,48 @@ function watchStats() { let watchBox = document.getElementById('watchBox'); clearLoading(watchBox); - for (const property in responseData) { - let tile = buildWatchTile(property, responseData[property]); - watchBox.appendChild(tile); - } + const { total, watched, unwatched } = responseData; + + let firstCard = buildWatchTile('total', total); + watchBox.appendChild(firstCard); + + let secondCard = buildWatchTile('watched', watched); + watchBox.appendChild(secondCard); + + let thirdCard = buildWatchTile('unwatched', unwatched); + watchBox.appendChild(thirdCard); +} + +function capitalizeFirstLetter(string) { + // source: https://stackoverflow.com/a/1026087 + return string.charAt(0).toUpperCase() + string.slice(1); } function buildWatchTile(title, watchDetail) { - let tile = buildTile(`Total ${title}`); - let message = document.createElement('p'); - message.innerHTML = ` - ${watchDetail.items} Videos
- ${watchDetail.duration} Seconds
- ${watchDetail.duration_str} Playback - `; - if (watchDetail.progress) { - message.innerHTML += `
${Number(watchDetail.progress * 100).toFixed(2)}%`; + const items = watchDetail.items || 0; + const duration = watchDetail.duration || 0; + const duration_str = watchDetail.duration_str || 0; + const hasProgess = !!watchDetail.progress; + const progress = Number(watchDetail.progress * 100).toFixed(2); + + let titleCapizalized = capitalizeFirstLetter(title); + + if (hasProgess) { + titleCapizalized = `${progress}% ` + titleCapizalized; } - tile.appendChild(message); + + let tile = buildTile(titleCapizalized); + + const content = { + Videos: items, + Seconds: duration, + Playback: duration_str, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); + return tile; } @@ -123,7 +234,15 @@ function downloadHist() { function buildDailyStat(dailyStat) { let tile = buildTile(dailyStat.date); let message = document.createElement('p'); - message.innerText = `new videos: ${dailyStat.count}`; + const isExactlyOne = dailyStat.count === 1; + + let text = 'Videos'; + if (isExactlyOne) { + text = 'Video'; + } + + message.innerText = `+${dailyStat.count} ${text}`; + tile.appendChild(message); return tile; }