diff --git a/deploy.sh b/deploy.sh index 3f258b7c..bbc27d5b 100755 --- a/deploy.sh +++ b/deploy.sh @@ -26,6 +26,7 @@ function sync_blackhole { --exclude "**/cache" \ --exclude "**/__pycache__/" \ --exclude "db.sqlite3" \ + --exclude ".mypy_cache" \ . -e ssh "$host":tubearchivist ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist' @@ -49,6 +50,7 @@ function sync_test { --exclude "**/cache" \ --exclude "**/__pycache__/" \ --exclude "db.sqlite3" \ + --exclude ".mypy_cache" \ . -e ssh "$host":tubearchivist # copy default docker-compose file if not exist diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 73aa7010..cf92db6d 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -18,6 +18,7 @@ import ldap from corsheaders.defaults import default_headers from django_auth_ldap.config import LDAPSearch from home.src.ta.config import AppConfig +from home.src.ta.helper import ta_host_parser # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -26,23 +27,13 @@ BASE_DIR = Path(__file__).resolve().parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ -PW_HASH = hashlib.sha256(environ.get("TA_PASSWORD").encode()) +PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode()) SECRET_KEY = PW_HASH.hexdigest() # SECURITY WARNING: don't run with debug turned on in production! DEBUG = bool(environ.get("DJANGO_DEBUG")) -ALLOWED_HOSTS = [] -if environ.get("TA_HOST"): - ALLOWED_HOSTS = [i.strip() for i in environ.get("TA_HOST").split()] - -CSRF_TRUSTED_ORIGINS = [] -for host in ALLOWED_HOSTS: - if host.startswith("http://") or host.startswith("https://"): - CSRF_TRUSTED_ORIGINS.append(host) - else: - CSRF_TRUSTED_ORIGINS.append(f"http://{host}") - +ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(environ["TA_HOST"]) # Application definition diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 0431842a..256a7464 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -358,6 +358,7 @@ class ThumbFilesystem: paginate = IndexPaginate( index_name=self.INDEX_NAME, data=data, + size=200, callback=EmbedCallback, task=self.task, total=self._get_total(), diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index fe31c03f..13678d86 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -189,7 +189,12 @@ class VideoDownloader: continue if self.task: - self.task.send_progress(["Add video metadata to index."]) + self.task.send_progress( + [ + f"Processing video {youtube_id}", + "Add video metadata to index.", + ] + ) vid_dict = index_new_video( youtube_id, @@ -200,7 +205,12 @@ class VideoDownloader: self.videos.add(vid_dict["youtube_id"]) if self.task: - self.task.send_progress(["Move downloaded file to archive."]) + self.task.send_progress( + [ + f"Processing video {youtube_id}", + "Move downloaded file to archive.", + ] + ) self.move_to_archive(vid_dict) diff --git a/tubearchivist/home/src/es/backup.py b/tubearchivist/home/src/es/backup.py index d3a76dcf..97092f0c 100644 --- a/tubearchivist/home/src/es/backup.py +++ b/tubearchivist/home/src/es/backup.py @@ -32,6 +32,7 @@ class ElasticBackup: if not self.reason: raise ValueError("missing backup reason in ElasticBackup") + self.task.send_progress(["Scanning your index."]) for index in self.index_config: index_name = index["index_name"] print(f"backup: export in progress for {index_name}") @@ -41,6 +42,7 @@ class ElasticBackup: self.backup_index(index_name) + self.task.send_progress(["Compress files to zip archive."]) self.zip_it() if self.reason == "auto": self.rotate_backup() diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 22797905..1fdf15a0 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -11,6 +11,7 @@ import string import subprocess import unicodedata from datetime import datetime +from urllib.parse import urlparse import requests @@ -148,6 +149,22 @@ def is_shorts(youtube_id): return response.status_code == 200 +def ta_host_parser(ta_host): + """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS""" + allowed_hosts = [] + csrf_trusted_origins = [] + for host in ta_host.split(): + host_clean = host.strip() + if not host_clean.startswith("http"): + host_clean = f"http://{host_clean}" + + parsed = urlparse(host_clean) + allowed_hosts.append(f"{parsed.hostname}") + csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}") + + return allowed_hosts, csrf_trusted_origins + + class DurationConverter: """ using ffmpeg to get and parse duration from filepath diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 5d25327f..345d8d78 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,4 +1,4 @@ -beautifulsoup4==4.12.0 +beautifulsoup4==4.12.2 celery==5.2.7 Django==4.2 django-auth-ldap==4.2.0