From 075054723660436d07ede5820d5186783f059f14 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 9 Apr 2023 12:42:26 +0700 Subject: [PATCH 1/7] bump bs4 --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 5d25327f..345d8d78 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,4 +1,4 @@ -beautifulsoup4==4.12.0 +beautifulsoup4==4.12.2 celery==5.2.7 Django==4.2 django-auth-ldap==4.2.0 From a924d648d67b8dbdfa46bbd13ffacdabae2aece9 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 9 Apr 2023 12:42:46 +0700 Subject: [PATCH 2/7] add video id to processing message --- tubearchivist/home/src/download/yt_dlp_handler.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index fe31c03f..13678d86 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -189,7 +189,12 @@ class VideoDownloader: continue if self.task: - self.task.send_progress(["Add video metadata to index."]) + self.task.send_progress( + [ + f"Processing video {youtube_id}", + "Add video metadata to index.", + ] + ) vid_dict = index_new_video( youtube_id, @@ -200,7 +205,12 @@ class VideoDownloader: self.videos.add(vid_dict["youtube_id"]) if self.task: - self.task.send_progress(["Move downloaded file to archive."]) + self.task.send_progress( + [ + f"Processing video {youtube_id}", + "Move downloaded file to archive.", + ] + ) self.move_to_archive(vid_dict) From a98a30cc85e3d5fae2bc7b9b3482f1bfe12ed5f7 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 9 Apr 2023 13:27:47 +0700 Subject: [PATCH 3/7] elaborate TA_HOST parsed, #441 --- tubearchivist/config/settings.py | 13 ++----------- tubearchivist/home/src/ta/helper.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 73aa7010..436f659d 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -18,6 +18,7 @@ import ldap from corsheaders.defaults import default_headers from django_auth_ldap.config import LDAPSearch from home.src.ta.config import AppConfig +from home.src.ta.helper import ta_host_parser # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -32,17 +33,7 @@ SECRET_KEY = PW_HASH.hexdigest() # SECURITY WARNING: don't run with debug turned on in production! DEBUG = bool(environ.get("DJANGO_DEBUG")) -ALLOWED_HOSTS = [] -if environ.get("TA_HOST"): - ALLOWED_HOSTS = [i.strip() for i in environ.get("TA_HOST").split()] - -CSRF_TRUSTED_ORIGINS = [] -for host in ALLOWED_HOSTS: - if host.startswith("http://") or host.startswith("https://"): - CSRF_TRUSTED_ORIGINS.append(host) - else: - CSRF_TRUSTED_ORIGINS.append(f"http://{host}") - +ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(environ["TA_HOST"]) # Application definition diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 22797905..2a9a95ff 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -11,6 +11,7 @@ import string import subprocess import unicodedata from datetime import datetime +from urllib.parse import urlparse import requests @@ -148,6 +149,22 @@ def is_shorts(youtube_id): return response.status_code == 200 +def ta_host_parser(ta_host): + """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS""" + allowed_hosts = [] + csrf_trusted_origins = [] + for host in ta_host.split(): + host_clean = host.strip() + if not host_clean.startswith("http"): + host_clean = f"http://{host}" + + parsed = urlparse(host_clean) + allowed_hosts.append(f"{parsed.hostname}") + csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}") + + return allowed_hosts, csrf_trusted_origins + + class DurationConverter: """ using ffmpeg to get and parse duration from filepath From 06c796807005e823e06008009dd4253a702ae4ac Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 9 Apr 2023 13:35:13 +0700 Subject: [PATCH 4/7] fix silly host_clean and direct access TA_PASSWORD --- tubearchivist/config/settings.py | 2 +- tubearchivist/home/src/ta/helper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 436f659d..cf92db6d 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -27,7 +27,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ -PW_HASH = hashlib.sha256(environ.get("TA_PASSWORD").encode()) +PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode()) SECRET_KEY = PW_HASH.hexdigest() # SECURITY WARNING: don't run with debug turned on in production! diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 2a9a95ff..1fdf15a0 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -156,7 +156,7 @@ def ta_host_parser(ta_host): for host in ta_host.split(): host_clean = host.strip() if not host_clean.startswith("http"): - host_clean = f"http://{host}" + host_clean = f"http://{host_clean}" parsed = urlparse(host_clean) allowed_hosts.append(f"{parsed.hostname}") From 4130a8c5c9586c7f0a0496a97b2d6b9b3ea6e382 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 9 Apr 2023 13:59:42 +0700 Subject: [PATCH 5/7] exclude mypy_cache from deploy --- deploy.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy.sh b/deploy.sh index 3f258b7c..bbc27d5b 100755 --- a/deploy.sh +++ b/deploy.sh @@ -26,6 +26,7 @@ function sync_blackhole { --exclude "**/cache" \ --exclude "**/__pycache__/" \ --exclude "db.sqlite3" \ + --exclude ".mypy_cache" \ . -e ssh "$host":tubearchivist ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist' @@ -49,6 +50,7 @@ function sync_test { --exclude "**/cache" \ --exclude "**/__pycache__/" \ --exclude "db.sqlite3" \ + --exclude ".mypy_cache" \ . -e ssh "$host":tubearchivist # copy default docker-compose file if not exist From 0b33edcb4cfe3fff515cf053a1742faddb3da299 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 9 Apr 2023 14:23:10 +0700 Subject: [PATCH 6/7] tweak thumb resync paginate size --- tubearchivist/home/src/download/thumbnails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 0431842a..256a7464 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -358,6 +358,7 @@ class ThumbFilesystem: paginate = IndexPaginate( index_name=self.INDEX_NAME, data=data, + size=200, callback=EmbedCallback, task=self.task, total=self._get_total(), From 3f7b38713fc808e633a9c957bee9c62bf5fabfaa Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 9 Apr 2023 14:29:13 +0700 Subject: [PATCH 7/7] add more progress notification for zip backup --- tubearchivist/home/src/es/backup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubearchivist/home/src/es/backup.py b/tubearchivist/home/src/es/backup.py index d3a76dcf..97092f0c 100644 --- a/tubearchivist/home/src/es/backup.py +++ b/tubearchivist/home/src/es/backup.py @@ -32,6 +32,7 @@ class ElasticBackup: if not self.reason: raise ValueError("missing backup reason in ElasticBackup") + self.task.send_progress(["Scanning your index."]) for index in self.index_config: index_name = index["index_name"] print(f"backup: export in progress for {index_name}") @@ -41,6 +42,7 @@ class ElasticBackup: self.backup_index(index_name) + self.task.send_progress(["Compress files to zip archive."]) self.zip_it() if self.reason == "auto": self.rotate_backup()