From e9619c12429d90dd6ba96f72fd896ed6e2dc5654 Mon Sep 17 00:00:00 2001 From: Dave Shoreman Date: Wed, 22 Jun 2022 12:29:46 +0100 Subject: [PATCH 1/5] Fix repeated superuser creations after rename/removal (#263) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Avoid creating superuser if variables are removed Enables the removal of `TA_USERNAME` and/or `TA_PASSWORD` as a way of preventing an additional superuser being created when another exists. In the old code, `ELASTIC_USER` would never error due to the default, so this replaces the loop with one requirement on `ELASTIC_PASSWORD`. * Use .superuser_created to prevent re-creation Creates an empty `.superuser_created` file after successful creation, and uses this to skip the command on future runs. If the file doesn't exist, then `TA_USERNAME` and `TA_PASSWORD` will also be required. * The superuser password can't be empty ...well—it *can*—but the password input of the login form requires a value so even though the account is created, you can't actually login. * Parse createsuperuser output for 'exists' error If a user already exists with the same name as the superuser being created, the command fails with "Error: That name is already taken". It's probably safe to assume that user is the one we expect. This parses the output, treating it as successful so it can be skipped next time. If it hits the catch-all, something probably failed so we exit. * Use the cache for persistence Updates the .superuser_created file to be placed in /cache so that it gets persisted in a volume between config tweaks and container rebuilds. Also makes `TA_PASSWORD` required always, because it seems to be needed by tubearchivist/config/settings.py to generate a secret key. * Let it work outside Docker If /cache doesn't exist (i.e. when it's not running in Docker), this falls back to a `cache` dir in the current working directory instead. Also renames the file to 'initsu.lock' rather than having it hidden. * Use flexible fallback path for `celery` command --- docker_assets/run.sh | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/docker_assets/run.sh b/docker_assets/run.sh index 31022c79..a3a11b1a 100644 --- a/docker_assets/run.sh +++ b/docker_assets/run.sh @@ -5,13 +5,14 @@ if [[ -z "$ELASTIC_USER" ]]; then export ELASTIC_USER=elastic fi -ENV_VARS=("TA_USERNAME" "TA_PASSWORD" "ELASTIC_PASSWORD" "ELASTIC_USER") -for each in "${ENV_VARS[@]}"; do - if ! [[ -v $each ]]; then - echo "missing environment variable $each" - exit 1 - fi -done +cachedir=/cache +[[ -d $cachedir ]] || cachedir=. +lockfile=${cachedir}/initsu.lock + +required="Missing required environment variable" +[[ -f $lockfile ]] || : "${TA_USERNAME:?$required}" +: "${TA_PASSWORD:?$required}" +: "${ELASTIC_PASSWORD:?$required}" # ugly nginx and uwsgi port overwrite with env vars if [[ -n "$TA_PORT" ]]; then @@ -39,12 +40,27 @@ done # start python application python manage.py makemigrations python manage.py migrate -export DJANGO_SUPERUSER_PASSWORD=$TA_PASSWORD && \ - python manage.py createsuperuser --noinput --name "$TA_USERNAME" + +if [[ -f $lockfile ]]; then + echo -e "\e[33;1m[WARNING]\e[0m This is not the first run! Skipping" \ + "superuser creation.\nTo force it, remove $lockfile" +else + export DJANGO_SUPERUSER_PASSWORD=$TA_PASSWORD + output="$(python manage.py createsuperuser --noinput --name "$TA_USERNAME" 2>&1)" + + case "$output" in + *"Superuser created successfully"*) + echo "$output" && touch $lockfile ;; + *"That name is already taken."*) + echo "Superuser already exists. Creation will be skipped on next start." + touch $lockfile ;; + *) echo "$output" && exit 1 + esac +fi python manage.py collectstatic --noinput -c nginx & celery -A home.tasks worker --loglevel=INFO & celery -A home beat --loglevel=INFO \ - -s "${BEAT_SCHEDULE_PATH:-/cache/celerybeat-schedule}" & + -s "${BEAT_SCHEDULE_PATH:-${cachedir}/celerybeat-schedule}" & uwsgi --ini uwsgi.ini From b4add20d101125bb886b52bed34c2d1196e55a97 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 22 Jun 2022 19:30:38 +0700 Subject: [PATCH 2/5] bump yt-dlp --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 3fe0767b..1372d028 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -9,4 +9,4 @@ requests==2.28.0 ryd-client==0.0.3 uWSGI==2.0.20 whitenoise==6.2.0 -yt_dlp==2022.5.18 +yt_dlp==2022.6.22.1 From 74d74d95a13f4d45e1acabea36284d47fb01a54e Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 23 Jun 2022 06:48:55 +0700 Subject: [PATCH 3/5] store validation result, return in API --- tubearchivist/api/README.md | 7 +++++-- tubearchivist/api/views.py | 6 ++++-- tubearchivist/home/src/download/yt_dlp_base.py | 17 +++++++++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index c155eac0..a5bc1b0a 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -233,11 +233,14 @@ List of valid task names: ## Cookie View -Check your youtube cookie settings +Check your youtube cookie settings, *status* turns to `true` if cookie has been validated. GET /api/cookie/ ```json { - "cookie_enabled": true + "cookie_enabled": true, + "status": true, + "validated": , + "validated_str": "timestamp" } ``` diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 9604ca58..b99ab6a6 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -488,9 +488,11 @@ class CookieView(ApiBaseView): """handle get request""" # pylint: disable=unused-argument config = AppConfig().config - cookie_enabled = config["downloads"]["cookie_import"] + valid = RedisArchivist().get_message("cookie:valid") + response = {"cookie_enabled": config["downloads"]["cookie_import"]} + response.update(valid) - return Response({"cookie_enabled": cookie_enabled}) + return Response(response) @staticmethod def post(request): diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index df929f57..089da10d 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -5,6 +5,7 @@ functionality: """ import os +from datetime import datetime from http import cookiejar from io import StringIO @@ -115,7 +116,8 @@ class CookieHandler: "extract_flat": True, } validator = YtWrap(obs_request, self.config) - response = validator.extract("LL") + response = bool(validator.extract("LL")) + self.store_validation(response) # update in redis to avoid expiring modified = validator.obs["cookiefile"].getvalue() @@ -134,4 +136,15 @@ class CookieHandler: ) print("cookie validation failed, exiting...") - return bool(response) + return response + + @staticmethod + def store_validation(response): + """remember last validation""" + now = datetime.now() + message = { + "status": response, + "validated": int(now.strftime("%s")), + "validated_str": now.strftime("%Y-%m-%d %H:%M"), + } + RedisArchivist().set_message("cookie:valid", message) From 8c07932071f4fbabf4e69f0ffd8e64916681ed95 Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 23 Jun 2022 19:21:13 +0700 Subject: [PATCH 4/5] delete cookie:valid key when revoking --- tubearchivist/home/src/download/yt_dlp_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index 089da10d..b3c94af4 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -103,6 +103,7 @@ class CookieHandler: def revoke(): """revoke cookie""" RedisArchivist().del_message("cookie") + RedisArchivist().del_message("cookie:valid") RedisArchivist().set_message( "config", False, path=".downloads.cookie_import" ) From 1857ac44ca0b62d300787f08610d1a744a7bf6de Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 27 Jun 2022 09:48:36 +0700 Subject: [PATCH 5/5] fix tablet css breakpoints and grid spacing --- tubearchivist/static/css/style.css | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index f97e2f27..45a7c0a1 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -1086,8 +1086,12 @@ button:hover { } /* tablet */ -@media screen and (max-width: 1000px) { - .boxed-content { +@media screen and (max-width: 1000px), screen and (max-height: 850px) { + .boxed-content, + .boxed-content.boxed-4, + .boxed-content.boxed-5, + .boxed-content.boxed-6, + .boxed-content.boxed-7 { width: 90%; } .video-list.grid.grid-3,