renamed django app folder to backend

backend/config/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for config project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+
+application = get_asgi_application()

backend/config/management/commands/ta_connection.py

@@ -0,0 +1,163 @@
+"""
+Functionality:
+- check that all connections are working
+"""
+
+from time import sleep
+
+import requests
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.ta_redis import RedisArchivist
+from django.core.management.base import BaseCommand, CommandError
+
+TOPIC = """
+
+#######################
+#  Connection check   #
+#######################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    TIMEOUT = 120
+    MIN_MAJOR, MAX_MAJOR = 8, 8
+    MIN_MINOR = 0
+
+    # pylint: disable=no-member
+    help = "Check connections"
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(TOPIC)
+        self._redis_connection_check()
+        self._redis_config_set()
+        self._es_connection_check()
+        self._es_version_check()
+        self._es_path_check()
+
+    def _redis_connection_check(self):
+        """check ir redis connection is established"""
+        self.stdout.write("[1] connect to Redis")
+        redis_conn = RedisArchivist().conn
+        for _ in range(5):
+            try:
+                pong = redis_conn.execute_command("PING")
+                if pong:
+                    self.stdout.write(
+                        self.style.SUCCESS("    ✓ Redis connection verified")
+                    )
+                    return
+
+            except Exception:  # pylint: disable=broad-except
+                self.stdout.write("    ... retry Redis connection")
+                sleep(2)
+
+        message = "    🗙 Redis connection failed"
+        self.stdout.write(self.style.ERROR(f"{message}"))
+        RedisArchivist().exec("PING")
+        sleep(60)
+        raise CommandError(message)
+
+    def _redis_config_set(self):
+        """set config for redis if not set already"""
+        self.stdout.write("[2] set Redis config")
+        redis_conn = RedisArchivist().conn
+        timeout_is = int(redis_conn.config_get("timeout").get("timeout"))
+        if not timeout_is:
+            redis_conn.config_set("timeout", 3600)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ Redis config set"))
+
+    def _es_connection_check(self):
+        """wait for elasticsearch connection"""
+        self.stdout.write("[3] connect to Elastic Search")
+        total = self.TIMEOUT // 5
+        for i in range(total):
+            self.stdout.write(f"    ... waiting for ES [{i}/{total}]")
+            try:
+                _, status_code = ElasticWrap("/").get(
+                    timeout=1, print_error=False
+                )
+            except (
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout,
+            ):
+                sleep(5)
+                continue
+
+            if status_code and status_code == 200:
+                path = "_cluster/health?wait_for_status=yellow&timeout=60s"
+                _, _ = ElasticWrap(path).get(timeout=60)
+                self.stdout.write(
+                    self.style.SUCCESS("    ✓ ES connection established")
+                )
+                return
+
+        response, status_code = ElasticWrap("/").get(
+            timeout=1, print_error=False
+        )
+
+        message = "    🗙 ES connection failed"
+        self.stdout.write(self.style.ERROR(f"{message}"))
+        self.stdout.write(f"    error message: {response}")
+        self.stdout.write(f"    status code: {status_code}")
+        sleep(60)
+        raise CommandError(message)
+
+    def _es_version_check(self):
+        """check for minimal elasticsearch version"""
+        self.stdout.write("[4] Elastic Search version check")
+        response, _ = ElasticWrap("/").get()
+        version = response["version"]["number"]
+        major = int(version.split(".")[0])
+
+        if self.MIN_MAJOR <= major <= self.MAX_MAJOR:
+            self.stdout.write(
+                self.style.SUCCESS("    ✓ ES version check passed")
+            )
+            return
+
+        message = (
+            "    🗙 ES version check failed. "
+            + f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}"
+        )
+        self.stdout.write(self.style.ERROR(f"{message}"))
+        sleep(60)
+        raise CommandError(message)
+
+    def _es_path_check(self):
+        """check that path.repo var is set"""
+        self.stdout.write("[5] check ES path.repo env var")
+        response, _ = ElasticWrap("_nodes/_all/settings").get()
+        snaphost_roles = [
+            "data",
+            "data_cold",
+            "data_content",
+            "data_frozen",
+            "data_hot",
+            "data_warm",
+            "master",
+        ]
+        for node in response["nodes"].values():
+            if not (set(node["roles"]) & set(snaphost_roles)):
+                continue
+
+            if node["settings"]["path"].get("repo"):
+                self.stdout.write(
+                    self.style.SUCCESS("    ✓ path.repo env var is set")
+                )
+                return
+
+            message = (
+                "    🗙 path.repo env var not found. "
+                + "set the following env var to the ES container:\n"
+                + "    path.repo="
+                + EnvironmentSettings.ES_SNAPSHOT_DIR
+            )
+            self.stdout.write(self.style.ERROR(message))
+            sleep(60)
+            raise CommandError(message)

backend/config/management/commands/ta_envcheck.py

@@ -0,0 +1,193 @@
+"""
+Functionality:
+- Check environment at startup
+- Process config file overwrites from env var
+- Stop startup on error
+- python management.py ta_envcheck
+"""
+
+import os
+import re
+
+from common.src.env_settings import EnvironmentSettings
+from django.core.management.base import BaseCommand, CommandError
+from user.models import Account
+
+LOGO = """
+
+                         ....  .....
+                  ...'',;:cc,. .;::;;,'...
+               ..,;:cccllclc,  .:ccllllcc;,..
+            ..,:cllcc:;,'.',.  ....'',;ccllc:,..
+          ..;cllc:,'..                ...,:cccc:'.
+         .;cccc;..                        ..,:ccc:'.
+       .ckkkOkxollllllllllllc.      .,:::;.  .,cclc;
+      .:0MMMMMMMMMMMMMMMMMMMX:     .cNMMMWx.   .;clc:
+     .;lOXK0000KNMMMMX00000KO;     ;KMMMMMNl.   .;ccl:,.
+     .;:c:'.....kMMMNo........    'OMMMWMMMK:    '::;;'.
+   .......     .xMMMNl           .dWMMXdOMMMO'   ........
+   .:cc:;.     .xMMMNc          .lNMMNo.:XMMWx.    .:cl:.
+   .:llc,.     .:xxxd,          ;KMMMk. .oWMMNl.   .:llc'
+   .cll:.     .;:;;:::,.       'OMMMK:';''kWMMK:   .;llc,
+   .cll:.     .,;;;;;;,.     .,xWMMNl.:l:.;KMMMO'  .;llc'
+   .:llc.      .cOOOk;      .lKNMMWx..:l:..lNMMWx. .:llc'
+   .;lcc,.     .xMMMNc      :KMMMM0, .:lc. .xWMMNl.'ccl:.
+    .cllc.     .xMMMNc     'OMMMMXc...:lc...,0MMMKl:lcc,.
+    .,ccl:.    .xMMMNc    .xWMMMWo.,;;:lc;;;.cXMMMXdcc;.
+     .,clc:.   .xMMMNc   .lNMMMWk. .':clc:,. .dWMMW0o;.
+      .,clcc,. .ckkkx;   .okkkOx,    .';,.    'kKKK0l.
+       .':lcc:'.....      .  ..            ..,;cllc,.
+         .,cclc,....                     ....;clc;..
+          ..,:,..,c:'..              ...';:,..,:,.
+            ....:lcccc:;,'''.....'',;;:clllc,....
+               .'',;:cllllllccccclllllcc:,'..
+                   ...'',,;;;;;;;;;,''...
+                            .....
+
+"""
+
+TOPIC = """
+#######################
+#  Environment Setup  #
+#######################
+
+"""
+
+EXPECTED_ENV_VARS = [
+    "TA_USERNAME",
+    "TA_PASSWORD",
+    "ELASTIC_PASSWORD",
+    "ES_URL",
+    "TA_HOST",
+]
+INST = "https://github.com/tubearchivist/tubearchivist#installing-and-updating"
+NGINX = "/etc/nginx/sites-available/default"
+UWSGI = "/app/uwsgi.ini"
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+    help = "Check environment before startup"
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(LOGO)
+        self.stdout.write(TOPIC)
+        self._expected_vars()
+        self._elastic_user_overwrite()
+        self._ta_port_overwrite()
+        self._ta_uwsgi_overwrite()
+        self._enable_cast_overwrite()
+        self._create_superuser()
+
+    def _expected_vars(self):
+        """check if expected env vars are set"""
+        self.stdout.write("[1] checking expected env vars")
+        env = os.environ
+        for var in EXPECTED_ENV_VARS:
+            if not env.get(var):
+                message = f"    🗙 expected env var {var} not set\n    {INST}"
+                self.stdout.write(self.style.ERROR(message))
+                raise CommandError(message)
+
+        message = "    ✓ all expected env vars are set"
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _elastic_user_overwrite(self):
+        """check for ELASTIC_USER overwrite"""
+        self.stdout.write("[2] check ES user overwrite")
+        env = EnvironmentSettings.ES_USER
+        self.stdout.write(self.style.SUCCESS(f"    ✓ ES user is set to {env}"))
+
+    def _ta_port_overwrite(self):
+        """set TA_PORT overwrite for nginx"""
+        self.stdout.write("[3] check TA_PORT overwrite")
+        overwrite = EnvironmentSettings.TA_PORT
+        if not overwrite:
+            self.stdout.write(self.style.SUCCESS("    TA_PORT is not set"))
+            return
+
+        regex = re.compile(r"listen [0-9]{1,5}")
+        to_overwrite = f"listen {overwrite}"
+        changed = file_overwrite(NGINX, regex, to_overwrite)
+        if changed:
+            message = f"    ✓ TA_PORT changed to {overwrite}"
+        else:
+            message = f"    ✓ TA_PORT already set to {overwrite}"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _ta_uwsgi_overwrite(self):
+        """set TA_UWSGI_PORT overwrite"""
+        self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
+        overwrite = EnvironmentSettings.TA_UWSGI_PORT
+        if not overwrite:
+            message = "    TA_UWSGI_PORT is not set"
+            self.stdout.write(self.style.SUCCESS(message))
+            return
+
+        # nginx
+        regex = re.compile(r"uwsgi_pass localhost:[0-9]{1,5}")
+        to_overwrite = f"uwsgi_pass localhost:{overwrite}"
+        changed = file_overwrite(NGINX, regex, to_overwrite)
+
+        # uwsgi
+        regex = re.compile(r"socket = :[0-9]{1,5}")
+        to_overwrite = f"socket = :{overwrite}"
+        changed = file_overwrite(UWSGI, regex, to_overwrite)
+
+        if changed:
+            message = f"    ✓ TA_UWSGI_PORT changed to {overwrite}"
+        else:
+            message = f"    ✓ TA_UWSGI_PORT already set to {overwrite}"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _enable_cast_overwrite(self):
+        """cast workaround, remove auth for static files in nginx"""
+        self.stdout.write("[5] check ENABLE_CAST overwrite")
+        overwrite = EnvironmentSettings.ENABLE_CAST
+        if not overwrite:
+            self.stdout.write(self.style.SUCCESS("    ENABLE_CAST is not set"))
+            return
+
+        regex = re.compile(r"[^\S\r\n]*auth_request /api/ping/;\n")
+        changed = file_overwrite(NGINX, regex, "")
+        if changed:
+            message = "    ✓ process nginx to enable Cast"
+        else:
+            message = "    ✓ Cast is already enabled in nginx"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _create_superuser(self):
+        """create superuser if not exist"""
+        self.stdout.write("[6] create superuser")
+        is_created = Account.objects.filter(is_superuser=True)
+        if is_created:
+            message = "    superuser already created"
+            self.stdout.write(self.style.SUCCESS(message))
+            return
+
+        name = EnvironmentSettings.TA_USERNAME
+        password = EnvironmentSettings.TA_PASSWORD
+        Account.objects.create_superuser(name, password)
+        message = f"    ✓ new superuser with name {name} created"
+        self.stdout.write(self.style.SUCCESS(message))
+
+
+def file_overwrite(file_path, regex, overwrite):
+    """change file content from old to overwrite, return true when changed"""
+    with open(file_path, "r", encoding="utf-8") as f:
+        file_content = f.read()
+
+    changed = re.sub(regex, overwrite, file_content)
+    if changed == file_content:
+        return False
+
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(changed)
+
+    return True

backend/config/management/commands/ta_startup.py

@@ -0,0 +1,251 @@
+"""
+Functionality:
+- Application startup
+- Apply migrations
+"""
+
+import os
+from datetime import datetime
+from random import randint
+from time import sleep
+
+from appsettings.src.config import ReleaseVersion
+from appsettings.src.index_setup import ElasitIndexWrap
+from appsettings.src.snapshot import ElasticSnapshot
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import clear_dl_cache
+from common.src.ta_redis import RedisArchivist
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import dateformat
+from django_celery_beat.models import CrontabSchedule, PeriodicTasks
+from task.models import CustomPeriodicTask
+from task.src.config_schedule import ScheduleBuilder
+from task.src.task_manager import TaskManager
+from task.tasks import version_check
+
+TOPIC = """
+
+#######################
+#  Application Start  #
+#######################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(TOPIC)
+        self._make_folders()
+        self._clear_redis_keys()
+        self._clear_tasks()
+        self._clear_dl_cache()
+        self._version_check()
+        self._index_setup()
+        self._snapshot_check()
+        self._create_default_schedules()
+        self._update_schedule_tz()
+
+    def _mig_app_settings(self) -> None:
+        """update from v0.4.10 to v0.5.0, migrate application settings"""
+        self.stdout.write("[MIGRATION] move appconfig to ES")
+        config = RedisArchivist().get_message("config")
+        if not config:
+            self.stdout.write(
+                self.style.SUCCESS("    no config values to migrate")
+            )
+            return
+
+        path = "ta_config/_doc/appsettings"
+        response, status_code = ElasticWrap(path).post(config)
+
+        if status_code in [200, 201]:
+            self.stdout.write(
+                self.style.SUCCESS("    ✓ migrated appconfig to ES")
+            )
+            RedisArchivist().del_message("config")
+            return
+
+        message = "    🗙 failed to migrate app config"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)
+
+    def _make_folders(self):
+        """make expected cache folders"""
+        self.stdout.write("[2] create expected cache folders")
+        folders = [
+            "backup",
+            "channels",
+            "download",
+            "import",
+            "playlists",
+            "videos",
+        ]
+        cache_dir = EnvironmentSettings.CACHE_DIR
+        for folder in folders:
+            folder_path = os.path.join(cache_dir, folder)
+            os.makedirs(folder_path, exist_ok=True)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ expected folders created"))
+
+    def _clear_redis_keys(self):
+        """make sure there are no leftover locks or keys set in redis"""
+        self.stdout.write("[3] clear leftover keys in redis")
+        all_keys = [
+            "dl_queue_id",
+            "dl_queue",
+            "downloading",
+            "manual_import",
+            "reindex",
+            "rescan",
+            "run_backup",
+            "startup_check",
+            "reindex:ta_video",
+            "reindex:ta_channel",
+            "reindex:ta_playlist",
+        ]
+
+        redis_con = RedisArchivist()
+        has_changed = False
+        for key in all_keys:
+            if redis_con.del_message(key):
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ cleared key {key}")
+                )
+                has_changed = True
+
+        if not has_changed:
+            self.stdout.write(self.style.SUCCESS("    no keys found"))
+
+    def _clear_tasks(self):
+        """clear tasks and messages"""
+        self.stdout.write("[4] clear task leftovers")
+        TaskManager().fail_pending()
+        redis_con = RedisArchivist()
+        to_delete = redis_con.list_keys("message:")
+        if to_delete:
+            for key in to_delete:
+                redis_con.del_message(key)
+
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {len(to_delete)} messages")
+            )
+
+    def _clear_dl_cache(self):
+        """clear leftover files from dl cache"""
+        self.stdout.write("[5] clear leftover files from dl cache")
+        leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
+        if leftover_files:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {leftover_files} files")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no files found"))
+
+    def _version_check(self):
+        """remove new release key if updated now"""
+        self.stdout.write("[6] check for first run after update")
+        new_version = ReleaseVersion().is_updated()
+        if new_version:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ update to {new_version} completed")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no new update found"))
+
+        version_task = CustomPeriodicTask.objects.filter(name="version_check")
+        if not version_task.exists():
+            return
+
+        if not version_task.first().last_run_at:
+            self.style.SUCCESS("    ✓ send initial version check task")
+            version_check.delay()
+
+    def _index_setup(self):
+        """migration: validate index mappings"""
+        self.stdout.write("[7] validate index mappings")
+        ElasitIndexWrap().setup()
+
+    def _snapshot_check(self):
+        """migration setup snapshots"""
+        self.stdout.write("[8] setup snapshots")
+        ElasticSnapshot().setup()
+
+    def _create_default_schedules(self) -> None:
+        """
+        create default schedules for new installations
+        needs to be called after _mig_schedule_store
+        """
+        self.stdout.write("[9] create initial schedules")
+        init_has_run = CustomPeriodicTask.objects.filter(
+            name="version_check"
+        ).exists()
+
+        if init_has_run:
+            self.stdout.write(
+                self.style.SUCCESS(
+                    "    schedule init already done, skipping..."
+                )
+            )
+            return
+
+        builder = ScheduleBuilder()
+        check_reindex = builder.get_set_task(
+            "check_reindex", schedule=builder.SCHEDULES["check_reindex"]
+        )
+        check_reindex.task_config.update({"days": 90})
+        check_reindex.last_run_at = dateformat.make_aware(datetime.now())
+        check_reindex.save()
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {check_reindex}"
+            )
+        )
+
+        thumbnail_check = builder.get_set_task(
+            "thumbnail_check", schedule=builder.SCHEDULES["thumbnail_check"]
+        )
+        thumbnail_check.last_run_at = dateformat.make_aware(datetime.now())
+        thumbnail_check.save()
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {thumbnail_check}"
+            )
+        )
+        daily_random = f"{randint(0, 59)} {randint(0, 23)} *"
+        version_check_task = builder.get_set_task(
+            "version_check", schedule=daily_random
+        )
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {version_check_task}"
+            )
+        )
+        self.stdout.write(
+            self.style.SUCCESS("    ✓ all default schedules created")
+        )
+
+    def _update_schedule_tz(self) -> None:
+        """update timezone for Schedule instances"""
+        self.stdout.write("[9] validate schedules TZ")
+        tz = EnvironmentSettings.TZ
+        to_update = CrontabSchedule.objects.exclude(timezone=tz)
+
+        if not to_update.exists():
+            self.stdout.write(
+                self.style.SUCCESS("    all schedules have correct TZ")
+            )
+            return
+
+        updated = to_update.update(timezone=tz)
+        self.stdout.write(
+            self.style.SUCCESS(f"    ✓ updated {updated} schedules to {tz}.")
+        )
+        PeriodicTasks.update_changed()

backend/config/settings.py

@@ -0,0 +1,294 @@
+"""
+Django settings for config project.
+
+Generated by 'django-admin startproject' using Django 3.2.5.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.2/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/3.2/ref/settings/
+"""
+
+import hashlib
+from os import environ, path
+from pathlib import Path
+
+import ldap
+from common.src.env_settings import EnvironmentSettings
+from common.src.helper import ta_host_parser
+from corsheaders.defaults import default_headers
+from django_auth_ldap.config import LDAPSearch
+
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv(".env")
+except ModuleNotFoundError:
+    pass
+
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
+
+PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode())
+SECRET_KEY = PW_HASH.hexdigest()
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = bool(environ.get("DJANGO_DEBUG"))
+
+ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(
+    environ.get("TA_HOST", "localhost")
+)
+
+# Application definition
+
+INSTALLED_APPS = [
+    "django_celery_beat",
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "corsheaders",
+    "whitenoise.runserver_nostatic",
+    "django.contrib.staticfiles",
+    "django.contrib.humanize",
+    "rest_framework",
+    "rest_framework.authtoken",
+    "common",
+    "video",
+    "channel",
+    "playlist",
+    "download",
+    "task",
+    "appsettings",
+    "stats",
+    "user",
+    "config",
+]
+
+MIDDLEWARE = [
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "corsheaders.middleware.CorsMiddleware",
+    "whitenoise.middleware.WhiteNoiseMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "common.src.health.HealthCheckMiddleware",
+]
+
+ROOT_URLCONF = "config.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = "config.wsgi.application"
+
+if bool(environ.get("TA_LDAP")):
+    # pylint: disable=global-at-module-level
+    global AUTH_LDAP_SERVER_URI
+    AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
+
+    global AUTH_LDAP_BIND_DN
+    AUTH_LDAP_BIND_DN = environ.get("TA_LDAP_BIND_DN")
+
+    global AUTH_LDAP_BIND_PASSWORD
+    AUTH_LDAP_BIND_PASSWORD = environ.get("TA_LDAP_BIND_PASSWORD")
+
+    """
+    Since these are new environment variables, taking the opporunity to use
+    more accurate env names.
+    Given Names are *_technically_* different from Personal names, as people
+    who change their names have different given names and personal names,
+    and they go by personal names. Additionally, "LastName" is actually
+    incorrect for many cultures, such as Korea, where the
+    family name comes first, and the personal name comes last.
+
+    But we all know people are going to try to guess at these, so still want
+    to include names that people will guess, hence using first/last as well.
+    """
+
+    # Attribute mapping options
+
+    global AUTH_LDAP_USER_ATTR_MAP_USERNAME
+    AUTH_LDAP_USER_ATTR_MAP_USERNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_USERNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_UID")
+        or "uid"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME
+    AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_PERSONALNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_FIRSTNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_GIVENNAME")
+        or "givenName"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_SURNAME
+    AUTH_LDAP_USER_ATTR_MAP_SURNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_SURNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_LASTNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_FAMILYNAME")
+        or "sn"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_EMAIL
+    AUTH_LDAP_USER_ATTR_MAP_EMAIL = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_EMAIL")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_MAIL")
+        or "mail"
+    )
+
+    global AUTH_LDAP_USER_BASE
+    AUTH_LDAP_USER_BASE = environ.get("TA_LDAP_USER_BASE")
+
+    global AUTH_LDAP_USER_FILTER
+    AUTH_LDAP_USER_FILTER = environ.get("TA_LDAP_USER_FILTER")
+
+    global AUTH_LDAP_USER_SEARCH
+    # pylint: disable=no-member
+    AUTH_LDAP_USER_SEARCH = LDAPSearch(
+        AUTH_LDAP_USER_BASE,
+        ldap.SCOPE_SUBTREE,
+        "(&("
+        + AUTH_LDAP_USER_ATTR_MAP_USERNAME
+        + "=%(user)s)"
+        + AUTH_LDAP_USER_FILTER
+        + ")",
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP
+    AUTH_LDAP_USER_ATTR_MAP = {
+        "username": AUTH_LDAP_USER_ATTR_MAP_USERNAME,
+        "first_name": AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME,
+        "last_name": AUTH_LDAP_USER_ATTR_MAP_SURNAME,
+        "email": AUTH_LDAP_USER_ATTR_MAP_EMAIL,
+    }
+
+    if bool(environ.get("TA_LDAP_DISABLE_CERT_CHECK")):
+        global AUTH_LDAP_GLOBAL_OPTIONS
+        AUTH_LDAP_GLOBAL_OPTIONS = {
+            ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
+        }
+
+    AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
+
+# Database
+# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
+
+CACHE_DIR = EnvironmentSettings.CACHE_DIR
+DB_PATH = path.join(CACHE_DIR, "db.sqlite3")
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": DB_PATH,
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",  # noqa: E501
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",  # noqa: E501
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",  # noqa: E501
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",  # noqa: E501
+    },
+]
+
+AUTH_USER_MODEL = "user.Account"
+
+# Forward-auth authentication
+if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
+    TA_AUTH_PROXY_USERNAME_HEADER = (
+        environ.get("TA_AUTH_PROXY_USERNAME_HEADER") or "HTTP_REMOTE_USER"
+    )
+    TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")
+
+    MIDDLEWARE.append("user.src.remote_user_auth.HttpRemoteUserMiddleware")
+
+    AUTHENTICATION_BACKENDS = (
+        "django.contrib.auth.backends.RemoteUserBackend",
+    )
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/3.2/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+TIME_ZONE = EnvironmentSettings.TZ
+USE_I18N = True
+USE_L10N = True
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.2/howto/static-files/
+
+STATIC_URL = "/static/"
+STATICFILES_DIRS = (str(BASE_DIR.joinpath("static")),)
+STATIC_ROOT = str(BASE_DIR.joinpath("staticfiles"))
+STORAGES = {
+    "staticfiles": {
+        "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage",
+    },
+}
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+LOGIN_URL = "/login/"
+LOGOUT_REDIRECT_URL = "/login/"
+
+# Cors needed for browser extension
+# background.js makes the request so HTTP_ORIGIN will be from extension
+if environ.get("DISABLE_CORS"):
+    # disable cors
+    CORS_ORIGIN_ALLOW_ALL = True
+else:
+    CORS_ALLOWED_ORIGIN_REGEXES = [
+        r"moz-extension://*",
+        r"chrome-extension://*",
+    ]
+    CORS_ALLOWED_ORIGINS = ["http://localhost:3000"]
+
+
+CORS_ALLOW_HEADERS = list(default_headers) + [
+    "mode",
+]
+
+# TA application settings
+TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
+TA_VERSION = "v0.4.10-unstable"

backend/config/urls.py

@@ -0,0 +1,31 @@
+"""config URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/3.2/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+
+from django.contrib import admin
+from django.urls import include, path
+
+urlpatterns = [
+    path("api/", include("common.urls")),
+    path("api/video/", include("video.urls")),
+    path("api/channel/", include("channel.urls")),
+    path("api/playlist/", include("playlist.urls")),
+    path("api/download/", include("download.urls")),
+    path("api/task/", include("task.urls")),
+    path("api/appsettings/", include("appsettings.urls")),
+    path("api/stats/", include("stats.urls")),
+    path("api/user/", include("user.urls")),
+    path("admin/", admin.site.urls),
+]

backend/config/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for config project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+
+application = get_wsgi_application()