New React Frontend, #build

Migration guide: https://gist.github.com/bbilly1/7b6abc52ab689f56671bf9011879379a

Changed:
- First testing build of the new React frontend

.dockerignore

@@ -18,4 +18,4 @@ venv/
 assets/*
 
 # for local testing only
-testing.sh
+testing.sh

.eslintrc.js

@@ -1,17 +0,0 @@
-'use strict';
-module.exports = {
-  extends: ['eslint:recommended', 'eslint-config-prettier'],
-  parserOptions: {
-    ecmaVersion: 2020,
-  },
-  env: {
-    browser: true,
-  },
-  rules: {
-    strict: ['error', 'global'],
-    'no-unused-vars': ['error', { vars: 'local' }],
-    eqeqeq: ['error', 'always', { null: 'ignore' }],
-    curly: ['error', 'multi-line'],
-    'no-var': 'error',
-  },
-};

.gitattributes

@@ -0,0 +1 @@
+docker_assets\run.sh eol=lf

.github/FUNDING.yml

@@ -1,3 +1,3 @@
 github: bbilly1
 ko_fi: bbilly1
-custom: https://paypal.me/bbilly1
+custom: https://paypal.me/bbilly1

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -6,7 +6,7 @@ body:
   - type: checkboxes
     id: block
     attributes:
-      label: "This project doesn't accept any new feature requests for the forseeable future. There is no shortage of ideas and the next development steps are clear for years to come."
+      label: "This project doesn't accept any new feature requests for the foreseeable future. There is no shortage of ideas and the next development steps are clear for years to come."
       options:
         - label: I understand that this issue will be closed without comment.
           required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/workflows/lint_js.yml

@@ -1,22 +0,0 @@
-name: lint_js
-
-on:
-  push:
-    paths:
-      - '**/*.js'
-  pull_request:
-    paths:
-      - '**/*.js'
-
-jobs:
-  check:
-    name: lint_js
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
-        with:
-          node-version: '22'
-      - run: npm ci
-      - run: npm run lint
-      - run: npm run format -- --check

.github/workflows/lint_python.yml

@@ -1,42 +0,0 @@
-name: lint_python
-
-on:
-  push:
-    paths:
-      - '**/*.py'
-  pull_request:
-    paths:
-      - '**/*.py'
-
-jobs:
-  lint_python:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y gcc libldap2-dev libsasl2-dev libssl-dev
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
-      - name: Cache pip
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Install python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r tubearchivist/requirements-dev.txt
-
-      - name: Run Linter
-        run: ./deploy.sh validate

.github/workflows/pre_commit.yml

@@ -0,0 +1,47 @@
+name: Lint, Test, Build, and Push Docker Image
+
+on:
+  push:
+    branches:
+      - '**'
+    tags:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '23'
+
+      - name: Install frontend dependencies
+        run: |
+          cd frontend
+          npm install
+
+      - name: Cache pre-commit environment
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pre-commit
+          key: ${{ runner.os }}-pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pre-commit-
+
+      - name: Install dependencies
+        run: |
+          pip install pre-commit
+          pre-commit install
+
+      - name: Run pre-commit
+        run: |
+          pre-commit run --all-files

.github/workflows/unit_tests.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r tubearchivist/requirements-dev.txt
+          pip install -r backend/requirements-dev.txt
 
       - name: Run unit tests
-        run: pytest tubearchivist
+        run: pytest backend

.gitignore

@@ -2,8 +2,9 @@
 __pycache__
 .venv
 
-# django testing db
+# django testing
-db.sqlite3
+backend/static
+backend/.env
 
 # vscode custom conf
 .vscode

.pre-commit-config.yaml

@@ -0,0 +1,49 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+  - repo: https://github.com/psf/black
+    rev: 24.10.0
+    hooks:
+      - id: black
+        alias: python
+        files: ^backend/
+        args: ["--line-length=79"]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)
+        alias: python
+        files: ^backend/
+        args: ["--profile", "black", "-l 79"]
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        alias: python
+        files: ^backend/
+        args: ["--max-complexity=10", "--max-line-length=79"]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.3.0
+    hooks:
+      - id: codespell
+        exclude: ^frontend/package-lock.json
+  - repo: https://github.com/pre-commit/mirrors-eslint
+    rev: v9.17.0
+    hooks:
+      - id: eslint
+        name: eslint
+        files: \.[jt]sx?$
+        types: [file]
+        entry: npm run --prefix ./frontend lint
+        pass_filenames: false
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v4.0.0-alpha.8
+    hooks:
+      - id: prettier
+        entry: npm run --prefix ./frontend format
+        pass_filenames: false
+
+exclude: '.*(\.svg|/migrations/).*'

CONTRIBUTING.md

@@ -138,9 +138,55 @@ The documentation available at [docs.tubearchivist.com](https://docs.tubearchivi
 
 ## Development Environment
 
-I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. You might be able to run the application directly, but this document assumes you're using docker.
+This codebase is set up to be developed natively outside of docker as well as in a docker container. Developing outside of a docker container can be convenient, as IDE and hot reload usually works out of the box. But testing inside of a container is still essential, as there are subtle differences, especially when working with the filesystem and networking between containers.
 
-### Instructions
+### Native Instruction
+
+For convenience, it's recommended to still run Redis and ES in a docker container. Make sure both containers can be reachable over the network.
+
+Set up your virtual environment and install the requirements defined in `requirements-dev.txt`.
+
+There are options built in to load environment variables from a file using `load_dotenv`. Example `.env` file to place in the same folder as `manage.py`:
+
+```
+TA_HOST="localhost"
+TA_USERNAME=tubearchivist
+TA_PASSWORD=verysecret
+TA_MEDIA_DIR="static/volume/media"
+TA_CACHE_DIR="static"
+TA_APP_DIR="."
+REDIS_CON=redis://localhost:6379
+ES_URL="http://localhost:9200"
+ELASTIC_PASSWORD=verysecret
+TZ=America/New_York
+DJANGO_DEBUG=True
+```
+
+Than from look at the container startup script `run.sh`, make sure all needed migrations and startup checks ran, then to start the dev backend server from the same folder as `manage.py` run:
+
+```bash
+python manage.py runserver
+```
+
+The backend will be available on [localhost:8000/api/](localhost:8000/api/).
+
+You'll probably also want to have a Celery worker instance running, refer to `run.sh` for that. The Beat Scheduler might not be needed.
+
+Then from the frontend folder, install the dependencies with:
+
+```bash
+npm install
+```
+
+Then to start the developlent server:
+
+```bash
+npm run dev
+```
+
+And the frontend should be available at [localhost:3000](localhost:3000).
+
+### Docker Instructions
 
 Set up docker on your development machine.
 

Dockerfile

@@ -1,6 +1,17 @@
 # multi stage to build tube archivist
 # build python wheel, download and extract ffmpeg, copy into final image
 
+FROM node:lts-alpine as node-builder
+
+# RUN npm config set registry https://registry.npmjs.org/
+
+COPY ./frontend /frontend
+
+WORKDIR /frontend
+RUN npm i
+RUN npm run build:deploy
+
+WORKDIR /
 
 # First stage to build python wheel
 FROM python:3.11.8-slim-bookworm AS builder
@@ -9,7 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential gcc libldap2-dev libsasl2-dev libssl-dev git
 
 # install requirements
-COPY ./tubearchivist/requirements.txt /requirements.txt
+COPY ./backend/requirements.txt /requirements.txt
 RUN pip install --user -r requirements.txt
 
 # build ffmpeg
@@ -56,9 +67,11 @@ COPY docker_assets/nginx.conf /etc/nginx/sites-available/default
 RUN sed -i 's/^user www\-data\;$/user root\;/' /etc/nginx/nginx.conf
 
 # copy application into container
-COPY ./tubearchivist /app
+COPY ./backend /app
 COPY ./docker_assets/run.sh /app
-COPY ./docker_assets/uwsgi.ini /app
+COPY ./docker_assets/backend_start.py /app
+
+COPY --from=node-builder ./frontend/dist /app/static
 
 # volumes
 VOLUME /cache

README.md

@@ -54,10 +54,10 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
 | TA_USERNAME | Initial username when logging into TA | Required |
 | TA_PASSWORD | Initial password when logging into TA | Required |
 | ELASTIC_PASSWORD | Password for ElasticSearch | Required |
-| REDIS_HOST | Hostname for Redis | Required |
+| REDIS_CON | Connection string to Redis | Required |
 | TZ | Set your timezone for the scheduler | Required |
 | TA_PORT | Overwrite Nginx port | Optional |
-| TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
+| TA_BACKEND_PORT | Overwrite container internal backend server port | Optional |
 | TA_ENABLE_AUTH_PROXY | Enables support for forwarding auth in reverse proxies | [Read more](https://docs.tubearchivist.com/configuration/forward-auth/) |
 | TA_AUTH_PROXY_USERNAME_HEADER | Header containing username to log in | Optional |
 | TA_AUTH_PROXY_LOGOUT_URL | Logout URL for forwarded auth | Optional |
@@ -67,7 +67,6 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
 | HOST_GID | Allow TA to own the video files instead of container user | Optional |
 | HOST_UID | Allow TA to own the video files instead of container user | Optional |
 | ELASTIC_USER | Change the default ElasticSearch user | Optional |
-| REDIS_PORT | Port that Redis runs on | Optional |
 | TA_LDAP | Configure TA to use LDAP Authentication | [Read more](https://docs.tubearchivist.com/configuration/ldap/) |
 | ENABLE_CAST | Enable casting support | [Read more](https://docs.tubearchivist.com/configuration/cast/) |
 | DJANGO_DEBUG | Return additional error messages, for debug only |  |

backend/README.md

@@ -0,0 +1,86 @@
+# Django Setup
+
+## Apps
+The backend is split up into the following apps.
+
+### config
+Root Django App. Doesn't define any views.
+
+- Has main `settings.py`
+- Has main `urls.py` responsible for routing to other apps
+
+### common
+Functionality shared between apps.
+
+Defines views on the root `/api/*` path. Has base views to inherit from.
+
+- Connections to ES and Redis
+- Searching
+- URL parser
+- Collection of helper functions
+
+### appsettings
+Responsible for functionality from the settings pages.
+
+Defines views at `/api/appsettings/*`.
+
+- Index setup
+- Reindexing
+- Snapshots
+- Filesystem Scan
+- Manual import
+
+### channel
+Responsible for Channel Indexing functionality.
+
+Defines views at `/api/channel/*` path.
+
+### download
+Implements download functionality with yt-dlp.
+
+Defines views at `/api/download/*`.
+
+- Download videos
+- Queue management
+- Thumbnails
+- Subscriptions
+
+### playlist
+Implements playlist functionality.
+
+Defines views at `/api/playlist/*`.
+
+- Index Playlists
+- Manual Playlists
+
+### stats
+Builds aggregations views for the statistics dashboard.
+
+Defines views at `/api/stats/*`.
+
+### task
+Defines tasks for Celery. 
+
+Defines views at `/api/task/*`.
+
+- Has main `tasks.py` with all shared_task definitions
+- Has `CustomPeriodicTask` model
+- Implements apprise notifications links
+- Implements schedule functionality
+
+### user
+Implements user and auth functionality.
+
+Defines views at `/api/config/*`.
+
+- Defines custom `Account` model
+
+### video
+Index functionality for videos.
+
+Defines views at `/api/video/*`.
+
+- Index videos
+- Index comments
+- Index/download subtitles
+- Media stream parsing

backend/appsettings/index_mapping.json

@@ -684,4 +684,4 @@
             }
         }
     ]
-}
+}

backend/appsettings/src/backup.py

@@ -10,11 +10,10 @@ import os
 import zipfile
 from datetime import datetime
 
-from home.models import CustomPeriodicTask
+from common.src.env_settings import EnvironmentSettings
-from home.src.es.connect import ElasticWrap, IndexPaginate
+from common.src.es_connect import ElasticWrap, IndexPaginate
-from home.src.ta.config import AppConfig
+from common.src.helper import get_mapping, ignore_filelist
-from home.src.ta.helper import get_mapping, ignore_filelist
+from task.models import CustomPeriodicTask
-from home.src.ta.settings import EnvironmentSettings
 
 
 class ElasticBackup:
@@ -24,8 +23,7 @@ class ElasticBackup:
     CACHE_DIR = EnvironmentSettings.CACHE_DIR
     BACKUP_DIR = os.path.join(CACHE_DIR, "backup")
 
-    def __init__(self, reason=False, task=False):
+    def __init__(self, reason=False, task=False) -> None:
-        self.config = AppConfig().config
         self.timestamp = datetime.now().strftime("%Y%m%d")
         self.index_config = get_mapping()
         self.reason = reason
@@ -139,6 +137,8 @@ class ElasticBackup:
         elif len(file_split) == 3:
             timestamp = file_split[1]
             reason = file_split[2].strip(".zip")
+        else:
+            raise ValueError
 
         data = {
             "filename": filename,

backend/appsettings/src/config.py

@@ -0,0 +1,253 @@
+"""
+Functionality:
+- read and write config
+- load config variables into redis
+"""
+
+from random import randint
+from time import sleep
+from typing import Literal, TypedDict
+
+import requests
+from appsettings.src.snapshot import ElasticSnapshot
+from common.src.es_connect import ElasticWrap
+from common.src.ta_redis import RedisArchivist
+from django.conf import settings
+
+
+class SubscriptionsConfigType(TypedDict):
+    """describes subscriptions config"""
+
+    channel_size: int
+    live_channel_size: int
+    shorts_channel_size: int
+    auto_start: bool
+
+
+class DownloadsConfigType(TypedDict):
+    """describes downloads config"""
+
+    limit_speed: int | None
+    sleep_interval: int | None
+    autodelete_days: int | None
+    format: str | None
+    format_sort: str | None
+    add_metadata: bool
+    add_thumbnail: bool
+    subtitle: str | None
+    subtitle_source: Literal["user", "auto"] | None
+    subtitle_index: bool
+    comment_max: str | None
+    comment_sort: Literal["top", "new"] | None
+    cookie_import: bool
+    potoken: bool
+    throttledratelimit: int | None
+    extractor_lang: str | None
+    integrate_ryd: bool
+    integrate_sponsorblock: bool
+
+
+class ApplicationConfigType(TypedDict):
+    """describes application config"""
+
+    enable_snapshot: bool
+
+
+class AppConfigType(TypedDict):
+    """combined app config type"""
+
+    subscriptions: SubscriptionsConfigType
+    downloads: DownloadsConfigType
+    application: ApplicationConfigType
+
+
+class AppConfig:
+    """handle application variables"""
+
+    ES_PATH = "ta_config/_doc/appsettings"
+    ES_UPDATE_PATH = "ta_config/_update/appsettings"
+    CONFIG_DEFAULTS: AppConfigType = {
+        "subscriptions": {
+            "channel_size": 50,
+            "live_channel_size": 50,
+            "shorts_channel_size": 50,
+            "auto_start": False,
+        },
+        "downloads": {
+            "limit_speed": None,
+            "sleep_interval": 10,
+            "autodelete_days": None,
+            "format": None,
+            "format_sort": None,
+            "add_metadata": False,
+            "add_thumbnail": False,
+            "subtitle": None,
+            "subtitle_source": None,
+            "subtitle_index": False,
+            "comment_max": None,
+            "comment_sort": "top",
+            "cookie_import": False,
+            "potoken": False,
+            "throttledratelimit": None,
+            "extractor_lang": None,
+            "integrate_ryd": False,
+            "integrate_sponsorblock": False,
+        },
+        "application": {"enable_snapshot": True},
+    }
+
+    def __init__(self):
+        self.config = self.get_config()
+
+    def get_config(self) -> AppConfigType:
+        """get config from ES"""
+        response, status_code = ElasticWrap(self.ES_PATH).get()
+        if not status_code == 200:
+            raise ValueError(f"no config found at {self.ES_PATH}")
+
+        return response["_source"]
+
+    def update_config(self, data: dict) -> AppConfigType:
+        """update single config value"""
+        for key, value in data.items():
+            key_map = key.split(".")
+            self._validate_key(key_map)
+            self.config[key_map[0]][key_map[1]] = value
+
+        response, status_code = ElasticWrap(self.ES_PATH).post(self.config)
+        if not status_code == 200:
+            print(response)
+
+        return self.config
+
+    def _update_config_dict(self, to_update) -> None:
+        """none validated partial update for defaults sync"""
+        data = {"doc": to_update}
+        response, status_code = ElasticWrap(self.ES_UPDATE_PATH).post(data)
+        if not status_code == 200:
+            print(f"update failed: {response}, {status_code}")
+
+    def _validate_key(self, key_map: list[str]) -> None:
+        """raise valueerror on invalid key"""
+        exists = key_map[1] in self.CONFIG_DEFAULTS.get(key_map[0], {})  # type: ignore  # noqa: E501
+        if exists is None:
+            raise ValueError(f"trying to access invalid config key: {key_map}")
+
+    def post_process_updated(self, data: dict) -> None:
+        """apply hooks for some config keys"""
+        for config_value, updated_value in data:
+            if config_value == "application.enable_snapshot" and updated_value:
+                ElasticSnapshot().setup()
+
+    @staticmethod
+    def _fail_message(message_line):
+        """notify our failure"""
+        key = "message:setting"
+        message = {
+            "status": key,
+            "group": "setting:application",
+            "level": "error",
+            "title": "Cookie import failed",
+            "messages": [message_line],
+            "id": "0000",
+        }
+        RedisArchivist().set_message(key, message=message, expire=True)
+
+    def sync_defaults(self):
+        """sync defaults at startup, needs to be called with __new__"""
+        return ElasticWrap(self.ES_PATH).post(self.CONFIG_DEFAULTS)
+
+    def add_new_defaults(self) -> list[str]:
+        """add new default config values to ES, called at startup"""
+        updated = []
+        for key, value in self.CONFIG_DEFAULTS.items():
+            if key not in self.config:
+                # complete new key
+                self._update_config_dict({key: value})
+                updated.append(str({key: value}))
+                continue
+
+            for sub_key, sub_value in value.items():  # type: ignore
+                if sub_key not in self.config[key]:
+                    # new partial key
+                    to_update = {key: {sub_key: sub_value}}
+                    self._update_config_dict(to_update)
+                    updated.append(str(to_update))
+
+        return updated
+
+
+class ReleaseVersion:
+    """compare local version with remote version"""
+
+    REMOTE_URL = "https://www.tubearchivist.com/api/release/latest/"
+    NEW_KEY = "versioncheck:new"
+
+    def __init__(self) -> None:
+        self.local_version: str = settings.TA_VERSION
+        self.is_unstable: bool = settings.TA_VERSION.endswith("-unstable")
+        self.remote_version: str = ""
+        self.is_breaking: bool = False
+
+    def check(self) -> None:
+        """check version"""
+        print(f"[{self.local_version}]: look for updates")
+        self.get_remote_version()
+        new_version = self._has_update()
+        if new_version:
+            message = {
+                "status": True,
+                "version": new_version,
+                "is_breaking": self.is_breaking,
+            }
+            RedisArchivist().set_message(self.NEW_KEY, message)
+            print(f"[{self.local_version}]: found new version {new_version}")
+
+    def get_local_version(self) -> str:
+        """read version from local"""
+        return self.local_version
+
+    def get_remote_version(self) -> None:
+        """read version from remote"""
+        sleep(randint(0, 60))
+        response = requests.get(self.REMOTE_URL, timeout=20).json()
+        self.remote_version = response["release_version"]
+        self.is_breaking = response["breaking_changes"]
+
+    def _has_update(self) -> str | bool:
+        """check if there is an update"""
+        remote_parsed = self._parse_version(self.remote_version)
+        local_parsed = self._parse_version(self.local_version)
+        if remote_parsed > local_parsed:
+            return self.remote_version
+
+        if self.is_unstable and local_parsed == remote_parsed:
+            return self.remote_version
+
+        return False
+
+    @staticmethod
+    def _parse_version(version) -> tuple[int, ...]:
+        """return version parts"""
+        clean = version.rstrip("-unstable").lstrip("v")
+        return tuple((int(i) for i in clean.split(".")))
+
+    def is_updated(self) -> str | bool:
+        """check if update happened in the mean time"""
+        message = self.get_update()
+        if not message:
+            return False
+
+        local_parsed = self._parse_version(self.local_version)
+        message_parsed = self._parse_version(message.get("version"))
+
+        if local_parsed >= message_parsed:
+            RedisArchivist().del_message(self.NEW_KEY)
+            return settings.TA_VERSION
+
+        return False
+
+    def get_update(self) -> dict:
+        """return new version dict if available"""
+        message = RedisArchivist().get_message_dict(self.NEW_KEY)
+        return message

backend/appsettings/src/filesystem.py

@@ -5,11 +5,11 @@ Functionality:
 
 import os
 
-from home.src.es.connect import ElasticWrap, IndexPaginate
+from common.src.env_settings import EnvironmentSettings
-from home.src.index.comments import CommentList
+from common.src.es_connect import ElasticWrap, IndexPaginate
-from home.src.index.video import YoutubeVideo, index_new_video
+from common.src.helper import ignore_filelist
-from home.src.ta.helper import ignore_filelist
+from video.src.comments import CommentList
-from home.src.ta.settings import EnvironmentSettings
+from video.src.index import YoutubeVideo, index_new_video
 
 
 class Scanner:

backend/appsettings/src/index_setup.py

@@ -5,11 +5,11 @@ functionality:
 - backup and restore metadata
 """
 
-from home.src.es.backup import ElasticBackup
+from appsettings.src.backup import ElasticBackup
-from home.src.es.connect import ElasticWrap
+from appsettings.src.config import AppConfig
-from home.src.es.snapshot import ElasticSnapshot
+from appsettings.src.snapshot import ElasticSnapshot
-from home.src.ta.config import AppConfig
+from common.src.es_connect import ElasticWrap
-from home.src.ta.helper import get_mapping
+from common.src.helper import get_mapping
 
 
 class ElasticIndex:
@@ -111,6 +111,8 @@ class ElasticIndex:
         elif method == "restore":
             source = f"ta_{self.index_name}_backup"
             destination = f"ta_{self.index_name}"
+        else:
+            raise ValueError("invalid method, expected 'backup' or 'restore'")
 
         data = {"source": {"index": source}, "dest": {"index": destination}}
         _, _ = ElasticWrap("_reindex?refresh=true").post(data=data)

backend/appsettings/src/manual.py

@@ -11,13 +11,13 @@ import re
 import shutil
 import subprocess
 
-from home.src.download.thumbnails import ThumbManager
+from appsettings.src.config import AppConfig
-from home.src.index.comments import CommentList
+from common.src.env_settings import EnvironmentSettings
-from home.src.index.video import YoutubeVideo
+from common.src.helper import ignore_filelist
-from home.src.ta.config import AppConfig
+from download.src.thumbnails import ThumbManager
-from home.src.ta.helper import ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
 from PIL import Image
+from video.src.comments import CommentList
+from video.src.index import YoutubeVideo
 from yt_dlp.utils import ISO639Utils
 
 
@@ -28,7 +28,6 @@ class ImportFolderScanner:
     - convert if needed
     """
 
-    CONFIG = AppConfig().config
     CACHE_DIR = EnvironmentSettings.CACHE_DIR
     IMPORT_DIR = os.path.join(CACHE_DIR, "import")
 
@@ -129,6 +128,7 @@ class ImportFolderScanner:
 
     def process_videos(self):
         """loop through all videos"""
+        config = AppConfig().config
         for idx, current_video in enumerate(self.to_import):
             if not current_video["media"]:
                 print(f"{current_video}: no matching media file found.")
@@ -144,7 +144,7 @@ class ImportFolderScanner:
             self._convert_video(current_video)
             print(f"manual import: {current_video}")
 
-            ManualImport(current_video, self.CONFIG).run()
+            ManualImport(current_video, config).run()
 
         video_ids = [i["video_id"] for i in self.to_import]
         comment_list = CommentList(task=self.task)
@@ -407,8 +407,11 @@ class ManualImport:
             media_path=self.current_video["media"],
         )
         if not video.json_data:
-            print(f"{video_id}: manual import failed, and no metadata found.")
+            message = (
-            raise ValueError
+                f"{video_id}: manual import failed, and no metadata found."
+            )
+            print(message)
+            raise ValueError(message)
 
         video.check_subtitles(subtitle_files=self.current_video["subtitle"])
         video.upload_to_es()

backend/appsettings/src/reindex.py

@@ -7,21 +7,21 @@ functionality:
 import json
 import os
 from datetime import datetime
-from time import sleep
 from typing import Callable, TypedDict
 
-from home.models import CustomPeriodicTask
+from appsettings.src.config import AppConfig
-from home.src.download.subscriptions import ChannelSubscription
+from channel.src.index import YoutubeChannel
-from home.src.download.thumbnails import ThumbManager
+from common.src.env_settings import EnvironmentSettings
-from home.src.download.yt_dlp_base import CookieHandler
+from common.src.es_connect import ElasticWrap, IndexPaginate
-from home.src.es.connect import ElasticWrap, IndexPaginate
+from common.src.helper import rand_sleep
-from home.src.index.channel import YoutubeChannel
+from common.src.ta_redis import RedisQueue
-from home.src.index.comments import Comments
+from download.src.subscriptions import ChannelSubscription
-from home.src.index.playlist import YoutubePlaylist
+from download.src.thumbnails import ThumbManager
-from home.src.index.video import YoutubeVideo
+from download.src.yt_dlp_base import CookieHandler
-from home.src.ta.config import AppConfig
+from playlist.src.index import YoutubePlaylist
-from home.src.ta.settings import EnvironmentSettings
+from task.models import CustomPeriodicTask
-from home.src.ta.ta_redis import RedisQueue
+from video.src.comments import Comments
+from video.src.index import YoutubeVideo
 
 
 class ReindexConfigType(TypedDict):
@@ -289,8 +289,7 @@ class Reindex(ReindexBase):
                 self._notify(name, total, idx)
 
             reindex(youtube_id)
-            sleep_interval = self.config["downloads"].get("sleep_interval", 0)
+            rand_sleep(self.config)
-            sleep(sleep_interval)
 
     def _get_reindex_map(self, index_name: str) -> Callable:
         """return def to run for index"""

backend/appsettings/src/snapshot.py

@@ -7,9 +7,9 @@ from datetime import datetime
 from time import sleep
 from zoneinfo import ZoneInfo
 
-from home.src.es.connect import ElasticWrap
+from common.src.env_settings import EnvironmentSettings
-from home.src.ta.helper import get_mapping
+from common.src.es_connect import ElasticWrap
-from home.src.ta.settings import EnvironmentSettings
+from common.src.helper import get_mapping
 
 
 class ElasticSnapshot:
@@ -150,7 +150,7 @@ class ElasticSnapshot:
         if statuscode == 200:
             print(f"snapshot: executing now: {response}")
 
-        if wait:
+        if wait and "snapshot_name" in response:
             self._wait_for_snapshot(response["snapshot_name"])
 
         return response
@@ -251,10 +251,9 @@ class ElasticSnapshot:
     @staticmethod
     def _date_converter(date_utc):
         """convert datetime string"""
-        expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
+        date = datetime.strptime(date_utc, "%Y-%m-%dT%H:%M:%S.%fZ")
-        date = datetime.strptime(date_utc, expected_format)
+        utc_date = date.replace(tzinfo=ZoneInfo("UTC"))
-        local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
+        converted = utc_date.astimezone(ZoneInfo(EnvironmentSettings.TZ))
-        converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ))
         converted_str = converted.strftime("%Y-%m-%d %H:%M")
 
         return converted_str

backend/appsettings/urls.py

@@ -0,0 +1,47 @@
+"""all app settings API urls"""
+
+from appsettings import views
+from django.urls import path
+
+urlpatterns = [
+    path(
+        "config/",
+        views.AppConfigApiView.as_view(),
+        name="api-config",
+    ),
+    path(
+        "snapshot/",
+        views.SnapshotApiListView.as_view(),
+        name="api-snapshot-list",
+    ),
+    path(
+        "snapshot/<slug:snapshot_id>/",
+        views.SnapshotApiView.as_view(),
+        name="api-snapshot",
+    ),
+    path(
+        "backup/",
+        views.BackupApiListView.as_view(),
+        name="api-backup-list",
+    ),
+    path(
+        "backup/<str:filename>/",
+        views.BackupApiView.as_view(),
+        name="api-backup",
+    ),
+    path(
+        "cookie/",
+        views.CookieView.as_view(),
+        name="api-cookie",
+    ),
+    path(
+        "potoken/",
+        views.POTokenView.as_view(),
+        name="api-potoken",
+    ),
+    path(
+        "token/",
+        views.TokenView.as_view(),
+        name="api-token",
+    ),
+]

backend/appsettings/views.py

@@ -0,0 +1,303 @@
+"""all app settings API views"""
+
+from appsettings.src.backup import ElasticBackup
+from appsettings.src.config import AppConfig
+from appsettings.src.snapshot import ElasticSnapshot
+from common.src.ta_redis import RedisArchivist
+from common.views_base import AdminOnly, ApiBaseView
+from download.src.yt_dlp_base import CookieHandler, POTokenHandler
+from rest_framework.authtoken.models import Token
+from rest_framework.response import Response
+from task.src.task_manager import TaskCommand
+from task.tasks import run_restore_backup
+
+
+class AppConfigApiView(ApiBaseView):
+    """resolves to /api/appsettings/config/
+    GET: return app settings
+    POST: update app settings
+    """
+
+    permission_classes = [AdminOnly]
+
+    @staticmethod
+    def get(request):
+        """get config"""
+        response = AppConfig().config
+        return Response(response)
+
+    @staticmethod
+    def post(request):
+        """
+        update config values
+        data object where key is flatted CONFIG_DEFAULTS separated by '.', e.g.
+        {"subscriptions.channel_size": 5, "subscriptions.live_channel_size": 5}
+        """
+        data = request.data
+        try:
+            config = AppConfig().update_config(data)
+        except ValueError as err:
+            return Response({"error": str(err)}, status=400)
+
+        return Response(config)
+
+
+class SnapshotApiListView(ApiBaseView):
+    """resolves to /api/appsettings/snapshot/
+    GET: returns snapshot config plus list of existing snapshots
+    POST: take snapshot now
+    """
+
+    permission_classes = [AdminOnly]
+
+    @staticmethod
+    def get(request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        snapshots = ElasticSnapshot().get_snapshot_stats()
+
+        return Response(snapshots)
+
+    @staticmethod
+    def post(request):
+        """take snapshot now with post request"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().take_snapshot_now()
+
+        return Response(response)
+
+
+class SnapshotApiView(ApiBaseView):
+    """resolves to /api/appsettings/snapshot/<snapshot-id>/
+    GET: return a single snapshot
+    POST: restore snapshot
+    DELETE: delete a snapshot
+    """
+
+    permission_classes = [AdminOnly]
+
+    @staticmethod
+    def get(request, snapshot_id):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
+
+        if not snapshot:
+            return Response({"message": "snapshot not found"}, status=404)
+
+        return Response(snapshot)
+
+    @staticmethod
+    def post(request, snapshot_id):
+        """restore snapshot with post request"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().restore_all(snapshot_id)
+        if not response:
+            message = {"message": "failed to restore snapshot"}
+            return Response(message, status=400)
+
+        return Response(response)
+
+    @staticmethod
+    def delete(request, snapshot_id):
+        """delete snapshot from index"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
+        if not response:
+            message = {"message": "failed to delete snapshot"}
+            return Response(message, status=400)
+
+        return Response(response)
+
+
+class BackupApiListView(ApiBaseView):
+    """resolves to /api/appsettings/backup/
+    GET: returns list of available zip backups
+    POST: take zip backup now
+    """
+
+    permission_classes = [AdminOnly]
+    task_name = "run_backup"
+
+    @staticmethod
+    def get(request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        backup_files = ElasticBackup().get_all_backup_files()
+        return Response(backup_files)
+
+    def post(self, request):
+        """handle post request"""
+        # pylint: disable=unused-argument
+        response = TaskCommand().start(self.task_name)
+        message = {
+            "message": "backup task started",
+            "task_id": response["task_id"],
+        }
+
+        return Response(message)
+
+
+class BackupApiView(ApiBaseView):
+    """resolves to /api/appsettings/backup/<filename>/
+    GET: return a single backup
+    POST: restore backup
+    DELETE: delete backup
+    """
+
+    permission_classes = [AdminOnly]
+    task_name = "restore_backup"
+
+    @staticmethod
+    def get(request, filename):
+        """get single backup"""
+        # pylint: disable=unused-argument
+        backup_file = ElasticBackup().build_backup_file_data(filename)
+        if not backup_file:
+            message = {"message": "file not found"}
+            return Response(message, status=404)
+
+        return Response(backup_file)
+
+    def post(self, request, filename):
+        """restore backup file"""
+        # pylint: disable=unused-argument
+        task = run_restore_backup.delay(filename)
+        message = {
+            "message": "backup restore task started",
+            "filename": filename,
+            "task_id": task.id,
+        }
+        return Response(message)
+
+    @staticmethod
+    def delete(request, filename):
+        """delete backup file"""
+        # pylint: disable=unused-argument
+
+        backup_file = ElasticBackup().delete_file(filename)
+        if not backup_file:
+            message = {"message": "file not found"}
+            return Response(message, status=404)
+
+        message = {"message": f"file {filename} deleted"}
+        return Response(message)
+
+
+class CookieView(ApiBaseView):
+    """resolves to /api/appsettings/cookie/
+    GET: check if cookie is enabled
+    POST: verify validity of cookie
+    PUT: import cookie
+    DELETE: revoke the cookie
+    """
+
+    permission_classes = [AdminOnly]
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        validation = self._get_cookie_validation()
+
+        return Response(validation)
+
+    def post(self, request):
+        """handle cookie validation request"""
+        # pylint: disable=unused-argument
+        config = AppConfig().config
+        _ = CookieHandler(config).validate()
+        validation = self._get_cookie_validation()
+
+        return Response(validation)
+
+    def put(self, request):
+        """handle put request"""
+        # pylint: disable=unused-argument
+        config = AppConfig().config
+        cookie = request.data.get("cookie")
+        if not cookie:
+            message = "missing cookie key in request data"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        print(f"cookie preview:\n\n{cookie[:300]}")
+        handler = CookieHandler(config)
+        handler.set_cookie(cookie)
+        validated = handler.validate()
+        if not validated:
+            handler.revoke()
+            print("cookie import failed, not valid")
+            status = 400
+        else:
+            status = 200
+
+        validation = self._get_cookie_validation()
+        return Response(validation, status=status)
+
+    def delete(self, request):
+        """delete the cookie"""
+        config = AppConfig().config
+        handler = CookieHandler(config)
+        handler.revoke()
+        return Response({"cookie_enabled": False})
+
+    @staticmethod
+    def _get_cookie_validation():
+        """get current cookie validation"""
+        config = AppConfig().config
+        validation = RedisArchivist().get_message_dict("cookie:valid")
+        is_enabled = {"cookie_enabled": config["downloads"]["cookie_import"]}
+        validation.update(is_enabled)
+
+        return validation
+
+
+class POTokenView(ApiBaseView):
+    """handle PO token"""
+
+    permission_classes = [AdminOnly]
+
+    def get(self, request):
+        """get token"""
+        config = AppConfig().config
+        potoken = POTokenHandler(config).get()
+        return Response({"potoken": potoken})
+
+    def post(self, request):
+        """post token"""
+        config = AppConfig().config
+        new_token = request.data.get("potoken")
+        if not new_token:
+            message = "missing potoken key in request data"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        POTokenHandler(config).set_token(new_token)
+        return Response({"potoken": new_token})
+
+    def delete(self, request):
+        """delete token"""
+        config = AppConfig().config
+        POTokenHandler(config).revoke_token()
+        return Response({"potoken": None})
+
+
+class TokenView(ApiBaseView):
+    """resolves to /api/appsettings/token/
+    DELETE: revoke the token
+    """
+
+    permission_classes = [AdminOnly]
+
+    @staticmethod
+    def get(request):
+        """get token"""
+        token, _ = Token.objects.get_or_create(user=request.user)
+        return Response({"token": token.key})
+
+    @staticmethod
+    def delete(request):
+        """delete the token, new will get created automatically"""
+        print("revoke API token")
+        request.user.auth_token.delete()
+        return Response({"success": True})

backend/channel/src/index.py

@@ -8,12 +8,12 @@ import json
 import os
 from datetime import datetime
 
-from home.src.download.thumbnails import ThumbManager
+from common.src.env_settings import EnvironmentSettings
-from home.src.download.yt_dlp_base import YtWrap
+from common.src.es_connect import ElasticWrap, IndexPaginate
-from home.src.es.connect import ElasticWrap, IndexPaginate
+from common.src.index_generic import YouTubeItem
-from home.src.index.generic import YouTubeItem
+from download.src.thumbnails import ThumbManager
-from home.src.index.playlist import YoutubePlaylist
+from download.src.yt_dlp_base import YtWrap
-from home.src.ta.settings import EnvironmentSettings
+from playlist.src.index import YoutubePlaylist
 
 
 class YoutubeChannel(YouTubeItem):
@@ -42,6 +42,10 @@ class YoutubeChannel(YouTubeItem):
         if not self.youtube_meta and fallback:
             self._video_fallback(fallback)
         else:
+            if not self.json_data:
+                message = f"{self.youtube_id}: Failed to get metadata"
+                raise ValueError(message)
+
             self.process_youtube_meta()
             self.get_channel_art()
 
@@ -59,21 +63,13 @@ class YoutubeChannel(YouTubeItem):
             "channel_name": self.youtube_meta["uploader"],
             "channel_subs": self.youtube_meta.get("channel_follower_count", 0),
             "channel_subscribed": False,
-            "channel_tags": self._parse_tags(self.youtube_meta.get("tags")),
+            "channel_tags": self.youtube_meta.get("tags", []),
             "channel_banner_url": self._get_banner_art(),
             "channel_thumb_url": self._get_thumb_art(),
             "channel_tvart_url": self._get_tv_art(),
             "channel_views": self.youtube_meta.get("view_count") or 0,
         }
 
-    def _parse_tags(self, tags):
-        """parse channel tags"""
-        if not tags:
-            return False
-
-        joined = " ".join(tags)
-        return [i.strip() for i in joined.split('"') if i and not i == " "]
-
     def _get_thumb_art(self):
         """extract thumb art"""
         for i in self.youtube_meta["thumbnails"]:
@@ -290,6 +286,9 @@ class YoutubeChannel(YouTubeItem):
         )
         obs = {"skip_download": True, "extract_flat": True}
         playlists = YtWrap(obs, self.config).extract(url)
+        if not playlists:
+            return
+
         all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
         self.all_playlists = all_entries
 
@@ -326,22 +325,12 @@ class YoutubeChannel(YouTubeItem):
         for key, value in overwrites.items():
             if key not in valid_keys:
                 raise ValueError(f"invalid overwrite key: {key}")
-            elif value == "disable":
+
-                to_write[key] = False
+            if value is None and key in to_write:
+                to_write.pop(key)
                 continue
-            elif value == "0":
+
-                if key in to_write:
+            to_write.update({key: value})
-                    del to_write[key]
-                continue
-            elif value == "1":
-                to_write[key] = True
-                continue
-            elif isinstance(value, int) and int(value) < 0:
-                if key in to_write:
-                    del to_write[key]
-                continue
-            elif value is not None and value != "":
-                to_write.update({key: value})
 
         self.json_data["channel_overwrites"] = to_write
 
@@ -353,3 +342,5 @@ def channel_overwrites(channel_id, overwrites):
     channel.set_overwrites(overwrites)
     channel.upload_to_es()
     channel.sync_to_videos()
+
+    return channel.json_data

backend/channel/src/nav.py

@@ -0,0 +1,78 @@
+"""build channel nav"""
+
+from common.src.es_connect import ElasticWrap
+
+
+class ChannelNav:
+    """get all nav items"""
+
+    def __init__(self, channel_id):
+        self.channel_id = channel_id
+
+    def get_nav(self):
+        """build nav items"""
+        nav = {
+            "has_pending": self._get_has_pending(),
+            "has_playlists": self._get_has_playlists(),
+        }
+        nav.update(self._get_vid_types())
+        return nav
+
+    def _get_vid_types(self):
+        """get available vid_types in given channel"""
+        data = {
+            "size": 0,
+            "query": {
+                "term": {"channel.channel_id": {"value": self.channel_id}}
+            },
+            "aggs": {"unique_values": {"terms": {"field": "vid_type"}}},
+        }
+        response, _ = ElasticWrap("ta_video/_search").get(data)
+        buckets = response["aggregations"]["unique_values"]["buckets"]
+
+        type_nav = {
+            "has_videos": False,
+            "has_streams": False,
+            "has_shorts": False,
+        }
+        for bucket in buckets:
+            if bucket["key"] == "videos":
+                type_nav["has_videos"] = True
+            if bucket["key"] == "streams":
+                type_nav["has_streams"] = True
+            if bucket["key"] == "shorts":
+                type_nav["has_shorts"] = True
+
+        return type_nav
+
+    def _get_has_pending(self):
+        """check if has pending videos in download queue"""
+        data = {
+            "size": 1,
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {"status": {"value": "pending"}}},
+                        {"term": {"channel_id": {"value": self.channel_id}}},
+                    ]
+                }
+            },
+            "_source": False,
+        }
+        response, _ = ElasticWrap("ta_download/_search").get(data=data)
+
+        return bool(response["hits"]["hits"])
+
+    def _get_has_playlists(self):
+        """check if channel has playlists"""
+        path = "ta_playlist/_search"
+        data = {
+            "size": 1,
+            "query": {
+                "term": {"playlist_channel_id": {"value": self.channel_id}}
+            },
+            "_source": False,
+        }
+        response, _ = ElasticWrap(path).get(data=data)
+
+        return bool(response["hits"]["hits"])

backend/channel/urls.py

@@ -0,0 +1,32 @@
+"""all channel API urls"""
+
+from channel import views
+from django.urls import path
+
+urlpatterns = [
+    path(
+        "",
+        views.ChannelApiListView.as_view(),
+        name="api-channel-list",
+    ),
+    path(
+        "search/",
+        views.ChannelApiSearchView.as_view(),
+        name="api-channel-search",
+    ),
+    path(
+        "<slug:channel_id>/",
+        views.ChannelApiView.as_view(),
+        name="api-channel",
+    ),
+    path(
+        "<slug:channel_id>/aggs/",
+        views.ChannelAggsApiView.as_view(),
+        name="api-channel-aggs",
+    ),
+    path(
+        "<slug:channel_id>/nav/",
+        views.ChannelNavApiView.as_view(),
+        name="api-channel-nav",
+    ),
+]

backend/channel/views.py

@@ -0,0 +1,198 @@
+"""all channel API views"""
+
+from channel.src.index import YoutubeChannel, channel_overwrites
+from channel.src.nav import ChannelNav
+from common.src.urlparser import Parser
+from common.views_base import AdminWriteOnly, ApiBaseView
+from download.src.subscriptions import ChannelSubscription
+from rest_framework.response import Response
+from task.tasks import index_channel_playlists, subscribe_to
+
+
+class ChannelApiListView(ApiBaseView):
+    """resolves to /api/channel/
+    GET: returns list of channels
+    POST: edit a list of channels
+    """
+
+    search_base = "ta_channel/_search/"
+    valid_filter = ["subscribed"]
+    permission_classes = [AdminWriteOnly]
+
+    def get(self, request):
+        """get request"""
+        self.data.update(
+            {"sort": [{"channel_name.keyword": {"order": "asc"}}]}
+        )
+
+        query_filter = request.GET.get("filter", False)
+        must_list = []
+        if query_filter:
+            if query_filter not in self.valid_filter:
+                message = f"invalid url query filter: {query_filter}"
+                print(message)
+                return Response({"message": message}, status=400)
+
+            must_list.append({"term": {"channel_subscribed": {"value": True}}})
+
+        self.data["query"] = {"bool": {"must": must_list}}
+        self.get_document_list(request)
+
+        return Response(self.response)
+
+    def post(self, request):
+        """subscribe/unsubscribe to list of channels"""
+        data = request.data
+        try:
+            to_add = data["data"]
+        except KeyError:
+            message = "missing expected data key"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        pending = []
+        for channel_item in to_add:
+            channel_id = channel_item["channel_id"]
+            if channel_item["channel_subscribed"]:
+                pending.append(channel_id)
+            else:
+                self._unsubscribe(channel_id)
+
+        if pending:
+            url_str = " ".join(pending)
+            subscribe_to.delay(url_str, expected_type="channel")
+
+        return Response(data)
+
+    @staticmethod
+    def _unsubscribe(channel_id: str):
+        """unsubscribe"""
+        print(f"[{channel_id}] unsubscribe from channel")
+        ChannelSubscription().change_subscribe(
+            channel_id, channel_subscribed=False
+        )
+
+
+class ChannelApiView(ApiBaseView):
+    """resolves to /api/channel/<channel_id>/
+    GET: returns metadata dict of channel
+    """
+
+    search_base = "ta_channel/_doc/"
+    permission_classes = [AdminWriteOnly]
+
+    def get(self, request, channel_id):
+        # pylint: disable=unused-argument
+        """get request"""
+        self.get_document(channel_id)
+        return Response(self.response, status=self.status_code)
+
+    def post(self, request, channel_id):
+        """modify channel overwrites"""
+        self.get_document(channel_id)
+        if not self.response["data"]:
+            return Response({"error": "channel not found"}, status=404)
+
+        data = request.data
+        subscribed = data.get("channel_subscribed")
+        if subscribed is not None:
+            channel_sub = ChannelSubscription()
+            json_data = channel_sub.change_subscribe(channel_id, subscribed)
+            return Response(json_data, status=200)
+
+        if "channel_overwrites" not in data:
+            return Response({"error": "invalid payload"}, status=400)
+
+        overwrites = data["channel_overwrites"]
+
+        try:
+            json_data = channel_overwrites(channel_id, overwrites)
+            if overwrites.get("index_playlists"):
+                index_channel_playlists.delay(channel_id)
+
+        except ValueError as err:
+            return Response({"error": str(err)}, status=400)
+
+        return Response(json_data, status=200)
+
+    def delete(self, request, channel_id):
+        # pylint: disable=unused-argument
+        """delete channel"""
+        message = {"channel": channel_id}
+        try:
+            YoutubeChannel(channel_id).delete_channel()
+            status_code = 200
+            message.update({"state": "delete"})
+        except FileNotFoundError:
+            status_code = 404
+            message.update({"state": "not found"})
+
+        return Response(message, status=status_code)
+
+
+class ChannelAggsApiView(ApiBaseView):
+    """resolves to /api/channel/<channel_id>/aggs/
+    GET: get channel aggregations
+    """
+
+    search_base = "ta_video/_search"
+
+    def get(self, request, channel_id):
+        """get aggs"""
+        self.data.update(
+            {
+                "query": {
+                    "term": {"channel.channel_id": {"value": channel_id}}
+                },
+                "aggs": {
+                    "total_items": {"value_count": {"field": "youtube_id"}},
+                    "total_size": {"sum": {"field": "media_size"}},
+                    "total_duration": {"sum": {"field": "player.duration"}},
+                },
+            }
+        )
+        self.get_aggs()
+
+        return Response(self.response)
+
+
+class ChannelNavApiView(ApiBaseView):
+    """resolves to /api/channel/<channel_id>/nav/
+    GET: get channel nav
+    """
+
+    def get(self, request, channel_id):
+        """get nav"""
+
+        nav = ChannelNav(channel_id).get_nav()
+        return Response(nav)
+
+
+class ChannelApiSearchView(ApiBaseView):
+    """resolves to /api/channel/search/
+    search for channel
+    """
+
+    search_base = "ta_channel/_doc/"
+
+    def get(self, request):
+        """handle get request, search with s parameter"""
+
+        query = request.GET.get("q")
+        if not query:
+            message = "missing expected q parameter"
+            return Response({"message": message, "data": False}, status=400)
+
+        try:
+            parsed = Parser(query).parse()[0]
+        except (ValueError, IndexError, AttributeError):
+            message = f"channel not found: {query}"
+            return Response({"message": message, "data": False}, status=404)
+
+        if not parsed["type"] == "channel":
+            message = "expected type channel"
+            return Response({"message": message, "data": False}, status=400)
+
+        self.get_document(parsed["url"])
+
+        return Response(self.response, status=self.status_code)

backend/common/src/env_settings.py

@@ -6,6 +6,14 @@ Functionality:
 
 from os import environ
 
+try:
+    from dotenv import load_dotenv
+
+    print("loading local dotenv")
+    load_dotenv(".env")
+except ModuleNotFoundError:
+    pass
+
 
 class EnvironmentSettings:
     """
@@ -19,7 +27,7 @@ class EnvironmentSettings:
     ENABLE_CAST: bool = bool(environ.get("ENABLE_CAST"))
     TZ: str = str(environ.get("TZ", "UTC"))
     TA_PORT: int = int(environ.get("TA_PORT", False))
-    TA_UWSGI_PORT: int = int(environ.get("TA_UWSGI_PORT", False))
+    TA_BACKEND_PORT: int = int(environ.get("TA_BACKEND_PORT", False))
     TA_USERNAME: str = str(environ.get("TA_USERNAME"))
     TA_PASSWORD: str = str(environ.get("TA_PASSWORD"))
 
@@ -29,8 +37,7 @@ class EnvironmentSettings:
     CACHE_DIR: str = str(environ.get("TA_CACHE_DIR", "/cache"))
 
     # Redis
-    REDIS_HOST: str = str(environ.get("REDIS_HOST"))
+    REDIS_CON: str = str(environ.get("REDIS_CON"))
-    REDIS_PORT: int = int(environ.get("REDIS_PORT", 6379))
     REDIS_NAME_SPACE: str = str(environ.get("REDIS_NAME_SPACE", "ta:"))
 
     # ElasticSearch
@@ -44,6 +51,20 @@ class EnvironmentSettings:
     )
     ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL"))
 
+    def get_cache_root(self):
+        """get root for web server"""
+        if self.CACHE_DIR.startswith("/"):
+            return self.CACHE_DIR
+
+        return f"/{self.CACHE_DIR}"
+
+    def get_media_root(self):
+        """get root for media folder"""
+        if self.MEDIA_DIR.startswith("/"):
+            return self.MEDIA_DIR
+
+        return f"/{self.MEDIA_DIR}"
+
     def print_generic(self):
         """print generic env vars"""
         print(
@@ -53,7 +74,7 @@ class EnvironmentSettings:
             TZ: {self.TZ}
             ENABLE_CAST: {self.ENABLE_CAST}
             TA_PORT: {self.TA_PORT}
-            TA_UWSGI_PORT: {self.TA_UWSGI_PORT}
+            TA_BACKEND_PORT: {self.TA_BACKEND_PORT}
             TA_USERNAME: {self.TA_USERNAME}
             TA_PASSWORD: *****"""
         )
@@ -71,8 +92,7 @@ class EnvironmentSettings:
         """debug redis conf paths"""
         print(
             f"""
-            REDIS_HOST: {self.REDIS_HOST}
+            REDIS_CON: {self.REDIS_CON}
-            REDIS_PORT: {self.REDIS_PORT}
             REDIS_NAME_SPACE: {self.REDIS_NAME_SPACE}"""
         )
 

backend/common/src/es_connect.py

@@ -11,7 +11,7 @@ from typing import Any
 
 import requests
 import urllib3
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
 
 
 class ElasticWrap:

backend/common/src/helper.py

@@ -1,265 +1,272 @@
-"""
+"""
-Loose collection of helper functions
+Loose collection of helper functions
-- don't import AppConfig class here to avoid circular imports
+- don't import AppConfig class here to avoid circular imports
-"""
+"""
-
+
-import json
+import json
-import os
+import os
-import random
+import random
-import string
+import string
-import subprocess
+import subprocess
-from datetime import datetime
+from datetime import datetime
-from typing import Any
+from time import sleep
-from urllib.parse import urlparse
+from typing import Any
-
+from urllib.parse import urlparse
-import requests
+
-from home.src.es.connect import IndexPaginate
+import requests
-from home.src.ta.settings import EnvironmentSettings
+from common.src.es_connect import IndexPaginate
-
+
-
+
-def ignore_filelist(filelist: list[str]) -> list[str]:
+def ignore_filelist(filelist: list[str]) -> list[str]:
-    """ignore temp files for os.listdir sanitizer"""
+    """ignore temp files for os.listdir sanitizer"""
-    to_ignore = [
+    to_ignore = [
-        "@eaDir",
+        "@eaDir",
-        "Icon\r\r",
+        "Icon\r\r",
-        "Network Trash Folder",
+        "Network Trash Folder",
-        "Temporary Items",
+        "Temporary Items",
-    ]
+    ]
-    cleaned: list[str] = []
+    cleaned: list[str] = []
-    for file_name in filelist:
+    for file_name in filelist:
-        if file_name.startswith(".") or file_name in to_ignore:
+        if file_name.startswith(".") or file_name in to_ignore:
-            continue
+            continue
-
+
-        cleaned.append(file_name)
+        cleaned.append(file_name)
-
+
-    return cleaned
+    return cleaned
-
+
-
+
-def randomizor(length: int) -> str:
+def randomizor(length: int) -> str:
-    """generate random alpha numeric string"""
+    """generate random alpha numeric string"""
-    pool: str = string.digits + string.ascii_letters
+    pool: str = string.digits + string.ascii_letters
-    return "".join(random.choice(pool) for i in range(length))
+    return "".join(random.choice(pool) for i in range(length))
-
+
-
+
-def requests_headers() -> dict[str, str]:
+def rand_sleep(config) -> None:
-    """build header with random user agent for requests outside of yt-dlp"""
+    """randomized sleep based on config"""
-
+    sleep_config = config["downloads"].get("sleep_interval")
-    chrome_versions = (
+    if not sleep_config:
-        "90.0.4430.212",
+        return
-        "90.0.4430.24",
+
-        "90.0.4430.70",
+    secs = random.randrange(int(sleep_config * 0.5), int(sleep_config * 1.5))
-        "90.0.4430.72",
+    sleep(secs)
-        "90.0.4430.85",
+
-        "90.0.4430.93",
+
-        "91.0.4472.101",
+def requests_headers() -> dict[str, str]:
-        "91.0.4472.106",
+    """build header with random user agent for requests outside of yt-dlp"""
-        "91.0.4472.114",
+
-        "91.0.4472.124",
+    chrome_versions = (
-        "91.0.4472.164",
+        "90.0.4430.212",
-        "91.0.4472.19",
+        "90.0.4430.24",
-        "91.0.4472.77",
+        "90.0.4430.70",
-        "92.0.4515.107",
+        "90.0.4430.72",
-        "92.0.4515.115",
+        "90.0.4430.85",
-        "92.0.4515.131",
+        "90.0.4430.93",
-        "92.0.4515.159",
+        "91.0.4472.101",
-        "92.0.4515.43",
+        "91.0.4472.106",
-        "93.0.4556.0",
+        "91.0.4472.114",
-        "93.0.4577.15",
+        "91.0.4472.124",
-        "93.0.4577.63",
+        "91.0.4472.164",
-        "93.0.4577.82",
+        "91.0.4472.19",
-        "94.0.4606.41",
+        "91.0.4472.77",
-        "94.0.4606.54",
+        "92.0.4515.107",
-        "94.0.4606.61",
+        "92.0.4515.115",
-        "94.0.4606.71",
+        "92.0.4515.131",
-        "94.0.4606.81",
+        "92.0.4515.159",
-        "94.0.4606.85",
+        "92.0.4515.43",
-        "95.0.4638.17",
+        "93.0.4556.0",
-        "95.0.4638.50",
+        "93.0.4577.15",
-        "95.0.4638.54",
+        "93.0.4577.63",
-        "95.0.4638.69",
+        "93.0.4577.82",
-        "95.0.4638.74",
+        "94.0.4606.41",
-        "96.0.4664.18",
+        "94.0.4606.54",
-        "96.0.4664.45",
+        "94.0.4606.61",
-        "96.0.4664.55",
+        "94.0.4606.71",
-        "96.0.4664.93",
+        "94.0.4606.81",
-        "97.0.4692.20",
+        "94.0.4606.85",
-    )
+        "95.0.4638.17",
-    template = (
+        "95.0.4638.50",
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "95.0.4638.54",
-        + "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "95.0.4638.69",
-        + f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
+        "95.0.4638.74",
-    )
+        "96.0.4664.18",
-
+        "96.0.4664.45",
-    return {"User-Agent": template}
+        "96.0.4664.55",
-
+        "96.0.4664.93",
-
+        "97.0.4692.20",
-def date_parser(timestamp: int | str) -> str:
+    )
-    """return formatted date string"""
+    template = (
-    if isinstance(timestamp, int):
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        date_obj = datetime.fromtimestamp(timestamp)
+        + "AppleWebKit/537.36 (KHTML, like Gecko) "
-    elif isinstance(timestamp, str):
+        + f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
-        date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
+    )
-    else:
+
-        raise TypeError(f"invalid timestamp: {timestamp}")
+    return {"User-Agent": template}
-
+
-    return date_obj.date().isoformat()
+
-
+def date_parser(timestamp: int | str) -> str:
-
+    """return formatted date string"""
-def time_parser(timestamp: str) -> float:
+    if isinstance(timestamp, int):
-    """return seconds from timestamp, false on empty"""
+        date_obj = datetime.fromtimestamp(timestamp)
-    if not timestamp:
+    elif isinstance(timestamp, str):
-        return False
+        date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
-
+    else:
-    if timestamp.isnumeric():
+        raise TypeError(f"invalid timestamp: {timestamp}")
-        return int(timestamp)
+
-
+    return date_obj.date().isoformat()
-    hours, minutes, seconds = timestamp.split(":", maxsplit=3)
+
-    return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
+
-
+def time_parser(timestamp: str) -> float:
-
+    """return seconds from timestamp, false on empty"""
-def clear_dl_cache(cache_dir: str) -> int:
+    if not timestamp:
-    """clear leftover files from dl cache"""
+        return False
-    print("clear download cache")
+
-    download_cache_dir = os.path.join(cache_dir, "download")
+    if timestamp.isnumeric():
-    leftover_files = ignore_filelist(os.listdir(download_cache_dir))
+        return int(timestamp)
-    for cached in leftover_files:
+
-        to_delete = os.path.join(download_cache_dir, cached)
+    hours, minutes, seconds = timestamp.split(":", maxsplit=3)
-        os.remove(to_delete)
+    return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
-
+
-    return len(leftover_files)
+
-
+def clear_dl_cache(cache_dir: str) -> int:
-
+    """clear leftover files from dl cache"""
-def get_mapping() -> dict:
+    print("clear download cache")
-    """read index_mapping.json and get expected mapping and settings"""
+    download_cache_dir = os.path.join(cache_dir, "download")
-    with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
+    leftover_files = ignore_filelist(os.listdir(download_cache_dir))
-        index_config: dict = json.load(f).get("index_config")
+    for cached in leftover_files:
-
+        to_delete = os.path.join(download_cache_dir, cached)
-    return index_config
+        os.remove(to_delete)
-
+
-
+    return len(leftover_files)
-def is_shorts(youtube_id: str) -> bool:
+
-    """check if youtube_id is a shorts video, bot not it it's not a shorts"""
+
-    shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
+def get_mapping() -> dict:
-    cookies = {"SOCS": "CAI"}
+    """read index_mapping.json and get expected mapping and settings"""
-    response = requests.head(
+    with open("appsettings/index_mapping.json", "r", encoding="utf-8") as f:
-        shorts_url, cookies=cookies, headers=requests_headers(), timeout=10
+        index_config: dict = json.load(f).get("index_config")
-    )
+
-
+    return index_config
-    return response.status_code == 200
+
-
+
-
+def is_shorts(youtube_id: str) -> bool:
-def get_duration_sec(file_path: str) -> int:
+    """check if youtube_id is a shorts video, bot not it it's not a shorts"""
-    """get duration of media file from file path"""
+    shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
-
+    cookies = {"SOCS": "CAI"}
-    duration = subprocess.run(
+    response = requests.head(
-        [
+        shorts_url, cookies=cookies, headers=requests_headers(), timeout=10
-            "ffprobe",
+    )
-            "-v",
+
-            "error",
+    return response.status_code == 200
-            "-show_entries",
+
-            "format=duration",
+
-            "-of",
+def get_duration_sec(file_path: str) -> int:
-            "default=noprint_wrappers=1:nokey=1",
+    """get duration of media file from file path"""
-            file_path,
+
-        ],
+    duration = subprocess.run(
-        capture_output=True,
+        [
-        check=True,
+            "ffprobe",
-    )
+            "-v",
-    duration_raw = duration.stdout.decode().strip()
+            "error",
-    if duration_raw == "N/A":
+            "-show_entries",
-        return 0
+            "format=duration",
-
+            "-of",
-    duration_sec = int(float(duration_raw))
+            "default=noprint_wrappers=1:nokey=1",
-    return duration_sec
+            file_path,
-
+        ],
-
+        capture_output=True,
-def get_duration_str(seconds: int) -> str:
+        check=True,
-    """Return a human-readable duration string from seconds."""
+    )
-    if not seconds:
+    duration_raw = duration.stdout.decode().strip()
-        return "NA"
+    if duration_raw == "N/A":
-
+        return 0
-    units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
+
-    duration_parts = []
+    duration_sec = int(float(duration_raw))
-
+    return duration_sec
-    for unit_label, unit_seconds in units:
+
-        if seconds >= unit_seconds:
+
-            unit_count, seconds = divmod(seconds, unit_seconds)
+def get_duration_str(seconds: int) -> str:
-            duration_parts.append(f"{unit_count:02}{unit_label}")
+    """Return a human-readable duration string from seconds."""
-
+    if not seconds:
-    duration_parts[0] = duration_parts[0].lstrip("0")
+        return "NA"
-
+
-    return " ".join(duration_parts)
+    units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
-
+    duration_parts = []
-
+
-def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
+    for unit_label, unit_seconds in units:
-    """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
+        if seconds >= unit_seconds:
-    allowed_hosts: list[str] = [
+            unit_count, seconds = divmod(seconds, unit_seconds)
-        "localhost",
+            duration_parts.append(f"{unit_count:02}{unit_label}")
-        "tubearchivist",
+
-    ]
+    duration_parts[0] = duration_parts[0].lstrip("0")
-    csrf_trusted_origins: list[str] = [
+
-        "http://localhost",
+    return " ".join(duration_parts)
-        "http://tubearchivist",
+
-    ]
+
-    for host in ta_host.split():
+def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
-        host_clean = host.strip()
+    """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
-        if not host_clean.startswith("http"):
+    allowed_hosts: list[str] = [
-            host_clean = f"http://{host_clean}"
+        "localhost",
-
+        "tubearchivist",
-        parsed = urlparse(host_clean)
+    ]
-        allowed_hosts.append(f"{parsed.hostname}")
+    csrf_trusted_origins: list[str] = [
-        csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}")
+        "http://localhost",
-
+        "http://tubearchivist",
-    return allowed_hosts, csrf_trusted_origins
+    ]
-
+    for host in ta_host.split():
-
+        host_clean = host.strip()
-def get_stylesheets():
+        if not host_clean.startswith("http"):
-    """Get all valid stylesheets from /static/css"""
+            host_clean = f"http://{host_clean}"
-    app_root = EnvironmentSettings.APP_DIR
+
-    stylesheets = os.listdir(os.path.join(app_root, "static/css"))
+        parsed = urlparse(host_clean)
-    stylesheets.remove("style.css")
+        allowed_hosts.append(f"{parsed.hostname}")
-    stylesheets.sort()
+        csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}")
-    stylesheets = list(filter(lambda x: x.endswith(".css"), stylesheets))
+
-    return stylesheets
+    return allowed_hosts, csrf_trusted_origins
-
+
-
+
-def check_stylesheet(stylesheet: str):
+def get_stylesheets() -> list:
-    """Check if a stylesheet exists. Return dark.css as a fallback"""
+    """Get all valid stylesheets from /static/css"""
-    if stylesheet in get_stylesheets():
+
-        return stylesheet
+    stylesheets = ["dark.css", "light.css", "matrix.css", "midnight.css"]
-
+    return stylesheets
-    return "dark.css"
+
-
+
-
+def check_stylesheet(stylesheet: str):
-def is_missing(
+    """Check if a stylesheet exists. Return dark.css as a fallback"""
-    to_check: str | list[str],
+    if stylesheet in get_stylesheets():
-    index_name: str = "ta_video,ta_download",
+        return stylesheet
-    on_key: str = "youtube_id",
+
-) -> list[str]:
+    return "dark.css"
-    """id or list of ids that are missing from index_name"""
+
-    if isinstance(to_check, str):
+
-        to_check = [to_check]
+def is_missing(
-
+    to_check: str | list[str],
-    data = {
+    index_name: str = "ta_video,ta_download",
-        "query": {"terms": {on_key: to_check}},
+    on_key: str = "youtube_id",
-        "_source": [on_key],
+) -> list[str]:
-    }
+    """id or list of ids that are missing from index_name"""
-    result = IndexPaginate(index_name, data=data).get_results()
+    if isinstance(to_check, str):
-    existing_ids = [i[on_key] for i in result]
+        to_check = [to_check]
-    dl = [i for i in to_check if i not in existing_ids]
+
-
+    data = {
-    return dl
+        "query": {"terms": {on_key: to_check}},
-
+        "_source": [on_key],
-
+    }
-def get_channel_overwrites() -> dict[str, dict[str, Any]]:
+    result = IndexPaginate(index_name, data=data).get_results()
-    """get overwrites indexed my channel_id"""
+    existing_ids = [i[on_key] for i in result]
-    data = {
+    dl = [i for i in to_check if i not in existing_ids]
-        "query": {
+
-            "bool": {"must": [{"exists": {"field": "channel_overwrites"}}]}
+    return dl
-        },
+
-        "_source": ["channel_id", "channel_overwrites"],
+
-    }
+def get_channel_overwrites() -> dict[str, dict[str, Any]]:
-    result = IndexPaginate("ta_channel", data).get_results()
+    """get overwrites indexed my channel_id"""
-    overwrites = {i["channel_id"]: i["channel_overwrites"] for i in result}
+    data = {
-
+        "query": {
-    return overwrites
+            "bool": {"must": [{"exists": {"field": "channel_overwrites"}}]}
+        },
+        "_source": ["channel_id", "channel_overwrites"],
+    }
+    result = IndexPaginate("ta_channel", data).get_results()
+    overwrites = {i["channel_id"]: i["channel_overwrites"] for i in result}
+
+    return overwrites

backend/common/src/index_generic.py

@@ -5,10 +5,10 @@ functionality:
 
 import math
 
-from home.src.download.yt_dlp_base import YtWrap
+from appsettings.src.config import AppConfig
-from home.src.es.connect import ElasticWrap
+from common.src.es_connect import ElasticWrap
-from home.src.ta.config import AppConfig
+from download.src.yt_dlp_base import YtWrap
-from home.src.ta.users import UserConfig
+from user.src.user_config import UserConfig
 
 
 class YouTubeItem:
@@ -104,8 +104,8 @@ class Pagination:
     def first_guess(self):
         """build first guess before api call"""
         page_get = self.page_get
+        page_from = 0
         if page_get in [0, 1]:
-            page_from = 0
             prev_pages = False
         elif page_get > 1:
             page_from = (page_get - 1) * self.page_size

backend/common/src/search_processor.py

@@ -6,19 +6,19 @@ Functionality:
 
 import urllib.parse
 
-from home.src.download.thumbnails import ThumbManager
+from common.src.env_settings import EnvironmentSettings
-from home.src.ta.helper import date_parser, get_duration_str
+from common.src.helper import date_parser, get_duration_str
-from home.src.ta.settings import EnvironmentSettings
+from common.src.ta_redis import RedisArchivist
+from download.src.thumbnails import ThumbManager
 
 
 class SearchProcess:
     """process search results"""
 
-    CACHE_DIR = EnvironmentSettings.CACHE_DIR
+    def __init__(self, response, match_video_user_progress: None | int = None):
-
-    def __init__(self, response):
         self.response = response
         self.processed = False
+        self.position_index = self.get_user_progress(match_video_user_progress)
 
     def process(self):
         """detect type and process"""
@@ -35,6 +35,19 @@ class SearchProcess:
 
         return self.processed
 
+    def get_user_progress(self, match_video_user_progress) -> dict | None:
+        """get user video watch progress"""
+        if not match_video_user_progress:
+            return None
+
+        query = f"{match_video_user_progress}:progress:*"
+        all_positions = RedisArchivist().list_items(query)
+        if not all_positions:
+            return None
+
+        pos_index = {i["youtube_id"]: i["position"] for i in all_positions}
+        return pos_index
+
     def _process_result(self, result):
         """detect which type of data to process"""
         index = result["_index"]
@@ -66,7 +79,8 @@ class SearchProcess:
     def _process_channel(channel_dict):
         """run on single channel"""
         channel_id = channel_dict["channel_id"]
-        art_base = f"/cache/channels/{channel_id}"
+        cache_root = EnvironmentSettings().get_cache_root()
+        art_base = f"{cache_root}/channels/{channel_id}"
         date_str = date_parser(channel_dict["channel_last_refresh"])
         channel_dict.update(
             {
@@ -93,16 +107,31 @@ class SearchProcess:
                 url = video_dict["subtitles"][idx]["media_url"]
                 video_dict["subtitles"][idx]["media_url"] = f"/media/{url}"
 
+        cache_root = EnvironmentSettings().get_cache_root()
+        media_root = EnvironmentSettings().get_media_root()
+
         video_dict.update(
             {
                 "channel": channel,
-                "media_url": f"/media/{media_url}",
+                "media_url": f"{media_root}/{media_url}",
                 "vid_last_refresh": vid_last_refresh,
                 "published": published,
-                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+                "vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
             }
         )
 
+        if self.position_index:
+            player_position = self.position_index.get(video_id)
+            total = video_dict["player"].get("duration")
+            if player_position and total:
+                progress = 100 * (player_position / total)
+                video_dict["player"].update(
+                    {
+                        "progress": progress,
+                        "position": player_position,
+                    }
+                )
+
         return dict(sorted(video_dict.items()))
 
     @staticmethod
@@ -112,9 +141,11 @@ class SearchProcess:
         playlist_last_refresh = date_parser(
             playlist_dict["playlist_last_refresh"]
         )
+        cache_root = EnvironmentSettings().get_cache_root()
+        playlist_thumbnail = f"{cache_root}/playlists/{playlist_id}.jpg"
         playlist_dict.update(
             {
-                "playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg",
+                "playlist_thumbnail": playlist_thumbnail,
                 "playlist_last_refresh": playlist_last_refresh,
             }
         )
@@ -124,12 +155,13 @@ class SearchProcess:
     def _process_download(self, download_dict):
         """run on single download item"""
         video_id = download_dict["youtube_id"]
+        cache_root = EnvironmentSettings().get_cache_root()
         vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
         published = date_parser(download_dict["published"])
 
         download_dict.update(
             {
-                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+                "vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
                 "published": published,
             }
         )

backend/common/src/searching.py

@@ -6,8 +6,8 @@ Functionality:
 - calculate pagination values
 """
 
-from api.src.search_processor import SearchProcess
+from common.src.es_connect import ElasticWrap
-from home.src.es.connect import ElasticWrap
+from common.src.search_processor import SearchProcess
 
 
 class SearchForm:

backend/common/src/ta_redis.py

@@ -8,7 +8,7 @@ functionality:
 import json
 
 import redis
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
 
 
 class RedisBase:
@@ -17,10 +17,8 @@ class RedisBase:
     NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE
 
     def __init__(self):
-        self.conn = redis.Redis(
+        self.conn = redis.from_url(
-            host=EnvironmentSettings.REDIS_HOST,
+            url=EnvironmentSettings.REDIS_CON, decode_responses=True
-            port=EnvironmentSettings.REDIS_PORT,
-            decode_responses=True,
         )
 
 
@@ -40,15 +38,15 @@ class RedisArchivist(RedisBase):
     def set_message(
         self,
         key: str,
-        message: dict,
+        message: dict | str,
-        path: str = ".",
         expire: bool | int = False,
         save: bool = False,
     ) -> None:
         """write new message to redis"""
-        self.conn.execute_command(
+        to_write = (
-            "JSON.SET", self.NAME_SPACE + key, path, json.dumps(message)
+            json.dumps(message) if isinstance(message, dict) else message
         )
+        self.conn.execute_command("SET", self.NAME_SPACE + key, to_write)
 
         if expire:
             if isinstance(expire, bool):
@@ -67,8 +65,24 @@ class RedisArchivist(RedisBase):
         except redis.exceptions.ResponseError:
             pass
 
-    def get_message(self, key: str) -> dict:
+    def get_message_str(self, key: str) -> str | None:
-        """get message dict from redis"""
+        """get message string"""
+        reply = self.conn.execute_command("GET", self.NAME_SPACE + key)
+        return reply
+
+    def get_message_dict(self, key: str) -> dict:
+        """get message dict"""
+        reply = self.conn.execute_command("GET", self.NAME_SPACE + key)
+        if not reply:
+            return {}
+
+        return json.loads(reply)
+
+    def get_message(self, key: str) -> dict | None:
+        """
+        get message dict from redis
+        old json get message, only used for migration, to be removed later
+        """
         reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
         if reply:
             return json.loads(reply)
@@ -91,7 +105,7 @@ class RedisArchivist(RedisBase):
         if not all_matches:
             return []
 
-        return [self.get_message(i) for i in all_matches]
+        return [self.get_message_dict(i) for i in all_matches]
 
     def del_message(self, key: str) -> bool:
         """delete key from redis"""

backend/common/src/urlparser.py

@@ -6,8 +6,8 @@ Functionality:
 
 from urllib.parse import parse_qs, urlparse
 
-from home.src.download.yt_dlp_base import YtWrap
+from download.src.yt_dlp_base import YtWrap
-from home.src.index.video_constants import VideoTypeEnum
+from video.src.constants import VideoTypeEnum
 
 
 class Parser:
@@ -67,6 +67,9 @@ class Parser:
         if all_paths[0] == "channel":
             return self._validate_expected(all_paths[1], "channel")
 
+        if all_paths[0] == "live":
+            return self._validate_expected(all_paths[1], "video")
+
         # detect channel
         channel_id = self._extract_channel_name(parsed.geturl())
         return {"type": "channel", "url": channel_id}
@@ -113,6 +116,9 @@ class Parser:
             "playlistend": 0,
         }
         url_info = YtWrap(obs_request).extract(url)
+        if not url_info:
+            raise ValueError(f"failed to retrieve content from URL: {url}")
+
         channel_id = url_info.get("channel_id", False)
         if channel_id:
             return channel_id

backend/common/src/watched.py

@@ -5,8 +5,8 @@ functionality:
 
 from datetime import datetime
 
-from home.src.es.connect import ElasticWrap
+from common.src.es_connect import ElasticWrap
-from home.src.ta.urlparser import Parser
+from common.src.urlparser import Parser
 
 
 class WatchState:

backend/common/tests/conftest.py

@@ -8,4 +8,4 @@ import pytest
 @pytest.fixture(scope="session", autouse=True)
 def change_test_dir(request):
     """change directory to project folder"""
-    os.chdir(request.config.rootdir / "tubearchivist")
+    os.chdir(request.config.rootdir / "backend")

backend/common/tests/test_src/test_helper.py

@@ -1,7 +1,7 @@
 """tests for helper functions"""
 
 import pytest
-from home.src.ta.helper import (
+from common.src.helper import (
     date_parser,
     get_duration_str,
     get_mapping,

backend/common/tests/test_src/test_urlparser.py

@@ -1,7 +1,7 @@
 """tests for url parser"""
 
 import pytest
-from home.src.ta.urlparser import Parser
+from common.src.urlparser import Parser
 
 # video id parsing
 VIDEO_URL_IN = [
@@ -10,6 +10,7 @@ VIDEO_URL_IN = [
     "https://www.youtube.com/watch?v=7DKv5H5Frt0&t=113&feature=shared",
     "https://www.youtube.com/watch?v=7DKv5H5Frt0&list=PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5&index=1&pp=iAQB"  # noqa: E501
     "https://youtu.be/7DKv5H5Frt0",
+    "https://www.youtube.com/live/7DKv5H5Frt0",
 ]
 VIDEO_OUT = [{"type": "video", "url": "7DKv5H5Frt0", "vid_type": "unknown"}]
 VIDEO_TEST_CASES = [(i, VIDEO_OUT) for i in VIDEO_URL_IN]

backend/common/urls.py

@@ -0,0 +1,28 @@
+"""all api urls"""
+
+from common import views
+from django.urls import path
+
+urlpatterns = [
+    path("ping/", views.PingView.as_view(), name="ping"),
+    path(
+        "refresh/",
+        views.RefreshView.as_view(),
+        name="api-refresh",
+    ),
+    path(
+        "watched/",
+        views.WatchedView.as_view(),
+        name="api-watched",
+    ),
+    path(
+        "search/",
+        views.SearchView.as_view(),
+        name="api-search",
+    ),
+    path(
+        "notification/",
+        views.NotificationView.as_view(),
+        name="api-notification",
+    ),
+]

backend/common/views.py

@@ -0,0 +1,116 @@
+"""all API views"""
+
+from appsettings.src.config import ReleaseVersion
+from appsettings.src.reindex import ReindexProgress
+from common.src.searching import SearchForm
+from common.src.ta_redis import RedisArchivist
+from common.src.watched import WatchState
+from common.views_base import AdminOnly, ApiBaseView
+from rest_framework.response import Response
+from task.tasks import check_reindex
+
+
+class PingView(ApiBaseView):
+    """resolves to /api/ping/
+    GET: test your connection
+    """
+
+    @staticmethod
+    def get(request):
+        """get pong"""
+        data = {
+            "response": "pong",
+            "user": request.user.id,
+            "version": ReleaseVersion().get_local_version(),
+            "ta_update": ReleaseVersion().get_update(),
+        }
+        return Response(data)
+
+
+class RefreshView(ApiBaseView):
+    """resolves to /api/refresh/
+    GET: get refresh progress
+    POST: start a manual refresh task
+    """
+
+    permission_classes = [AdminOnly]
+
+    def get(self, request):
+        """handle get request"""
+        request_type = request.GET.get("type")
+        request_id = request.GET.get("id")
+
+        if request_id and not request_type:
+            return Response({"status": "Bad Request"}, status=400)
+
+        try:
+            progress = ReindexProgress(
+                request_type=request_type, request_id=request_id
+            ).get_progress()
+        except ValueError:
+            return Response({"status": "Bad Request"}, status=400)
+
+        return Response(progress)
+
+    def post(self, request):
+        """handle post request"""
+        data = request.data
+        extract_videos = bool(request.GET.get("extract_videos", False))
+        check_reindex.delay(data=data, extract_videos=extract_videos)
+
+        return Response(data)
+
+
+class WatchedView(ApiBaseView):
+    """resolves to /api/watched/
+    POST: change watched state of video, channel or playlist
+    """
+
+    def post(self, request):
+        """change watched state"""
+        youtube_id = request.data.get("id")
+        is_watched = request.data.get("is_watched")
+
+        if not youtube_id or is_watched is None:
+            message = {"message": "missing id or is_watched"}
+            return Response(message, status=400)
+
+        WatchState(youtube_id, is_watched).change()
+        return Response({"message": "success"}, status=200)
+
+
+class SearchView(ApiBaseView):
+    """resolves to /api/search/
+    GET: run a search with the string in the ?query parameter
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request
+        search through all indexes"""
+        search_query = request.GET.get("query", None)
+        if search_query is None:
+            return Response(
+                {"message": "no search query specified"}, status=400
+            )
+
+        search_results = SearchForm().multi_search(search_query)
+        return Response(search_results)
+
+
+class NotificationView(ApiBaseView):
+    """resolves to /api/notification/
+    GET: returns a list of notifications
+    filter query to filter messages by group
+    """
+
+    valid_filters = ["download", "settings", "channel"]
+
+    def get(self, request):
+        """get all notifications"""
+        query = "message"
+        filter_by = request.GET.get("filter", None)
+        if filter_by in self.valid_filters:
+            query = f"{query}:{filter_by}"
+
+        return Response(RedisArchivist().list_items(query))

backend/common/views_base.py

@@ -0,0 +1,110 @@
+"""base classes to inherit from"""
+
+from appsettings.src.config import AppConfig
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.index_generic import Pagination
+from common.src.search_processor import SearchProcess, process_aggs
+from rest_framework import permissions
+from rest_framework.authentication import (
+    SessionAuthentication,
+    TokenAuthentication,
+)
+from rest_framework.views import APIView
+
+
+def check_admin(user):
+    """check for admin permission for restricted views"""
+    return user.is_staff or user.groups.filter(name="admin").exists()
+
+
+class AdminOnly(permissions.BasePermission):
+    """allow only admin"""
+
+    def has_permission(self, request, view):
+        return check_admin(request.user)
+
+
+class AdminWriteOnly(permissions.BasePermission):
+    """allow only admin writes"""
+
+    def has_permission(self, request, view):
+        if request.method in permissions.SAFE_METHODS:
+            return permissions.IsAuthenticated().has_permission(request, view)
+
+        return check_admin(request.user)
+
+
+class ApiBaseView(APIView):
+    """base view to inherit from"""
+
+    authentication_classes = [SessionAuthentication, TokenAuthentication]
+    permission_classes = [permissions.IsAuthenticated]
+    search_base = ""
+    data = ""
+
+    def __init__(self):
+        super().__init__()
+        self.response = {
+            "data": False,
+            "config": {
+                "enable_cast": EnvironmentSettings.ENABLE_CAST,
+                "downloads": AppConfig().config["downloads"],
+            },
+        }
+        self.data = {"query": {"match_all": {}}}
+        self.status_code = False
+        self.context = False
+        self.pagination_handler = False
+
+    def get_document(self, document_id, progress_match=None):
+        """get single document from es"""
+        path = f"{self.search_base}{document_id}"
+        response, status_code = ElasticWrap(path).get()
+        try:
+            self.response["data"] = SearchProcess(
+                response, match_video_user_progress=progress_match
+            ).process()
+        except KeyError:
+            print(f"item not found: {document_id}")
+            self.response["data"] = False
+        self.status_code = status_code
+
+    def initiate_pagination(self, request):
+        """set initial pagination values"""
+        self.pagination_handler = Pagination(request)
+        self.data.update(
+            {
+                "size": self.pagination_handler.pagination["page_size"],
+                "from": self.pagination_handler.pagination["page_from"],
+            }
+        )
+
+    def get_document_list(self, request, pagination=True, progress_match=None):
+        """get a list of results"""
+        if pagination:
+            self.initiate_pagination(request)
+
+        es_handler = ElasticWrap(self.search_base)
+        response, status_code = es_handler.get(data=self.data)
+        self.response["data"] = SearchProcess(
+            response, match_video_user_progress=progress_match
+        ).process()
+        if self.response["data"]:
+            self.status_code = status_code
+        else:
+            self.status_code = 404
+
+        if pagination and response.get("hits"):
+            self.pagination_handler.validate(
+                response["hits"]["total"]["value"]
+            )
+            self.response["paginate"] = self.pagination_handler.pagination
+
+    def get_aggs(self):
+        """get aggs alone"""
+        self.data["size"] = 0
+        response, _ = ElasticWrap(self.search_base).get(data=self.data)
+        process_aggs(response)
+
+        self.response = response.get("aggregations")

backend/config/management/commands/ta_config_backup.py

@@ -0,0 +1,76 @@
+"""backup config for sqlite reset and restore"""
+
+import json
+from pathlib import Path
+
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand
+from home.models import CustomPeriodicTask
+from home.src.ta.settings import EnvironmentSettings
+from rest_framework.authtoken.models import Token
+
+User = get_user_model()
+
+
+class Command(BaseCommand):
+    """export"""
+
+    help = "Exports all users and their auth tokens to a JSON file"
+    FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
+
+    def handle(self, *args, **kwargs):
+        """entry point"""
+
+        data = {
+            "user_data": self.get_users(),
+            "schedule_data": self.get_schedules(),
+        }
+
+        with open(self.FILE, "w", encoding="utf-8") as json_file:
+            json_file.write(json.dumps(data))
+
+    def get_users(self):
+        """get users"""
+
+        users = User.objects.all()
+
+        user_data = []
+
+        for user in users:
+            user_info = {
+                "username": user.name,
+                "is_staff": user.is_staff,
+                "is_superuser": user.is_superuser,
+                "password": user.password,
+                "tokens": [],
+            }
+
+            try:
+                token = Token.objects.get(user=user)
+                user_info["tokens"] = [token.key]
+            except Token.DoesNotExist:
+                user_info["tokens"] = []
+
+            user_data.append(user_info)
+
+        return user_data
+
+    def get_schedules(self):
+        """get schedules"""
+
+        all_schedules = CustomPeriodicTask.objects.all()
+        schedule_data = []
+
+        for schedule in all_schedules:
+            schedule_info = {
+                "name": schedule.name,
+                "crontab": {
+                    "minute": schedule.crontab.minute,
+                    "hour": schedule.crontab.hour,
+                    "day_of_week": schedule.crontab.day_of_week,
+                },
+            }
+
+            schedule_data.append(schedule_info)
+
+        return schedule_data

backend/config/management/commands/ta_config_restore.py

@@ -0,0 +1,89 @@
+"""restore config from backup"""
+
+import json
+from pathlib import Path
+
+from common.src.env_settings import EnvironmentSettings
+from django.core.management.base import BaseCommand
+from django_celery_beat.models import CrontabSchedule
+from rest_framework.authtoken.models import Token
+from task.models import CustomPeriodicTask
+from task.src.task_config import TASK_CONFIG
+from user.models import Account
+
+
+class Command(BaseCommand):
+    """export"""
+
+    help = "Exports all users and their auth tokens to a JSON file"
+    FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
+
+    def handle(self, *args, **options):
+        """handle"""
+        self.stdout.write("restore users and schedules")
+        data = self.get_config()
+        self.restore_users(data["user_data"])
+        self.restore_schedules(data["schedule_data"])
+        self.stdout.write(
+            self.style.SUCCESS(
+                "    ✓ restore completed. Please restart the container."
+            )
+        )
+
+    def get_config(self) -> dict:
+        """get config from backup"""
+        with open(self.FILE, "r", encoding="utf-8") as json_file:
+            data = json.loads(json_file.read())
+
+        self.stdout.write(
+            self.style.SUCCESS(f"    ✓ json file found: {self.FILE}")
+        )
+
+        return data
+
+    def restore_users(self, user_data: list[dict]) -> None:
+        """restore users from config"""
+        self.stdout.write("delete existing users")
+        Account.objects.all().delete()
+
+        self.stdout.write("recreate users")
+        for user_info in user_data:
+            user = Account.objects.create(
+                name=user_info["username"],
+                is_staff=user_info["is_staff"],
+                is_superuser=user_info["is_superuser"],
+                password=user_info["password"],
+            )
+            for token in user_info["tokens"]:
+                Token.objects.create(user=user, key=token)
+
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"    ✓ recreated user with name: {user_info['username']}"
+                )
+            )
+
+    def restore_schedules(self, schedule_data: list[dict]) -> None:
+        """restore schedules"""
+        self.stdout.write("delete existing schedules")
+        CustomPeriodicTask.objects.all().delete()
+
+        self.stdout.write("recreate schedules")
+        for schedule in schedule_data:
+            task_name = schedule["name"]
+            description = TASK_CONFIG[task_name].get("title")
+            crontab, _ = CrontabSchedule.objects.get_or_create(
+                minute=schedule["crontab"]["minute"],
+                hour=schedule["crontab"]["hour"],
+                day_of_week=schedule["crontab"]["day_of_week"],
+                timezone=EnvironmentSettings.TZ,
+            )
+            task = CustomPeriodicTask.objects.create(
+                name=task_name,
+                task=task_name,
+                description=description,
+                crontab=crontab,
+            )
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ recreated schedule: {task}")
+            )

backend/config/management/commands/ta_connection.py

@@ -6,10 +6,10 @@ Functionality:
 from time import sleep
 
 import requests
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.ta_redis import RedisArchivist
 from django.core.management.base import BaseCommand, CommandError
-from home.src.es.connect import ElasticWrap
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisArchivist
 
 TOPIC = """
 

backend/config/management/commands/ta_envcheck.py

@@ -8,10 +8,11 @@ Functionality:
 
 import os
 import re
+from time import sleep
 
+from common.src.env_settings import EnvironmentSettings
 from django.core.management.base import BaseCommand, CommandError
-from home.models import Account
+from user.models import Account
-from home.src.ta.settings import EnvironmentSettings
 
 LOGO = """
 
@@ -60,9 +61,13 @@ EXPECTED_ENV_VARS = [
     "ES_URL",
     "TA_HOST",
 ]
+UNEXPECTED_ENV_VARS = {
+    "TA_UWSGI_PORT": "Has been replaced with 'TA_BACKEND_PORT'",
+    "REDIS_HOST": "Has been replaced with 'REDIS_CON' connection string",
+    "REDIS_PORT": "Has been consolidated in 'REDIS_CON' connection string",
+}
 INST = "https://github.com/tubearchivist/tubearchivist#installing-and-updating"
 NGINX = "/etc/nginx/sites-available/default"
-UWSGI = "/app/uwsgi.ini"
 
 
 class Command(BaseCommand):
@@ -76,9 +81,10 @@ class Command(BaseCommand):
         self.stdout.write(LOGO)
         self.stdout.write(TOPIC)
         self._expected_vars()
+        self._unexpected_vars()
         self._elastic_user_overwrite()
         self._ta_port_overwrite()
-        self._ta_uwsgi_overwrite()
+        self._ta_backend_port_overwrite()
         self._enable_cast_overwrite()
         self._create_superuser()
 
@@ -90,20 +96,41 @@ class Command(BaseCommand):
             if not env.get(var):
                 message = f"    🗙 expected env var {var} not set\n    {INST}"
                 self.stdout.write(self.style.ERROR(message))
+                sleep(60)
                 raise CommandError(message)
 
         message = "    ✓ all expected env vars are set"
         self.stdout.write(self.style.SUCCESS(message))
 
+    def _unexpected_vars(self):
+        """check for unexpected env vars"""
+        self.stdout.write("[2] checking for unexpected env vars")
+        for var, message in UNEXPECTED_ENV_VARS.items():
+            if not os.environ.get(var):
+                continue
+
+            message = (
+                f"    🗙 unexpected env var {var} found\n"
+                f"    {message} \n"
+                "    see release notes for a list of all changes."
+            )
+
+            self.stdout.write(self.style.ERROR(message))
+            sleep(60)
+            raise CommandError(message)
+
+        message = "    ✓ no unexpected env vars found"
+        self.stdout.write(self.style.SUCCESS(message))
+
     def _elastic_user_overwrite(self):
         """check for ELASTIC_USER overwrite"""
-        self.stdout.write("[2] check ES user overwrite")
+        self.stdout.write("[3] check ES user overwrite")
         env = EnvironmentSettings.ES_USER
         self.stdout.write(self.style.SUCCESS(f"    ✓ ES user is set to {env}"))
 
     def _ta_port_overwrite(self):
         """set TA_PORT overwrite for nginx"""
-        self.stdout.write("[3] check TA_PORT overwrite")
+        self.stdout.write("[4] check TA_PORT overwrite")
         overwrite = EnvironmentSettings.TA_PORT
         if not overwrite:
             self.stdout.write(self.style.SUCCESS("    TA_PORT is not set"))
@@ -119,35 +146,30 @@ class Command(BaseCommand):
 
         self.stdout.write(self.style.SUCCESS(message))
 
-    def _ta_uwsgi_overwrite(self):
+    def _ta_backend_port_overwrite(self):
-        """set TA_UWSGI_PORT overwrite"""
+        """set TA_BACKEND_PORT overwrite"""
-        self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
+        self.stdout.write("[5] check TA_BACKEND_PORT overwrite")
-        overwrite = EnvironmentSettings.TA_UWSGI_PORT
+        overwrite = EnvironmentSettings.TA_BACKEND_PORT
         if not overwrite:
-            message = "    TA_UWSGI_PORT is not set"
+            message = "    TA_BACKEND_PORT is not set"
             self.stdout.write(self.style.SUCCESS(message))
             return
 
-        # nginx
+        # modify nginx conf
-        regex = re.compile(r"uwsgi_pass localhost:[0-9]{1,5}")
+        regex = re.compile(r"proxy_pass http://localhost:[0-9]{1,5}")
-        to_overwrite = f"uwsgi_pass localhost:{overwrite}"
+        to_overwrite = f"proxy_pass http://localhost:{overwrite}"
         changed = file_overwrite(NGINX, regex, to_overwrite)
 
-        # uwsgi
-        regex = re.compile(r"socket = :[0-9]{1,5}")
-        to_overwrite = f"socket = :{overwrite}"
-        changed = file_overwrite(UWSGI, regex, to_overwrite)
-
         if changed:
-            message = f"    ✓ TA_UWSGI_PORT changed to {overwrite}"
+            message = f"    ✓ TA_BACKEND_PORT changed to {overwrite}"
         else:
-            message = f"    ✓ TA_UWSGI_PORT already set to {overwrite}"
+            message = f"    ✓ TA_BACKEND_PORT already set to {overwrite}"
 
         self.stdout.write(self.style.SUCCESS(message))
 
     def _enable_cast_overwrite(self):
         """cast workaround, remove auth for static files in nginx"""
-        self.stdout.write("[5] check ENABLE_CAST overwrite")
+        self.stdout.write("[6] check ENABLE_CAST overwrite")
         overwrite = EnvironmentSettings.ENABLE_CAST
         if not overwrite:
             self.stdout.write(self.style.SUCCESS("    ENABLE_CAST is not set"))
@@ -164,7 +186,7 @@ class Command(BaseCommand):
 
     def _create_superuser(self):
         """create superuser if not exist"""
-        self.stdout.write("[6] create superuser")
+        self.stdout.write("[7] create superuser")
         is_created = Account.objects.filter(is_superuser=True)
         if is_created:
             message = "    superuser already created"

backend/config/management/commands/ta_startup.py

@@ -0,0 +1,281 @@
+"""
+Functionality:
+- Application startup
+- Apply migrations
+"""
+
+import os
+from datetime import datetime
+from random import randint
+from time import sleep
+
+from appsettings.src.config import AppConfig, ReleaseVersion
+from appsettings.src.index_setup import ElasitIndexWrap
+from appsettings.src.snapshot import ElasticSnapshot
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import clear_dl_cache
+from common.src.ta_redis import RedisArchivist
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import dateformat
+from django_celery_beat.models import CrontabSchedule, PeriodicTasks
+from redis.exceptions import ResponseError
+from task.models import CustomPeriodicTask
+from task.src.config_schedule import ScheduleBuilder
+from task.src.task_manager import TaskManager
+from task.tasks import version_check
+
+TOPIC = """
+
+#######################
+#  Application Start  #
+#######################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(TOPIC)
+        self._make_folders()
+        self._clear_redis_keys()
+        self._clear_tasks()
+        self._clear_dl_cache()
+        self._version_check()
+        self._index_setup()
+        self._snapshot_check()
+        self._create_default_schedules()
+        self._update_schedule_tz()
+        self._init_app_config()
+
+    def _mig_app_settings(self) -> None:
+        """update from v0.4.13 to v0.5.0, migrate application settings"""
+        self.stdout.write("[MIGRATION] move appconfig to ES")
+        try:
+            config = RedisArchivist().get_message("config")
+        except ResponseError:
+            self.stdout.write(
+                self.style.SUCCESS("    Redis does not support JSON decoding")
+            )
+            return
+
+        if not config:
+            self.stdout.write(
+                self.style.SUCCESS("    no config values to migrate")
+            )
+            return
+
+        path = "ta_config/_doc/appsettings"
+        response, status_code = ElasticWrap(path).post(config)
+
+        if status_code in [200, 201]:
+            self.stdout.write(
+                self.style.SUCCESS("    ✓ migrated appconfig to ES")
+            )
+            RedisArchivist().del_message("config")
+            return
+
+        message = "    🗙 failed to migrate app config"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)
+
+    def _make_folders(self):
+        """make expected cache folders"""
+        self.stdout.write("[2] create expected cache folders")
+        folders = [
+            "backup",
+            "channels",
+            "download",
+            "import",
+            "playlists",
+            "videos",
+        ]
+        cache_dir = EnvironmentSettings.CACHE_DIR
+        for folder in folders:
+            folder_path = os.path.join(cache_dir, folder)
+            os.makedirs(folder_path, exist_ok=True)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ expected folders created"))
+
+    def _clear_redis_keys(self):
+        """make sure there are no leftover locks or keys set in redis"""
+        self.stdout.write("[3] clear leftover keys in redis")
+        all_keys = [
+            "dl_queue_id",
+            "dl_queue",
+            "downloading",
+            "manual_import",
+            "reindex",
+            "rescan",
+            "run_backup",
+            "startup_check",
+            "reindex:ta_video",
+            "reindex:ta_channel",
+            "reindex:ta_playlist",
+        ]
+
+        redis_con = RedisArchivist()
+        has_changed = False
+        for key in all_keys:
+            if redis_con.del_message(key):
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ cleared key {key}")
+                )
+                has_changed = True
+
+        if not has_changed:
+            self.stdout.write(self.style.SUCCESS("    no keys found"))
+
+    def _clear_tasks(self):
+        """clear tasks and messages"""
+        self.stdout.write("[4] clear task leftovers")
+        TaskManager().fail_pending()
+        redis_con = RedisArchivist()
+        to_delete = redis_con.list_keys("message:")
+        if to_delete:
+            for key in to_delete:
+                redis_con.del_message(key)
+
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {len(to_delete)} messages")
+            )
+
+    def _clear_dl_cache(self):
+        """clear leftover files from dl cache"""
+        self.stdout.write("[5] clear leftover files from dl cache")
+        leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
+        if leftover_files:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {leftover_files} files")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no files found"))
+
+    def _version_check(self):
+        """remove new release key if updated now"""
+        self.stdout.write("[6] check for first run after update")
+        new_version = ReleaseVersion().is_updated()
+        if new_version:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ update to {new_version} completed")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no new update found"))
+
+        version_task = CustomPeriodicTask.objects.filter(name="version_check")
+        if not version_task.exists():
+            return
+
+        if not version_task.first().last_run_at:
+            self.style.SUCCESS("    ✓ send initial version check task")
+            version_check.delay()
+
+    def _index_setup(self):
+        """migration: validate index mappings"""
+        self.stdout.write("[7] validate index mappings")
+        ElasitIndexWrap().setup()
+
+    def _snapshot_check(self):
+        """migration setup snapshots"""
+        self.stdout.write("[8] setup snapshots")
+        ElasticSnapshot().setup()
+
+    def _create_default_schedules(self) -> None:
+        """create default schedules for new installations"""
+        self.stdout.write("[9] create initial schedules")
+        init_has_run = CustomPeriodicTask.objects.filter(
+            name="version_check"
+        ).exists()
+
+        if init_has_run:
+            self.stdout.write(
+                self.style.SUCCESS(
+                    "    schedule init already done, skipping..."
+                )
+            )
+            return
+
+        builder = ScheduleBuilder()
+        check_reindex = builder.get_set_task(
+            "check_reindex", schedule=builder.SCHEDULES["check_reindex"]
+        )
+        check_reindex.task_config.update({"days": 90})
+        check_reindex.last_run_at = dateformat.make_aware(datetime.now())
+        check_reindex.save()
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {check_reindex}"
+            )
+        )
+
+        thumbnail_check = builder.get_set_task(
+            "thumbnail_check", schedule=builder.SCHEDULES["thumbnail_check"]
+        )
+        thumbnail_check.last_run_at = dateformat.make_aware(datetime.now())
+        thumbnail_check.save()
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {thumbnail_check}"
+            )
+        )
+        daily_random = f"{randint(0, 59)} {randint(0, 23)} *"
+        version_check_task = builder.get_set_task(
+            "version_check", schedule=daily_random
+        )
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {version_check_task}"
+            )
+        )
+        self.stdout.write(
+            self.style.SUCCESS("    ✓ all default schedules created")
+        )
+
+    def _update_schedule_tz(self) -> None:
+        """update timezone for Schedule instances"""
+        self.stdout.write("[10] validate schedules TZ")
+        tz = EnvironmentSettings.TZ
+        to_update = CrontabSchedule.objects.exclude(timezone=tz)
+
+        if not to_update.exists():
+            self.stdout.write(
+                self.style.SUCCESS("    all schedules have correct TZ")
+            )
+            return
+
+        updated = to_update.update(timezone=tz)
+        self.stdout.write(
+            self.style.SUCCESS(f"    ✓ updated {updated} schedules to {tz}.")
+        )
+        PeriodicTasks.update_changed()
+
+    def _init_app_config(self) -> None:
+        """init default app config to ES"""
+        self.stdout.write("[11] Check AppConfig")
+        try:
+            _ = AppConfig().config
+            self.stdout.write(
+                self.style.SUCCESS("    skip completed appsettings init")
+            )
+            updated_defaults = AppConfig().add_new_defaults()
+            for new_default in updated_defaults:
+                self.stdout.write(
+                    self.style.SUCCESS(f"    added new default: {new_default}")
+                )
+
+        except ValueError:
+            handler = AppConfig.__new__(AppConfig)
+            _, status_code = handler.sync_defaults()
+            self.stdout.write(
+                self.style.SUCCESS("    ✓ Created default appsettings.")
+            )
+            self.stdout.write(
+                self.style.SUCCESS(f"      Status code: {status_code}")
+            )

backend/config/management/commands/ta_stop_on_error.py

@@ -0,0 +1,40 @@
+"""stop on unexpected table"""
+
+from time import sleep
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import connection
+
+ERROR_MESSAGE = """
+    🗙 Database is incompatible, see latest release notes for instructions:
+    🗙 https://github.com/tubearchivist/tubearchivist/releases/tag/v0.5.0
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """handle"""
+        self.stdout.write("[MIGRATION] Confirming v0.5.0 table layout")
+        all_tables = self.list_tables()
+        for table in all_tables:
+            if table == "home_account":
+
+                self.stdout.write(self.style.ERROR(ERROR_MESSAGE))
+                sleep(60)
+                raise CommandError(ERROR_MESSAGE)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ local DB is up-to-date."))
+
+    def list_tables(self):
+        """raw list all tables"""
+        with connection.cursor() as cursor:
+            cursor.execute(
+                "SELECT name FROM sqlite_master WHERE type='table';"
+            )
+            tables = cursor.fetchall()
+
+        return [table[0] for table in tables]

backend/config/settings.py

@@ -14,11 +14,17 @@ import hashlib
 from os import environ, path
 from pathlib import Path
 
-import ldap
+from common.src.env_settings import EnvironmentSettings
+from common.src.helper import ta_host_parser
 from corsheaders.defaults import default_headers
-from django_auth_ldap.config import LDAPSearch
+
-from home.src.ta.helper import ta_host_parser
+try:
-from home.src.ta.settings import EnvironmentSettings
+    from dotenv import load_dotenv
+
+    load_dotenv(".env")
+except ModuleNotFoundError:
+    pass
+
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -41,7 +47,6 @@ ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(
 
 INSTALLED_APPS = [
     "django_celery_beat",
-    "home.apps.HomeConfig",
     "django.contrib.admin",
     "django.contrib.auth",
     "django.contrib.contenttypes",
@@ -53,7 +58,15 @@ INSTALLED_APPS = [
     "django.contrib.humanize",
     "rest_framework",
     "rest_framework.authtoken",
-    "api",
+    "common",
+    "video",
+    "channel",
+    "playlist",
+    "download",
+    "task",
+    "appsettings",
+    "stats",
+    "user",
     "config",
 ]
 
@@ -67,7 +80,7 @@ MIDDLEWARE = [
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
-    "home.src.ta.health.HealthCheckMiddleware",
+    "common.src.health.HealthCheckMiddleware",
 ]
 
 ROOT_URLCONF = "config.urls"
@@ -92,6 +105,9 @@ WSGI_APPLICATION = "config.wsgi.application"
 
 if bool(environ.get("TA_LDAP")):
     # pylint: disable=global-at-module-level
+    import ldap
+    from django_auth_ldap.config import LDAPSearch
+
     global AUTH_LDAP_SERVER_URI
     AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
 
@@ -211,7 +227,7 @@ AUTH_PASSWORD_VALIDATORS = [
     },
 ]
 
-AUTH_USER_MODEL = "home.Account"
+AUTH_USER_MODEL = "user.Account"
 
 # Forward-auth authentication
 if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
@@ -220,7 +236,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
     )
     TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")
 
-    MIDDLEWARE.append("home.src.ta.auth.HttpRemoteUserMiddleware")
+    MIDDLEWARE.append("user.src.remote_user_auth.HttpRemoteUserMiddleware")
 
     AUTHENTICATION_BACKENDS = (
         "django.contrib.auth.backends.RemoteUserBackend",
@@ -261,14 +277,18 @@ LOGOUT_REDIRECT_URL = "/login/"
 # background.js makes the request so HTTP_ORIGIN will be from extension
 if environ.get("DISABLE_CORS"):
     # disable cors
-    CORS_ORIGIN_ALLOW_ALL = True
+    CORS_ALLOW_ALL_ORIGINS = True
 else:
     CORS_ALLOWED_ORIGIN_REGEXES = [
         r"moz-extension://*",
         r"chrome-extension://*",
     ]
+    CORS_ORIGIN_WHITELIST = ["http://localhost:3000", "http://localhost:8000"]
+    CSRF_TRUSTED_ORIGINS = ["http://localhost:3000", "http://localhost:8000"]
     CORS_ALLOWED_ORIGINS = ["http://localhost:3000"]
 
+CORS_ALLOW_CREDENTIALS = True
+
 
 CORS_ALLOW_HEADERS = list(default_headers) + [
     "mode",
@@ -276,4 +296,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.4.11"
+TA_VERSION = "v0.4.13"

backend/config/urls.py

@@ -18,7 +18,14 @@ from django.contrib import admin
 from django.urls import include, path
 
 urlpatterns = [
-    path("", include("home.urls")),
+    path("api/", include("common.urls")),
-    path("api/", include("api.urls")),
+    path("api/video/", include("video.urls")),
+    path("api/channel/", include("channel.urls")),
+    path("api/playlist/", include("playlist.urls")),
+    path("api/download/", include("download.urls")),
+    path("api/task/", include("task.urls")),
+    path("api/appsettings/", include("appsettings.urls")),
+    path("api/stats/", include("stats.urls")),
+    path("api/user/", include("user.urls")),
     path("admin/", admin.site.urls),
 ]

backend/download/src/queue.py

@@ -4,17 +4,16 @@ Functionality:
 - linked with ta_dowload index
 """
 
-import json
 from datetime import datetime
 
-from home.src.download.subscriptions import ChannelSubscription
+from appsettings.src.config import AppConfig
-from home.src.download.thumbnails import ThumbManager
+from common.src.es_connect import ElasticWrap, IndexPaginate
-from home.src.download.yt_dlp_base import YtWrap
+from common.src.helper import get_duration_str, is_shorts, rand_sleep
-from home.src.es.connect import ElasticWrap, IndexPaginate
+from download.src.subscriptions import ChannelSubscription
-from home.src.index.playlist import YoutubePlaylist
+from download.src.thumbnails import ThumbManager
-from home.src.index.video_constants import VideoTypeEnum
+from download.src.yt_dlp_base import YtWrap
-from home.src.ta.config import AppConfig
+from playlist.src.index import YoutubePlaylist
-from home.src.ta.helper import get_duration_str, is_shorts
+from video.src.constants import VideoTypeEnum
 
 
 class PendingIndex:
@@ -241,7 +240,6 @@ class PendingList(PendingIndex):
     def add_to_pending(self, status="pending", auto_start=False):
         """add missing videos to pending list"""
         self.get_channels()
-        bulk_list = []
 
         total = len(self.missing_videos)
         videos_added = []
@@ -253,6 +251,7 @@ class PendingList(PendingIndex):
             self._notify_add(idx, total)
             video_details = self.get_youtube_details(youtube_id, vid_type)
             if not video_details:
+                rand_sleep(self.config)
                 continue
 
             video_details.update(
@@ -262,32 +261,17 @@ class PendingList(PendingIndex):
                 }
             )
 
-            action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(video_details))
-
             url = video_details["vid_thumb_url"]
             ThumbManager(youtube_id).download_video_thumb(url)
+            es_url = f"ta_download/_doc/{youtube_id}"
+            _, _ = ElasticWrap(es_url).put(video_details)
             videos_added.append(youtube_id)
 
-            if len(bulk_list) >= 20:
+            if idx != total:
-                self._ingest_bulk(bulk_list)
+                rand_sleep(self.config)
-                bulk_list = []
-
-        self._ingest_bulk(bulk_list)
 
         return videos_added
 
-    def _ingest_bulk(self, bulk_list):
-        """add items to queue in bulk"""
-        if not bulk_list:
-            return
-
-        # add last newline
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-        _, _ = ElasticWrap("_bulk?refresh=true").post(query_str, ndjson=True)
-
     def _notify_add(self, idx, total):
         """send notification for adding videos to download queue"""
         if not self.task:

backend/download/src/subscriptions.py

@@ -4,16 +4,16 @@ Functionality:
 - handle playlist subscriptions
 """
 
-from home.src.download.thumbnails import ThumbManager
+from appsettings.src.config import AppConfig
-from home.src.download.yt_dlp_base import YtWrap
+from channel.src.index import YoutubeChannel
-from home.src.es.connect import IndexPaginate
+from common.src.es_connect import IndexPaginate
-from home.src.index.channel import YoutubeChannel
+from common.src.helper import is_missing, rand_sleep
-from home.src.index.playlist import YoutubePlaylist
+from common.src.urlparser import Parser
-from home.src.index.video import YoutubeVideo
+from download.src.thumbnails import ThumbManager
-from home.src.index.video_constants import VideoTypeEnum
+from download.src.yt_dlp_base import YtWrap
-from home.src.ta.config import AppConfig
+from playlist.src.index import YoutubePlaylist
-from home.src.ta.helper import is_missing
+from video.src.constants import VideoTypeEnum
-from home.src.ta.urlparser import Parser
+from video.src.index import YoutubeVideo
 
 
 class ChannelSubscription:
@@ -108,6 +108,7 @@ class ChannelSubscription:
                 message_lines=[f"Scanning Channel {idx + 1}/{total}"],
                 progress=(idx + 1) / total,
             )
+            rand_sleep(self.config)
 
         return missing_videos
 
@@ -120,6 +121,8 @@ class ChannelSubscription:
         channel.upload_to_es()
         channel.sync_to_videos()
 
+        return channel.json_data
+
 
 class VideoQueryBuilder:
     """Build queries for yt-dlp."""
@@ -277,6 +280,7 @@ class PlaylistSubscription:
         playlist.build_json()
         playlist.json_data["playlist_subscribed"] = subscribe_status
         playlist.upload_to_es()
+        return playlist.json_data
 
     def find_missing(self):
         """find videos in subscribed playlists not downloaded yet"""
@@ -317,6 +321,7 @@ class PlaylistSubscription:
                 message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
                 progress=(idx + 1) / total,
             )
+            rand_sleep(self.config)
 
         return missing_videos
 
@@ -422,7 +427,7 @@ class SubscriptionHandler:
 
     def _subscribe(self, channel_id):
         """subscribe to channel"""
-        ChannelSubscription().change_subscribe(
+        _ = ChannelSubscription().change_subscribe(
             channel_id, channel_subscribed=True
         )
 

backend/download/src/thumbnails.py

@@ -10,9 +10,9 @@ from io import BytesIO
 from time import sleep
 
 import requests
-from home.src.es.connect import ElasticWrap, IndexPaginate
+from common.src.env_settings import EnvironmentSettings
-from home.src.ta.helper import is_missing
+from common.src.es_connect import ElasticWrap, IndexPaginate
-from home.src.ta.settings import EnvironmentSettings
+from common.src.helper import is_missing
 from mutagen.mp4 import MP4, MP4Cover
 from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
 

backend/download/src/yt_dlp_base.py

@@ -4,14 +4,14 @@ functionality:
 - handle yt-dlp errors
 """
 
-import os
 from datetime import datetime
 from http import cookiejar
 from io import StringIO
 
 import yt_dlp
-from home.src.ta.settings import EnvironmentSettings
+from appsettings.src.config import AppConfig
-from home.src.ta.ta_redis import RedisArchivist
+from common.src.ta_redis import RedisArchivist
+from django.conf import settings
 
 
 class YtWrap:
@@ -36,14 +36,33 @@ class YtWrap:
         self.obs = self.OBS_BASE.copy()
         self.obs.update(self.obs_request)
         if self.config:
-            self.add_cookie()
+            self._add_cookie()
+            self._add_potoken()
 
-    def add_cookie(self):
+        if getattr(settings, "DEBUG", False):
+            print(self.obs)
+
+    def _add_cookie(self):
         """add cookie if enabled"""
         if self.config["downloads"]["cookie_import"]:
             cookie_io = CookieHandler(self.config).get()
             self.obs["cookiefile"] = cookie_io
 
+    def _add_potoken(self):
+        """add potoken if enabled"""
+        if self.config["downloads"].get("potoken"):
+            potoken = POTokenHandler(self.config).get()
+            self.obs.update(
+                {
+                    "extractor_args": {
+                        "youtube": {
+                            "po_token": [potoken],
+                            "player-client": ["web", "default"],
+                        },
+                    }
+                }
+            )
+
     def download(self, url):
         """make download request"""
         with yt_dlp.YoutubeDL(self.obs) as ydl:
@@ -56,6 +75,8 @@ class YtWrap:
 
                 return False, str(err)
 
+        self._validate_cookie()
+
         return True, True
 
     def extract(self, url):
@@ -78,8 +99,21 @@ class YtWrap:
 
             return False
 
+        self._validate_cookie()
+
         return response
 
+    def _validate_cookie(self):
+        """check cookie and write it back for next use"""
+        if not self.obs.get("cookiefile"):
+            return
+
+        new_cookie = self.obs["cookiefile"].read()
+        old_cookie = RedisArchivist().get_message_str("cookie")
+        if new_cookie and old_cookie != new_cookie:
+            print("refreshed stored cookie")
+            RedisArchivist().set_message("cookie", new_cookie, save=True)
+
 
 class CookieHandler:
     """handle youtube cookie for yt-dlp"""
@@ -87,37 +121,17 @@ class CookieHandler:
     def __init__(self, config):
         self.cookie_io = False
         self.config = config
-        self.cache_dir = EnvironmentSettings.CACHE_DIR
 
     def get(self):
         """get cookie io stream"""
-        cookie = RedisArchivist().get_message("cookie")
+        cookie = RedisArchivist().get_message_str("cookie")
         self.cookie_io = StringIO(cookie)
         return self.cookie_io
 
-    def import_cookie(self):
-        """import cookie from file"""
-        import_path = os.path.join(
-            self.cache_dir, "import", "cookies.google.txt"
-        )
-
-        try:
-            with open(import_path, encoding="utf-8") as cookie_file:
-                cookie = cookie_file.read()
-        except FileNotFoundError as err:
-            print(f"cookie: {import_path} file not found")
-            raise err
-
-        self.set_cookie(cookie)
-
-        os.remove(import_path)
-        print("cookie: import successful")
-
     def set_cookie(self, cookie):
         """set cookie str and activate in config"""
         RedisArchivist().set_message("cookie", cookie, save=True)
-        path = ".downloads.cookie_import"
+        AppConfig().update_config({"downloads.cookie_import": True})
-        RedisArchivist().set_message("config", True, path=path, save=True)
         self.config["downloads"]["cookie_import"] = True
         print("cookie: activated and stored in Redis")
 
@@ -126,9 +140,7 @@ class CookieHandler:
         """revoke cookie"""
         RedisArchivist().del_message("cookie")
         RedisArchivist().del_message("cookie:valid")
-        RedisArchivist().set_message(
+        AppConfig().update_config({"downloads.cookie_import": False})
-            "config", False, path=".downloads.cookie_import"
-        )
         print("cookie: revoked")
 
     def validate(self):
@@ -171,3 +183,27 @@ class CookieHandler:
             "validated_str": now.strftime("%Y-%m-%d %H:%M"),
         }
         RedisArchivist().set_message("cookie:valid", message)
+
+
+class POTokenHandler:
+    """handle po token"""
+
+    REDIS_KEY = "potoken"
+
+    def __init__(self, config):
+        self.config = config
+
+    def get(self) -> str | None:
+        """get PO token"""
+        potoken = RedisArchivist().get_message_str(self.REDIS_KEY)
+        return potoken
+
+    def set_token(self, new_token: str) -> None:
+        """set new PO token"""
+        RedisArchivist().set_message(self.REDIS_KEY, new_token)
+        AppConfig().update_config({"downloads.potoken": True})
+
+    def revoke_token(self) -> None:
+        """revoke token"""
+        RedisArchivist().del_message(self.REDIS_KEY)
+        AppConfig().update_config({"downloads.potoken": False})

backend/download/src/yt_dlp_handler.py

@@ -10,19 +10,23 @@ import os
 import shutil
 from datetime import datetime
 
-from home.src.download.queue import PendingList
+from appsettings.src.config import AppConfig
-from home.src.download.subscriptions import PlaylistSubscription
+from channel.src.index import YoutubeChannel
-from home.src.download.yt_dlp_base import YtWrap
+from common.src.env_settings import EnvironmentSettings
-from home.src.es.connect import ElasticWrap, IndexPaginate
+from common.src.es_connect import ElasticWrap, IndexPaginate
-from home.src.index.channel import YoutubeChannel
+from common.src.helper import (
-from home.src.index.comments import CommentList
+    get_channel_overwrites,
-from home.src.index.playlist import YoutubePlaylist
+    ignore_filelist,
-from home.src.index.video import YoutubeVideo, index_new_video
+    rand_sleep,
-from home.src.index.video_constants import VideoTypeEnum
+)
-from home.src.ta.config import AppConfig
+from common.src.ta_redis import RedisQueue
-from home.src.ta.helper import get_channel_overwrites, ignore_filelist
+from download.src.queue import PendingList
-from home.src.ta.settings import EnvironmentSettings
+from download.src.subscriptions import PlaylistSubscription
-from home.src.ta.ta_redis import RedisQueue
+from download.src.yt_dlp_base import YtWrap
+from playlist.src.index import YoutubePlaylist
+from video.src.comments import CommentList
+from video.src.constants import VideoTypeEnum
+from video.src.index import YoutubeVideo, index_new_video
 
 
 class DownloaderBase:
@@ -370,6 +374,7 @@ class DownloadPostProcess(DownloaderBase):
             ]
             progress = idx / total
             self.task.send_progress(message, progress=progress)
+            rand_sleep(self.config)
 
     def add_playlists_to_refresh(self) -> None:
         """add playlists to refresh"""

backend/download/urls.py

@@ -0,0 +1,18 @@
+"""all download API urls"""
+
+from django.urls import path
+from download import views
+
+urlpatterns = [
+    path("", views.DownloadApiListView.as_view(), name="api-download-list"),
+    path(
+        "aggs/",
+        views.DownloadAggsApiView.as_view(),
+        name="api-download-aggs",
+    ),
+    path(
+        "<slug:video_id>/",
+        views.DownloadApiView.as_view(),
+        name="api-download",
+    ),
+]

backend/download/views.py

@@ -0,0 +1,170 @@
+"""all download API views"""
+
+from common.views_base import AdminOnly, ApiBaseView
+from download.src.queue import PendingInteract
+from rest_framework.response import Response
+from task.tasks import download_pending, extrac_dl
+
+
+class DownloadApiListView(ApiBaseView):
+    """resolves to /api/download/
+    GET: returns latest videos in the download queue
+    POST: add a list of videos to download queue
+    DELETE: remove items based on query filter
+    """
+
+    search_base = "ta_download/_search/"
+    valid_filter = ["pending", "ignore"]
+    permission_classes = [AdminOnly]
+
+    def get(self, request):
+        """get request"""
+        query_filter = request.GET.get("filter", False)
+        self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
+
+        must_list = []
+        if query_filter:
+            if query_filter not in self.valid_filter:
+                message = f"invalid url query filter: {query_filter}"
+                print(message)
+                return Response({"message": message}, status=400)
+
+            must_list.append({"term": {"status": {"value": query_filter}}})
+
+        filter_channel = request.GET.get("channel", False)
+        if filter_channel:
+            must_list.append(
+                {"term": {"channel_id": {"value": filter_channel}}}
+            )
+
+        self.data["query"] = {"bool": {"must": must_list}}
+
+        self.get_document_list(request)
+        return Response(self.response)
+
+    @staticmethod
+    def post(request):
+        """add list of videos to download queue"""
+        data = request.data
+        auto_start = bool(request.GET.get("autostart"))
+        try:
+            to_add = data["data"]
+        except KeyError:
+            message = "missing expected data key"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
+        url_str = " ".join(pending)
+        extrac_dl.delay(url_str, auto_start=auto_start)
+
+        return Response(data)
+
+    def delete(self, request):
+        """delete download queue"""
+        query_filter = request.GET.get("filter", False)
+        if query_filter not in self.valid_filter:
+            message = f"invalid url query filter: {query_filter}"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        message = f"delete queue by status: {query_filter}"
+        print(message)
+        PendingInteract(status=query_filter).delete_by_status()
+
+        return Response({"message": message})
+
+
+class DownloadAggsApiView(ApiBaseView):
+    """resolves to /api/download/aggs/
+    GET: get download aggregations
+    """
+
+    search_base = "ta_download/_search"
+    valid_filter_view = ["ignore", "pending"]
+
+    def get(self, request):
+        """get aggs"""
+        filter_view = request.GET.get("filter")
+        if filter_view:
+            if filter_view not in self.valid_filter_view:
+                message = f"invalid filter: {filter_view}"
+                return Response({"message": message}, status=400)
+
+            self.data.update(
+                {
+                    "query": {"term": {"status": {"value": filter_view}}},
+                }
+            )
+
+        self.data.update(
+            {
+                "aggs": {
+                    "channel_downloads": {
+                        "multi_terms": {
+                            "size": 30,
+                            "terms": [
+                                {"field": "channel_name.keyword"},
+                                {"field": "channel_id"},
+                            ],
+                            "order": {"_count": "desc"},
+                        }
+                    }
+                }
+            }
+        )
+        self.get_aggs()
+
+        return Response(self.response)
+
+
+class DownloadApiView(ApiBaseView):
+    """resolves to /api/download/<video_id>/
+    GET: returns metadata dict of an item in the download queue
+    POST: update status of item to pending or ignore
+    DELETE: forget from download queue
+    """
+
+    search_base = "ta_download/_doc/"
+    valid_status = ["pending", "ignore", "ignore-force", "priority"]
+    permission_classes = [AdminOnly]
+
+    def get(self, request, video_id):
+        # pylint: disable=unused-argument
+        """get request"""
+        self.get_document(video_id)
+        return Response(self.response, status=self.status_code)
+
+    def post(self, request, video_id):
+        """post to video to change status"""
+        item_status = request.data.get("status")
+        if item_status not in self.valid_status:
+            message = f"{video_id}: invalid status {item_status}"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        if item_status == "ignore-force":
+            extrac_dl.delay(video_id, status="ignore")
+            message = f"{video_id}: set status to ignore"
+            return Response(request.data)
+
+        _, status_code = PendingInteract(video_id).get_item()
+        if status_code == 404:
+            message = f"{video_id}: item not found {status_code}"
+            return Response({"message": message}, status=404)
+
+        print(f"{video_id}: change status to {item_status}")
+        PendingInteract(video_id, item_status).update_status()
+        if item_status == "priority":
+            download_pending.delay(auto_only=True)
+
+        return Response(request.data)
+
+    @staticmethod
+    def delete(request, video_id):
+        # pylint: disable=unused-argument
+        """delete single video from queue"""
+        print(f"{video_id}: delete from queue")
+        PendingInteract(video_id).delete_item()
+
+        return Response({"success": True})

backend/playlist/src/constants.py

@@ -0,0 +1,10 @@
+"""playlist constants"""
+
+import enum
+
+
+class PlaylistTypesEnum(enum.Enum):
+    """all playlist_type options"""
+
+    REGULAR = "regular"
+    CUSTOM = "custom"

backend/playlist/src/index.py

@@ -7,11 +7,12 @@ functionality:
 import json
 from datetime import datetime
 
-from home.src.download.thumbnails import ThumbManager
+from channel.src import index as channel
-from home.src.es.connect import ElasticWrap, IndexPaginate
+from common.src.env_settings import EnvironmentSettings
-from home.src.index import channel
+from common.src.es_connect import ElasticWrap, IndexPaginate
-from home.src.index.generic import YouTubeItem
+from common.src.index_generic import YouTubeItem
-from home.src.index.video import YoutubeVideo
+from download.src.thumbnails import ThumbManager
+from video.src import index as ta_video
 
 
 class YoutubePlaylist(YouTubeItem):
@@ -93,13 +94,10 @@ class YoutubePlaylist(YouTubeItem):
         """get all videos in playlist, match downloaded with ids_found"""
         all_members = []
         for idx, entry in enumerate(self.youtube_meta["entries"]):
-            if not entry["channel"]:
-                continue
-
             to_append = {
                 "youtube_id": entry["id"],
                 "title": entry["title"],
-                "uploader": entry["channel"],
+                "uploader": entry.get("channel"),
                 "idx": idx,
                 "downloaded": entry["id"] in ids_found,
             }
@@ -149,6 +147,9 @@ class YoutubePlaylist(YouTubeItem):
             "query": {"match": {"playlist": self.youtube_id}},
             "_source": ["youtube_id"],
         }
+        data = {
+            "query": {"term": {"playlist.keyword": {"value": self.youtube_id}}}
+        }
         result = IndexPaginate("ta_video", data).get_results()
         to_remove = [
             i["youtube_id"] for i in result if i["youtube_id"] not in needed
@@ -190,6 +191,7 @@ class YoutubePlaylist(YouTubeItem):
 
     def build_nav(self, youtube_id):
         """find next and previous in playlist of a given youtube_id"""
+        cache_root = EnvironmentSettings().get_cache_root()
         all_entries_available = self.json_data["playlist_entries"]
         all_entries = [i for i in all_entries_available if i["downloaded"]]
         current = [i for i in all_entries if i["youtube_id"] == youtube_id]
@@ -203,14 +205,16 @@ class YoutubePlaylist(YouTubeItem):
         else:
             previous_item = all_entries[current_idx - 1]
             prev_id = previous_item["youtube_id"]
-            previous_item["vid_thumb"] = ThumbManager(prev_id).vid_thumb_path()
+            prev_thumb_path = ThumbManager(prev_id).vid_thumb_path()
+            previous_item["vid_thumb"] = f"{cache_root}/{prev_thumb_path}"
 
         if current_idx == len(all_entries) - 1:
             next_item = False
         else:
             next_item = all_entries[current_idx + 1]
             next_id = next_item["youtube_id"]
-            next_item["vid_thumb"] = ThumbManager(next_id).vid_thumb_path()
+            next_thumb_path = ThumbManager(next_id).vid_thumb_path()
+            next_item["vid_thumb"] = f"{cache_root}/{next_thumb_path}"
 
         self.nav = {
             "playlist_meta": {
@@ -255,7 +259,7 @@ class YoutubePlaylist(YouTubeItem):
         i = 0
         while i < len(playlist):
             video_id = playlist[i]["youtube_id"]
-            video = YoutubeVideo(video_id)
+            video = ta_video.YoutubeVideo(video_id)
             video.get_from_es()
             if (
                 channel_id is None
@@ -278,7 +282,7 @@ class YoutubePlaylist(YouTubeItem):
             if i["downloaded"]
         ]
         for youtube_id in all_youtube_id:
-            YoutubeVideo(youtube_id).delete_media_file()
+            ta_video.YoutubeVideo(youtube_id).delete_media_file()
 
         self.delete_metadata()
 
@@ -312,7 +316,7 @@ class YoutubePlaylist(YouTubeItem):
             )
             self.set_playlist_thumbnail()
             self.upload_to_es()
-            video = YoutubeVideo(video_id)
+            video = ta_video.YoutubeVideo(video_id)
             video.get_from_es()
             if "playlist" not in video.json_data:
                 video.json_data["playlist"] = []
@@ -321,7 +325,7 @@ class YoutubePlaylist(YouTubeItem):
         return True
 
     def remove_playlist_from_video(self, video_id):
-        video = YoutubeVideo(video_id)
+        video = ta_video.YoutubeVideo(video_id)
         video.get_from_es()
         if video.json_data is not None and "playlist" in video.json_data:
             video.json_data["playlist"].remove(self.youtube_id)
@@ -410,7 +414,7 @@ class YoutubePlaylist(YouTubeItem):
         )
 
     def get_video_is_watched(self, video_id):
-        video = YoutubeVideo(video_id)
+        video = ta_video.YoutubeVideo(video_id)
         video.get_from_es()
         return video.json_data["player"]["watched"]
 
@@ -426,7 +430,7 @@ class YoutubePlaylist(YouTubeItem):
         self.get_playlist_art()
 
     def get_video_metadata(self, video_id):
-        video = YoutubeVideo(video_id)
+        video = ta_video.YoutubeVideo(video_id)
         video.get_from_es()
         video_json_data = {
             "youtube_id": video.json_data["youtube_id"],

backend/playlist/src/query_building.py

@@ -0,0 +1,53 @@
+"""build query for playlists"""
+
+from playlist.src.constants import PlaylistTypesEnum
+
+
+class QueryBuilder:
+    """contain functionality"""
+
+    def __init__(self, **kwargs):
+        self.request_params = kwargs
+
+    def build_data(self) -> dict:
+        """build data dict"""
+        data = {}
+        data["query"] = self.build_query()
+        if sort := self.parse_sort():
+            data.update(sort)
+
+        return data
+
+    def build_query(self) -> dict:
+        """build query key"""
+        must_list = []
+        channel = self.request_params.get("channel")
+        if channel:
+            must_list.append({"match": {"playlist_channel_id": channel[0]}})
+
+        subscribed = self.request_params.get("subscribed")
+        if subscribed:
+            subed_bool = subscribed[0] == "true"
+            must_list.append({"match": {"playlist_subscribed": subed_bool}})
+
+        playlist_type = self.request_params.get("type")
+        if playlist_type:
+            type_list = self.parse_type(playlist_type[0])
+            must_list.append(type_list)
+
+        query = {"bool": {"must": must_list}}
+
+        return query
+
+    def parse_type(self, playlist_type: str) -> dict:
+        """parse playlist type"""
+        if not hasattr(PlaylistTypesEnum, playlist_type.upper()):
+            raise ValueError(f"'{playlist_type}' not in PlaylistTypesEnum")
+
+        type_parsed = getattr(PlaylistTypesEnum, playlist_type.upper()).value
+
+        return {"match": {"playlist_type.keyword": type_parsed}}
+
+    def parse_sort(self) -> dict:
+        """return sort"""
+        return {"sort": [{"playlist_name.keyword": {"order": "asc"}}]}

backend/playlist/tests/test_src/test_query_building.py

@@ -0,0 +1,30 @@
+"""test playlist query building"""
+
+import pytest
+from playlist.src.query_building import QueryBuilder
+
+
+def test_build_data():
+    """test for correct key building"""
+    qb = QueryBuilder(
+        channel=["test_channel"],
+        subscribed=["true"],
+        type=["regular"],
+    )
+    result = qb.build_data()
+    must_list = result["query"]["bool"]["must"]
+    assert "query" in result
+    assert "sort" in result
+    assert result["sort"] == [{"playlist_name.keyword": {"order": "asc"}}]
+    assert {"match": {"playlist_channel_id": "test_channel"}} in must_list
+    assert {"match": {"playlist_subscribed": True}} in must_list
+
+
+def test_parse_type():
+    """validate type"""
+    qb = QueryBuilder(type=["regular"])
+    with pytest.raises(ValueError):
+        qb.parse_type("invalid")
+
+    result = qb.parse_type("custom")
+    assert result == {"match": {"playlist_type.keyword": "custom"}}

backend/playlist/urls.py

@@ -0,0 +1,17 @@
+"""all playlist API urls"""
+
+from django.urls import path
+from playlist import views
+
+urlpatterns = [
+    path(
+        "",
+        views.PlaylistApiListView.as_view(),
+        name="api-playlist-list",
+    ),
+    path(
+        "<slug:playlist_id>/",
+        views.PlaylistApiView.as_view(),
+        name="api-playlist",
+    ),
+]

backend/playlist/views.py

@@ -0,0 +1,138 @@
+"""all playlist API views"""
+
+import uuid
+
+from common.views_base import AdminWriteOnly, ApiBaseView
+from download.src.subscriptions import PlaylistSubscription
+from playlist.src.index import YoutubePlaylist
+from playlist.src.query_building import QueryBuilder
+from rest_framework import status
+from rest_framework.response import Response
+from task.tasks import subscribe_to
+from user.src.user_config import UserConfig
+
+
+class PlaylistApiListView(ApiBaseView):
+    """resolves to /api/playlist/
+    GET: returns list of indexed playlists
+    params:
+    - channel:str=<channel-id>
+    - subscribed: bool
+    - type:enum=regular|custom
+    POST: change subscribe state
+    """
+
+    search_base = "ta_playlist/_search/"
+    permission_classes = [AdminWriteOnly]
+
+    def get(self, request):
+        """get request"""
+        try:
+            data = QueryBuilder(**request.GET).build_data()
+        except ValueError as err:
+            return Response({"error": str(err)}, status=400)
+
+        self.data = data
+        self.get_document_list(request)
+
+        return Response(self.response)
+
+    def post(self, request):
+        """subscribe/unsubscribe to list of playlists"""
+        data = request.data
+        try:
+            to_add = data["data"]
+        except KeyError:
+            message = "missing expected data key"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        data = data["data"]
+        if isinstance(data, dict):
+            custom_name = data.get("create")
+            if custom_name:
+                playlist_id = f"TA_playlist_{uuid.uuid4()}"
+                custom_playlist = YoutubePlaylist(playlist_id)
+                custom_playlist.create(custom_name)
+                return Response(custom_playlist.json_data)
+
+        pending = []
+        for playlist_item in to_add:
+            playlist_id = playlist_item["playlist_id"]
+            if playlist_item["playlist_subscribed"]:
+                pending.append(playlist_id)
+            else:
+                self._unsubscribe(playlist_id)
+
+        if pending:
+            url_str = " ".join(pending)
+            subscribe_to.delay(url_str, expected_type="playlist")
+
+        return Response(data)
+
+    @staticmethod
+    def _unsubscribe(playlist_id: str):
+        """unsubscribe"""
+        print(f"[{playlist_id}] unsubscribe from playlist")
+        _ = PlaylistSubscription().change_subscribe(
+            playlist_id, subscribe_status=False
+        )
+
+
+class PlaylistApiView(ApiBaseView):
+    """resolves to /api/playlist/<playlist_id>/
+    GET: returns metadata dict of playlist
+    """
+
+    search_base = "ta_playlist/_doc/"
+    permission_classes = [AdminWriteOnly]
+    valid_custom_actions = ["create", "remove", "up", "down", "top", "bottom"]
+
+    def get(self, request, playlist_id):
+        # pylint: disable=unused-argument
+        """get request"""
+        self.get_document(playlist_id)
+        return Response(self.response, status=self.status_code)
+
+    def post(self, request, playlist_id):
+        """post to custom playlist to add a video to list"""
+        self.get_document(playlist_id)
+        if not self.response["data"]:
+            return Response({"error": "playlist not found"}, status=404)
+
+        data = request.data
+        subscribed = data.get("playlist_subscribed")
+        if subscribed is not None:
+            playlist_sub = PlaylistSubscription()
+            json_data = playlist_sub.change_subscribe(playlist_id, subscribed)
+            return Response(json_data, status=200)
+
+        if not self.response["data"]["playlist_type"] == "custom":
+            message = f"playlist with ID {playlist_id} is not custom"
+            return Response({"message": message}, status=400)
+
+        action = request.data.get("action")
+        if action not in self.valid_custom_actions:
+            message = f"invalid action: {action}"
+            return Response({"message": message}, status=400)
+
+        playlist = YoutubePlaylist(playlist_id)
+        video_id = request.data.get("video_id")
+        if action == "create":
+            playlist.add_video_to_playlist(video_id)
+        else:
+            hide = UserConfig(request.user.id).get_value("hide_watched")
+            playlist.move_video(video_id, action, hide_watched=hide)
+
+        return Response({"success": True}, status=status.HTTP_201_CREATED)
+
+    def delete(self, request, playlist_id):
+        """delete playlist"""
+        print(f"{playlist_id}: delete playlist")
+        delete_videos = request.GET.get("delete-videos", False)
+        if delete_videos:
+            YoutubePlaylist(playlist_id).delete_videos_playlist()
+        else:
+            YoutubePlaylist(playlist_id).delete_metadata()
+
+        return Response({"success": True})

backend/requirements-dev.txt

@@ -0,0 +1,10 @@
+-r requirements.txt
+ipython==8.31.0
+pre-commit==4.1.0
+pylint-django==2.6.1
+pylint==3.3.3
+pytest-django==4.9.0
+pytest==8.3.4
+python-dotenv==1.0.1
+requirementscheck==0.0.5
+types-requests==2.32.0.20241016

backend/requirements.txt

@@ -1,14 +1,14 @@
-apprise==1.9.0
+apprise==1.9.2
 celery==5.4.0
 django-auth-ldap==5.1.0
 django-celery-beat==2.7.0
 django-cors-headers==4.6.0
-Django==5.1.3
+Django==5.1.5
 djangorestframework==3.15.2
-Pillow==11.0.0
+Pillow==11.1.0
-redis==5.2.0
+redis==5.2.1
 requests==2.32.3
 ryd-client==0.0.6
-uWSGI==2.0.28
+uvicorn==0.34.0
 whitenoise==6.8.2
-yt-dlp[default]==2024.11.4
+yt-dlp[default]==2025.1.26

backend/stats/src/aggs.py

@@ -1,8 +1,8 @@
 """aggregations"""
 
-from home.src.es.connect import ElasticWrap
+from common.src.env_settings import EnvironmentSettings
-from home.src.ta.helper import get_duration_str
+from common.src.es_connect import ElasticWrap
-from home.src.ta.settings import EnvironmentSettings
+from common.src.helper import get_duration_str
 
 
 class AggBase:
@@ -55,6 +55,8 @@ class Video(AggBase):
     def process(self):
         """process aggregation"""
         aggregations = self.get()
+        if not aggregations:
+            return None
 
         duration = int(aggregations["duration"]["value"])
         response = {
@@ -109,6 +111,8 @@ class Channel(AggBase):
     def process(self):
         """process aggregation"""
         aggregations = self.get()
+        if not aggregations:
+            return None
 
         response = {
             "doc_count": aggregations["channel_count"].get("value"),
@@ -140,6 +144,9 @@ class Playlist(AggBase):
     def process(self):
         """process aggregation"""
         aggregations = self.get()
+        if not aggregations:
+            return None
+
         response = {"doc_count": aggregations["playlist_count"].get("value")}
         for bucket in aggregations["playlist_active"]["buckets"]:
             key = f"active_{bucket['key_as_string']}"
@@ -171,6 +178,9 @@ class Download(AggBase):
         """process aggregation"""
         aggregations = self.get()
         response = {}
+        if not aggregations:
+            return None
+
         for bucket in aggregations["status"]["buckets"]:
             response.update({bucket["key"]: bucket.get("doc_count")})
 
@@ -209,9 +219,11 @@ class WatchProgress(AggBase):
     def process(self):
         """make the call"""
         aggregations = self.get()
-        buckets = aggregations[self.name]["buckets"]
-
         response = {}
+        if not aggregations:
+            return None
+
+        buckets = aggregations[self.name]["buckets"]
         all_duration = int(aggregations["total_duration"].get("value"))
         response.update(
             {
@@ -287,6 +299,9 @@ class DownloadHist(AggBase):
     def process(self):
         """process query"""
         aggregations = self.get()
+        if not aggregations:
+            return None
+
         buckets = aggregations[self.name]["buckets"]
 
         response = [
@@ -334,6 +349,9 @@ class BiggestChannel(AggBase):
         """process aggregation, order_by validated in the view"""
 
         aggregations = self.get()
+        if not aggregations:
+            return None
+
         buckets = aggregations[self.name]["buckets"]
 
         response = [

backend/stats/urls.py

@@ -0,0 +1,42 @@
+"""all stats API urls"""
+
+from django.urls import path
+from stats import views
+
+urlpatterns = [
+    path(
+        "video/",
+        views.StatVideoView.as_view(),
+        name="api-stats-video",
+    ),
+    path(
+        "channel/",
+        views.StatChannelView.as_view(),
+        name="api-stats-channel",
+    ),
+    path(
+        "playlist/",
+        views.StatPlaylistView.as_view(),
+        name="api-stats-playlist",
+    ),
+    path(
+        "download/",
+        views.StatDownloadView.as_view(),
+        name="api-stats-download",
+    ),
+    path(
+        "watch/",
+        views.StatWatchProgress.as_view(),
+        name="api-stats-watch",
+    ),
+    path(
+        "downloadhist/",
+        views.StatDownloadHist.as_view(),
+        name="api-stats-downloadhist",
+    ),
+    path(
+        "biggestchannels/",
+        views.StatBiggestChannel.as_view(),
+        name="api-stats-biggestchannels",
+    ),
+]

backend/stats/views.py

@@ -0,0 +1,104 @@
+"""all stats API views"""
+
+from common.views_base import ApiBaseView
+from rest_framework.response import Response
+from stats.src.aggs import (
+    BiggestChannel,
+    Channel,
+    Download,
+    DownloadHist,
+    Playlist,
+    Video,
+    WatchProgress,
+)
+
+
+class StatVideoView(ApiBaseView):
+    """resolves to /api/stats/video/
+    GET: return video stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Video().process())
+
+
+class StatChannelView(ApiBaseView):
+    """resolves to /api/stats/channel/
+    GET: return channel stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Channel().process())
+
+
+class StatPlaylistView(ApiBaseView):
+    """resolves to /api/stats/playlist/
+    GET: return playlist stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Playlist().process())
+
+
+class StatDownloadView(ApiBaseView):
+    """resolves to /api/stats/download/
+    GET: return download stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Download().process())
+
+
+class StatWatchProgress(ApiBaseView):
+    """resolves to /api/stats/watchprogress/
+    GET: return watch/unwatch progress stats
+    """
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+
+        return Response(WatchProgress().process())
+
+
+class StatDownloadHist(ApiBaseView):
+    """resolves to /api/stats/downloadhist/
+    GET: return download video count histogram for last days
+    """
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+
+        return Response(DownloadHist().process())
+
+
+class StatBiggestChannel(ApiBaseView):
+    """resolves to /api/stats/biggestchannels/
+    GET: return biggest channels
+    param: order
+    """
+
+    order_choices = ["doc_count", "duration", "media_size"]
+
+    def get(self, request):
+        """handle get request"""
+
+        order = request.GET.get("order", "doc_count")
+        if order and order not in self.order_choices:
+            message = {"message": f"invalid order parameter {order}"}
+            return Response(message, status=400)
+
+        return Response(BiggestChannel(order).process())