bump TA_VERSION

Changed:
- Fixed watched state date tracking
- Fixed channel fallback extraction error handling

.dockerignore

@@ -5,6 +5,13 @@
 # Docker
 .docker
 
+# Backend development
+backend/static
+backend/staticfiles
+
+# Frontend development
+frontend/node_modules
+
 # Python
 tubearchivist/__pycache__/
 tubearchivist/*/__pycache__/
@@ -18,4 +25,4 @@ venv/
 assets/*
 
 # for local testing only
-testing.sh
+testing.sh

.eslintrc.js

@@ -1,17 +0,0 @@
-'use strict';
-module.exports = {
-  extends: ['eslint:recommended', 'eslint-config-prettier'],
-  parserOptions: {
-    ecmaVersion: 2020,
-  },
-  env: {
-    browser: true,
-  },
-  rules: {
-    strict: ['error', 'global'],
-    'no-unused-vars': ['error', { vars: 'local' }],
-    eqeqeq: ['error', 'always', { null: 'ignore' }],
-    curly: ['error', 'multi-line'],
-    'no-var': 'error',
-  },
-};

.gitattributes

@@ -0,0 +1 @@
+docker_assets\run.sh eol=lf

.github/FUNDING.yml

@@ -1,3 +1,3 @@
 github: bbilly1
 ko_fi: bbilly1
-custom: https://paypal.me/bbilly1
+custom: https://paypal.me/bbilly1

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -6,7 +6,7 @@ body:
   - type: checkboxes
     id: block
     attributes:
-      label: "This project doesn't accept any new feature requests for the forseeable future. There is no shortage of ideas and the next development steps are clear for years to come."
+      label: "This project doesn't accept any new feature requests for the foreseeable future. There is no shortage of ideas and the next development steps are clear for years to come."
       options:
         - label: I understand that this issue will be closed without comment.
           required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/workflows/lint_js.yml

@@ -1,22 +0,0 @@
-name: lint_js
-
-on:
-  push:
-    paths:
-      - '**/*.js'
-  pull_request:
-    paths:
-      - '**/*.js'
-
-jobs:
-  check:
-    name: lint_js
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
-        with:
-          node-version: '22'
-      - run: npm ci
-      - run: npm run lint
-      - run: npm run format -- --check

.github/workflows/lint_python.yml

@@ -1,42 +0,0 @@
-name: lint_python
-
-on:
-  push:
-    paths:
-      - '**/*.py'
-  pull_request:
-    paths:
-      - '**/*.py'
-
-jobs:
-  lint_python:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y gcc libldap2-dev libsasl2-dev libssl-dev
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
-      - name: Cache pip
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Install python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r tubearchivist/requirements-dev.txt
-
-      - name: Run Linter
-        run: ./deploy.sh validate

.github/workflows/pre_commit.yml

@@ -0,0 +1,47 @@
+name: Lint, Test, Build, and Push Docker Image
+
+on:
+  push:
+    branches:
+      - '**'
+    tags:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '23'
+
+      - name: Install frontend dependencies
+        run: |
+          cd frontend
+          npm install
+
+      - name: Cache pre-commit environment
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pre-commit
+          key: ${{ runner.os }}-pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pre-commit-
+
+      - name: Install dependencies
+        run: |
+          pip install pre-commit
+          pre-commit install
+
+      - name: Run pre-commit
+        run: |
+          pre-commit run --all-files

.github/workflows/unit_tests.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r tubearchivist/requirements-dev.txt
+          pip install -r backend/requirements-dev.txt
 
       - name: Run unit tests
-        run: pytest tubearchivist
+        run: pytest backend

.gitignore

@@ -2,8 +2,10 @@
 __pycache__
 .venv
 
-# django testing db
-db.sqlite3
+# django testing
+backend/static
+backend/staticfiles
+backend/.env
 
 # vscode custom conf
 .vscode

.pre-commit-config.yaml

@@ -0,0 +1,49 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+        alias: python
+        files: ^backend/
+        args: ["--line-length=79"]
+  - repo: https://github.com/pycqa/isort
+    rev: 6.0.1
+    hooks:
+      - id: isort
+        name: isort (python)
+        alias: python
+        files: ^backend/
+        args: ["--profile", "black", "-l 79"]
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.2
+    hooks:
+      - id: flake8
+        alias: python
+        files: ^backend/
+        args: ["--max-complexity=10", "--max-line-length=79"]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        exclude: ^frontend/package-lock.json
+  - repo: https://github.com/pre-commit/mirrors-eslint
+    rev: v9.22.0
+    hooks:
+      - id: eslint
+        name: eslint
+        files: \.[jt]sx?$
+        types: [file]
+        entry: npm run --prefix ./frontend lint
+        pass_filenames: false
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v4.0.0-alpha.8
+    hooks:
+      - id: prettier
+        entry: npm run --prefix ./frontend format
+        pass_filenames: false
+
+exclude: '.*(\.svg|/migrations/).*'

ARCHITECTURE.md

@@ -1,48 +0,0 @@
-# The Inner Workings of Tube Archivist
-This is a high level overview of the architecture of Tube Archivist, intended for interested contributors to find your way around quickly.
-
-```
-                        Tube Archivist
-        ______________________|_____________________
-        |                     |                    |
--------------------    ---------------    -------------------
-|                 |    |             |    |                 |
-|  DjangoProject  |    |  RedisJson  |    |  ElasticSearch  |
-|                 |    |             |    |                 |
--------------------    ---------------    -------------------
-```
-
-## DjangoProject
-This is the main Python application. Django serves its data container internally with **Uwsgi** on port 8080, the interface is served with **Nginx** on the public port 8000.
-
-Users created static files like media files and artwork as well as application artwork like logos and fonts are served directly from Nginx, while the rest of the application uses uwsgi_pass to proxy the requests to uwsgi.
-
-Config files are located in the `docker_assets` folder. The script `run.sh` is the container `CMD` command and entry point, validating env vars, connection to ElasticSearch (ES) and will start the application.
-
-Compared to other Django projects, this application doesn't make use of the database models, due to a lack of integration with ES. This project has its own abstractions and integrations, treating ES as a REST API.
-
-Long running application tasks are handed off to **Celery** - using **Redis** as a broker - to run asynchronously from the main threads. 
-- All tasks are defined in the `home.tasks.py` module.
-
-There are three Django apps:
-- **config**: The root app, routing the main endpoints and the main `settings.py` file
-- **api**: The API app with its views and functionality
-- **home**: Most of the application logic, templates and views, will probably get split up further in the future.
-
-The *home* app is split up into packages in the `src` directory:
-- **download**: All download related classes, interact with yt-dlp, download artwork, handle the download queue and post processing tasks.
-- **es**: All index setup and validation classes, handles mapping validations and makes mapping changes, wrapper functions to simplify interactions with Elasticsearch, backup and restore.
-- **frontend**: All direct interactions with the frontend, like Django forms, searching, watched state changes, and legacy api_calls in the process of moving to the api app.
-- **index**: Contains all functionality for scraping and indexing videos, channels, playlists, comments, subtitles, etc...
-- **ta**: Loose collection of functions and classes, handle application config and contains redis wrapper classes.
-
-## RedisJson
-Holds the main application config json object that gets dynamically edited from the frontend, serves as a message broker for **Celery**. Redis serves as a temporary and thread safe link between Django and the frontend, storing progress messages and temporary queues for processing. Used to store locking keys for threads and execution details for tasks.
-
-- Wrapper classes to interact with Redis are located in the `home.src.ta.ta_redis.py` module.
-
-## ElasticSearch (ES)
-Is used to store and index all metadata, functions as an application database and makes it all searchable. The mapping defines which fields are indexed as searchable text fields and which fields are used for match filtering.
-
-- The index setup and validation is handled in the `home.src.es.index_setup.py` module.
-- Wrapper classes for making requests to ES are located in the `home.src.es.connect.py` module.

CONTRIBUTING.md

@@ -3,7 +3,6 @@
 Welcome, and thanks for showing interest in improving Tube Archivist!  
 
 ## Table of Content
-- [Next Steps](#next-steps)
 - [Beta Testing](#beta-testing)
 - [How to open an issue](#how-to-open-an-issue)
   - [Bug Report](#bug-report)
@@ -16,16 +15,6 @@ Welcome, and thanks for showing interest in improving Tube Archivist!
 - [Development Environment](#development-environment)
 ---
 
-## Next Steps
-Going forward, this project will focus on developing a new modern frontend.
-
-- For the time being, don't open any new PRs that are not towards the new frontend.
-- New features requests likely won't get accepted during this process.
-- Depending on the severity, bug reports may or may not get fixed during this time.
-- When in doubt, reach out.
-
-Join us on [Discord](https://tubearchivist.com/discord) if you want to help with that process.
-
 ## Beta Testing
 Be the first to help test new features and improvements and provide feedback! There are regular `:unstable` builds for easy access. That's for the tinkerers and the breave. Ideally use a testing environment first, before a release be the first to install it on your main system.
 
@@ -87,19 +76,16 @@ This is a quick checklist to help streamline the process:
 - Show off your progress, even if not yet complete, by creating a [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) PR first and switch it as *ready* when you are ready.
 - Make sure all your code is linted and formatted correctly, see below. The automatic GH action unfortunately needs to be triggered manually by a maintainer for first time contributors, but will trigger automatically for existing contributors.
 
-### Making changes to the JavaScript
-
-The JavaScript does not require any build step; you just edit the files directly. However, there is config for eslint and prettier (a linter and formatter respectively); their use is recommended but not required. To use them, install `node`, run `npm i` from the root directory of this repository to install dependencies, then run `npm run lint` and `npm run format` to run eslint and prettier respectively.
-
 ### Code formatting and linting
 
-To keep things clean and consistent for everybody, there is a github action setup to lint and check the changes. You can test your code locally first if you want. For example if you made changes in the **video** module, run
+This project uses the excellent [pre-commit](https://github.com/pre-commit/pre-commit) library. The [pre-commit-config.yml](https://github.com/tubearchivist/tubearchivist/blob/master/.pre-commit-config.yaml) file is part of this repo.
 
-```shell
-./deploy.sh validate tubearchivist/home/src/index/video.py
-```
+**Quick Start**
+- Run `pre-commit install` from the root of the repo.
+- Next time you commit to your local git repo, the defined hooks will run.
+- On first run, this will download and install the needed environments to your local machine, that can take some time. But that will be reused on sunsequent commits. 
 
-to validate your changes. If you omit the path, all the project files will get checked. This is subject to change as the codebase improves.
+That is also running as a Git Hub action.
 
 ---
 
@@ -132,15 +118,64 @@ Some of you might have created useful scripts or API integrations around this pr
 
 ## Improve to the Documentation
 
-The documentation available at [docs.tubearchivist.com](https://docs.tubearchivist.com/) and is build from a separate repo [tubearchivist/docs](https://github.com/tubearchivist/docs). The Readme has additional instructions on how to make changes.
+The documentation available at [docs.tubearchivist.com](https://docs.tubearchivist.com/) and is build from a separate repo [tubearchivist/docs](https://github.com/tubearchivist/docs). The Readme there has additional instructions on how to make changes.
 
 ---
 
 ## Development Environment
 
-I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. You might be able to run the application directly, but this document assumes you're using docker.
+This codebase is set up to be developed natively outside of docker as well as in a docker container. Developing outside of a docker container can be convenient, as IDE and hot reload usually works out of the box. But testing inside of a container is still essential, as there are subtle differences, especially when working with the filesystem and networking between containers.
 
-### Instructions
+Note:
+- Subtitles currently fail to load with `DJANGO_DEBUG=True`, that is due to incorrect `Content-Type` error set by Django's static file implementation. That's only if you run the Django dev server, Nginx sets the correct headers.
+
+### Native Instruction
+
+For convenience, it's recommended to still run Redis and ES in a docker container. Make sure both containers can be reachable over the network.
+
+Set up your virtual environment and install the requirements defined in `requirements-dev.txt`.
+
+There are options built in to load environment variables from a file using `load_dotenv`. Example `.env` file to place in the same folder as `manage.py`:
+
+```
+TA_HOST="localhost"
+TA_USERNAME=tubearchivist
+TA_PASSWORD=verysecret
+TA_MEDIA_DIR="static/volume/media"
+TA_CACHE_DIR="static"
+TA_APP_DIR="."
+REDIS_CON=redis://localhost:6379
+ES_URL="http://localhost:9200"
+ELASTIC_PASSWORD=verysecret
+TZ=America/New_York
+DJANGO_DEBUG=True
+```
+
+Then look at the container startup script `run.sh`, make sure all needed migrations and startup checks ran. To start the dev backend server from the same folder as `manage.py` run:
+
+```bash
+python manage.py runserver
+```
+
+The backend will be available on [localhost:8000/api/](localhost:8000/api/).
+
+You'll probably also want to have a Celery worker instance running, refer to `run.sh` for that. The Beat Scheduler might not be needed.
+
+Then from the frontend folder, install the dependencies with:
+
+```bash
+npm install
+```
+
+Then to start the frontend development server:
+
+```bash
+npm run dev
+```
+
+And the frontend should be available at [localhost:3000](localhost:3000).
+
+### Docker Instructions
 
 Set up docker on your development machine.
 
@@ -156,11 +191,11 @@ Make your changes locally and re-run `docker compose up --build`. The `Dockerfil
 
 ### Develop environment inside a VM
 
-You may find it nice to run everything inside of a VM, though this is not necessary. There's a `deploy.sh` script which has some helpers for this use case. YMMV, this is what one of the developers does:
+You may find it nice to run everything inside of a VM for complete environment snapshots and encapsulation, though this is not strictly necessary. There's a `deploy.sh` script which has some helpers for this use case:
 
-- Clone the repo, work on it with your favorite code editor in your local filesystem. *testing* branch is where all the changes are happening, might be unstable and is WIP.
-- Then I have a VM running standard Ubuntu Server LTS with docker installed. The VM keeps my projects separate and offers convenient snapshot functionality. The VM also offers ways to simulate low end environments by limiting CPU cores and memory. You can use this [Ansible Docker Ubuntu](https://github.com/bbilly1/ansible-playbooks) playbook to get started quickly. But you could also just run docker on your host system.
-- I have my local DNS resolve `tubearchivist.local` to the IP of the VM for convenience. To deploy the latest changes and rebuild the application to the testing VM run:
+- This assumes a standard Ubuntu Server VM with docker and docker compose already installed.
+- Configure your local DNS to resolve `tubearchivist.local` to the IP of the VM.
+- To deploy the latest changes and rebuild the application to the testing VM run:
 ```bash
 ./deploy.sh test
 ```
@@ -191,3 +226,19 @@ services:
 ```
 
 If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.
+
+**Persist Token**  
+The token will get stored in ES in the `config` folder, and not in the `data` folder. To persist the token between ES container rebuilds, you'll need to persist the config folder as an additional volume:
+
+1. Create the token as described above
+2. While the container is running, copy the current config folder out of the container, e.g.: 
+```
+docker cp archivist-es:/usr/share/elasticsearch/config/ volume/es_config
+```
+3. Then stop all containers and mount this folder into the container as an additional volume:
+```yml
+- ./volume/es_config:/usr/share/elasticsearch/config
+```
+4. Start all containers back up.
+
+Now your token will persist between ES container rebuilds.

Dockerfile

@@ -1,6 +1,17 @@
 # multi stage to build tube archivist
 # build python wheel, download and extract ffmpeg, copy into final image
 
+FROM node:lts-alpine AS node-builder
+
+# RUN npm config set registry https://registry.npmjs.org/
+
+COPY ./frontend /frontend
+
+WORKDIR /frontend
+RUN npm i
+RUN npm run build:deploy
+
+WORKDIR /
 
 # First stage to build python wheel
 FROM python:3.11.8-slim-bookworm AS builder
@@ -9,11 +20,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential gcc libldap2-dev libsasl2-dev libssl-dev git
 
 # install requirements
-COPY ./tubearchivist/requirements.txt /requirements.txt
+COPY ./backend/requirements.txt /requirements.txt
 RUN pip install --user -r requirements.txt
 
 # build ffmpeg
-FROM python:3.11.8-slim-bookworm as ffmpeg-builder
+FROM python:3.11.8-slim-bookworm AS ffmpeg-builder
 
 ARG TARGETPLATFORM
 
@@ -21,11 +32,11 @@ COPY docker_assets/ffmpeg_download.py ffmpeg_download.py
 RUN python ffmpeg_download.py $TARGETPLATFORM
 
 # build final image
-FROM python:3.11.8-slim-bookworm as tubearchivist
+FROM python:3.11.8-slim-bookworm AS tubearchivist
 
 ARG INSTALL_DEBUG
 
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 
 # copy build requirements
 COPY --from=builder /root/.local /root/.local
@@ -44,7 +55,7 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm
 # install debug tools for testing environment
 RUN if [ "$INSTALL_DEBUG" ] ; then \
         apt-get -y update && apt-get -y install --no-install-recommends \
-        vim htop bmon net-tools iputils-ping procps \
+        vim htop bmon net-tools iputils-ping procps lsof \
         && pip install --user ipython pytest pytest-django \
     ; fi
 
@@ -56,9 +67,11 @@ COPY docker_assets/nginx.conf /etc/nginx/sites-available/default
 RUN sed -i 's/^user www\-data\;$/user root\;/' /etc/nginx/nginx.conf
 
 # copy application into container
-COPY ./tubearchivist /app
+COPY ./backend /app
 COPY ./docker_assets/run.sh /app
-COPY ./docker_assets/uwsgi.ini /app
+COPY ./docker_assets/backend_start.py /app
+
+COPY --from=node-builder ./frontend/dist /app/static
 
 # volumes
 VOLUME /cache

README.md

@@ -2,9 +2,9 @@
 [*more screenshots and video*](SHOWCASE.MD)
 
 <div align="center">
-<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-docker.png" alt="tubearchivist-docker" title="Tube Archivist Docker Pulls" height="50" width="190"/></a>
-<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-star.png" alt="tubearchivist-github-star" title="Tube Archivist GitHub Stars" height="50" width="190"/></a>
-<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-forks.png" alt="tubearchivist-github-forks" title="Tube Archivist GitHub Forks" height="50" width="190"/></a>
+<a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-docker.png" alt="tubearchivist-docker" title="Tube Archivist Docker Pulls" height="50" width="190"/></a>
+<a href="https://github.com/tubearchivist/tubearchivist/stargazers" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-star.png" alt="tubearchivist-github-star" title="Tube Archivist GitHub Stars" height="50" width="190"/></a>
+<a href="https://github.com/tubearchivist/tubearchivist/forks" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-forks.png" alt="tubearchivist-github-forks" title="Tube Archivist GitHub Forks" height="50" width="190"/></a>
 <a href="https://www.tubearchivist.com/discord" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-discord.png" alt="tubearchivist-discord" title="TA Discord Server Members" height="50" width="190"/></a>
 </div>
 
@@ -47,17 +47,19 @@ The instructions here should get you up and running quickly, for Docker beginner
 
 Take a look at the example [docker-compose.yml](https://github.com/tubearchivist/tubearchivist/blob/master/docker-compose.yml) and configure the required environment variables.
 
+All environment variables are explained in detail in the docs [here](https://docs.tubearchivist.com/installation/env-vars/).
+
 **TubeArchivist**:  
 | Environment Var | Value |  |
 | ----------- | ----------- | ----------- |
-| TA_HOST | Server IP or hostname | Required |
+| TA_HOST | Server IP or hostname `http://tubearchivist.local:8000` | Required |
 | TA_USERNAME | Initial username when logging into TA | Required |
 | TA_PASSWORD | Initial password when logging into TA | Required |
 | ELASTIC_PASSWORD | Password for ElasticSearch | Required |
-| REDIS_HOST | Hostname for Redis | Required |
+| REDIS_CON | Connection string to Redis | Required |
 | TZ | Set your timezone for the scheduler | Required |
 | TA_PORT | Overwrite Nginx port | Optional |
-| TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
+| TA_BACKEND_PORT | Overwrite container internal backend server port | Optional |
 | TA_ENABLE_AUTH_PROXY | Enables support for forwarding auth in reverse proxies | [Read more](https://docs.tubearchivist.com/configuration/forward-auth/) |
 | TA_AUTH_PROXY_USERNAME_HEADER | Header containing username to log in | Optional |
 | TA_AUTH_PROXY_LOGOUT_URL | Logout URL for forwarded auth | Optional |
@@ -67,10 +69,9 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
 | HOST_GID | Allow TA to own the video files instead of container user | Optional |
 | HOST_UID | Allow TA to own the video files instead of container user | Optional |
 | ELASTIC_USER | Change the default ElasticSearch user | Optional |
-| REDIS_PORT | Port that Redis runs on | Optional |
 | TA_LDAP | Configure TA to use LDAP Authentication | [Read more](https://docs.tubearchivist.com/configuration/ldap/) |
-| ENABLE_CAST | Enable casting support | [Read more](https://docs.tubearchivist.com/configuration/cast/) |
-| DJANGO_DEBUG | Return additional error messages, for debug only |  |
+| DISABLE_STATIC_AUTH | Remove authentication from media files, (Google Cast...) | [Read more](https://docs.tubearchivist.com/installation/env-vars/#disable_static_auth) |
+| DJANGO_DEBUG | Return additional error messages, for debug only | Optional |
 
 **ElasticSearch**  
 | Environment Var | Value | State |
@@ -202,6 +203,7 @@ This is your time to shine, [read this](https://github.com/tubearchivist/tubearc
 - [lamusmaser/ta_fix_for_video_redirection](https://github.com/lamusmaser/ta_fix_for_video_redirection): Script to fix videos that were incorrectly indexed by YouTube's "Video is Unavailable" response. 
 - [RoninTech/ta-helper](https://github.com/RoninTech/ta-helper): Helper script to provide a symlink association to reference TubeArchivist videos with their original titles.
 - [tangyjoust/Tautulli-Notify-TubeArchivist-of-Plex-Watched-State](https://github.com/tangyjoust/Tautulli-Notify-TubeArchivist-of-Plex-Watched-State) Mark videos watched in Plex (through streaming not manually) through Tautulli back to TubeArchivist
+- [Dhs92/delete_shorts](https://github.com/Dhs92/delete_shorts): A script to delete ALL YouTube Shorts from TubeArchivist
 
 ## Donate
 The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.  

backend/README.md

@@ -0,0 +1,86 @@
+# Django Setup
+
+## Apps
+The backend is split up into the following apps.
+
+### config
+Root Django App. Doesn't define any views.
+
+- Has main `settings.py`
+- Has main `urls.py` responsible for routing to other apps
+
+### common
+Functionality shared between apps.
+
+Defines views on the root `/api/*` path. Has base views to inherit from.
+
+- Connections to ES and Redis
+- Searching
+- URL parser
+- Collection of helper functions
+
+### appsettings
+Responsible for functionality from the settings pages.
+
+Defines views at `/api/appsettings/*`.
+
+- Index setup
+- Reindexing
+- Snapshots
+- Filesystem Scan
+- Manual import
+
+### channel
+Responsible for Channel Indexing functionality.
+
+Defines views at `/api/channel/*` path.
+
+### download
+Implements download functionality with yt-dlp.
+
+Defines views at `/api/download/*`.
+
+- Download videos
+- Queue management
+- Thumbnails
+- Subscriptions
+
+### playlist
+Implements playlist functionality.
+
+Defines views at `/api/playlist/*`.
+
+- Index Playlists
+- Manual Playlists
+
+### stats
+Builds aggregations views for the statistics dashboard.
+
+Defines views at `/api/stats/*`.
+
+### task
+Defines tasks for Celery. 
+
+Defines views at `/api/task/*`.
+
+- Has main `tasks.py` with all shared_task definitions
+- Has `CustomPeriodicTask` model
+- Implements apprise notifications links
+- Implements schedule functionality
+
+### user
+Implements user and auth functionality.
+
+Defines views at `/api/config/*`.
+
+- Defines custom `Account` model
+
+### video
+Index functionality for videos.
+
+Defines views at `/api/video/*`.
+
+- Index videos
+- Index comments
+- Index/download subtitles
+- Media stream parsing

backend/appsettings/index_mapping.json

@@ -684,4 +684,4 @@
             }
         }
     ]
-}
+}

backend/appsettings/serializers.py

@@ -0,0 +1,133 @@
+"""appsettings erializers"""
+
+# pylint: disable=abstract-method
+
+from common.serializers import ValidateUnknownFieldsMixin
+from rest_framework import serializers
+
+
+class BackupFileSerializer(serializers.Serializer):
+    """serialize backup file"""
+
+    filename = serializers.CharField()
+    file_path = serializers.CharField()
+    file_size = serializers.IntegerField()
+    timestamp = serializers.CharField()
+    reason = serializers.CharField()
+
+
+class AppConfigSubSerializer(
+    ValidateUnknownFieldsMixin, serializers.Serializer
+):
+    """serialize app config subscriptions"""
+
+    channel_size = serializers.IntegerField(required=False)
+    live_channel_size = serializers.IntegerField(required=False)
+    shorts_channel_size = serializers.IntegerField(required=False)
+    auto_start = serializers.BooleanField(required=False)
+
+
+class AppConfigDownloadsSerializer(
+    ValidateUnknownFieldsMixin, serializers.Serializer
+):
+    """serialize app config downloads config"""
+
+    limit_speed = serializers.IntegerField(allow_null=True)
+    sleep_interval = serializers.IntegerField(allow_null=True)
+    autodelete_days = serializers.IntegerField(allow_null=True)
+    format = serializers.CharField(allow_null=True)
+    format_sort = serializers.CharField(allow_null=True)
+    add_metadata = serializers.BooleanField()
+    add_thumbnail = serializers.BooleanField()
+    subtitle = serializers.CharField(allow_null=True)
+    subtitle_source = serializers.ChoiceField(
+        choices=["auto", "user"], allow_null=True
+    )
+    subtitle_index = serializers.BooleanField()
+    comment_max = serializers.CharField(allow_null=True)
+    comment_sort = serializers.ChoiceField(
+        choices=["top", "new"], allow_null=True
+    )
+    cookie_import = serializers.BooleanField()
+    potoken = serializers.BooleanField()
+    throttledratelimit = serializers.IntegerField(allow_null=True)
+    extractor_lang = serializers.CharField(allow_null=True)
+    integrate_ryd = serializers.BooleanField()
+    integrate_sponsorblock = serializers.BooleanField()
+
+
+class AppConfigAppSerializer(
+    ValidateUnknownFieldsMixin, serializers.Serializer
+):
+    """serialize app config"""
+
+    enable_snapshot = serializers.BooleanField()
+    enable_cast = serializers.BooleanField()
+
+
+class AppConfigSerializer(ValidateUnknownFieldsMixin, serializers.Serializer):
+    """serialize appconfig"""
+
+    subscriptions = AppConfigSubSerializer(required=False)
+    downloads = AppConfigDownloadsSerializer(required=False)
+    application = AppConfigAppSerializer(required=False)
+
+
+class CookieValidationSerializer(serializers.Serializer):
+    """serialize cookie validation response"""
+
+    cookie_enabled = serializers.BooleanField()
+    status = serializers.BooleanField(required=False)
+    validated = serializers.IntegerField(required=False)
+    validated_str = serializers.CharField(required=False)
+
+
+class CookieUpdateSerializer(serializers.Serializer):
+    """serialize cookie to update"""
+
+    cookie = serializers.CharField()
+
+
+class PoTokenSerializer(serializers.Serializer):
+    """serialize PO token"""
+
+    potoken = serializers.CharField()
+
+
+class SnapshotItemSerializer(serializers.Serializer):
+    """serialize snapshot response"""
+
+    id = serializers.CharField()
+    state = serializers.CharField()
+    es_version = serializers.CharField()
+    start_date = serializers.CharField()
+    end_date = serializers.CharField()
+    end_stamp = serializers.IntegerField()
+    duration_s = serializers.IntegerField()
+
+
+class SnapshotListSerializer(serializers.Serializer):
+    """serialize snapshot list response"""
+
+    next_exec = serializers.IntegerField()
+    next_exec_str = serializers.CharField()
+    expire_after = serializers.CharField()
+    snapshots = SnapshotItemSerializer(many=True)
+
+
+class SnapshotCreateResponseSerializer(serializers.Serializer):
+    """serialize new snapshot creating response"""
+
+    snapshot_name = serializers.CharField()
+
+
+class SnapshotRestoreResponseSerializer(serializers.Serializer):
+    """serialize snapshot restore response"""
+
+    accepted = serializers.BooleanField()
+
+
+class TokenResponseSerializer(serializers.Serializer):
+    """serialize token response"""
+
+    token = serializers.CharField()

backend/appsettings/src/backup.py

@@ -10,11 +10,10 @@ import os
 import zipfile
 from datetime import datetime
 
-from home.models import CustomPeriodicTask
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_mapping, ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import get_mapping, ignore_filelist
+from task.models import CustomPeriodicTask
 
 
 class ElasticBackup:
@@ -24,8 +23,7 @@ class ElasticBackup:
     CACHE_DIR = EnvironmentSettings.CACHE_DIR
     BACKUP_DIR = os.path.join(CACHE_DIR, "backup")
 
-    def __init__(self, reason=False, task=False):
-        self.config = AppConfig().config
+    def __init__(self, reason=False, task=False) -> None:
         self.timestamp = datetime.now().strftime("%Y%m%d")
         self.index_config = get_mapping()
         self.reason = reason
@@ -139,6 +137,8 @@ class ElasticBackup:
         elif len(file_split) == 3:
             timestamp = file_split[1]
             reason = file_split[2].strip(".zip")
+        else:
+            raise ValueError
 
         data = {
             "filename": filename,

backend/appsettings/src/config.py

@@ -0,0 +1,252 @@
+"""
+Functionality:
+- read and write config
+- load config variables into redis
+"""
+
+from random import randint
+from time import sleep
+from typing import Literal, TypedDict
+
+import requests
+from appsettings.src.snapshot import ElasticSnapshot
+from common.src.es_connect import ElasticWrap
+from common.src.ta_redis import RedisArchivist
+from django.conf import settings
+
+
+class SubscriptionsConfigType(TypedDict):
+    """describes subscriptions config"""
+
+    channel_size: int
+    live_channel_size: int
+    shorts_channel_size: int
+    auto_start: bool
+
+
+class DownloadsConfigType(TypedDict):
+    """describes downloads config"""
+
+    limit_speed: int | None
+    sleep_interval: int | None
+    autodelete_days: int | None
+    format: str | None
+    format_sort: str | None
+    add_metadata: bool
+    add_thumbnail: bool
+    subtitle: str | None
+    subtitle_source: Literal["user", "auto"] | None
+    subtitle_index: bool
+    comment_max: str | None
+    comment_sort: Literal["top", "new"] | None
+    cookie_import: bool
+    potoken: bool
+    throttledratelimit: int | None
+    extractor_lang: str | None
+    integrate_ryd: bool
+    integrate_sponsorblock: bool
+
+
+class ApplicationConfigType(TypedDict):
+    """describes application config"""
+
+    enable_snapshot: bool
+    enable_cast: bool
+
+
+class AppConfigType(TypedDict):
+    """combined app config type"""
+
+    subscriptions: SubscriptionsConfigType
+    downloads: DownloadsConfigType
+    application: ApplicationConfigType
+
+
+class AppConfig:
+    """handle application variables"""
+
+    ES_PATH = "ta_config/_doc/appsettings"
+    ES_UPDATE_PATH = "ta_config/_update/appsettings"
+    CONFIG_DEFAULTS: AppConfigType = {
+        "subscriptions": {
+            "channel_size": 50,
+            "live_channel_size": 50,
+            "shorts_channel_size": 50,
+            "auto_start": False,
+        },
+        "downloads": {
+            "limit_speed": None,
+            "sleep_interval": 10,
+            "autodelete_days": None,
+            "format": None,
+            "format_sort": None,
+            "add_metadata": False,
+            "add_thumbnail": False,
+            "subtitle": None,
+            "subtitle_source": None,
+            "subtitle_index": False,
+            "comment_max": None,
+            "comment_sort": "top",
+            "cookie_import": False,
+            "potoken": False,
+            "throttledratelimit": None,
+            "extractor_lang": None,
+            "integrate_ryd": False,
+            "integrate_sponsorblock": False,
+        },
+        "application": {
+            "enable_snapshot": True,
+            "enable_cast": False,
+        },
+    }
+
+    def __init__(self):
+        self.config = self.get_config()
+
+    def get_config(self) -> AppConfigType:
+        """get config from ES"""
+        response, status_code = ElasticWrap(self.ES_PATH).get()
+        if not status_code == 200:
+            raise ValueError(f"no config found at {self.ES_PATH}")
+
+        return response["_source"]
+
+    def update_config(self, data: dict) -> AppConfigType:
+        """update single config value"""
+        new_config = self.config.copy()
+        for key, value in data.items():
+            if (
+                isinstance(value, dict)
+                and key in new_config
+                and isinstance(new_config[key], dict)
+            ):
+                new_config[key].update(value)
+            else:
+                new_config[key] = value
+
+        response, status_code = ElasticWrap(self.ES_PATH).post(new_config)
+        if not status_code == 200:
+            print(response)
+
+        self.config = new_config
+
+        return new_config
+
+    def post_process_updated(self, data: dict) -> None:
+        """apply hooks for some config keys"""
+        for config_value, updated_value in data:
+            if config_value == "application.enable_snapshot" and updated_value:
+                ElasticSnapshot().setup()
+
+    @staticmethod
+    def _fail_message(message_line):
+        """notify our failure"""
+        key = "message:setting"
+        message = {
+            "status": key,
+            "group": "setting:application",
+            "level": "error",
+            "title": "Cookie import failed",
+            "messages": [message_line],
+            "id": "0000",
+        }
+        RedisArchivist().set_message(key, message=message, expire=True)
+
+    def sync_defaults(self):
+        """sync defaults at startup, needs to be called with __new__"""
+        return ElasticWrap(self.ES_PATH).post(self.CONFIG_DEFAULTS)
+
+    def add_new_defaults(self) -> list[str]:
+        """add new default config values to ES, called at startup"""
+        updated = []
+        for key, value in self.CONFIG_DEFAULTS.items():
+            if key not in self.config:
+                # complete new key
+                self.update_config({key: value})
+                updated.append(str({key: value}))
+                continue
+
+            for sub_key, sub_value in value.items():  # type: ignore
+                if sub_key not in self.config[key]:
+                    # new partial key
+                    to_update = {key: {sub_key: sub_value}}
+                    self.update_config(to_update)
+                    updated.append(str(to_update))
+
+        return updated
+
+
+class ReleaseVersion:
+    """compare local version with remote version"""
+
+    REMOTE_URL = "https://www.tubearchivist.com/api/release/latest/"
+    NEW_KEY = "versioncheck:new"
+
+    def __init__(self) -> None:
+        self.local_version: str = settings.TA_VERSION
+        self.is_unstable: bool = settings.TA_VERSION.endswith("-unstable")
+        self.remote_version: str = ""
+        self.is_breaking: bool = False
+
+    def check(self) -> None:
+        """check version"""
+        print(f"[{self.local_version}]: look for updates")
+        self.get_remote_version()
+        new_version = self._has_update()
+        if new_version:
+            message = {
+                "status": True,
+                "version": new_version,
+                "is_breaking": self.is_breaking,
+            }
+            RedisArchivist().set_message(self.NEW_KEY, message)
+            print(f"[{self.local_version}]: found new version {new_version}")
+
+    def get_local_version(self) -> str:
+        """read version from local"""
+        return self.local_version
+
+    def get_remote_version(self) -> None:
+        """read version from remote"""
+        sleep(randint(0, 60))
+        response = requests.get(self.REMOTE_URL, timeout=20).json()
+        self.remote_version = response["release_version"]
+        self.is_breaking = response["breaking_changes"]
+
+    def _has_update(self) -> str | bool:
+        """check if there is an update"""
+        remote_parsed = self._parse_version(self.remote_version)
+        local_parsed = self._parse_version(self.local_version)
+        if remote_parsed > local_parsed:
+            return self.remote_version
+
+        if self.is_unstable and local_parsed == remote_parsed:
+            return self.remote_version
+
+        return False
+
+    @staticmethod
+    def _parse_version(version) -> tuple[int, ...]:
+        """return version parts"""
+        clean = version.rstrip("-unstable").lstrip("v")
+        return tuple((int(i) for i in clean.split(".")))
+
+    def is_updated(self) -> str | bool:
+        """check if update happened in the mean time"""
+        message = self.get_update()
+        if not message:
+            return False
+
+        local_parsed = self._parse_version(self.local_version)
+        message_parsed = self._parse_version(message.get("version"))
+
+        if local_parsed >= message_parsed:
+            RedisArchivist().del_message(self.NEW_KEY)
+            return settings.TA_VERSION
+
+        return False
+
+    def get_update(self) -> dict | None:
+        """return new version dict if available"""
+        message = RedisArchivist().get_message_dict(self.NEW_KEY)
+        return message or None

backend/appsettings/src/filesystem.py

@@ -5,11 +5,11 @@ Functionality:
 
 import os
 
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.index.comments import CommentList
-from home.src.index.video import YoutubeVideo, index_new_video
-from home.src.ta.helper import ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import IndexPaginate
+from common.src.helper import ignore_filelist
+from video.src.comments import CommentList
+from video.src.index import YoutubeVideo, index_new_video
 
 
 class Scanner:
@@ -56,7 +56,6 @@ class Scanner:
         """apply all changes"""
         self.delete()
         self.index()
-        self.url_fix()
 
     def delete(self) -> None:
         """delete videos from index"""
@@ -92,35 +91,3 @@ class Scanner:
         comment_list = CommentList(task=self.task)
         comment_list.add(video_ids=list(self.to_index))
         comment_list.index()
-
-    def url_fix(self) -> None:
-        """
-        update path v0.3.6 to v0.3.7
-        fix url not matching channel-videoid pattern
-        """
-        bool_must = (
-            "doc['media_url'].value == "
-            + "(doc['channel.channel_id'].value + '/' + "
-            + "doc['youtube_id'].value) + '.mp4'"
-        )
-        to_update = (
-            "ctx._source['media_url'] = "
-            + "ctx._source.channel['channel_id'] + '/' + "
-            + "ctx._source['youtube_id'] + '.mp4'"
-        )
-        data = {
-            "query": {
-                "bool": {
-                    "must_not": [{"script": {"script": {"source": bool_must}}}]
-                }
-            },
-            "script": {"source": to_update},
-        }
-        response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data)
-        updated = response.get("updates")
-        if updated:
-            print(f"updated {updated} bad media_url")
-            if self.task:
-                self.task.send_progress(
-                    [f"Updated {updated} wrong media urls."]
-                )

backend/appsettings/src/index_setup.py

@@ -5,11 +5,11 @@ functionality:
 - backup and restore metadata
 """
 
-from home.src.es.backup import ElasticBackup
-from home.src.es.connect import ElasticWrap
-from home.src.es.snapshot import ElasticSnapshot
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_mapping
+from appsettings.src.backup import ElasticBackup
+from appsettings.src.config import AppConfig
+from appsettings.src.snapshot import ElasticSnapshot
+from common.src.es_connect import ElasticWrap
+from common.src.helper import get_mapping
 
 
 class ElasticIndex:
@@ -111,6 +111,8 @@ class ElasticIndex:
         elif method == "restore":
             source = f"ta_{self.index_name}_backup"
             destination = f"ta_{self.index_name}"
+        else:
+            raise ValueError("invalid method, expected 'backup' or 'restore'")
 
         data = {"source": {"index": source}, "dest": {"index": destination}}
         _, _ = ElasticWrap("_reindex?refresh=true").post(data=data)

backend/appsettings/src/manual.py

@@ -11,13 +11,13 @@ import re
 import shutil
 import subprocess
 
-from home.src.download.thumbnails import ThumbManager
-from home.src.index.comments import CommentList
-from home.src.index.video import YoutubeVideo
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
+from appsettings.src.config import AppConfig
+from common.src.env_settings import EnvironmentSettings
+from common.src.helper import ignore_filelist
+from download.src.thumbnails import ThumbManager
 from PIL import Image
+from video.src.comments import CommentList
+from video.src.index import YoutubeVideo
 from yt_dlp.utils import ISO639Utils
 
 
@@ -28,7 +28,6 @@ class ImportFolderScanner:
     - convert if needed
     """
 
-    CONFIG = AppConfig().config
     CACHE_DIR = EnvironmentSettings.CACHE_DIR
     IMPORT_DIR = os.path.join(CACHE_DIR, "import")
 
@@ -129,6 +128,7 @@ class ImportFolderScanner:
 
     def process_videos(self):
         """loop through all videos"""
+        config = AppConfig().config
         for idx, current_video in enumerate(self.to_import):
             if not current_video["media"]:
                 print(f"{current_video}: no matching media file found.")
@@ -144,7 +144,7 @@ class ImportFolderScanner:
             self._convert_video(current_video)
             print(f"manual import: {current_video}")
 
-            ManualImport(current_video, self.CONFIG).run()
+            ManualImport(current_video, config).run()
 
         video_ids = [i["video_id"] for i in self.to_import]
         comment_list = CommentList(task=self.task)
@@ -407,8 +407,11 @@ class ManualImport:
             media_path=self.current_video["media"],
         )
         if not video.json_data:
-            print(f"{video_id}: manual import failed, and no metadata found.")
-            raise ValueError
+            message = (
+                f"{video_id}: manual import failed, and no metadata found."
+            )
+            print(message)
+            raise ValueError(message)
 
         video.check_subtitles(subtitle_files=self.current_video["subtitle"])
         video.upload_to_es()

backend/appsettings/src/reindex.py

@@ -7,21 +7,21 @@ functionality:
 import json
 import os
 from datetime import datetime
-from time import sleep
 from typing import Callable, TypedDict
 
-from home.models import CustomPeriodicTask
-from home.src.download.subscriptions import ChannelSubscription
-from home.src.download.thumbnails import ThumbManager
-from home.src.download.yt_dlp_base import CookieHandler
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.index.channel import YoutubeChannel
-from home.src.index.comments import Comments
-from home.src.index.playlist import YoutubePlaylist
-from home.src.index.video import YoutubeVideo
-from home.src.ta.config import AppConfig
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisQueue
+from appsettings.src.config import AppConfig
+from channel.src.index import YoutubeChannel
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import rand_sleep
+from common.src.ta_redis import RedisQueue
+from download.src.subscriptions import ChannelSubscription
+from download.src.thumbnails import ThumbManager
+from download.src.yt_dlp_base import CookieHandler
+from playlist.src.index import YoutubePlaylist
+from task.models import CustomPeriodicTask
+from video.src.comments import Comments
+from video.src.index import YoutubeVideo
 
 
 class ReindexConfigType(TypedDict):
@@ -289,8 +289,7 @@ class Reindex(ReindexBase):
                 self._notify(name, total, idx)
 
             reindex(youtube_id)
-            sleep_interval = self.config["downloads"].get("sleep_interval", 0)
-            sleep(sleep_interval)
+            rand_sleep(self.config)
 
     def _get_reindex_map(self, index_name: str) -> Callable:
         """return def to run for index"""
@@ -334,6 +333,7 @@ class Reindex(ReindexBase):
         # add back
         video.json_data["player"] = es_meta.get("player")
         video.json_data["date_downloaded"] = es_meta.get("date_downloaded")
+        video.json_data["vid_type"] = es_meta.get("vid_type")
         video.json_data["channel"] = es_meta.get("channel")
         if es_meta.get("playlist"):
             video.json_data["playlist"] = es_meta.get("playlist")

backend/appsettings/src/snapshot.py

@@ -7,9 +7,9 @@ from datetime import datetime
 from time import sleep
 from zoneinfo import ZoneInfo
 
-from home.src.es.connect import ElasticWrap
-from home.src.ta.helper import get_mapping
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import get_mapping
 
 
 class ElasticSnapshot:
@@ -150,7 +150,7 @@ class ElasticSnapshot:
         if statuscode == 200:
             print(f"snapshot: executing now: {response}")
 
-        if wait:
+        if wait and "snapshot_name" in response:
             self._wait_for_snapshot(response["snapshot_name"])
 
         return response
@@ -251,10 +251,9 @@ class ElasticSnapshot:
     @staticmethod
     def _date_converter(date_utc):
         """convert datetime string"""
-        expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
-        date = datetime.strptime(date_utc, expected_format)
-        local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
-        converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ))
+        date = datetime.strptime(date_utc, "%Y-%m-%dT%H:%M:%S.%fZ")
+        utc_date = date.replace(tzinfo=ZoneInfo("UTC"))
+        converted = utc_date.astimezone(ZoneInfo(EnvironmentSettings.TZ))
         converted_str = converted.strftime("%Y-%m-%d %H:%M")
 
         return converted_str

backend/appsettings/urls.py

@@ -0,0 +1,47 @@
+"""all app settings API urls"""
+
+from appsettings import views
+from django.urls import path
+
+urlpatterns = [
+    path(
+        "config/",
+        views.AppConfigApiView.as_view(),
+        name="api-config",
+    ),
+    path(
+        "snapshot/",
+        views.SnapshotApiListView.as_view(),
+        name="api-snapshot-list",
+    ),
+    path(
+        "snapshot/<slug:snapshot_id>/",
+        views.SnapshotApiView.as_view(),
+        name="api-snapshot",
+    ),
+    path(
+        "backup/",
+        views.BackupApiListView.as_view(),
+        name="api-backup-list",
+    ),
+    path(
+        "backup/<str:filename>/",
+        views.BackupApiView.as_view(),
+        name="api-backup",
+    ),
+    path(
+        "cookie/",
+        views.CookieView.as_view(),
+        name="api-cookie",
+    ),
+    path(
+        "potoken/",
+        views.POTokenView.as_view(),
+        name="api-potoken",
+    ),
+    path(
+        "token/",
+        views.TokenView.as_view(),
+        name="api-token",
+    ),
+]

backend/appsettings/views.py

@@ -0,0 +1,493 @@
+"""all app settings API views"""
+
+from appsettings.serializers import (
+    AppConfigSerializer,
+    BackupFileSerializer,
+    CookieUpdateSerializer,
+    CookieValidationSerializer,
+    PoTokenSerializer,
+    SnapshotCreateResponseSerializer,
+    SnapshotItemSerializer,
+    SnapshotListSerializer,
+    SnapshotRestoreResponseSerializer,
+    TokenResponseSerializer,
+)
+from appsettings.src.backup import ElasticBackup
+from appsettings.src.config import AppConfig
+from appsettings.src.snapshot import ElasticSnapshot
+from common.serializers import (
+    AsyncTaskResponseSerializer,
+    ErrorResponseSerializer,
+)
+from common.src.ta_redis import RedisArchivist
+from common.views_base import AdminOnly, AdminWriteOnly, ApiBaseView
+from django.conf import settings
+from download.src.yt_dlp_base import CookieHandler, POTokenHandler
+from drf_spectacular.utils import OpenApiResponse, extend_schema
+from rest_framework.authtoken.models import Token
+from rest_framework.response import Response
+from task.src.task_manager import TaskCommand
+from task.tasks import run_restore_backup
+
+
+class BackupApiListView(ApiBaseView):
+    """resolves to /api/appsettings/backup/
+    GET: returns list of available zip backups
+    POST: take zip backup now
+    """
+
+    permission_classes = [AdminOnly]
+    task_name = "run_backup"
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(BackupFileSerializer(many=True)),
+        },
+    )
+    def get(request):
+        """get list of available backup files"""
+        # pylint: disable=unused-argument
+        backup_files = ElasticBackup().get_all_backup_files()
+        serializer = BackupFileSerializer(backup_files, many=True)
+        return Response(serializer.data)
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(AsyncTaskResponseSerializer()),
+        },
+    )
+    def post(self, request):
+        """start new backup file task"""
+        # pylint: disable=unused-argument
+        response = TaskCommand().start(self.task_name)
+        message = {
+            "message": "backup task started",
+            "task_id": response["task_id"],
+        }
+        serializer = AsyncTaskResponseSerializer(message)
+
+        return Response(serializer.data)
+
+
+class BackupApiView(ApiBaseView):
+    """resolves to /api/appsettings/backup/<filename>/
+    GET: return a single backup
+    POST: restore backup
+    DELETE: delete backup
+    """
+
+    permission_classes = [AdminOnly]
+    task_name = "restore_backup"
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(BackupFileSerializer()),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="file not found"
+            ),
+        }
+    )
+    def get(request, filename):
+        """get single backup"""
+        # pylint: disable=unused-argument
+        backup_file = ElasticBackup().build_backup_file_data(filename)
+        if not backup_file:
+            error = ErrorResponseSerializer({"error": "file not found"})
+            return Response(error.data, status=404)
+
+        serializer = BackupFileSerializer(backup_file)
+
+        return Response(serializer.data)
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(AsyncTaskResponseSerializer()),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="file not found"
+            ),
+        }
+    )
+    def post(self, request, filename):
+        """start new task to restore backup file"""
+        # pylint: disable=unused-argument
+        backup_file = ElasticBackup().build_backup_file_data(filename)
+        if not backup_file:
+            error = ErrorResponseSerializer({"error": "file not found"})
+            return Response(error.data, status=404)
+
+        task = run_restore_backup.delay(filename)
+        message = {
+            "message": "backup restore task started",
+            "filename": filename,
+            "task_id": task.id,
+        }
+        return Response(message)
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            204: OpenApiResponse(description="file deleted"),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="file not found"
+            ),
+        }
+    )
+    def delete(request, filename):
+        """delete backup file"""
+        # pylint: disable=unused-argument
+
+        backup_file = ElasticBackup().delete_file(filename)
+        if not backup_file:
+            error = ErrorResponseSerializer({"error": "file not found"})
+            return Response(error.data, status=404)
+
+        return Response(status=204)
+
+
+class AppConfigApiView(ApiBaseView):
+    """resolves to /api/appsettings/config/
+    GET: return app settings
+    POST: update app settings
+    """
+
+    permission_classes = [AdminWriteOnly]
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(AppConfigSerializer()),
+        }
+    )
+    def get(request):
+        """get app config"""
+        response = AppConfig().config
+        serializer = AppConfigSerializer(response)
+        return Response(serializer.data)
+
+    @staticmethod
+    @extend_schema(
+        request=AppConfigSerializer(),
+        responses={
+            200: OpenApiResponse(AppConfigSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        },
+    )
+    def post(request):
+        """update config values, allows partial update"""
+        serializer = AppConfigSerializer(data=request.data, partial=True)
+        serializer.is_valid(raise_exception=True)
+        validated_data = serializer.validated_data
+        updated_config = AppConfig().update_config(validated_data)
+        updated_serializer = AppConfigSerializer(updated_config)
+        return Response(updated_serializer.data)
+
+
+class CookieView(ApiBaseView):
+    """resolves to /api/appsettings/cookie/
+    GET: check if cookie is enabled
+    POST: verify validity of cookie
+    PUT: import cookie
+    DELETE: revoke the cookie
+    """
+
+    permission_classes = [AdminOnly]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(CookieValidationSerializer()),
+        }
+    )
+    def get(self, request):
+        """get cookie validation status"""
+        # pylint: disable=unused-argument
+        validation = self._get_cookie_validation()
+        serializer = CookieValidationSerializer(validation)
+
+        return Response(serializer.data)
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(CookieValidationSerializer()),
+        }
+    )
+    def post(self, request):
+        """validate cookie"""
+        # pylint: disable=unused-argument
+        config = AppConfig().config
+        _ = CookieHandler(config).validate()
+        validation = self._get_cookie_validation()
+        serializer = CookieValidationSerializer(validation)
+
+        return Response(serializer.data)
+
+    @extend_schema(
+        request=CookieUpdateSerializer(),
+        responses={
+            200: OpenApiResponse(CookieValidationSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        },
+    )
+    def put(self, request):
+        """handle put request"""
+        # pylint: disable=unused-argument
+
+        serializer = CookieUpdateSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        validated_data = serializer.validated_data
+
+        cookie = validated_data.get("cookie")
+        if not cookie:
+            message = "missing cookie key in request data"
+            print(message)
+            error = ErrorResponseSerializer({"error": message})
+            return Response(error.data, status=400)
+
+        if settings.DEBUG:
+            print(f"[cookie] preview:\n\n{cookie[:300]}")
+
+        config = AppConfig().config
+        handler = CookieHandler(config)
+        handler.set_cookie(cookie)
+        validated = handler.validate()
+        if not validated:
+            message = "[cookie]: import failed, not valid"
+            print(message)
+            error = ErrorResponseSerializer({"error": message})
+            handler.revoke()
+            return Response(error.data, status=400)
+
+        validation = self._get_cookie_validation()
+        serializer = CookieValidationSerializer(validation)
+
+        return Response(serializer.data)
+
+    @extend_schema(
+        responses={
+            204: OpenApiResponse(description="Cookie revoked"),
+        },
+    )
+    def delete(self, request):
+        """delete the cookie"""
+        config = AppConfig().config
+        handler = CookieHandler(config)
+        handler.revoke()
+        return Response(status=204)
+
+    @staticmethod
+    def _get_cookie_validation():
+        """get current cookie validation"""
+        config = AppConfig().config
+        validation = RedisArchivist().get_message_dict("cookie:valid")
+        is_enabled = {"cookie_enabled": config["downloads"]["cookie_import"]}
+        validation.update(is_enabled)
+
+        return validation
+
+
+class POTokenView(ApiBaseView):
+    """handle PO token"""
+
+    permission_classes = [AdminOnly]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(PoTokenSerializer()),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="PO token not found"
+            ),
+        }
+    )
+    def get(self, request):
+        """get PO token"""
+        config = AppConfig().config
+        potoken = POTokenHandler(config).get()
+        if not potoken:
+            error = ErrorResponseSerializer({"error": "PO token not found"})
+            return Response(error.data, status=404)
+
+        serializer = PoTokenSerializer(data={"potoken": potoken})
+        serializer.is_valid(raise_exception=True)
+
+        return Response(serializer.data)
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(PoTokenSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        }
+    )
+    def post(self, request):
+        """Update PO token"""
+        serializer = PoTokenSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        validated_data = serializer.validated_data
+        if not validated_data:
+            error = ErrorResponseSerializer(
+                {"error": "missing PO token key in request data"}
+            )
+            return Response(error.data, status=400)
+
+        config = AppConfig().config
+        new_token = validated_data["potoken"]
+
+        POTokenHandler(config).set_token(new_token)
+        return Response(serializer.data)
+
+    @extend_schema(
+        responses={
+            204: OpenApiResponse(description="PO token revoked"),
+        },
+    )
+    def delete(self, request):
+        """delete PO token"""
+        config = AppConfig().config
+        POTokenHandler(config).revoke_token()
+        return Response(status=204)
+
+
+class SnapshotApiListView(ApiBaseView):
+    """resolves to /api/appsettings/snapshot/
+    GET: returns snapshot config plus list of existing snapshots
+    POST: take snapshot now
+    """
+
+    permission_classes = [AdminOnly]
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(SnapshotListSerializer()),
+        }
+    )
+    def get(request):
+        """get available snapshots with metadata"""
+        # pylint: disable=unused-argument
+        snapshots = ElasticSnapshot().get_snapshot_stats()
+        serializer = SnapshotListSerializer(snapshots)
+        return Response(serializer.data)
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(SnapshotCreateResponseSerializer()),
+        }
+    )
+    def post(request):
+        """take snapshot now"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().take_snapshot_now()
+        serializer = SnapshotCreateResponseSerializer(response)
+        return Response(serializer.data)
+
+
+class SnapshotApiView(ApiBaseView):
+    """resolves to /api/appsettings/snapshot/<snapshot-id>/
+    GET: return a single snapshot
+    POST: restore snapshot
+    DELETE: delete a snapshot
+    """
+
+    permission_classes = [AdminOnly]
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(SnapshotItemSerializer()),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="snapshot not found"
+            ),
+        }
+    )
+    def get(request, snapshot_id):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
+
+        if not snapshot:
+            error = ErrorResponseSerializer({"error": "snapshot not found"})
+            return Response(error.data, status=404)
+
+        serializer = SnapshotItemSerializer(snapshot)
+        return Response(serializer.data)
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(SnapshotRestoreResponseSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="bad request"
+            ),
+        }
+    )
+    def post(request, snapshot_id):
+        """restore snapshot"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().restore_all(snapshot_id)
+        if not response:
+            error = ErrorResponseSerializer(
+                {"error": "failed to restore snapshot"}
+            )
+            return Response(error.data, status=400)
+
+        serializer = SnapshotRestoreResponseSerializer(response)
+
+        return Response(serializer.data)
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            204: OpenApiResponse(description="delete snapshot from index"),
+        }
+    )
+    def delete(request, snapshot_id):
+        """delete snapshot from index"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
+        if not response:
+            error = ErrorResponseSerializer(
+                {"error": "failed to delete snapshot"}
+            )
+            return Response(error.data, status=400)
+
+        return Response(status=204)
+
+
+class TokenView(ApiBaseView):
+    """resolves to /api/appsettings/token/
+    GET: get API token
+    DELETE: revoke the token
+    """
+
+    permission_classes = [AdminOnly]
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(TokenResponseSerializer()),
+        }
+    )
+    def get(request):
+        """get your API token"""
+        token, _ = Token.objects.get_or_create(user=request.user)
+        serializer = TokenResponseSerializer({"token": token.key})
+        return Response(serializer.data)
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            204: OpenApiResponse(description="delete token"),
+        }
+    )
+    def delete(request):
+        """delete your API token, new will get created on next get"""
+        print("revoke API token")
+        request.user.auth_token.delete()
+        return Response(status=204)

backend/channel/serializers.py

@@ -0,0 +1,102 @@
+"""channel serializers"""
+
+# pylint: disable=abstract-method
+
+from common.serializers import PaginationSerializer, ValidateUnknownFieldsMixin
+from rest_framework import serializers
+
+
+class ChannelOverwriteSerializer(
+    ValidateUnknownFieldsMixin, serializers.Serializer
+):
+    """serialize channel overwrites"""
+
+    download_format = serializers.CharField(required=False, allow_null=True)
+    autodelete_days = serializers.IntegerField(required=False, allow_null=True)
+    index_playlists = serializers.BooleanField(required=False, allow_null=True)
+    integrate_sponsorblock = serializers.BooleanField(
+        required=False, allow_null=True
+    )
+    subscriptions_channel_size = serializers.IntegerField(
+        required=False, allow_null=True
+    )
+    subscriptions_live_channel_size = serializers.IntegerField(
+        required=False, allow_null=True
+    )
+    subscriptions_shorts_channel_size = serializers.IntegerField(
+        required=False, allow_null=True
+    )
+
+
+class ChannelSerializer(serializers.Serializer):
+    """serialize channel"""
+
+    channel_id = serializers.CharField()
+    channel_active = serializers.BooleanField()
+    channel_banner_url = serializers.CharField()
+    channel_thumb_url = serializers.CharField()
+    channel_tvart_url = serializers.CharField()
+    channel_description = serializers.CharField()
+    channel_last_refresh = serializers.CharField()
+    channel_name = serializers.CharField()
+    channel_overwrites = ChannelOverwriteSerializer(required=False)
+    channel_subs = serializers.IntegerField()
+    channel_subscribed = serializers.BooleanField()
+    channel_tags = serializers.ListField(
+        child=serializers.CharField(), required=False
+    )
+    channel_views = serializers.IntegerField()
+    _index = serializers.CharField(required=False)
+    _score = serializers.IntegerField(required=False)
+
+
+class ChannelListSerializer(serializers.Serializer):
+    """serialize channel list"""
+
+    data = ChannelSerializer(many=True)
+    paginate = PaginationSerializer()
+
+
+class ChannelListQuerySerializer(serializers.Serializer):
+    """serialize list query"""
+
+    filter = serializers.ChoiceField(choices=["subscribed"], required=False)
+    page = serializers.IntegerField(required=False)
+
+
+class ChannelUpdateSerializer(serializers.Serializer):
+    """update channel"""
+
+    channel_subscribed = serializers.BooleanField(required=False)
+    channel_overwrites = ChannelOverwriteSerializer(required=False)
+
+
+class ChannelAggBucketSerializer(serializers.Serializer):
+    """serialize channel agg bucket"""
+
+    value = serializers.IntegerField()
+    value_str = serializers.CharField(required=False)
+
+
+class ChannelAggSerializer(serializers.Serializer):
+    """serialize channel aggregation"""
+
+    total_items = ChannelAggBucketSerializer()
+    total_size = ChannelAggBucketSerializer()
+    total_duration = ChannelAggBucketSerializer()
+
+
+class ChannelNavSerializer(serializers.Serializer):
+    """serialize channel navigation"""
+
+    has_pending = serializers.BooleanField()
+    has_playlists = serializers.BooleanField()
+    has_videos = serializers.BooleanField()
+    has_streams = serializers.BooleanField()
+    has_shorts = serializers.BooleanField()
+
+
+class ChannelSearchQuerySerializer(serializers.Serializer):
+    """serialize query parameters for searching"""
+
+    q = serializers.CharField()

backend/channel/src/index.py

@@ -8,12 +8,13 @@ import json
 import os
 from datetime import datetime
 
-from home.src.download.thumbnails import ThumbManager
-from home.src.download.yt_dlp_base import YtWrap
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.index.generic import YouTubeItem
-from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import rand_sleep
+from common.src.index_generic import YouTubeItem
+from download.src.thumbnails import ThumbManager
+from download.src.yt_dlp_base import YtWrap
+from playlist.src.index import YoutubePlaylist
 
 
 class YoutubeChannel(YouTubeItem):
@@ -23,7 +24,7 @@ class YoutubeChannel(YouTubeItem):
     index_name = "ta_channel"
     yt_base = "https://www.youtube.com/channel/"
     yt_obs = {
-        "playlist_items": "1,0",
+        "playlist_items": "0,0",
         "skip_download": True,
     }
 
@@ -42,6 +43,10 @@ class YoutubeChannel(YouTubeItem):
         if not self.youtube_meta and fallback:
             self._video_fallback(fallback)
         else:
+            if not self.youtube_meta:
+                message = f"{self.youtube_id}: Failed to get metadata"
+                raise ValueError(message)
+
             self.process_youtube_meta()
             self.get_channel_art()
 
@@ -51,29 +56,22 @@ class YoutubeChannel(YouTubeItem):
     def process_youtube_meta(self):
         """extract relevant fields"""
         self.youtube_meta["thumbnails"].reverse()
+        channel_name = self.youtube_meta["uploader"] or self.youtube_meta["id"]
         self.json_data = {
             "channel_active": True,
-            "channel_description": self.youtube_meta.get("description", False),
+            "channel_description": self.youtube_meta.get("description", ""),
             "channel_id": self.youtube_id,
             "channel_last_refresh": int(datetime.now().timestamp()),
-            "channel_name": self.youtube_meta["uploader"],
+            "channel_name": channel_name,
             "channel_subs": self.youtube_meta.get("channel_follower_count", 0),
             "channel_subscribed": False,
-            "channel_tags": self._parse_tags(self.youtube_meta.get("tags")),
+            "channel_tags": self.youtube_meta.get("tags", []),
             "channel_banner_url": self._get_banner_art(),
             "channel_thumb_url": self._get_thumb_art(),
             "channel_tvart_url": self._get_tv_art(),
             "channel_views": self.youtube_meta.get("view_count") or 0,
         }
 
-    def _parse_tags(self, tags):
-        """parse channel tags"""
-        if not tags:
-            return False
-
-        joined = " ".join(tags)
-        return [i.strip() for i in joined.split('"') if i and not i == " "]
-
     def _get_thumb_art(self):
         """extract thumb art"""
         for i in self.youtube_meta["thumbnails"]:
@@ -119,8 +117,8 @@ class YoutubeChannel(YouTubeItem):
             "channel_tvart_url": False,
             "channel_id": self.youtube_id,
             "channel_subscribed": False,
-            "channel_tags": False,
-            "channel_description": False,
+            "channel_tags": [],
+            "channel_description": "",
             "channel_thumb_url": False,
             "channel_views": 0,
         }
@@ -160,8 +158,18 @@ class YoutubeChannel(YouTubeItem):
         # add ingest pipeline
         processors = []
         for field, value in self.json_data.items():
-            line = {"set": {"field": "channel." + field, "value": value}}
+            if value is None:
+                line = {
+                    "script": {
+                        "lang": "painless",
+                        "source": f"ctx['{field}'] = null;",
+                    }
+                }
+            else:
+                line = {"set": {"field": "channel." + field, "value": value}}
+
             processors.append(line)
+
         data = {"description": self.youtube_id, "processors": processors}
         ingest_path = f"_ingest/pipeline/{self.youtube_id}"
         _, _ = ElasticWrap(ingest_path).put(data)
@@ -255,6 +263,7 @@ class YoutubeChannel(YouTubeItem):
 
             self._index_single_playlist(playlist)
             print("add playlist: " + playlist[1])
+            rand_sleep(self.config)
 
     def _notify_single_playlist(self, idx, total):
         """send notification"""
@@ -290,6 +299,10 @@ class YoutubeChannel(YouTubeItem):
         )
         obs = {"skip_download": True, "extract_flat": True}
         playlists = YtWrap(obs, self.config).extract(url)
+        if not playlists:
+            self.all_playlists = []
+            return
+
         all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
         self.all_playlists = all_entries
 
@@ -326,22 +339,12 @@ class YoutubeChannel(YouTubeItem):
         for key, value in overwrites.items():
             if key not in valid_keys:
                 raise ValueError(f"invalid overwrite key: {key}")
-            elif value == "disable":
-                to_write[key] = False
+
+            if value is None and key in to_write:
+                to_write.pop(key)
                 continue
-            elif value == "0":
-                if key in to_write:
-                    del to_write[key]
-                continue
-            elif value == "1":
-                to_write[key] = True
-                continue
-            elif isinstance(value, int) and int(value) < 0:
-                if key in to_write:
-                    del to_write[key]
-                continue
-            elif value is not None and value != "":
-                to_write.update({key: value})
+
+            to_write.update({key: value})
 
         self.json_data["channel_overwrites"] = to_write
 
@@ -353,3 +356,5 @@ def channel_overwrites(channel_id, overwrites):
     channel.set_overwrites(overwrites)
     channel.upload_to_es()
     channel.sync_to_videos()
+
+    return channel.json_data

backend/channel/src/nav.py

@@ -0,0 +1,78 @@
+"""build channel nav"""
+
+from common.src.es_connect import ElasticWrap
+
+
+class ChannelNav:
+    """get all nav items"""
+
+    def __init__(self, channel_id):
+        self.channel_id = channel_id
+
+    def get_nav(self):
+        """build nav items"""
+        nav = {
+            "has_pending": self._get_has_pending(),
+            "has_playlists": self._get_has_playlists(),
+        }
+        nav.update(self._get_vid_types())
+        return nav
+
+    def _get_vid_types(self):
+        """get available vid_types in given channel"""
+        data = {
+            "size": 0,
+            "query": {
+                "term": {"channel.channel_id": {"value": self.channel_id}}
+            },
+            "aggs": {"unique_values": {"terms": {"field": "vid_type"}}},
+        }
+        response, _ = ElasticWrap("ta_video/_search").get(data)
+        buckets = response["aggregations"]["unique_values"]["buckets"]
+
+        type_nav = {
+            "has_videos": False,
+            "has_streams": False,
+            "has_shorts": False,
+        }
+        for bucket in buckets:
+            if bucket["key"] == "videos":
+                type_nav["has_videos"] = True
+            if bucket["key"] == "streams":
+                type_nav["has_streams"] = True
+            if bucket["key"] == "shorts":
+                type_nav["has_shorts"] = True
+
+        return type_nav
+
+    def _get_has_pending(self):
+        """check if has pending videos in download queue"""
+        data = {
+            "size": 1,
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {"status": {"value": "pending"}}},
+                        {"term": {"channel_id": {"value": self.channel_id}}},
+                    ]
+                }
+            },
+            "_source": False,
+        }
+        response, _ = ElasticWrap("ta_download/_search").get(data=data)
+
+        return bool(response["hits"]["hits"])
+
+    def _get_has_playlists(self):
+        """check if channel has playlists"""
+        path = "ta_playlist/_search"
+        data = {
+            "size": 1,
+            "query": {
+                "term": {"playlist_channel_id": {"value": self.channel_id}}
+            },
+            "_source": False,
+        }
+        response, _ = ElasticWrap(path).get(data=data)
+
+        return bool(response["hits"]["hits"])

backend/channel/urls.py

@@ -0,0 +1,32 @@
+"""all channel API urls"""
+
+from channel import views
+from django.urls import path
+
+urlpatterns = [
+    path(
+        "",
+        views.ChannelApiListView.as_view(),
+        name="api-channel-list",
+    ),
+    path(
+        "search/",
+        views.ChannelApiSearchView.as_view(),
+        name="api-channel-search",
+    ),
+    path(
+        "<slug:channel_id>/",
+        views.ChannelApiView.as_view(),
+        name="api-channel",
+    ),
+    path(
+        "<slug:channel_id>/aggs/",
+        views.ChannelAggsApiView.as_view(),
+        name="api-channel-aggs",
+    ),
+    path(
+        "<slug:channel_id>/nav/",
+        views.ChannelNavApiView.as_view(),
+        name="api-channel-nav",
+    ),
+]

backend/channel/views.py

@@ -0,0 +1,281 @@
+"""all channel API views"""
+
+from channel.serializers import (
+    ChannelAggSerializer,
+    ChannelListQuerySerializer,
+    ChannelListSerializer,
+    ChannelNavSerializer,
+    ChannelSearchQuerySerializer,
+    ChannelSerializer,
+    ChannelUpdateSerializer,
+)
+from channel.src.index import YoutubeChannel, channel_overwrites
+from channel.src.nav import ChannelNav
+from common.serializers import ErrorResponseSerializer
+from common.src.urlparser import Parser
+from common.views_base import AdminWriteOnly, ApiBaseView
+from download.src.subscriptions import ChannelSubscription
+from drf_spectacular.utils import (
+    OpenApiParameter,
+    OpenApiResponse,
+    extend_schema,
+)
+from rest_framework.response import Response
+from task.tasks import index_channel_playlists, subscribe_to
+
+
+class ChannelApiListView(ApiBaseView):
+    """resolves to /api/channel/
+    GET: returns list of channels
+    POST: edit a list of channels
+    """
+
+    search_base = "ta_channel/_search/"
+    valid_filter = ["subscribed"]
+    permission_classes = [AdminWriteOnly]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(ChannelListSerializer()),
+        },
+        parameters=[ChannelListQuerySerializer()],
+    )
+    def get(self, request):
+        """get request"""
+        self.data.update(
+            {"sort": [{"channel_name.keyword": {"order": "asc"}}]}
+        )
+
+        serializer = ChannelListQuerySerializer(data=request.query_params)
+        serializer.is_valid(raise_exception=True)
+        validated_data = serializer.validated_data
+
+        must_list = []
+        query_filter = validated_data.get("filter")
+        if query_filter:
+            must_list.append({"term": {"channel_subscribed": {"value": True}}})
+
+        self.data["query"] = {"bool": {"must": must_list}}
+        self.get_document_list(request)
+        serializer = ChannelListSerializer(self.response)
+
+        return Response(serializer.data)
+
+    def post(self, request):
+        """subscribe/unsubscribe to list of channels"""
+        data = request.data
+        try:
+            to_add = data["data"]
+        except KeyError:
+            message = "missing expected data key"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        pending = []
+        for channel_item in to_add:
+            channel_id = channel_item["channel_id"]
+            if channel_item["channel_subscribed"]:
+                pending.append(channel_id)
+            else:
+                self._unsubscribe(channel_id)
+
+        if pending:
+            url_str = " ".join(pending)
+            subscribe_to.delay(url_str, expected_type="channel")
+
+        return Response(data)
+
+    @staticmethod
+    def _unsubscribe(channel_id: str):
+        """unsubscribe"""
+        print(f"[{channel_id}] unsubscribe from channel")
+        ChannelSubscription().change_subscribe(
+            channel_id, channel_subscribed=False
+        )
+
+
+class ChannelApiView(ApiBaseView):
+    """resolves to /api/channel/<channel_id>/
+    GET: returns metadata dict of channel
+    """
+
+    search_base = "ta_channel/_doc/"
+    permission_classes = [AdminWriteOnly]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(ChannelSerializer()),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="Channel not found"
+            ),
+        }
+    )
+    def get(self, request, channel_id):
+        # pylint: disable=unused-argument
+        """get channel detail"""
+        self.get_document(channel_id)
+        if not self.response:
+            error = ErrorResponseSerializer({"error": "channel not found"})
+            return Response(error.data, status=404)
+
+        response_serializer = ChannelSerializer(self.response)
+        return Response(response_serializer.data, status=self.status_code)
+
+    @extend_schema(
+        request=ChannelUpdateSerializer(),
+        responses={
+            200: OpenApiResponse(ChannelUpdateSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="Channel not found"
+            ),
+        },
+    )
+    def post(self, request, channel_id):
+        """modify channel"""
+        self.get_document(channel_id)
+        if not self.response:
+            error = ErrorResponseSerializer({"error": "channel not found"})
+            return Response(error.data, status=404)
+
+        serializer = ChannelUpdateSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        validated_data = serializer.validated_data
+
+        subscribed = validated_data.get("channel_subscribed")
+        if subscribed is not None:
+            ChannelSubscription().change_subscribe(channel_id, subscribed)
+
+        overwrites = validated_data.get("channel_overwrites")
+        if overwrites:
+            channel_overwrites(channel_id, overwrites)
+            if overwrites.get("index_playlists"):
+                index_channel_playlists.delay(channel_id)
+
+        return Response(serializer.data, status=200)
+
+    @extend_schema(
+        responses={
+            204: OpenApiResponse(description="Channel deleted"),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="Channel not found"
+            ),
+        },
+    )
+    def delete(self, request, channel_id):
+        # pylint: disable=unused-argument
+        """delete channel"""
+        try:
+            YoutubeChannel(channel_id).delete_channel()
+            return Response(status=204)
+        except FileNotFoundError:
+            pass
+
+        error = ErrorResponseSerializer({"error": "channel not found"})
+        return Response(error.data, status=404)
+
+
+class ChannelAggsApiView(ApiBaseView):
+    """resolves to /api/channel/<channel_id>/aggs/
+    GET: get channel aggregations
+    """
+
+    search_base = "ta_video/_search"
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(ChannelAggSerializer()),
+        },
+    )
+    def get(self, request, channel_id):
+        """get channel aggregations"""
+        self.data.update(
+            {
+                "query": {
+                    "term": {"channel.channel_id": {"value": channel_id}}
+                },
+                "aggs": {
+                    "total_items": {"value_count": {"field": "youtube_id"}},
+                    "total_size": {"sum": {"field": "media_size"}},
+                    "total_duration": {"sum": {"field": "player.duration"}},
+                },
+            }
+        )
+        self.get_aggs()
+        serializer = ChannelAggSerializer(self.response)
+
+        return Response(serializer.data)
+
+
+class ChannelNavApiView(ApiBaseView):
+    """resolves to /api/channel/<channel_id>/nav/
+    GET: get channel nav
+    """
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(ChannelNavSerializer()),
+        },
+    )
+    def get(self, request, channel_id):
+        """get navigation"""
+
+        nav = ChannelNav(channel_id).get_nav()
+        serializer = ChannelNavSerializer(nav)
+        return Response(serializer.data)
+
+
+class ChannelApiSearchView(ApiBaseView):
+    """resolves to /api/channel/search/
+    search for channel
+    """
+
+    search_base = "ta_channel/_doc/"
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(ChannelSerializer()),
+            400: OpenApiResponse(description="Bad Request"),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="Channel not found"
+            ),
+        },
+        parameters=[
+            OpenApiParameter(
+                name="q",
+                description="Search query string",
+                required=True,
+                type=str,
+            ),
+        ],
+    )
+    def get(self, request):
+        """search for local channel ID"""
+
+        serializer = ChannelSearchQuerySerializer(data=request.query_params)
+        serializer.is_valid(raise_exception=True)
+        validated_data = serializer.validated_data
+
+        query = validated_data.get("q")
+        if not query:
+            message = "missing expected q parameter"
+            return Response({"message": message, "data": False}, status=400)
+
+        try:
+            parsed = Parser(query).parse()[0]
+        except (ValueError, IndexError, AttributeError):
+            error = ErrorResponseSerializer(
+                {"error": f"channel not found: {query}"}
+            )
+            return Response(error.data, status=404)
+
+        if not parsed["type"] == "channel":
+            error = ErrorResponseSerializer({"error": "expected channel data"})
+            return Response(error.data, status=400)
+
+        self.get_document(parsed["url"])
+        serializer = ChannelSerializer(self.response)
+
+        return Response(serializer.data, status=self.status_code)

backend/common/serializers.py

@@ -0,0 +1,142 @@
+"""common serializers"""
+
+# pylint: disable=abstract-method
+
+from rest_framework import serializers
+
+
+class ValidateUnknownFieldsMixin:
+    """
+    Mixin to validate and reject unknown fields in a serializer.
+    """
+
+    def to_internal_value(self, data):
+        """check expected keys"""
+        allowed_fields = set(self.fields.keys())
+        input_fields = set(data.keys())
+
+        # Find unknown fields
+        unknown_fields = input_fields - allowed_fields
+        if unknown_fields:
+            raise serializers.ValidationError(
+                {"error": f"Unknown fields: {', '.join(unknown_fields)}"}
+            )
+
+        return super().to_internal_value(data)
+
+
+class ErrorResponseSerializer(serializers.Serializer):
+    """error message"""
+
+    error = serializers.CharField()
+
+
+class PaginationSerializer(serializers.Serializer):
+    """serialize paginate response"""
+
+    page_size = serializers.IntegerField()
+    page_from = serializers.IntegerField()
+    prev_pages = serializers.ListField(
+        child=serializers.IntegerField(), allow_null=True
+    )
+    current_page = serializers.IntegerField()
+    max_hits = serializers.BooleanField()
+    params = serializers.CharField()
+    last_page = serializers.IntegerField()
+    next_pages = serializers.ListField(
+        child=serializers.IntegerField(), allow_null=True
+    )
+    total_hits = serializers.IntegerField()
+
+
+class AsyncTaskResponseSerializer(serializers.Serializer):
+    """serialize new async task"""
+
+    message = serializers.CharField(required=False)
+    task_id = serializers.CharField()
+    status = serializers.CharField(required=False)
+    filename = serializers.CharField(required=False)
+
+
+class NotificationSerializer(serializers.Serializer):
+    """serialize notification messages"""
+
+    id = serializers.CharField()
+    title = serializers.CharField()
+    group = serializers.CharField()
+    api_start = serializers.BooleanField()
+    api_stop = serializers.BooleanField()
+    level = serializers.ChoiceField(choices=["info", "error"])
+    messages = serializers.ListField(child=serializers.CharField())
+    progress = serializers.FloatField(required=False)
+
+
+class NotificationQueryFilterSerializer(serializers.Serializer):
+    """serialize notification query filter"""
+
+    filter = serializers.ChoiceField(
+        choices=["download", "settings", "channel"], required=False
+    )
+
+
+class PingUpdateSerializer(serializers.Serializer):
+    """serialize update notification"""
+
+    status = serializers.BooleanField()
+    version = serializers.CharField()
+    is_breaking = serializers.BooleanField()
+
+
+class PingSerializer(serializers.Serializer):
+    """serialize ping response"""
+
+    response = serializers.ChoiceField(choices=["pong"])
+    user = serializers.IntegerField()
+    version = serializers.CharField()
+    ta_update = PingUpdateSerializer(required=False)
+
+
+class WatchedDataSerializer(serializers.Serializer):
+    """mark as watched serializer"""
+
+    id = serializers.CharField()
+    is_watched = serializers.BooleanField()
+
+
+class RefreshQuerySerializer(serializers.Serializer):
+    """refresh query filtering"""
+
+    type = serializers.ChoiceField(
+        choices=["video", "channel", "playlist"], required=False
+    )
+    id = serializers.CharField(required=False)
+
+
+class RefreshResponseSerializer(serializers.Serializer):
+    """serialize refresh response"""
+
+    state = serializers.ChoiceField(
+        choices=["running", "queued", "empty", False]
+    )
+    total_queued = serializers.IntegerField()
+    in_queue_name = serializers.CharField(required=False)
+
+
+class RefreshAddQuerySerializer(serializers.Serializer):
+    """serialize add to refresh queue"""
+
+    extract_videos = serializers.BooleanField(required=False)
+
+
+class RefreshAddDataSerializer(serializers.Serializer):
+    """add to refresh queue serializer"""
+
+    video = serializers.ListField(
+        child=serializers.CharField(), required=False
+    )
+    channel = serializers.ListField(
+        child=serializers.CharField(), required=False
+    )
+    playlist = serializers.ListField(
+        child=serializers.CharField(), required=False
+    )

backend/common/src/env_settings.py

@@ -6,6 +6,14 @@ Functionality:
 
 from os import environ
 
+try:
+    from dotenv import load_dotenv
+
+    print("loading local dotenv")
+    load_dotenv(".env")
+except ModuleNotFoundError:
+    pass
+
 
 class EnvironmentSettings:
     """
@@ -16,10 +24,10 @@ class EnvironmentSettings:
 
     HOST_UID: int = int(environ.get("HOST_UID", False))
     HOST_GID: int = int(environ.get("HOST_GID", False))
-    ENABLE_CAST: bool = bool(environ.get("ENABLE_CAST"))
+    DISABLE_STATIC_AUTH: bool = bool(environ.get("DISABLE_STATIC_AUTH"))
     TZ: str = str(environ.get("TZ", "UTC"))
     TA_PORT: int = int(environ.get("TA_PORT", False))
-    TA_UWSGI_PORT: int = int(environ.get("TA_UWSGI_PORT", False))
+    TA_BACKEND_PORT: int = int(environ.get("TA_BACKEND_PORT", False))
     TA_USERNAME: str = str(environ.get("TA_USERNAME"))
     TA_PASSWORD: str = str(environ.get("TA_PASSWORD"))
 
@@ -29,8 +37,7 @@ class EnvironmentSettings:
     CACHE_DIR: str = str(environ.get("TA_CACHE_DIR", "/cache"))
 
     # Redis
-    REDIS_HOST: str = str(environ.get("REDIS_HOST"))
-    REDIS_PORT: int = int(environ.get("REDIS_PORT", 6379))
+    REDIS_CON: str = str(environ.get("REDIS_CON"))
     REDIS_NAME_SPACE: str = str(environ.get("REDIS_NAME_SPACE", "ta:"))
 
     # ElasticSearch
@@ -44,6 +51,20 @@ class EnvironmentSettings:
     )
     ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL"))
 
+    def get_cache_root(self):
+        """get root for web server"""
+        if self.CACHE_DIR.startswith("/"):
+            return self.CACHE_DIR
+
+        return f"/{self.CACHE_DIR}"
+
+    def get_media_root(self):
+        """get root for media folder"""
+        if self.MEDIA_DIR.startswith("/"):
+            return self.MEDIA_DIR
+
+        return f"/{self.MEDIA_DIR}"
+
     def print_generic(self):
         """print generic env vars"""
         print(
@@ -51,9 +72,9 @@ class EnvironmentSettings:
             HOST_UID: {self.HOST_UID}
             HOST_GID: {self.HOST_GID}
             TZ: {self.TZ}
-            ENABLE_CAST: {self.ENABLE_CAST}
+            DISABLE_STATIC_AUTH: {self.DISABLE_STATIC_AUTH}
             TA_PORT: {self.TA_PORT}
-            TA_UWSGI_PORT: {self.TA_UWSGI_PORT}
+            TA_BACKEND_PORT: {self.TA_BACKEND_PORT}
             TA_USERNAME: {self.TA_USERNAME}
             TA_PASSWORD: *****"""
         )
@@ -71,8 +92,7 @@ class EnvironmentSettings:
         """debug redis conf paths"""
         print(
             f"""
-            REDIS_HOST: {self.REDIS_HOST}
-            REDIS_PORT: {self.REDIS_PORT}
+            REDIS_CON: {self.REDIS_CON}
             REDIS_NAME_SPACE: {self.REDIS_NAME_SPACE}"""
         )
 

backend/common/src/es_connect.py

@@ -11,7 +11,7 @@ from typing import Any
 
 import requests
 import urllib3
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
 
 
 class ElasticWrap:

backend/common/src/helper.py

@@ -1,265 +1,299 @@
-"""
-Loose collection of helper functions
-- don't import AppConfig class here to avoid circular imports
-"""
-
-import json
-import os
-import random
-import string
-import subprocess
-from datetime import datetime
-from typing import Any
-from urllib.parse import urlparse
-
-import requests
-from home.src.es.connect import IndexPaginate
-from home.src.ta.settings import EnvironmentSettings
-
-
-def ignore_filelist(filelist: list[str]) -> list[str]:
-    """ignore temp files for os.listdir sanitizer"""
-    to_ignore = [
-        "@eaDir",
-        "Icon\r\r",
-        "Network Trash Folder",
-        "Temporary Items",
-    ]
-    cleaned: list[str] = []
-    for file_name in filelist:
-        if file_name.startswith(".") or file_name in to_ignore:
-            continue
-
-        cleaned.append(file_name)
-
-    return cleaned
-
-
-def randomizor(length: int) -> str:
-    """generate random alpha numeric string"""
-    pool: str = string.digits + string.ascii_letters
-    return "".join(random.choice(pool) for i in range(length))
-
-
-def requests_headers() -> dict[str, str]:
-    """build header with random user agent for requests outside of yt-dlp"""
-
-    chrome_versions = (
-        "90.0.4430.212",
-        "90.0.4430.24",
-        "90.0.4430.70",
-        "90.0.4430.72",
-        "90.0.4430.85",
-        "90.0.4430.93",
-        "91.0.4472.101",
-        "91.0.4472.106",
-        "91.0.4472.114",
-        "91.0.4472.124",
-        "91.0.4472.164",
-        "91.0.4472.19",
-        "91.0.4472.77",
-        "92.0.4515.107",
-        "92.0.4515.115",
-        "92.0.4515.131",
-        "92.0.4515.159",
-        "92.0.4515.43",
-        "93.0.4556.0",
-        "93.0.4577.15",
-        "93.0.4577.63",
-        "93.0.4577.82",
-        "94.0.4606.41",
-        "94.0.4606.54",
-        "94.0.4606.61",
-        "94.0.4606.71",
-        "94.0.4606.81",
-        "94.0.4606.85",
-        "95.0.4638.17",
-        "95.0.4638.50",
-        "95.0.4638.54",
-        "95.0.4638.69",
-        "95.0.4638.74",
-        "96.0.4664.18",
-        "96.0.4664.45",
-        "96.0.4664.55",
-        "96.0.4664.93",
-        "97.0.4692.20",
-    )
-    template = (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        + "AppleWebKit/537.36 (KHTML, like Gecko) "
-        + f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
-    )
-
-    return {"User-Agent": template}
-
-
-def date_parser(timestamp: int | str) -> str:
-    """return formatted date string"""
-    if isinstance(timestamp, int):
-        date_obj = datetime.fromtimestamp(timestamp)
-    elif isinstance(timestamp, str):
-        date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
-    else:
-        raise TypeError(f"invalid timestamp: {timestamp}")
-
-    return date_obj.date().isoformat()
-
-
-def time_parser(timestamp: str) -> float:
-    """return seconds from timestamp, false on empty"""
-    if not timestamp:
-        return False
-
-    if timestamp.isnumeric():
-        return int(timestamp)
-
-    hours, minutes, seconds = timestamp.split(":", maxsplit=3)
-    return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
-
-
-def clear_dl_cache(cache_dir: str) -> int:
-    """clear leftover files from dl cache"""
-    print("clear download cache")
-    download_cache_dir = os.path.join(cache_dir, "download")
-    leftover_files = ignore_filelist(os.listdir(download_cache_dir))
-    for cached in leftover_files:
-        to_delete = os.path.join(download_cache_dir, cached)
-        os.remove(to_delete)
-
-    return len(leftover_files)
-
-
-def get_mapping() -> dict:
-    """read index_mapping.json and get expected mapping and settings"""
-    with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
-        index_config: dict = json.load(f).get("index_config")
-
-    return index_config
-
-
-def is_shorts(youtube_id: str) -> bool:
-    """check if youtube_id is a shorts video, bot not it it's not a shorts"""
-    shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
-    cookies = {"SOCS": "CAI"}
-    response = requests.head(
-        shorts_url, cookies=cookies, headers=requests_headers(), timeout=10
-    )
-
-    return response.status_code == 200
-
-
-def get_duration_sec(file_path: str) -> int:
-    """get duration of media file from file path"""
-
-    duration = subprocess.run(
-        [
-            "ffprobe",
-            "-v",
-            "error",
-            "-show_entries",
-            "format=duration",
-            "-of",
-            "default=noprint_wrappers=1:nokey=1",
-            file_path,
-        ],
-        capture_output=True,
-        check=True,
-    )
-    duration_raw = duration.stdout.decode().strip()
-    if duration_raw == "N/A":
-        return 0
-
-    duration_sec = int(float(duration_raw))
-    return duration_sec
-
-
-def get_duration_str(seconds: int) -> str:
-    """Return a human-readable duration string from seconds."""
-    if not seconds:
-        return "NA"
-
-    units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
-    duration_parts = []
-
-    for unit_label, unit_seconds in units:
-        if seconds >= unit_seconds:
-            unit_count, seconds = divmod(seconds, unit_seconds)
-            duration_parts.append(f"{unit_count:02}{unit_label}")
-
-    duration_parts[0] = duration_parts[0].lstrip("0")
-
-    return " ".join(duration_parts)
-
-
-def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
-    """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
-    allowed_hosts: list[str] = [
-        "localhost",
-        "tubearchivist",
-    ]
-    csrf_trusted_origins: list[str] = [
-        "http://localhost",
-        "http://tubearchivist",
-    ]
-    for host in ta_host.split():
-        host_clean = host.strip()
-        if not host_clean.startswith("http"):
-            host_clean = f"http://{host_clean}"
-
-        parsed = urlparse(host_clean)
-        allowed_hosts.append(f"{parsed.hostname}")
-        csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}")
-
-    return allowed_hosts, csrf_trusted_origins
-
-
-def get_stylesheets():
-    """Get all valid stylesheets from /static/css"""
-    app_root = EnvironmentSettings.APP_DIR
-    stylesheets = os.listdir(os.path.join(app_root, "static/css"))
-    stylesheets.remove("style.css")
-    stylesheets.sort()
-    stylesheets = list(filter(lambda x: x.endswith(".css"), stylesheets))
-    return stylesheets
-
-
-def check_stylesheet(stylesheet: str):
-    """Check if a stylesheet exists. Return dark.css as a fallback"""
-    if stylesheet in get_stylesheets():
-        return stylesheet
-
-    return "dark.css"
-
-
-def is_missing(
-    to_check: str | list[str],
-    index_name: str = "ta_video,ta_download",
-    on_key: str = "youtube_id",
-) -> list[str]:
-    """id or list of ids that are missing from index_name"""
-    if isinstance(to_check, str):
-        to_check = [to_check]
-
-    data = {
-        "query": {"terms": {on_key: to_check}},
-        "_source": [on_key],
-    }
-    result = IndexPaginate(index_name, data=data).get_results()
-    existing_ids = [i[on_key] for i in result]
-    dl = [i for i in to_check if i not in existing_ids]
-
-    return dl
-
-
-def get_channel_overwrites() -> dict[str, dict[str, Any]]:
-    """get overwrites indexed my channel_id"""
-    data = {
-        "query": {
-            "bool": {"must": [{"exists": {"field": "channel_overwrites"}}]}
-        },
-        "_source": ["channel_id", "channel_overwrites"],
-    }
-    result = IndexPaginate("ta_channel", data).get_results()
-    overwrites = {i["channel_id"]: i["channel_overwrites"] for i in result}
-
-    return overwrites
+"""
+Loose collection of helper functions
+- don't import AppConfig class here to avoid circular imports
+"""
+
+import json
+import os
+import random
+import string
+import subprocess
+from datetime import datetime
+from time import sleep
+from typing import Any
+from urllib.parse import urlparse
+
+import requests
+from common.src.es_connect import IndexPaginate
+
+
+def ignore_filelist(filelist: list[str]) -> list[str]:
+    """ignore temp files for os.listdir sanitizer"""
+    to_ignore = [
+        "@eaDir",
+        "Icon\r\r",
+        "Network Trash Folder",
+        "Temporary Items",
+    ]
+    cleaned: list[str] = []
+    for file_name in filelist:
+        if file_name.startswith(".") or file_name in to_ignore:
+            continue
+
+        cleaned.append(file_name)
+
+    return cleaned
+
+
+def randomizor(length: int) -> str:
+    """generate random alpha numeric string"""
+    pool: str = string.digits + string.ascii_letters
+    return "".join(random.choice(pool) for i in range(length))
+
+
+def rand_sleep(config) -> None:
+    """randomized sleep based on config"""
+    sleep_config = config["downloads"].get("sleep_interval")
+    if not sleep_config:
+        return
+
+    secs = random.randrange(int(sleep_config * 0.5), int(sleep_config * 1.5))
+    sleep(secs)
+
+
+def requests_headers() -> dict[str, str]:
+    """build header with random user agent for requests outside of yt-dlp"""
+
+    chrome_versions = (
+        "90.0.4430.212",
+        "90.0.4430.24",
+        "90.0.4430.70",
+        "90.0.4430.72",
+        "90.0.4430.85",
+        "90.0.4430.93",
+        "91.0.4472.101",
+        "91.0.4472.106",
+        "91.0.4472.114",
+        "91.0.4472.124",
+        "91.0.4472.164",
+        "91.0.4472.19",
+        "91.0.4472.77",
+        "92.0.4515.107",
+        "92.0.4515.115",
+        "92.0.4515.131",
+        "92.0.4515.159",
+        "92.0.4515.43",
+        "93.0.4556.0",
+        "93.0.4577.15",
+        "93.0.4577.63",
+        "93.0.4577.82",
+        "94.0.4606.41",
+        "94.0.4606.54",
+        "94.0.4606.61",
+        "94.0.4606.71",
+        "94.0.4606.81",
+        "94.0.4606.85",
+        "95.0.4638.17",
+        "95.0.4638.50",
+        "95.0.4638.54",
+        "95.0.4638.69",
+        "95.0.4638.74",
+        "96.0.4664.18",
+        "96.0.4664.45",
+        "96.0.4664.55",
+        "96.0.4664.93",
+        "97.0.4692.20",
+    )
+    template = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        + "AppleWebKit/537.36 (KHTML, like Gecko) "
+        + f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
+    )
+
+    return {"User-Agent": template}
+
+
+def date_parser(timestamp: int | str) -> str:
+    """return formatted date string"""
+    if isinstance(timestamp, int):
+        date_obj = datetime.fromtimestamp(timestamp)
+    elif isinstance(timestamp, str):
+        date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
+    else:
+        raise TypeError(f"invalid timestamp: {timestamp}")
+
+    return date_obj.date().isoformat()
+
+
+def time_parser(timestamp: str) -> float:
+    """return seconds from timestamp, false on empty"""
+    if not timestamp:
+        return False
+
+    if timestamp.isnumeric():
+        return int(timestamp)
+
+    hours, minutes, seconds = timestamp.split(":", maxsplit=3)
+    return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
+
+
+def clear_dl_cache(cache_dir: str) -> int:
+    """clear leftover files from dl cache"""
+    print("clear download cache")
+    download_cache_dir = os.path.join(cache_dir, "download")
+    leftover_files = ignore_filelist(os.listdir(download_cache_dir))
+    for cached in leftover_files:
+        to_delete = os.path.join(download_cache_dir, cached)
+        os.remove(to_delete)
+
+    return len(leftover_files)
+
+
+def get_mapping() -> dict:
+    """read index_mapping.json and get expected mapping and settings"""
+    with open("appsettings/index_mapping.json", "r", encoding="utf-8") as f:
+        index_config: dict = json.load(f).get("index_config")
+
+    return index_config
+
+
+def is_shorts(youtube_id: str) -> bool:
+    """check if youtube_id is a shorts video, bot not it it's not a shorts"""
+    shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
+    cookies = {"SOCS": "CAI"}
+    response = requests.head(
+        shorts_url, cookies=cookies, headers=requests_headers(), timeout=10
+    )
+
+    return response.status_code == 200
+
+
+def get_duration_sec(file_path: str) -> int:
+    """get duration of media file from file path"""
+
+    duration = subprocess.run(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            file_path,
+        ],
+        capture_output=True,
+        check=True,
+    )
+    duration_raw = duration.stdout.decode().strip()
+    if duration_raw == "N/A":
+        return 0
+
+    duration_sec = int(float(duration_raw))
+    return duration_sec
+
+
+def get_duration_str(seconds: int) -> str:
+    """Return a human-readable duration string from seconds."""
+    if not seconds:
+        return "NA"
+
+    units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
+    duration_parts = []
+
+    for unit_label, unit_seconds in units:
+        if seconds >= unit_seconds:
+            unit_count, seconds = divmod(seconds, unit_seconds)
+            duration_parts.append(f"{unit_count:02}{unit_label}")
+
+    duration_parts[0] = duration_parts[0].lstrip("0")
+
+    return " ".join(duration_parts)
+
+
+def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
+    """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
+    allowed_hosts: list[str] = [
+        "localhost",
+        "tubearchivist",
+    ]
+    csrf_trusted_origins: list[str] = [
+        "http://localhost",
+        "http://tubearchivist",
+    ]
+    for host in ta_host.split():
+        host_clean = host.strip()
+        if not host_clean.startswith("http"):
+            host_clean = f"http://{host_clean}"
+
+        parsed = urlparse(host_clean)
+        allowed_hosts.append(f"{parsed.hostname}")
+        cors_url = f"{parsed.scheme}://{parsed.hostname}"
+
+        if parsed.port:
+            cors_url = f"{cors_url}:{parsed.port}"
+
+        csrf_trusted_origins.append(cors_url)
+
+    return allowed_hosts, csrf_trusted_origins
+
+
+def get_stylesheets() -> list:
+    """Get all valid stylesheets from /static/css"""
+
+    stylesheets = [
+        "dark.css",
+        "light.css",
+        "matrix.css",
+        "midnight.css",
+        "custom.css",
+    ]
+    return stylesheets
+
+
+def check_stylesheet(stylesheet: str):
+    """Check if a stylesheet exists. Return dark.css as a fallback"""
+    if stylesheet in get_stylesheets():
+        return stylesheet
+
+    return "dark.css"
+
+
+def is_missing(
+    to_check: str | list[str],
+    index_name: str = "ta_video,ta_download",
+    on_key: str = "youtube_id",
+) -> list[str]:
+    """id or list of ids that are missing from index_name"""
+    if isinstance(to_check, str):
+        to_check = [to_check]
+
+    data = {
+        "query": {"terms": {on_key: to_check}},
+        "_source": [on_key],
+    }
+    result = IndexPaginate(index_name, data=data).get_results()
+    existing_ids = [i[on_key] for i in result]
+    dl = [i for i in to_check if i not in existing_ids]
+
+    return dl
+
+
+def get_channel_overwrites() -> dict[str, dict[str, Any]]:
+    """get overwrites indexed my channel_id"""
+    data = {
+        "query": {
+            "bool": {"must": [{"exists": {"field": "channel_overwrites"}}]}
+        },
+        "_source": ["channel_id", "channel_overwrites"],
+    }
+    result = IndexPaginate("ta_channel", data).get_results()
+    overwrites = {i["channel_id"]: i["channel_overwrites"] for i in result}
+
+    return overwrites
+
+
+def calc_is_watched(duration: float, position: float) -> bool:
+    """considered watched based on duration position"""
+
+    if not duration or duration <= 0:
+        return False
+
+    if duration < 60:
+        threshold = 0.5
+    elif duration > 900:
+        threshold = 1 - (180 / duration)
+    else:
+        threshold = 0.9
+
+    return position >= duration * threshold

backend/common/src/index_generic.py

@@ -5,10 +5,10 @@ functionality:
 
 import math
 
-from home.src.download.yt_dlp_base import YtWrap
-from home.src.es.connect import ElasticWrap
-from home.src.ta.config import AppConfig
-from home.src.ta.users import UserConfig
+from appsettings.src.config import AppConfig
+from common.src.es_connect import ElasticWrap
+from download.src.yt_dlp_base import YtWrap
+from user.src.user_config import UserConfig
 
 
 class YouTubeItem:
@@ -104,9 +104,9 @@ class Pagination:
     def first_guess(self):
         """build first guess before api call"""
         page_get = self.page_get
+        page_from = 0
         if page_get in [0, 1]:
-            page_from = 0
-            prev_pages = False
+            prev_pages = None
         elif page_get > 1:
             page_from = (page_get - 1) * self.page_size
             prev_pages = [

backend/common/src/search_processor.py

@@ -6,19 +6,19 @@ Functionality:
 
 import urllib.parse
 
-from home.src.download.thumbnails import ThumbManager
-from home.src.ta.helper import date_parser, get_duration_str
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.helper import date_parser, get_duration_str
+from common.src.ta_redis import RedisArchivist
+from download.src.thumbnails import ThumbManager
 
 
 class SearchProcess:
     """process search results"""
 
-    CACHE_DIR = EnvironmentSettings.CACHE_DIR
-
-    def __init__(self, response):
+    def __init__(self, response, match_video_user_progress: None | int = None):
         self.response = response
         self.processed = False
+        self.position_index = self.get_user_progress(match_video_user_progress)
 
     def process(self):
         """detect type and process"""
@@ -35,6 +35,23 @@ class SearchProcess:
 
         return self.processed
 
+    def get_user_progress(self, match_video_user_progress) -> dict | None:
+        """get user video watch progress"""
+        if not match_video_user_progress:
+            return None
+
+        query = f"{match_video_user_progress}:progress:*"
+        all_positions = RedisArchivist().list_items(query)
+        if not all_positions:
+            return None
+
+        pos_index = {
+            i["youtube_id"]: i["position"]
+            for i in all_positions
+            if not i.get("watched")
+        }
+        return pos_index
+
     def _process_result(self, result):
         """detect which type of data to process"""
         index = result["_index"]
@@ -66,7 +83,8 @@ class SearchProcess:
     def _process_channel(channel_dict):
         """run on single channel"""
         channel_id = channel_dict["channel_id"]
-        art_base = f"/cache/channels/{channel_id}"
+        cache_root = EnvironmentSettings().get_cache_root()
+        art_base = f"{cache_root}/channels/{channel_id}"
         date_str = date_parser(channel_dict["channel_last_refresh"])
         channel_dict.update(
             {
@@ -88,21 +106,43 @@ class SearchProcess:
         vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
         channel = self._process_channel(video_dict["channel"])
 
+        cache_root = EnvironmentSettings().get_cache_root()
+        media_root = EnvironmentSettings().get_media_root()
+
         if "subtitles" in video_dict:
             for idx, _ in enumerate(video_dict["subtitles"]):
                 url = video_dict["subtitles"][idx]["media_url"]
-                video_dict["subtitles"][idx]["media_url"] = f"/media/{url}"
+                video_dict["subtitles"][idx][
+                    "media_url"
+                ] = f"{media_root}/{url}"
+        else:
+            video_dict["subtitles"] = []
 
         video_dict.update(
             {
                 "channel": channel,
-                "media_url": f"/media/{media_url}",
+                "media_url": f"{media_root}/{media_url}",
                 "vid_last_refresh": vid_last_refresh,
                 "published": published,
-                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+                "vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
             }
         )
 
+        if self.position_index:
+            player_position = self.position_index.get(video_id)
+            total = video_dict["player"].get("duration")
+            if player_position and total:
+                progress = 100 * (player_position / total)
+                video_dict["player"].update(
+                    {
+                        "progress": progress,
+                        "position": player_position,
+                    }
+                )
+
+        if "playlist" not in video_dict:
+            video_dict["playlist"] = []
+
         return dict(sorted(video_dict.items()))
 
     @staticmethod
@@ -112,9 +152,11 @@ class SearchProcess:
         playlist_last_refresh = date_parser(
             playlist_dict["playlist_last_refresh"]
         )
+        cache_root = EnvironmentSettings().get_cache_root()
+        playlist_thumbnail = f"{cache_root}/playlists/{playlist_id}.jpg"
         playlist_dict.update(
             {
-                "playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg",
+                "playlist_thumbnail": playlist_thumbnail,
                 "playlist_last_refresh": playlist_last_refresh,
             }
         )
@@ -124,12 +166,13 @@ class SearchProcess:
     def _process_download(self, download_dict):
         """run on single download item"""
         video_id = download_dict["youtube_id"]
+        cache_root = EnvironmentSettings().get_cache_root()
         vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
         published = date_parser(download_dict["published"])
 
         download_dict.update(
             {
-                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+                "vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
                 "published": published,
             }
         )

backend/common/src/searching.py

@@ -6,8 +6,8 @@ Functionality:
 - calculate pagination values
 """
 
-from api.src.search_processor import SearchProcess
-from home.src.es.connect import ElasticWrap
+from common.src.es_connect import ElasticWrap
+from common.src.search_processor import SearchProcess
 
 
 class SearchForm:

backend/common/src/ta_redis.py

@@ -8,7 +8,7 @@ functionality:
 import json
 
 import redis
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
 
 
 class RedisBase:
@@ -17,10 +17,8 @@ class RedisBase:
     NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE
 
     def __init__(self):
-        self.conn = redis.Redis(
-            host=EnvironmentSettings.REDIS_HOST,
-            port=EnvironmentSettings.REDIS_PORT,
-            decode_responses=True,
+        self.conn = redis.from_url(
+            url=EnvironmentSettings.REDIS_CON, decode_responses=True
         )
 
 
@@ -40,15 +38,15 @@ class RedisArchivist(RedisBase):
     def set_message(
         self,
         key: str,
-        message: dict,
-        path: str = ".",
+        message: dict | str,
         expire: bool | int = False,
         save: bool = False,
     ) -> None:
         """write new message to redis"""
-        self.conn.execute_command(
-            "JSON.SET", self.NAME_SPACE + key, path, json.dumps(message)
+        to_write = (
+            json.dumps(message) if isinstance(message, dict) else message
         )
+        self.conn.execute_command("SET", self.NAME_SPACE + key, to_write)
 
         if expire:
             if isinstance(expire, bool):
@@ -67,8 +65,24 @@ class RedisArchivist(RedisBase):
         except redis.exceptions.ResponseError:
             pass
 
-    def get_message(self, key: str) -> dict:
-        """get message dict from redis"""
+    def get_message_str(self, key: str) -> str | None:
+        """get message string"""
+        reply = self.conn.execute_command("GET", self.NAME_SPACE + key)
+        return reply
+
+    def get_message_dict(self, key: str) -> dict:
+        """get message dict"""
+        reply = self.conn.execute_command("GET", self.NAME_SPACE + key)
+        if not reply:
+            return {}
+
+        return json.loads(reply)
+
+    def get_message(self, key: str) -> dict | None:
+        """
+        get message dict from redis
+        old json get message, only used for migration, to be removed later
+        """
         reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
         if reply:
             return json.loads(reply)
@@ -91,11 +105,14 @@ class RedisArchivist(RedisBase):
         if not all_matches:
             return []
 
-        return [self.get_message(i) for i in all_matches]
+        return [self.get_message_dict(i) for i in all_matches]
 
-    def del_message(self, key: str) -> bool:
+    def del_message(self, key: str, save: bool = False) -> bool:
         """delete key from redis"""
         response = self.conn.execute_command("DEL", self.NAME_SPACE + key)
+        if save:
+            self.bg_save()
+
         return response
 
 

backend/common/src/urlparser.py

@@ -6,15 +6,20 @@ Functionality:
 
 from urllib.parse import parse_qs, urlparse
 
-from home.src.download.yt_dlp_base import YtWrap
-from home.src.index.video_constants import VideoTypeEnum
+from common.src.ta_redis import RedisArchivist
+from download.src.yt_dlp_base import YtWrap
+from video.src.constants import VideoTypeEnum
 
 
 class Parser:
-    """take a multi line string and detect valid youtube ids"""
+    """
+    take a multi line string and detect valid youtube ids
+    channel handle lookup is cached, can be disabled for unittests
+    """
 
-    def __init__(self, url_str):
+    def __init__(self, url_str, use_cache=True):
         self.url_list = [i.strip() for i in url_str.split()]
+        self.use_cache = use_cache
 
     def parse(self):
         """parse the list"""
@@ -67,6 +72,9 @@ class Parser:
         if all_paths[0] == "channel":
             return self._validate_expected(all_paths[1], "channel")
 
+        if all_paths[0] == "live":
+            return self._validate_expected(all_paths[1], "video")
+
         # detect channel
         channel_id = self._extract_channel_name(parsed.geturl())
         return {"type": "channel", "url": channel_id}
@@ -103,9 +111,13 @@ class Parser:
 
         return {"type": item_type, "url": id_str}
 
-    @staticmethod
-    def _extract_channel_name(url):
-        """find channel id from channel name with yt-dlp help"""
+    def _extract_channel_name(self, url):
+        """find channel id from channel name with yt-dlp help, cache result"""
+        if self.use_cache:
+            cached = self._get_cached(url)
+            if cached:
+                return cached
+
         obs_request = {
             "check_formats": None,
             "skip_download": True,
@@ -113,8 +125,14 @@ class Parser:
             "playlistend": 0,
         }
         url_info = YtWrap(obs_request).extract(url)
+        if not url_info:
+            raise ValueError(f"failed to retrieve content from URL: {url}")
+
         channel_id = url_info.get("channel_id", False)
         if channel_id:
+            if self.use_cache:
+                self._set_cache(url, channel_id)
+
             return channel_id
 
         url = url_info.get("url", False)
@@ -127,6 +145,42 @@ class Parser:
         print(f"failed to extract channel id from {url}")
         raise ValueError
 
+    @staticmethod
+    def _get_cached(url) -> str | None:
+        """get cached channel ID, if available"""
+        path = urlparse(url).path.lstrip("/")
+        if not path.startswith("@"):
+            return None
+
+        handle = path.split("/")[0]
+        if not handle:
+            return None
+
+        cache_key = f"channel:handlesearch:{handle.lower()}"
+        cached = RedisArchivist().get_message_dict(cache_key)
+        if cached:
+            return cached["channel_id"]
+
+        return None
+
+    @staticmethod
+    def _set_cache(url, channel_id) -> None:
+        """set cache"""
+        path = urlparse(url).path.lstrip("/")
+        if not path.startswith("@"):
+            return
+
+        handle = path.split("/")[0]
+        if not handle:
+            return
+
+        cache_key = f"channel:handlesearch:{handle.lower()}"
+        message = {
+            "channel_id": channel_id,
+            "handle": handle,
+        }
+        RedisArchivist().set_message(cache_key, message, expire=3600 * 24 * 7)
+
     def _detect_vid_type(self, path):
         """try to match enum from path, needs to be serializable"""
         last = path.strip("/").split("/")[-1]

backend/common/src/watched.py

@@ -5,16 +5,18 @@ functionality:
 
 from datetime import datetime
 
-from home.src.es.connect import ElasticWrap
-from home.src.ta.urlparser import Parser
+from common.src.es_connect import ElasticWrap
+from common.src.ta_redis import RedisArchivist
+from common.src.urlparser import Parser
 
 
 class WatchState:
     """handle watched checkbox for videos and channels"""
 
-    def __init__(self, youtube_id, is_watched):
+    def __init__(self, youtube_id: str, is_watched: bool, user_id: int):
         self.youtube_id = youtube_id
         self.is_watched = is_watched
+        self.user_id = user_id
         self.stamp = int(datetime.now().timestamp())
         self.pipeline = f"_ingest/pipeline/watch_{youtube_id}"
 
@@ -41,15 +43,12 @@ class WatchState:
     def change_vid_state(self):
         """change watched state of video"""
         path = f"ta_video/_update/{self.youtube_id}"
-        data = {
-            "doc": {
-                "player": {
-                    "watched": self.is_watched,
-                    "watched_date": self.stamp,
-                }
-            }
-        }
+        data = {"doc": {"player": {"watched": self.is_watched}}}
+        if self.is_watched:
+            data["doc"]["player"]["watched_date"] = self.stamp
         response, status_code = ElasticWrap(path).post(data=data)
+        key = f"{self.user_id}:progress:{self.youtube_id}"
+        RedisArchivist().del_message(key)
         if status_code != 200:
             print(response)
             raise ValueError("failed to mark video as watched")

backend/common/tests/conftest.py

@@ -8,4 +8,4 @@ import pytest
 @pytest.fixture(scope="session", autouse=True)
 def change_test_dir(request):
     """change directory to project folder"""
-    os.chdir(request.config.rootdir / "tubearchivist")
+    os.chdir(request.config.rootdir / "backend")

backend/common/tests/test_src/test_helper.py

@@ -1,7 +1,7 @@
 """tests for helper functions"""
 
 import pytest
-from home.src.ta.helper import (
+from common.src.helper import (
     date_parser,
     get_duration_str,
     get_mapping,

backend/common/tests/test_src/test_urlparser.py

@@ -1,7 +1,7 @@
 """tests for url parser"""
 
 import pytest
-from home.src.ta.urlparser import Parser
+from common.src.urlparser import Parser
 
 # video id parsing
 VIDEO_URL_IN = [
@@ -10,6 +10,7 @@ VIDEO_URL_IN = [
     "https://www.youtube.com/watch?v=7DKv5H5Frt0&t=113&feature=shared",
     "https://www.youtube.com/watch?v=7DKv5H5Frt0&list=PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5&index=1&pp=iAQB"  # noqa: E501
     "https://youtu.be/7DKv5H5Frt0",
+    "https://www.youtube.com/live/7DKv5H5Frt0",
 ]
 VIDEO_OUT = [{"type": "video", "url": "7DKv5H5Frt0", "vid_type": "unknown"}]
 VIDEO_TEST_CASES = [(i, VIDEO_OUT) for i in VIDEO_URL_IN]
@@ -109,7 +110,7 @@ PASSTING_TESTS.extend(PERSONAL_PLAYLISTS_TEST_CASES)
 @pytest.mark.parametrize("url_str, expected_result", PASSTING_TESTS)
 def test_passing_parse(url_str, expected_result):
     """test parser"""
-    parser = Parser(url_str)
+    parser = Parser(url_str, use_cache=False)
     parsed = parser.parse()
     assert parsed == expected_result
 
@@ -126,7 +127,7 @@ INVALID_IDS_ERRORS = [
 def test_invalid_ids(invalid_value):
     """test for invalid IDs"""
     with pytest.raises(ValueError, match="not a valid id_str"):
-        parser = Parser(invalid_value)
+        parser = Parser(invalid_value, use_cache=False)
         parser.parse()
 
 
@@ -139,6 +140,6 @@ INVALID_DOMAINS = [
 @pytest.mark.parametrize("invalid_value", INVALID_DOMAINS)
 def test_invalid_domains(invalid_value):
     """raise error on none YT domains"""
-    parser = Parser(invalid_value)
+    parser = Parser(invalid_value, use_cache=False)
     with pytest.raises(ValueError, match="invalid domain"):
         parser.parse()

backend/common/urls.py

@@ -0,0 +1,33 @@
+"""all api urls"""
+
+from common import views
+from django.urls import path
+
+urlpatterns = [
+    path("ping/", views.PingView.as_view(), name="ping"),
+    path(
+        "refresh/",
+        views.RefreshView.as_view(),
+        name="api-refresh",
+    ),
+    path(
+        "watched/",
+        views.WatchedView.as_view(),
+        name="api-watched",
+    ),
+    path(
+        "search/",
+        views.SearchView.as_view(),
+        name="api-search",
+    ),
+    path(
+        "notification/",
+        views.NotificationView.as_view(),
+        name="api-notification",
+    ),
+    path(
+        "health/",
+        views.HealthCheck.as_view(),
+        name="api-health",
+    ),
+]

backend/common/views.py

@@ -0,0 +1,210 @@
+"""all API views"""
+
+from appsettings.src.config import ReleaseVersion
+from appsettings.src.reindex import ReindexProgress
+from common.serializers import (
+    AsyncTaskResponseSerializer,
+    ErrorResponseSerializer,
+    NotificationQueryFilterSerializer,
+    NotificationSerializer,
+    PingSerializer,
+    RefreshAddDataSerializer,
+    RefreshAddQuerySerializer,
+    RefreshQuerySerializer,
+    RefreshResponseSerializer,
+    WatchedDataSerializer,
+)
+from common.src.searching import SearchForm
+from common.src.ta_redis import RedisArchivist
+from common.src.watched import WatchState
+from common.views_base import AdminOnly, ApiBaseView
+from drf_spectacular.utils import OpenApiResponse, extend_schema
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from task.tasks import check_reindex
+
+
+class PingView(ApiBaseView):
+    """resolves to /api/ping/
+    GET: test your connection
+    """
+
+    @staticmethod
+    @extend_schema(
+        responses={200: OpenApiResponse(PingSerializer())},
+    )
+    def get(request):
+        """get pong"""
+        data = {
+            "response": "pong",
+            "user": request.user.id,
+            "version": ReleaseVersion().get_local_version(),
+            "ta_update": ReleaseVersion().get_update(),
+        }
+        serializer = PingSerializer(data)
+        return Response(serializer.data)
+
+
+class RefreshView(ApiBaseView):
+    """resolves to /api/refresh/
+    GET: get refresh progress
+    POST: start a manual refresh task
+    """
+
+    permission_classes = [AdminOnly]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(RefreshResponseSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        },
+        parameters=[RefreshQuerySerializer()],
+    )
+    def get(self, request):
+        """get refresh status"""
+        query_serializer = RefreshQuerySerializer(data=request.query_params)
+        query_serializer.is_valid(raise_exception=True)
+        validated_query = query_serializer.validated_data
+        request_type = validated_query.get("type")
+        request_id = validated_query.get("id")
+
+        if request_id and not request_type:
+            error = ErrorResponseSerializer(
+                {"error": "specified id also needs type"}
+            )
+            return Response(error.data, status=400)
+
+        try:
+            progress = ReindexProgress(
+                request_type=request_type, request_id=request_id
+            ).get_progress()
+        except ValueError:
+            error = ErrorResponseSerializer({"error": "bad request"})
+            return Response(error.data, status=400)
+
+        response_serializer = RefreshResponseSerializer(progress)
+
+        return Response(response_serializer.data)
+
+    @extend_schema(
+        request=RefreshAddDataSerializer(),
+        responses={
+            200: OpenApiResponse(AsyncTaskResponseSerializer()),
+        },
+        parameters=[RefreshAddQuerySerializer()],
+    )
+    def post(self, request):
+        """add to reindex queue"""
+        query_serializer = RefreshAddQuerySerializer(data=request.query_params)
+        query_serializer.is_valid(raise_exception=True)
+        validated_query = query_serializer.validated_data
+
+        data_serializer = RefreshAddDataSerializer(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+
+        extract_videos = validated_query.get("extract_videos")
+        task = check_reindex.delay(
+            data=validated_data, extract_videos=extract_videos
+        )
+        message = {
+            "message": "reindex task started",
+            "task_id": task.id,
+        }
+        serializer = AsyncTaskResponseSerializer(message)
+
+        return Response(serializer.data)
+
+
+class WatchedView(ApiBaseView):
+    """resolves to /api/watched/
+    POST: change watched state of video, channel or playlist
+    """
+
+    @extend_schema(
+        request=WatchedDataSerializer(),
+        responses={
+            200: OpenApiResponse(WatchedDataSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        },
+    )
+    def post(self, request):
+        """change watched state"""
+        data_serializer = WatchedDataSerializer(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+        youtube_id = validated_data.get("id")
+        is_watched = validated_data.get("is_watched")
+
+        if not youtube_id or is_watched is None:
+            error = ErrorResponseSerializer(
+                {"error": "missing id or is_watched"}
+            )
+            return Response(error.data, status=400)
+
+        WatchState(youtube_id, is_watched, request.user.id).change()
+        return Response(data_serializer.data)
+
+
+class SearchView(ApiBaseView):
+    """resolves to /api/search/
+    GET: run a search with the string in the ?query parameter
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request
+        search through all indexes"""
+        search_query = request.GET.get("query", None)
+        if search_query is None:
+            return Response(
+                {"message": "no search query specified"}, status=400
+            )
+
+        search_results = SearchForm().multi_search(search_query)
+        return Response(search_results)
+
+
+class NotificationView(ApiBaseView):
+    """resolves to /api/notification/
+    GET: returns a list of notifications
+    filter query to filter messages by group
+    """
+
+    valid_filters = ["download", "settings", "channel"]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(NotificationSerializer(many=True)),
+        },
+        parameters=[NotificationQueryFilterSerializer],
+    )
+    def get(self, request):
+        """get all notifications"""
+        query_serializer = NotificationQueryFilterSerializer(
+            data=request.query_params
+        )
+        query_serializer.is_valid(raise_exception=True)
+        validated_query = query_serializer.validated_data
+        filter_by = validated_query.get("filter")
+
+        query = "message"
+        if filter_by in self.valid_filters:
+            query = f"{query}:{filter_by}"
+
+        notifications = RedisArchivist().list_items(query)
+        response_serializer = NotificationSerializer(notifications, many=True)
+
+        return Response(response_serializer.data)
+
+
+class HealthCheck(APIView):
+    """health check view, no auth needed"""
+
+    def get(self, request):
+        """health check, no auth needed"""
+        return Response("OK", status=200)

backend/common/views_base.py

@@ -0,0 +1,102 @@
+"""base classes to inherit from"""
+
+from common.src.es_connect import ElasticWrap
+from common.src.index_generic import Pagination
+from common.src.search_processor import SearchProcess, process_aggs
+from rest_framework import permissions
+from rest_framework.authentication import (
+    SessionAuthentication,
+    TokenAuthentication,
+)
+from rest_framework.views import APIView
+
+
+def check_admin(user):
+    """check for admin permission for restricted views"""
+    return user.is_staff or user.groups.filter(name="admin").exists()
+
+
+class AdminOnly(permissions.BasePermission):
+    """allow only admin"""
+
+    def has_permission(self, request, view):
+        return check_admin(request.user)
+
+
+class AdminWriteOnly(permissions.BasePermission):
+    """allow only admin writes"""
+
+    def has_permission(self, request, view):
+        if request.method in permissions.SAFE_METHODS:
+            return permissions.IsAuthenticated().has_permission(request, view)
+
+        return check_admin(request.user)
+
+
+class ApiBaseView(APIView):
+    """base view to inherit from"""
+
+    authentication_classes = [SessionAuthentication, TokenAuthentication]
+    permission_classes = [permissions.IsAuthenticated]
+    search_base = ""
+    data = ""
+
+    def __init__(self):
+        super().__init__()
+        self.response = {}
+        self.data = {"query": {"match_all": {}}}
+        self.status_code = False
+        self.context = False
+        self.pagination_handler = False
+
+    def get_document(self, document_id, progress_match=None):
+        """get single document from es"""
+        path = f"{self.search_base}{document_id}"
+        response, status_code = ElasticWrap(path).get()
+        try:
+            self.response = SearchProcess(
+                response, match_video_user_progress=progress_match
+            ).process()
+        except KeyError:
+            print(f"item not found: {document_id}")
+
+        self.status_code = status_code
+
+    def initiate_pagination(self, request):
+        """set initial pagination values"""
+        self.pagination_handler = Pagination(request)
+        self.data.update(
+            {
+                "size": self.pagination_handler.pagination["page_size"],
+                "from": self.pagination_handler.pagination["page_from"],
+            }
+        )
+
+    def get_document_list(self, request, pagination=True, progress_match=None):
+        """get a list of results"""
+        if pagination:
+            self.initiate_pagination(request)
+
+        es_handler = ElasticWrap(self.search_base)
+        response, status_code = es_handler.get(data=self.data)
+        self.response["data"] = SearchProcess(
+            response, match_video_user_progress=progress_match
+        ).process()
+        if self.response["data"]:
+            self.status_code = status_code
+        else:
+            self.status_code = 404
+
+        if pagination and response.get("hits"):
+            self.pagination_handler.validate(
+                response["hits"]["total"]["value"]
+            )
+            self.response["paginate"] = self.pagination_handler.pagination
+
+    def get_aggs(self):
+        """get aggs alone"""
+        self.data["size"] = 0
+        response, _ = ElasticWrap(self.search_base).get(data=self.data)
+        process_aggs(response)
+
+        self.response = response.get("aggregations")

backend/config/management/commands/ta_change_password.py

@@ -0,0 +1,36 @@
+"""change user password"""
+
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand, CommandError
+
+User = get_user_model()
+
+
+class Command(BaseCommand):
+    """change password"""
+
+    help = "Change Password of user"
+
+    def add_arguments(self, parser):
+        parser.add_argument("username", type=str)
+        parser.add_argument("password", type=str)
+
+    def handle(self, *args, **kwargs):
+        """entry point"""
+        username = kwargs["username"]
+        new_password = kwargs["password"]
+        self.stdout.write(f"Changing password for user '{username}'")
+        try:
+            user = User.objects.get(name=username)
+        except User.DoesNotExist as err:
+            message = f"Username '{username}' does not exist. "
+            message += "Available username(s) are:\n"
+            message += ", ".join([i.name for i in User.objects.all()])
+            raise CommandError(message) from err
+
+        user.set_password(new_password)
+        user.save()
+
+        self.stdout.write(
+            self.style.SUCCESS(f"    ✓ updated password for user '{username}'")
+        )

backend/config/management/commands/ta_config_backup.py

@@ -0,0 +1,76 @@
+"""backup config for sqlite reset and restore"""
+
+import json
+from pathlib import Path
+
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand
+from home.models import CustomPeriodicTask
+from home.src.ta.settings import EnvironmentSettings
+from rest_framework.authtoken.models import Token
+
+User = get_user_model()
+
+
+class Command(BaseCommand):
+    """export"""
+
+    help = "Exports all users and their auth tokens to a JSON file"
+    FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
+
+    def handle(self, *args, **kwargs):
+        """entry point"""
+
+        data = {
+            "user_data": self.get_users(),
+            "schedule_data": self.get_schedules(),
+        }
+
+        with open(self.FILE, "w", encoding="utf-8") as json_file:
+            json_file.write(json.dumps(data))
+
+    def get_users(self):
+        """get users"""
+
+        users = User.objects.all()
+
+        user_data = []
+
+        for user in users:
+            user_info = {
+                "username": user.name,
+                "is_staff": user.is_staff,
+                "is_superuser": user.is_superuser,
+                "password": user.password,
+                "tokens": [],
+            }
+
+            try:
+                token = Token.objects.get(user=user)
+                user_info["tokens"] = [token.key]
+            except Token.DoesNotExist:
+                user_info["tokens"] = []
+
+            user_data.append(user_info)
+
+        return user_data
+
+    def get_schedules(self):
+        """get schedules"""
+
+        all_schedules = CustomPeriodicTask.objects.all()
+        schedule_data = []
+
+        for schedule in all_schedules:
+            schedule_info = {
+                "name": schedule.name,
+                "crontab": {
+                    "minute": schedule.crontab.minute,
+                    "hour": schedule.crontab.hour,
+                    "day_of_week": schedule.crontab.day_of_week,
+                },
+            }
+
+            schedule_data.append(schedule_info)
+
+        return schedule_data

backend/config/management/commands/ta_config_restore.py

@@ -0,0 +1,89 @@
+"""restore config from backup"""
+
+import json
+from pathlib import Path
+
+from common.src.env_settings import EnvironmentSettings
+from django.core.management.base import BaseCommand
+from django_celery_beat.models import CrontabSchedule
+from rest_framework.authtoken.models import Token
+from task.models import CustomPeriodicTask
+from task.src.task_config import TASK_CONFIG
+from user.models import Account
+
+
+class Command(BaseCommand):
+    """export"""
+
+    help = "Exports all users and their auth tokens to a JSON file"
+    FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
+
+    def handle(self, *args, **options):
+        """handle"""
+        self.stdout.write("restore users and schedules")
+        data = self.get_config()
+        self.restore_users(data["user_data"])
+        self.restore_schedules(data["schedule_data"])
+        self.stdout.write(
+            self.style.SUCCESS(
+                "    ✓ restore completed. Please restart the container."
+            )
+        )
+
+    def get_config(self) -> dict:
+        """get config from backup"""
+        with open(self.FILE, "r", encoding="utf-8") as json_file:
+            data = json.loads(json_file.read())
+
+        self.stdout.write(
+            self.style.SUCCESS(f"    ✓ json file found: {self.FILE}")
+        )
+
+        return data
+
+    def restore_users(self, user_data: list[dict]) -> None:
+        """restore users from config"""
+        self.stdout.write("delete existing users")
+        Account.objects.all().delete()
+
+        self.stdout.write("recreate users")
+        for user_info in user_data:
+            user = Account.objects.create(
+                name=user_info["username"],
+                is_staff=user_info["is_staff"],
+                is_superuser=user_info["is_superuser"],
+                password=user_info["password"],
+            )
+            for token in user_info["tokens"]:
+                Token.objects.create(user=user, key=token)
+
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"    ✓ recreated user with name: {user_info['username']}"
+                )
+            )
+
+    def restore_schedules(self, schedule_data: list[dict]) -> None:
+        """restore schedules"""
+        self.stdout.write("delete existing schedules")
+        CustomPeriodicTask.objects.all().delete()
+
+        self.stdout.write("recreate schedules")
+        for schedule in schedule_data:
+            task_name = schedule["name"]
+            description = TASK_CONFIG[task_name].get("title")
+            crontab, _ = CrontabSchedule.objects.get_or_create(
+                minute=schedule["crontab"]["minute"],
+                hour=schedule["crontab"]["hour"],
+                day_of_week=schedule["crontab"]["day_of_week"],
+                timezone=EnvironmentSettings.TZ,
+            )
+            task = CustomPeriodicTask.objects.create(
+                name=task_name,
+                task=task_name,
+                description=description,
+                crontab=crontab,
+            )
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ recreated schedule: {task}")
+            )

backend/config/management/commands/ta_connection.py

@@ -6,10 +6,10 @@ Functionality:
 from time import sleep
 
 import requests
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.ta_redis import RedisArchivist
 from django.core.management.base import BaseCommand, CommandError
-from home.src.es.connect import ElasticWrap
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisArchivist
 
 TOPIC = """
 
@@ -58,7 +58,12 @@ class Command(BaseCommand):
 
         message = "    🗙 Redis connection failed"
         self.stdout.write(self.style.ERROR(f"{message}"))
-        RedisArchivist().exec("PING")
+        try:
+            redis_conn.execute_command("PING")
+        except Exception as err:  # pylint: disable=broad-except
+            message = f"    🗙 {type(err).__name__}: {err}"
+            self.stdout.write(self.style.ERROR(f"{message}"))
+
         sleep(60)
         raise CommandError(message)
 
@@ -89,6 +94,10 @@ class Command(BaseCommand):
                 sleep(5)
                 continue
 
+            if status_code and status_code == 401:
+                sleep(5)
+                continue
+
             if status_code and status_code == 200:
                 path = "_cluster/health?wait_for_status=yellow&timeout=60s"
                 _, _ = ElasticWrap(path).get(timeout=60)

backend/config/management/commands/ta_envcheck.py

@@ -8,10 +8,11 @@ Functionality:
 
 import os
 import re
+from time import sleep
 
+from common.src.env_settings import EnvironmentSettings
 from django.core.management.base import BaseCommand, CommandError
-from home.models import Account
-from home.src.ta.settings import EnvironmentSettings
+from user.models import Account
 
 LOGO = """
 
@@ -60,9 +61,14 @@ EXPECTED_ENV_VARS = [
     "ES_URL",
     "TA_HOST",
 ]
+UNEXPECTED_ENV_VARS = {
+    "TA_UWSGI_PORT": "Has been replaced with 'TA_BACKEND_PORT'",
+    "REDIS_HOST": "Has been replaced with 'REDIS_CON' connection string",
+    "REDIS_PORT": "Has been consolidated in 'REDIS_CON' connection string",
+    "ENABLE_CAST": "That is now a toggle in setting and DISABLE_STATIC_AUTH",
+}
 INST = "https://github.com/tubearchivist/tubearchivist#installing-and-updating"
 NGINX = "/etc/nginx/sites-available/default"
-UWSGI = "/app/uwsgi.ini"
 
 
 class Command(BaseCommand):
@@ -76,10 +82,11 @@ class Command(BaseCommand):
         self.stdout.write(LOGO)
         self.stdout.write(TOPIC)
         self._expected_vars()
+        self._unexpected_vars()
         self._elastic_user_overwrite()
         self._ta_port_overwrite()
-        self._ta_uwsgi_overwrite()
-        self._enable_cast_overwrite()
+        self._ta_backend_port_overwrite()
+        self._disable_static_auth()
         self._create_superuser()
 
     def _expected_vars(self):
@@ -90,20 +97,41 @@ class Command(BaseCommand):
             if not env.get(var):
                 message = f"    🗙 expected env var {var} not set\n    {INST}"
                 self.stdout.write(self.style.ERROR(message))
+                sleep(60)
                 raise CommandError(message)
 
         message = "    ✓ all expected env vars are set"
         self.stdout.write(self.style.SUCCESS(message))
 
+    def _unexpected_vars(self):
+        """check for unexpected env vars"""
+        self.stdout.write("[2] checking for unexpected env vars")
+        for var, message in UNEXPECTED_ENV_VARS.items():
+            if not os.environ.get(var):
+                continue
+
+            message = (
+                f"    🗙 unexpected env var {var} found\n"
+                f"    {message} \n"
+                "    see release notes for a list of all changes."
+            )
+
+            self.stdout.write(self.style.ERROR(message))
+            sleep(60)
+            raise CommandError(message)
+
+        message = "    ✓ no unexpected env vars found"
+        self.stdout.write(self.style.SUCCESS(message))
+
     def _elastic_user_overwrite(self):
         """check for ELASTIC_USER overwrite"""
-        self.stdout.write("[2] check ES user overwrite")
+        self.stdout.write("[3] check ES user overwrite")
         env = EnvironmentSettings.ES_USER
         self.stdout.write(self.style.SUCCESS(f"    ✓ ES user is set to {env}"))
 
     def _ta_port_overwrite(self):
         """set TA_PORT overwrite for nginx"""
-        self.stdout.write("[3] check TA_PORT overwrite")
+        self.stdout.write("[4] check TA_PORT overwrite")
         overwrite = EnvironmentSettings.TA_PORT
         if not overwrite:
             self.stdout.write(self.style.SUCCESS("    TA_PORT is not set"))
@@ -119,52 +147,49 @@ class Command(BaseCommand):
 
         self.stdout.write(self.style.SUCCESS(message))
 
-    def _ta_uwsgi_overwrite(self):
-        """set TA_UWSGI_PORT overwrite"""
-        self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
-        overwrite = EnvironmentSettings.TA_UWSGI_PORT
+    def _ta_backend_port_overwrite(self):
+        """set TA_BACKEND_PORT overwrite"""
+        self.stdout.write("[5] check TA_BACKEND_PORT overwrite")
+        overwrite = EnvironmentSettings.TA_BACKEND_PORT
         if not overwrite:
-            message = "    TA_UWSGI_PORT is not set"
+            message = "    TA_BACKEND_PORT is not set"
             self.stdout.write(self.style.SUCCESS(message))
             return
 
-        # nginx
-        regex = re.compile(r"uwsgi_pass localhost:[0-9]{1,5}")
-        to_overwrite = f"uwsgi_pass localhost:{overwrite}"
+        # modify nginx conf
+        regex = re.compile(r"proxy_pass http://localhost:[0-9]{1,5}")
+        to_overwrite = f"proxy_pass http://localhost:{overwrite}"
         changed = file_overwrite(NGINX, regex, to_overwrite)
 
-        # uwsgi
-        regex = re.compile(r"socket = :[0-9]{1,5}")
-        to_overwrite = f"socket = :{overwrite}"
-        changed = file_overwrite(UWSGI, regex, to_overwrite)
-
         if changed:
-            message = f"    ✓ TA_UWSGI_PORT changed to {overwrite}"
+            message = f"    ✓ TA_BACKEND_PORT changed to {overwrite}"
         else:
-            message = f"    ✓ TA_UWSGI_PORT already set to {overwrite}"
+            message = f"    ✓ TA_BACKEND_PORT already set to {overwrite}"
 
         self.stdout.write(self.style.SUCCESS(message))
 
-    def _enable_cast_overwrite(self):
+    def _disable_static_auth(self):
         """cast workaround, remove auth for static files in nginx"""
-        self.stdout.write("[5] check ENABLE_CAST overwrite")
-        overwrite = EnvironmentSettings.ENABLE_CAST
+        self.stdout.write("[7] check DISABLE_STATIC_AUTH overwrite")
+        overwrite = EnvironmentSettings.DISABLE_STATIC_AUTH
         if not overwrite:
-            self.stdout.write(self.style.SUCCESS("    ENABLE_CAST is not set"))
+            self.stdout.write(
+                self.style.SUCCESS("    DISABLE_STATIC_AUTH is not set")
+            )
             return
 
         regex = re.compile(r"[^\S\r\n]*auth_request /api/ping/;\n")
         changed = file_overwrite(NGINX, regex, "")
         if changed:
-            message = "    ✓ process nginx to enable Cast"
+            message = "    ✓ process nginx to disable static auth"
         else:
-            message = "    ✓ Cast is already enabled in nginx"
+            message = "    ✓ static auth is already disabled in nginx"
 
         self.stdout.write(self.style.SUCCESS(message))
 
     def _create_superuser(self):
         """create superuser if not exist"""
-        self.stdout.write("[6] create superuser")
+        self.stdout.write("[8] create superuser")
         is_created = Account.objects.filter(is_superuser=True)
         if is_created:
             message = "    superuser already created"

backend/config/management/commands/ta_startup.py

@@ -0,0 +1,349 @@
+"""
+Functionality:
+- Application startup
+- Apply migrations
+"""
+
+import os
+from datetime import datetime
+from random import randint
+from time import sleep
+
+from appsettings.src.config import AppConfig, ReleaseVersion
+from appsettings.src.index_setup import ElasitIndexWrap
+from appsettings.src.snapshot import ElasticSnapshot
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import clear_dl_cache
+from common.src.ta_redis import RedisArchivist
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import dateformat
+from django_celery_beat.models import CrontabSchedule, PeriodicTasks
+from redis.exceptions import ResponseError
+from task.models import CustomPeriodicTask
+from task.src.config_schedule import ScheduleBuilder
+from task.src.task_manager import TaskManager
+from task.tasks import version_check
+
+TOPIC = """
+
+#######################
+#  Application Start  #
+#######################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(TOPIC)
+        self._make_folders()
+        self._clear_redis_keys()
+        self._clear_tasks()
+        self._clear_dl_cache()
+        self._version_check()
+        self._index_setup()
+        self._snapshot_check()
+        self._mig_app_settings()
+        self._create_default_schedules()
+        self._update_schedule_tz()
+        self._init_app_config()
+        self._mig_channel_tags()
+        self._mig_video_channel_tags()
+
+    def _make_folders(self):
+        """make expected cache folders"""
+        self.stdout.write("[1] create expected cache folders")
+        folders = [
+            "backup",
+            "channels",
+            "download",
+            "import",
+            "playlists",
+            "videos",
+        ]
+        cache_dir = EnvironmentSettings.CACHE_DIR
+        for folder in folders:
+            folder_path = os.path.join(cache_dir, folder)
+            os.makedirs(folder_path, exist_ok=True)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ expected folders created"))
+
+    def _clear_redis_keys(self):
+        """make sure there are no leftover locks or keys set in redis"""
+        self.stdout.write("[2] clear leftover keys in redis")
+        all_keys = [
+            "dl_queue_id",
+            "dl_queue",
+            "downloading",
+            "manual_import",
+            "reindex",
+            "rescan",
+            "run_backup",
+            "startup_check",
+            "reindex:ta_video",
+            "reindex:ta_channel",
+            "reindex:ta_playlist",
+        ]
+
+        redis_con = RedisArchivist()
+        has_changed = False
+        for key in all_keys:
+            if redis_con.del_message(key):
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ cleared key {key}")
+                )
+                has_changed = True
+
+        if not has_changed:
+            self.stdout.write(self.style.SUCCESS("    no keys found"))
+
+    def _clear_tasks(self):
+        """clear tasks and messages"""
+        self.stdout.write("[3] clear task leftovers")
+        TaskManager().fail_pending()
+        redis_con = RedisArchivist()
+        to_delete = redis_con.list_keys("message:")
+        if to_delete:
+            for key in to_delete:
+                redis_con.del_message(key)
+
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {len(to_delete)} messages")
+            )
+
+    def _clear_dl_cache(self):
+        """clear leftover files from dl cache"""
+        self.stdout.write("[4] clear leftover files from dl cache")
+        leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
+        if leftover_files:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {leftover_files} files")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no files found"))
+
+    def _version_check(self):
+        """remove new release key if updated now"""
+        self.stdout.write("[5] check for first run after update")
+        new_version = ReleaseVersion().is_updated()
+        if new_version:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ update to {new_version} completed")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no new update found"))
+
+        version_task = CustomPeriodicTask.objects.filter(name="version_check")
+        if not version_task.exists():
+            return
+
+        if not version_task.first().last_run_at:
+            self.style.SUCCESS("    ✓ send initial version check task")
+            version_check.delay()
+
+    def _index_setup(self):
+        """migration: validate index mappings"""
+        self.stdout.write("[6] validate index mappings")
+        ElasitIndexWrap().setup()
+
+    def _snapshot_check(self):
+        """migration setup snapshots"""
+        self.stdout.write("[7] setup snapshots")
+        ElasticSnapshot().setup()
+
+    def _mig_app_settings(self) -> None:
+        """update from v0.4.13 to v0.5.0, migrate application settings"""
+        self.stdout.write("[MIGRATION] move appconfig to ES")
+        try:
+            config = RedisArchivist().get_message("config")
+        except ResponseError:
+            self.stdout.write(
+                self.style.SUCCESS("    Redis does not support JSON decoding")
+            )
+            return
+
+        if not config or config == {"status": False}:
+            self.stdout.write(
+                self.style.SUCCESS("    no config values to migrate")
+            )
+            return
+
+        path = "ta_config/_doc/appsettings"
+        response, status_code = ElasticWrap(path).post(config)
+
+        if status_code in [200, 201]:
+            self.stdout.write(
+                self.style.SUCCESS("    ✓ migrated appconfig to ES")
+            )
+            RedisArchivist().del_message("config", save=True)
+            return
+
+        message = "    🗙 failed to migrate app config"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)
+
+    def _create_default_schedules(self) -> None:
+        """create default schedules for new installations"""
+        self.stdout.write("[8] create initial schedules")
+        init_has_run = CustomPeriodicTask.objects.filter(
+            name="version_check"
+        ).exists()
+
+        if init_has_run:
+            self.stdout.write(
+                self.style.SUCCESS(
+                    "    schedule init already done, skipping..."
+                )
+            )
+            return
+
+        builder = ScheduleBuilder()
+        check_reindex = builder.get_set_task(
+            "check_reindex", schedule=builder.SCHEDULES["check_reindex"]
+        )
+        check_reindex.task_config.update({"days": 90})
+        check_reindex.last_run_at = dateformat.make_aware(datetime.now())
+        check_reindex.save()
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {check_reindex}"
+            )
+        )
+
+        thumbnail_check = builder.get_set_task(
+            "thumbnail_check", schedule=builder.SCHEDULES["thumbnail_check"]
+        )
+        thumbnail_check.last_run_at = dateformat.make_aware(datetime.now())
+        thumbnail_check.save()
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {thumbnail_check}"
+            )
+        )
+        daily_random = f"{randint(0, 59)} {randint(0, 23)} *"
+        version_check_task = builder.get_set_task(
+            "version_check", schedule=daily_random
+        )
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {version_check_task}"
+            )
+        )
+        self.stdout.write(
+            self.style.SUCCESS("    ✓ all default schedules created")
+        )
+
+    def _update_schedule_tz(self) -> None:
+        """update timezone for Schedule instances"""
+        self.stdout.write("[9] validate schedules TZ")
+        tz = EnvironmentSettings.TZ
+        to_update = CrontabSchedule.objects.exclude(timezone=tz)
+
+        if not to_update.exists():
+            self.stdout.write(
+                self.style.SUCCESS("    all schedules have correct TZ")
+            )
+            return
+
+        updated = to_update.update(timezone=tz)
+        self.stdout.write(
+            self.style.SUCCESS(f"    ✓ updated {updated} schedules to {tz}.")
+        )
+        PeriodicTasks.update_changed()
+
+    def _init_app_config(self) -> None:
+        """init default app config to ES"""
+        self.stdout.write("[10] Check AppConfig")
+        _, status_code = ElasticWrap("ta_config/_doc/appsettings").get()
+        if status_code in [200, 201]:
+            self.stdout.write(
+                self.style.SUCCESS("    skip completed appsettings init")
+            )
+            updated_defaults = AppConfig().add_new_defaults()
+            for new_default in updated_defaults:
+                self.stdout.write(
+                    self.style.SUCCESS(f"    added new default: {new_default}")
+                )
+
+            return
+
+        handler = AppConfig.__new__(AppConfig)
+        _, status_code = handler.sync_defaults()
+        self.stdout.write(
+            self.style.SUCCESS("    ✓ Created default appsettings.")
+        )
+        self.stdout.write(
+            self.style.SUCCESS(f"      Status code: {status_code}")
+        )
+
+    def _mig_channel_tags(self) -> None:
+        """update from v0.4.13 to v0.5.0, migrate incorrect data types"""
+        self.stdout.write("[MIGRATION] fix incorrect channel tags types")
+        path = "ta_channel/_update_by_query"
+        data = {
+            "query": {"match": {"channel_tags": False}},
+            "script": {
+                "source": "ctx._source.channel_tags = []",
+                "lang": "painless",
+            },
+        }
+        response, status_code = ElasticWrap(path).post(data)
+        if status_code in [200, 201]:
+            updated = response.get("updated")
+            if updated:
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ fixed {updated} channel tags")
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS("    no channel tags needed fixing")
+                )
+            return
+
+        message = "    🗙 failed to fix channel tags"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)
+
+    def _mig_video_channel_tags(self) -> None:
+        """update from v0.4.13 to v0.5.0, migrate incorrect data types"""
+        self.stdout.write("[MIGRATION] fix incorrect video channel tags types")
+        path = "ta_video/_update_by_query"
+        data = {
+            "query": {"match": {"channel.channel_tags": False}},
+            "script": {
+                "source": "ctx._source.channel.channel_tags = []",
+                "lang": "painless",
+            },
+        }
+        response, status_code = ElasticWrap(path).post(data)
+        if status_code in [200, 201]:
+            updated = response.get("updated")
+            if updated:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"    ✓ fixed {updated} video channel tags"
+                    )
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        "    no video channel tags needed fixing"
+                    )
+                )
+            return
+
+        message = "    🗙 failed to fix video channel tags"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)

backend/config/management/commands/ta_stop_on_error.py

@@ -0,0 +1,40 @@
+"""stop on unexpected table"""
+
+from time import sleep
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import connection
+
+ERROR_MESSAGE = """
+    🗙 Database is incompatible, see latest release notes for instructions:
+    🗙 https://github.com/tubearchivist/tubearchivist/releases/tag/v0.5.0
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """handle"""
+        self.stdout.write("[MIGRATION] Confirming v0.5.0 table layout")
+        all_tables = self.list_tables()
+        for table in all_tables:
+            if table == "home_account":
+
+                self.stdout.write(self.style.ERROR(ERROR_MESSAGE))
+                sleep(60)
+                raise CommandError(ERROR_MESSAGE)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ local DB is up-to-date."))
+
+    def list_tables(self):
+        """raw list all tables"""
+        with connection.cursor() as cursor:
+            cursor.execute(
+                "SELECT name FROM sqlite_master WHERE type='table';"
+            )
+            tables = cursor.fetchall()
+
+        return [table[0] for table in tables]

backend/config/settings.py

@@ -14,11 +14,17 @@ import hashlib
 from os import environ, path
 from pathlib import Path
 
-import ldap
+from common.src.env_settings import EnvironmentSettings
+from common.src.helper import ta_host_parser
 from corsheaders.defaults import default_headers
-from django_auth_ldap.config import LDAPSearch
-from home.src.ta.helper import ta_host_parser
-from home.src.ta.settings import EnvironmentSettings
+
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv(".env")
+except ModuleNotFoundError:
+    pass
+
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -36,12 +42,12 @@ DEBUG = bool(environ.get("DJANGO_DEBUG"))
 ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(
     environ.get("TA_HOST", "localhost")
 )
+CORS_ALLOWED_ORIGINS = CSRF_TRUSTED_ORIGINS
 
 # Application definition
 
 INSTALLED_APPS = [
     "django_celery_beat",
-    "home.apps.HomeConfig",
     "django.contrib.admin",
     "django.contrib.auth",
     "django.contrib.contenttypes",
@@ -53,7 +59,16 @@ INSTALLED_APPS = [
     "django.contrib.humanize",
     "rest_framework",
     "rest_framework.authtoken",
-    "api",
+    "drf_spectacular",
+    "common",
+    "video",
+    "channel",
+    "playlist",
+    "download",
+    "task",
+    "appsettings",
+    "stats",
+    "user",
     "config",
 ]
 
@@ -67,7 +82,6 @@ MIDDLEWARE = [
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
-    "home.src.ta.health.HealthCheckMiddleware",
 ]
 
 ROOT_URLCONF = "config.urls"
@@ -92,6 +106,9 @@ WSGI_APPLICATION = "config.wsgi.application"
 
 if bool(environ.get("TA_LDAP")):
     # pylint: disable=global-at-module-level
+    import ldap
+    from django_auth_ldap.config import LDAPSearch
+
     global AUTH_LDAP_SERVER_URI
     AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
 
@@ -211,7 +228,7 @@ AUTH_PASSWORD_VALIDATORS = [
     },
 ]
 
-AUTH_USER_MODEL = "home.Account"
+AUTH_USER_MODEL = "user.Account"
 
 # Forward-auth authentication
 if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
@@ -220,7 +237,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
     )
     TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")
 
-    MIDDLEWARE.append("home.src.ta.auth.HttpRemoteUserMiddleware")
+    MIDDLEWARE.append("user.src.remote_user_auth.HttpRemoteUserMiddleware")
 
     AUTHENTICATION_BACKENDS = (
         "django.contrib.auth.backends.RemoteUserBackend",
@@ -261,13 +278,14 @@ LOGOUT_REDIRECT_URL = "/login/"
 # background.js makes the request so HTTP_ORIGIN will be from extension
 if environ.get("DISABLE_CORS"):
     # disable cors
-    CORS_ORIGIN_ALLOW_ALL = True
+    CORS_ALLOW_ALL_ORIGINS = True
 else:
     CORS_ALLOWED_ORIGIN_REGEXES = [
         r"moz-extension://*",
         r"chrome-extension://*",
     ]
-    CORS_ALLOWED_ORIGINS = ["http://localhost:3000"]
+
+CORS_ALLOW_CREDENTIALS = True
 
 
 CORS_ALLOW_HEADERS = list(default_headers) + [
@@ -276,4 +294,16 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.4.11"
+TA_VERSION = "v0.5.2"
+
+# API
+REST_FRAMEWORK = {
+    "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema",
+}
+
+SPECTACULAR_SETTINGS = {
+    "TITLE": "Tube Archivist API",
+    "DESCRIPTION": "API documentation for Tube Archivist backend.",
+    "VERSION": TA_VERSION,
+    "SERVE_INCLUDE_SCHEMA": False,
+}

backend/config/urls.py

@@ -16,9 +16,23 @@ Including another URLconf
 
 from django.contrib import admin
 from django.urls import include, path
+from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView
 
 urlpatterns = [
-    path("", include("home.urls")),
-    path("api/", include("api.urls")),
+    path("api/", include("common.urls")),
+    path("api/video/", include("video.urls")),
+    path("api/channel/", include("channel.urls")),
+    path("api/playlist/", include("playlist.urls")),
+    path("api/download/", include("download.urls")),
+    path("api/task/", include("task.urls")),
+    path("api/appsettings/", include("appsettings.urls")),
+    path("api/stats/", include("stats.urls")),
+    path("api/user/", include("user.urls")),
+    path("api/schema/", SpectacularAPIView.as_view(), name="schema"),
+    path(
+        "api/docs/",
+        SpectacularSwaggerView.as_view(url_name="schema"),
+        name="swagger-ui",
+    ),
     path("admin/", admin.site.urls),
 ]

backend/download/serializers.py

@@ -0,0 +1,95 @@
+"""download serializers"""
+
+# pylint: disable=abstract-method
+
+from common.serializers import PaginationSerializer, ValidateUnknownFieldsMixin
+from rest_framework import serializers
+from video.src.constants import VideoTypeEnum
+
+
+class DownloadItemSerializer(serializers.Serializer):
+    """serialize download item"""
+
+    auto_start = serializers.BooleanField()
+    channel_id = serializers.CharField()
+    channel_indexed = serializers.BooleanField()
+    channel_name = serializers.CharField()
+    duration = serializers.CharField()
+    published = serializers.CharField()
+    status = serializers.ChoiceField(choices=["pending", "ignore"])
+    timestamp = serializers.IntegerField()
+    title = serializers.CharField()
+    vid_thumb_url = serializers.CharField()
+    vid_type = serializers.ChoiceField(choices=VideoTypeEnum.values())
+    youtube_id = serializers.CharField()
+    message = serializers.CharField(required=False)
+    _index = serializers.CharField(required=False)
+    _score = serializers.IntegerField(required=False)
+
+
+class DownloadListSerializer(serializers.Serializer):
+    """serialize download list"""
+
+    data = DownloadItemSerializer(many=True)
+    paginate = PaginationSerializer()
+
+
+class DownloadListQuerySerializer(
+    ValidateUnknownFieldsMixin, serializers.Serializer
+):
+    """serialize query params for download list"""
+
+    filter = serializers.ChoiceField(
+        choices=["pending", "ignore"], required=False
+    )
+    channel = serializers.CharField(required=False, help_text="channel ID")
+    page = serializers.IntegerField(required=False)
+
+
+class DownloadListQueueDeleteQuerySerializer(serializers.Serializer):
+    """serialize bulk delete download queue query string"""
+
+    filter = serializers.ChoiceField(choices=["pending", "ignore"])
+
+
+class AddDownloadItemSerializer(serializers.Serializer):
+    """serialize single item to add"""
+
+    youtube_id = serializers.CharField()
+    status = serializers.ChoiceField(choices=["pending", "ignore-force"])
+
+
+class AddToDownloadListSerializer(serializers.Serializer):
+    """serialize add to download queue data"""
+
+    data = AddDownloadItemSerializer(many=True)
+
+
+class AddToDownloadQuerySerializer(serializers.Serializer):
+    """add to queue query serializer"""
+
+    autostart = serializers.BooleanField(required=False)
+
+
+class DownloadQueueItemUpdateSerializer(serializers.Serializer):
+    """update single download queue item"""
+
+    status = serializers.ChoiceField(
+        choices=["pending", "ignore", "ignore-force", "priority"]
+    )
+
+
+class DownloadAggBucketSerializer(serializers.Serializer):
+    """serialize bucket"""
+
+    key = serializers.ListField(child=serializers.CharField())
+    key_as_string = serializers.CharField()
+    doc_count = serializers.IntegerField()
+
+
+class DownloadAggsSerializer(serializers.Serializer):
+    """serialize download channel bucket aggregations"""
+
+    doc_count_error_upper_bound = serializers.IntegerField()
+    sum_other_doc_count = serializers.IntegerField()
+    buckets = DownloadAggBucketSerializer(many=True)

backend/download/src/queue.py

@@ -4,17 +4,16 @@ Functionality:
 - linked with ta_dowload index
 """
 
-import json
 from datetime import datetime
 
-from home.src.download.subscriptions import ChannelSubscription
-from home.src.download.thumbnails import ThumbManager
-from home.src.download.yt_dlp_base import YtWrap
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.index.playlist import YoutubePlaylist
-from home.src.index.video_constants import VideoTypeEnum
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_duration_str, is_shorts
+from appsettings.src.config import AppConfig
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import get_duration_str, is_shorts, rand_sleep
+from download.src.subscriptions import ChannelSubscription
+from download.src.thumbnails import ThumbManager
+from download.src.yt_dlp_base import YtWrap
+from playlist.src.index import YoutubePlaylist
+from video.src.constants import VideoTypeEnum
 
 
 class PendingIndex:
@@ -241,7 +240,6 @@ class PendingList(PendingIndex):
     def add_to_pending(self, status="pending", auto_start=False):
         """add missing videos to pending list"""
         self.get_channels()
-        bulk_list = []
 
         total = len(self.missing_videos)
         videos_added = []
@@ -253,6 +251,7 @@ class PendingList(PendingIndex):
             self._notify_add(idx, total)
             video_details = self.get_youtube_details(youtube_id, vid_type)
             if not video_details:
+                rand_sleep(self.config)
                 continue
 
             video_details.update(
@@ -262,32 +261,17 @@ class PendingList(PendingIndex):
                 }
             )
 
-            action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(video_details))
-
             url = video_details["vid_thumb_url"]
             ThumbManager(youtube_id).download_video_thumb(url)
+            es_url = f"ta_download/_doc/{youtube_id}"
+            _, _ = ElasticWrap(es_url).put(video_details)
             videos_added.append(youtube_id)
 
-            if len(bulk_list) >= 20:
-                self._ingest_bulk(bulk_list)
-                bulk_list = []
-
-        self._ingest_bulk(bulk_list)
+            if idx != total:
+                rand_sleep(self.config)
 
         return videos_added
 
-    def _ingest_bulk(self, bulk_list):
-        """add items to queue in bulk"""
-        if not bulk_list:
-            return
-
-        # add last newline
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-        _, _ = ElasticWrap("_bulk?refresh=true").post(query_str, ndjson=True)
-
     def _notify_add(self, idx, total):
         """send notification for adding videos to download queue"""
         if not self.task:
@@ -338,7 +322,7 @@ class PendingList(PendingIndex):
 
         duration = vid.get("duration")
         if duration and isinstance(duration, int):
-            if duration > 60:
+            if duration > 3 * 60:
                 return False
 
         return is_shorts(vid["id"])

backend/download/src/subscriptions.py

@@ -4,16 +4,16 @@ Functionality:
 - handle playlist subscriptions
 """
 
-from home.src.download.thumbnails import ThumbManager
-from home.src.download.yt_dlp_base import YtWrap
-from home.src.es.connect import IndexPaginate
-from home.src.index.channel import YoutubeChannel
-from home.src.index.playlist import YoutubePlaylist
-from home.src.index.video import YoutubeVideo
-from home.src.index.video_constants import VideoTypeEnum
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import is_missing
-from home.src.ta.urlparser import Parser
+from appsettings.src.config import AppConfig
+from channel.src.index import YoutubeChannel
+from common.src.es_connect import IndexPaginate
+from common.src.helper import is_missing, rand_sleep
+from common.src.urlparser import Parser
+from download.src.thumbnails import ThumbManager
+from download.src.yt_dlp_base import YtWrap
+from playlist.src.index import YoutubePlaylist
+from video.src.constants import VideoTypeEnum
+from video.src.index import YoutubeVideo
 
 
 class ChannelSubscription:
@@ -108,6 +108,7 @@ class ChannelSubscription:
                 message_lines=[f"Scanning Channel {idx + 1}/{total}"],
                 progress=(idx + 1) / total,
             )
+            rand_sleep(self.config)
 
         return missing_videos
 
@@ -120,6 +121,8 @@ class ChannelSubscription:
         channel.upload_to_es()
         channel.sync_to_videos()
 
+        return channel.json_data
+
 
 class VideoQueryBuilder:
     """Build queries for yt-dlp."""
@@ -277,6 +280,7 @@ class PlaylistSubscription:
         playlist.build_json()
         playlist.json_data["playlist_subscribed"] = subscribe_status
         playlist.upload_to_es()
+        return playlist.json_data
 
     def find_missing(self):
         """find videos in subscribed playlists not downloaded yet"""
@@ -317,6 +321,7 @@ class PlaylistSubscription:
                 message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
                 progress=(idx + 1) / total,
             )
+            rand_sleep(self.config)
 
         return missing_videos
 
@@ -422,7 +427,7 @@ class SubscriptionHandler:
 
     def _subscribe(self, channel_id):
         """subscribe to channel"""
-        ChannelSubscription().change_subscribe(
+        _ = ChannelSubscription().change_subscribe(
             channel_id, channel_subscribed=True
         )
 

backend/download/src/thumbnails.py

@@ -10,9 +10,9 @@ from io import BytesIO
 from time import sleep
 
 import requests
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.ta.helper import is_missing
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import is_missing
 from mutagen.mp4 import MP4, MP4Cover
 from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
 

backend/download/src/yt_dlp_base.py

@@ -0,0 +1,219 @@
+"""
+functionality:
+- base class to make all calls to yt-dlp
+- handle yt-dlp errors
+"""
+
+from datetime import datetime
+from http import cookiejar
+from io import StringIO
+
+import yt_dlp
+from appsettings.src.config import AppConfig
+from common.src.ta_redis import RedisArchivist
+from django.conf import settings
+
+
+class YtWrap:
+    """wrap calls to yt"""
+
+    OBS_BASE = {
+        "default_search": "ytsearch",
+        "quiet": True,
+        "check_formats": "selected",
+        "socket_timeout": 10,
+        "extractor_retries": 3,
+        "retries": 10,
+    }
+
+    def __init__(self, obs_request, config=False):
+        self.obs_request = obs_request
+        self.config = config
+        self.build_obs()
+
+    def build_obs(self):
+        """build yt-dlp obs"""
+        self.obs = self.OBS_BASE.copy()
+        self.obs.update(self.obs_request)
+        if self.config:
+            self._add_cookie()
+            self._add_potoken()
+
+        if getattr(settings, "DEBUG", False):
+            del self.obs["quiet"]
+            print(self.obs)
+
+    def _add_cookie(self):
+        """add cookie if enabled"""
+        if self.config["downloads"]["cookie_import"]:
+            cookie_io = CookieHandler(self.config).get()
+            self.obs["cookiefile"] = cookie_io
+
+    def _add_potoken(self):
+        """add potoken if enabled"""
+        if self.config["downloads"].get("potoken"):
+            potoken = POTokenHandler(self.config).get()
+            self.obs.update(
+                {
+                    "extractor_args": {
+                        "youtube": {
+                            "po_token": [potoken],
+                            "player-client": ["web", "default"],
+                        },
+                    }
+                }
+            )
+
+    def download(self, url):
+        """make download request"""
+        with yt_dlp.YoutubeDL(self.obs) as ydl:
+            try:
+                ydl.download([url])
+            except yt_dlp.utils.DownloadError as err:
+                print(f"{url}: failed to download with message {err}")
+                if "Temporary failure in name resolution" in str(err):
+                    raise ConnectionError("lost the internet, abort!") from err
+
+                return False, str(err)
+
+        self._validate_cookie()
+
+        return True, True
+
+    def extract(self, url):
+        """make extract request"""
+        with yt_dlp.YoutubeDL(self.obs) as ydl:
+            try:
+                response = ydl.extract_info(url)
+            except cookiejar.LoadError as err:
+                print(f"cookie file is invalid: {err}")
+                return False
+            except yt_dlp.utils.ExtractorError as err:
+                print(f"{url}: failed to extract: {err}, continue...")
+                return False
+            except yt_dlp.utils.DownloadError as err:
+                if "This channel does not have a" in str(err):
+                    return False
+
+                print(f"{url}: failed to get info from youtube: {err}")
+                if "Temporary failure in name resolution" in str(err):
+                    raise ConnectionError("lost the internet, abort!") from err
+
+                return False
+
+        self._validate_cookie()
+
+        return response
+
+    def _validate_cookie(self):
+        """check cookie and write it back for next use"""
+        if not self.obs.get("cookiefile"):
+            return
+
+        new_cookie = self.obs["cookiefile"].read()
+        old_cookie = RedisArchivist().get_message_str("cookie")
+        if new_cookie and old_cookie != new_cookie:
+            print("refreshed stored cookie")
+            RedisArchivist().set_message("cookie", new_cookie, save=True)
+
+
+class CookieHandler:
+    """handle youtube cookie for yt-dlp"""
+
+    def __init__(self, config):
+        self.cookie_io = False
+        self.config = config
+
+    def get(self):
+        """get cookie io stream"""
+        cookie = RedisArchivist().get_message_str("cookie")
+        self.cookie_io = StringIO(cookie)
+        return self.cookie_io
+
+    def set_cookie(self, cookie):
+        """set cookie str and activate in config"""
+        cookie_clean = cookie.strip("\x00")
+        RedisArchivist().set_message("cookie", cookie_clean, save=True)
+        AppConfig().update_config({"downloads": {"cookie_import": True}})
+        self.config["downloads"]["cookie_import"] = True
+        print("[cookie]: activated and stored in Redis")
+
+    @staticmethod
+    def revoke():
+        """revoke cookie"""
+        RedisArchivist().del_message("cookie")
+        RedisArchivist().del_message("cookie:valid")
+        AppConfig().update_config({"downloads": {"cookie_import": False}})
+        print("[cookie]: revoked")
+
+    def validate(self):
+        """validate cookie using the liked videos playlist"""
+        validation = RedisArchivist().get_message_dict("cookie:valid")
+        if validation:
+            print("[cookie]: used cached cookie validation")
+            return True
+
+        print("[cookie] validating cookie")
+        obs_request = {
+            "skip_download": True,
+            "extract_flat": True,
+        }
+        validator = YtWrap(obs_request, self.config)
+        response = bool(validator.extract("LL"))
+        self.store_validation(response)
+
+        # update in redis to avoid expiring
+        modified = validator.obs["cookiefile"].getvalue().strip("\x00")
+        if modified:
+            cookie_clean = modified.strip("\x00")
+            RedisArchivist().set_message("cookie", cookie_clean)
+
+        if not response:
+            mess_dict = {
+                "status": "message:download",
+                "level": "error",
+                "title": "Cookie validation failed, exiting...",
+                "message": "",
+            }
+            RedisArchivist().set_message(
+                "message:download", mess_dict, expire=4
+            )
+            print("[cookie]: validation failed, exiting...")
+
+        print(f"[cookie]: validation success: {response}")
+        return response
+
+    @staticmethod
+    def store_validation(response):
+        """remember last validation"""
+        now = datetime.now()
+        message = {
+            "status": response,
+            "validated": int(now.timestamp()),
+            "validated_str": now.strftime("%Y-%m-%d %H:%M"),
+        }
+        RedisArchivist().set_message("cookie:valid", message, expire=3600)
+
+
+class POTokenHandler:
+    """handle po token"""
+
+    REDIS_KEY = "potoken"
+
+    def __init__(self, config):
+        self.config = config
+
+    def get(self) -> str | None:
+        """get PO token"""
+        potoken = RedisArchivist().get_message_str(self.REDIS_KEY)
+        return potoken
+
+    def set_token(self, new_token: str) -> None:
+        """set new PO token"""
+        RedisArchivist().set_message(self.REDIS_KEY, new_token)
+        AppConfig().update_config({"downloads": {"potoken": True}})
+
+    def revoke_token(self) -> None:
+        """revoke token"""
+        RedisArchivist().del_message(self.REDIS_KEY)
+        AppConfig().update_config({"downloads": {"potoken": False}})

backend/download/src/yt_dlp_handler.py

@@ -10,19 +10,23 @@ import os
 import shutil
 from datetime import datetime
 
-from home.src.download.queue import PendingList
-from home.src.download.subscriptions import PlaylistSubscription
-from home.src.download.yt_dlp_base import YtWrap
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.index.channel import YoutubeChannel
-from home.src.index.comments import CommentList
-from home.src.index.playlist import YoutubePlaylist
-from home.src.index.video import YoutubeVideo, index_new_video
-from home.src.index.video_constants import VideoTypeEnum
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_channel_overwrites, ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisQueue
+from appsettings.src.config import AppConfig
+from channel.src.index import YoutubeChannel
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import (
+    get_channel_overwrites,
+    ignore_filelist,
+    rand_sleep,
+)
+from common.src.ta_redis import RedisQueue
+from download.src.queue import PendingList
+from download.src.subscriptions import PlaylistSubscription
+from download.src.yt_dlp_base import YtWrap
+from playlist.src.index import YoutubePlaylist
+from video.src.comments import CommentList
+from video.src.constants import VideoTypeEnum
+from video.src.index import YoutubeVideo, index_new_video
 
 
 class DownloaderBase:
@@ -60,6 +64,9 @@ class VideoDownloader(DownloaderBase):
                 self._reset_auto()
                 break
 
+            if downloaded > 0:
+                rand_sleep(self.config)
+
             youtube_id = video_data["youtube_id"]
             channel_id = video_data["channel_id"]
             print(f"{youtube_id}: Downloading video")
@@ -304,8 +311,19 @@ class DownloadPostProcess(DownloaderBase):
 
         print(f"auto delete older than {autodelete_days} days")
         now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
+        channel_overwrite = "channel.channel_overwrites.autodelete_days"
         data = {
-            "query": {"range": {"player.watched_date": {"lte": now_lte}}},
+            "query": {
+                "bool": {
+                    "must": [
+                        {"range": {"player.watched_date": {"lte": now_lte}}},
+                        {"term": {"player.watched": True}},
+                    ],
+                    "must_not": [
+                        {"exists": {"field": channel_overwrite}},
+                    ],
+                }
+            },
             "sort": [{"player.watched_date": {"order": "asc"}}],
         }
         self._auto_delete_watched(data)
@@ -320,6 +338,7 @@ class DownloadPostProcess(DownloaderBase):
                 must_list = [
                     {"range": {"player.watched_date": {"lte": now_lte}}},
                     {"term": {"channel.channel_id": {"value": channel_id}}},
+                    {"term": {"player.watched": True}},
                 ]
                 data = {
                     "query": {"bool": {"must": must_list}},
@@ -370,6 +389,7 @@ class DownloadPostProcess(DownloaderBase):
             ]
             progress = idx / total
             self.task.send_progress(message, progress=progress)
+            rand_sleep(self.config)
 
     def add_playlists_to_refresh(self) -> None:
         """add playlists to refresh"""

backend/download/urls.py

@@ -0,0 +1,18 @@
+"""all download API urls"""
+
+from django.urls import path
+from download import views
+
+urlpatterns = [
+    path("", views.DownloadApiListView.as_view(), name="api-download-list"),
+    path(
+        "aggs/",
+        views.DownloadAggsApiView.as_view(),
+        name="api-download-aggs",
+    ),
+    path(
+        "<slug:video_id>/",
+        views.DownloadApiView.as_view(),
+        name="api-download",
+    ),
+]

backend/download/views.py

@@ -0,0 +1,292 @@
+"""all download API views"""
+
+from common.serializers import (
+    AsyncTaskResponseSerializer,
+    ErrorResponseSerializer,
+)
+from common.views_base import AdminOnly, ApiBaseView
+from download.serializers import (
+    AddToDownloadListSerializer,
+    AddToDownloadQuerySerializer,
+    DownloadAggsSerializer,
+    DownloadItemSerializer,
+    DownloadListQuerySerializer,
+    DownloadListQueueDeleteQuerySerializer,
+    DownloadListSerializer,
+    DownloadQueueItemUpdateSerializer,
+)
+from download.src.queue import PendingInteract
+from drf_spectacular.utils import OpenApiResponse, extend_schema
+from rest_framework.response import Response
+from task.tasks import download_pending, extrac_dl
+
+
+class DownloadApiListView(ApiBaseView):
+    """resolves to /api/download/
+    GET: returns latest videos in the download queue
+    POST: add a list of videos to download queue
+    DELETE: remove items based on query filter
+    """
+
+    search_base = "ta_download/_search/"
+    valid_filter = ["pending", "ignore"]
+    permission_classes = [AdminOnly]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(DownloadListSerializer()),
+        },
+        parameters=[DownloadListQuerySerializer()],
+    )
+    def get(self, request):
+        """get download queue list"""
+        query_filter = request.GET.get("filter", False)
+        self.data.update(
+            {
+                "sort": [
+                    {"auto_start": {"order": "desc"}},
+                    {"timestamp": {"order": "asc"}},
+                ],
+            }
+        )
+
+        serializer = DownloadListQuerySerializer(data=request.query_params)
+        serializer.is_valid(raise_exception=True)
+        validated_data = serializer.validated_data
+
+        must_list = []
+        query_filter = validated_data.get("filter")
+        if query_filter:
+            must_list.append({"term": {"status": {"value": query_filter}}})
+
+        filter_channel = validated_data.get("channel")
+        if filter_channel:
+            must_list.append(
+                {"term": {"channel_id": {"value": filter_channel}}}
+            )
+
+        self.data["query"] = {"bool": {"must": must_list}}
+
+        self.get_document_list(request)
+        serializer = DownloadListSerializer(self.response)
+
+        return Response(serializer.data)
+
+    @staticmethod
+    @extend_schema(
+        request=AddToDownloadListSerializer(),
+        parameters=[AddToDownloadQuerySerializer()],
+        responses={
+            200: OpenApiResponse(
+                AsyncTaskResponseSerializer(),
+                description="New async task started",
+            ),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        },
+    )
+    def post(request):
+        """add list of videos to download queue"""
+        data_serializer = AddToDownloadListSerializer(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+
+        query_serializer = AddToDownloadQuerySerializer(
+            data=request.query_params
+        )
+        query_serializer.is_valid(raise_exception=True)
+        validated_query = query_serializer.validated_data
+
+        auto_start = validated_query.get("autostart")
+        print(f"auto_start: {auto_start}")
+        to_add = validated_data["data"]
+
+        pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
+        url_str = " ".join(pending)
+        task = extrac_dl.delay(url_str, auto_start=auto_start)
+
+        message = {
+            "message": "add to queue task started",
+            "task_id": task.id,
+        }
+        response_serializer = AsyncTaskResponseSerializer(message)
+
+        return Response(response_serializer.data)
+
+    @extend_schema(
+        parameters=[DownloadListQueueDeleteQuerySerializer()],
+        responses={
+            204: OpenApiResponse(description="Download items deleted"),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        },
+    )
+    def delete(self, request):
+        """bulk delete download queue items by filter"""
+        serializer = DownloadListQueueDeleteQuerySerializer(
+            data=request.query_params
+        )
+        serializer.is_valid(raise_exception=True)
+        validated_query = serializer.validated_data
+
+        query_filter = validated_query["filter"]
+        message = f"delete queue by status: {query_filter}"
+        print(message)
+        PendingInteract(status=query_filter).delete_by_status()
+
+        return Response(status=204)
+
+
+class DownloadApiView(ApiBaseView):
+    """resolves to /api/download/<video_id>/
+    GET: returns metadata dict of an item in the download queue
+    POST: update status of item to pending or ignore
+    DELETE: forget from download queue
+    """
+
+    search_base = "ta_download/_doc/"
+    valid_status = ["pending", "ignore", "ignore-force", "priority"]
+    permission_classes = [AdminOnly]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(DownloadItemSerializer()),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(),
+                description="Download item not found",
+            ),
+        },
+    )
+    def get(self, request, video_id):
+        # pylint: disable=unused-argument
+        """get download queue item"""
+        self.get_document(video_id)
+        if not self.response:
+            error = ErrorResponseSerializer(
+                {"error": "Download item not found"}
+            )
+            return Response(error.data, status=404)
+
+        response_serializer = DownloadItemSerializer(self.response)
+
+        return Response(response_serializer.data, status=self.status_code)
+
+    @extend_schema(
+        request=DownloadQueueItemUpdateSerializer(),
+        responses={
+            200: OpenApiResponse(
+                DownloadQueueItemUpdateSerializer(),
+                description="Download item update",
+            ),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(),
+                description="Download item not found",
+            ),
+        },
+    )
+    def post(self, request, video_id):
+        """post to video to change status"""
+        data_serializer = DownloadQueueItemUpdateSerializer(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+        item_status = validated_data["status"]
+
+        if item_status == "ignore-force":
+            extrac_dl.delay(video_id, status="ignore")
+            return Response(data_serializer.data)
+
+        _, status_code = PendingInteract(video_id).get_item()
+        if status_code == 404:
+            error = ErrorResponseSerializer(
+                {"error": "Download item not found"}
+            )
+            return Response(error.data, status=404)
+
+        print(f"{video_id}: change status to {item_status}")
+        PendingInteract(video_id, item_status).update_status()
+        if item_status == "priority":
+            download_pending.delay(auto_only=True)
+
+        return Response(data_serializer.data)
+
+    @staticmethod
+    @extend_schema(
+        responses={
+            204: OpenApiResponse(description="delete download item"),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(),
+                description="Download item not found",
+            ),
+        },
+    )
+    def delete(request, video_id):
+        # pylint: disable=unused-argument
+        """delete single video from queue"""
+        print(f"{video_id}: delete from queue")
+        PendingInteract(video_id).delete_item()
+
+        return Response(status=204)
+
+
+class DownloadAggsApiView(ApiBaseView):
+    """resolves to /api/download/aggs/
+    GET: get download aggregations
+    """
+
+    search_base = "ta_download/_search"
+    valid_filter_view = ["ignore", "pending"]
+
+    @extend_schema(
+        parameters=[DownloadListQueueDeleteQuerySerializer()],
+        responses={
+            200: OpenApiResponse(DownloadAggsSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="bad request"
+            ),
+        },
+    )
+    def get(self, request):
+        """get aggs"""
+        serializer = DownloadListQueueDeleteQuerySerializer(
+            data=request.query_params
+        )
+        serializer.is_valid(raise_exception=True)
+        validated_query = serializer.validated_data
+
+        filter_view = validated_query.get("filter")
+        if filter_view:
+            if filter_view not in self.valid_filter_view:
+                message = f"invalid filter: {filter_view}"
+                return Response({"message": message}, status=400)
+
+            self.data.update(
+                {
+                    "query": {"term": {"status": {"value": filter_view}}},
+                }
+            )
+
+        self.data.update(
+            {
+                "aggs": {
+                    "channel_downloads": {
+                        "multi_terms": {
+                            "size": 30,
+                            "terms": [
+                                {"field": "channel_name.keyword"},
+                                {"field": "channel_id"},
+                            ],
+                            "order": {"_count": "desc"},
+                        }
+                    }
+                }
+            }
+        )
+        self.get_aggs()
+        serializer = DownloadAggsSerializer(self.response["channel_downloads"])
+
+        return Response(serializer.data)

backend/playlist/serializers.py

@@ -0,0 +1,92 @@
+"""playlist serializers"""
+
+# pylint: disable=abstract-method
+
+from common.serializers import PaginationSerializer
+from rest_framework import serializers
+
+
+class PlaylistEntrySerializer(serializers.Serializer):
+    """serialize single playlist entry"""
+
+    youtube_id = serializers.CharField()
+    title = serializers.CharField()
+    uploader = serializers.CharField()
+    idx = serializers.IntegerField()
+    downloaded = serializers.BooleanField()
+
+
+class PlaylistSerializer(serializers.Serializer):
+    """serialize playlist"""
+
+    playlist_active = serializers.BooleanField()
+    playlist_channel = serializers.CharField()
+    playlist_channel_id = serializers.CharField()
+    playlist_description = serializers.CharField()
+    playlist_entries = PlaylistEntrySerializer(many=True)
+    playlist_id = serializers.CharField()
+    playlist_last_refresh = serializers.CharField()
+    playlist_name = serializers.CharField()
+    playlist_subscribed = serializers.BooleanField()
+    playlist_thumbnail = serializers.CharField()
+    playlist_type = serializers.ChoiceField(choices=["regular", "custom"])
+    _index = serializers.CharField(required=False)
+    _score = serializers.IntegerField(required=False)
+
+
+class PlaylistListSerializer(serializers.Serializer):
+    """serialize list of playlists"""
+
+    data = PlaylistSerializer(many=True)
+    paginate = PaginationSerializer()
+
+
+class PlaylistListQuerySerializer(serializers.Serializer):
+    """serialize playlist list query params"""
+
+    channel = serializers.CharField(required=False)
+    subscribed = serializers.BooleanField(required=False)
+    type = serializers.ChoiceField(
+        choices=["regular", "custom"], required=False
+    )
+    page = serializers.IntegerField(required=False)
+
+
+class PlaylistSingleAddSerializer(serializers.Serializer):
+    """single item to add"""
+
+    playlist_id = serializers.CharField()
+    playlist_subscribed = serializers.ChoiceField(choices=[True])
+
+
+class PlaylistBulkAddSerializer(serializers.Serializer):
+    """bulk add playlists serializers"""
+
+    data = PlaylistSingleAddSerializer(many=True)
+
+
+class PlaylistSingleUpdate(serializers.Serializer):
+    """update state of single playlist"""
+
+    playlist_subscribed = serializers.BooleanField()
+
+
+class PlaylistListCustomPostSerializer(serializers.Serializer):
+    """serialize list post custom playlist"""
+
+    playlist_name = serializers.CharField()
+
+
+class PlaylistCustomPostSerializer(serializers.Serializer):
+    """serialize playlist custom action"""
+
+    action = serializers.ChoiceField(
+        choices=["create", "remove", "up", "down", "top", "bottom"]
+    )
+    video_id = serializers.CharField()
+
+
+class PlaylistDeleteQuerySerializer(serializers.Serializer):
+    """serialize playlist delete query params"""
+
+    delete_videos = serializers.BooleanField(required=False)

backend/playlist/src/constants.py

@@ -0,0 +1,10 @@
+"""playlist constants"""
+
+import enum
+
+
+class PlaylistTypesEnum(enum.Enum):
+    """all playlist_type options"""
+
+    REGULAR = "regular"
+    CUSTOM = "custom"

backend/playlist/src/index.py

@@ -7,11 +7,12 @@ functionality:
 import json
 from datetime import datetime
 
-from home.src.download.thumbnails import ThumbManager
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.index import channel
-from home.src.index.generic import YouTubeItem
-from home.src.index.video import YoutubeVideo
+from channel.src import index as channel
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.index_generic import YouTubeItem
+from download.src.thumbnails import ThumbManager
+from video.src import index as ta_video
 
 
 class YoutubePlaylist(YouTubeItem):
@@ -93,13 +94,10 @@ class YoutubePlaylist(YouTubeItem):
         """get all videos in playlist, match downloaded with ids_found"""
         all_members = []
         for idx, entry in enumerate(self.youtube_meta["entries"]):
-            if not entry["channel"]:
-                continue
-
             to_append = {
                 "youtube_id": entry["id"],
                 "title": entry["title"],
-                "uploader": entry["channel"],
+                "uploader": entry.get("channel"),
                 "idx": idx,
                 "downloaded": entry["id"] in ids_found,
             }
@@ -149,6 +147,9 @@ class YoutubePlaylist(YouTubeItem):
             "query": {"match": {"playlist": self.youtube_id}},
             "_source": ["youtube_id"],
         }
+        data = {
+            "query": {"term": {"playlist.keyword": {"value": self.youtube_id}}}
+        }
         result = IndexPaginate("ta_video", data).get_results()
         to_remove = [
             i["youtube_id"] for i in result if i["youtube_id"] not in needed
@@ -190,6 +191,7 @@ class YoutubePlaylist(YouTubeItem):
 
     def build_nav(self, youtube_id):
         """find next and previous in playlist of a given youtube_id"""
+        cache_root = EnvironmentSettings().get_cache_root()
         all_entries_available = self.json_data["playlist_entries"]
         all_entries = [i for i in all_entries_available if i["downloaded"]]
         current = [i for i in all_entries if i["youtube_id"] == youtube_id]
@@ -199,18 +201,20 @@ class YoutubePlaylist(YouTubeItem):
 
         current_idx = all_entries.index(current[0])
         if current_idx == 0:
-            previous_item = False
+            previous_item = None
         else:
             previous_item = all_entries[current_idx - 1]
             prev_id = previous_item["youtube_id"]
-            previous_item["vid_thumb"] = ThumbManager(prev_id).vid_thumb_path()
+            prev_thumb_path = ThumbManager(prev_id).vid_thumb_path()
+            previous_item["vid_thumb"] = f"{cache_root}/{prev_thumb_path}"
 
         if current_idx == len(all_entries) - 1:
-            next_item = False
+            next_item = None
         else:
             next_item = all_entries[current_idx + 1]
             next_id = next_item["youtube_id"]
-            next_item["vid_thumb"] = ThumbManager(next_id).vid_thumb_path()
+            next_thumb_path = ThumbManager(next_id).vid_thumb_path()
+            next_item["vid_thumb"] = f"{cache_root}/{next_thumb_path}"
 
         self.nav = {
             "playlist_meta": {
@@ -255,7 +259,7 @@ class YoutubePlaylist(YouTubeItem):
         i = 0
         while i < len(playlist):
             video_id = playlist[i]["youtube_id"]
-            video = YoutubeVideo(video_id)
+            video = ta_video.YoutubeVideo(video_id)
             video.get_from_es()
             if (
                 channel_id is None
@@ -278,7 +282,7 @@ class YoutubePlaylist(YouTubeItem):
             if i["downloaded"]
         ]
         for youtube_id in all_youtube_id:
-            YoutubeVideo(youtube_id).delete_media_file()
+            ta_video.YoutubeVideo(youtube_id).delete_media_file()
 
         self.delete_metadata()
 
@@ -312,7 +316,7 @@ class YoutubePlaylist(YouTubeItem):
             )
             self.set_playlist_thumbnail()
             self.upload_to_es()
-            video = YoutubeVideo(video_id)
+            video = ta_video.YoutubeVideo(video_id)
             video.get_from_es()
             if "playlist" not in video.json_data:
                 video.json_data["playlist"] = []
@@ -321,7 +325,7 @@ class YoutubePlaylist(YouTubeItem):
         return True
 
     def remove_playlist_from_video(self, video_id):
-        video = YoutubeVideo(video_id)
+        video = ta_video.YoutubeVideo(video_id)
         video.get_from_es()
         if video.json_data is not None and "playlist" in video.json_data:
             video.json_data["playlist"].remove(self.youtube_id)
@@ -410,7 +414,7 @@ class YoutubePlaylist(YouTubeItem):
         )
 
     def get_video_is_watched(self, video_id):
-        video = YoutubeVideo(video_id)
+        video = ta_video.YoutubeVideo(video_id)
         video.get_from_es()
         return video.json_data["player"]["watched"]
 
@@ -426,7 +430,7 @@ class YoutubePlaylist(YouTubeItem):
         self.get_playlist_art()
 
     def get_video_metadata(self, video_id):
-        video = YoutubeVideo(video_id)
+        video = ta_video.YoutubeVideo(video_id)
         video.get_from_es()
         video_json_data = {
             "youtube_id": video.json_data["youtube_id"],

backend/playlist/src/query_building.py

@@ -0,0 +1,52 @@
+"""build query for playlists"""
+
+from playlist.src.constants import PlaylistTypesEnum
+
+
+class QueryBuilder:
+    """contain functionality"""
+
+    def __init__(self, **kwargs):
+        self.request_params = kwargs
+
+    def build_data(self) -> dict:
+        """build data dict"""
+        data = {}
+        data["query"] = self.build_query()
+        if sort := self.parse_sort():
+            data.update(sort)
+
+        return data
+
+    def build_query(self) -> dict:
+        """build query key"""
+        must_list = []
+        channel = self.request_params.get("channel")
+        if channel:
+            must_list.append({"match": {"playlist_channel_id": channel}})
+
+        subscribed = self.request_params.get("subscribed")
+        if subscribed:
+            must_list.append({"match": {"playlist_subscribed": subscribed}})
+
+        playlist_type = self.request_params.get("type")
+        if playlist_type:
+            type_list = self.parse_type(playlist_type)
+            must_list.append(type_list)
+
+        query = {"bool": {"must": must_list}}
+
+        return query
+
+    def parse_type(self, playlist_type: str) -> dict:
+        """parse playlist type"""
+        if not hasattr(PlaylistTypesEnum, playlist_type.upper()):
+            raise ValueError(f"'{playlist_type}' not in PlaylistTypesEnum")
+
+        type_parsed = getattr(PlaylistTypesEnum, playlist_type.upper()).value
+
+        return {"match": {"playlist_type.keyword": type_parsed}}
+
+    def parse_sort(self) -> dict:
+        """return sort"""
+        return {"sort": [{"playlist_name.keyword": {"order": "asc"}}]}

backend/playlist/tests/test_src/test_query_building.py

@@ -0,0 +1,30 @@
+"""test playlist query building"""
+
+import pytest
+from playlist.src.query_building import QueryBuilder
+
+
+def test_build_data():
+    """test for correct key building"""
+    qb = QueryBuilder(
+        channel="test_channel",
+        subscribed=True,
+        type="regular",
+    )
+    result = qb.build_data()
+    must_list = result["query"]["bool"]["must"]
+    assert "query" in result
+    assert "sort" in result
+    assert result["sort"] == [{"playlist_name.keyword": {"order": "asc"}}]
+    assert {"match": {"playlist_channel_id": "test_channel"}} in must_list
+    assert {"match": {"playlist_subscribed": True}} in must_list
+
+
+def test_parse_type():
+    """validate type"""
+    qb = QueryBuilder(type="regular")
+    with pytest.raises(ValueError):
+        qb.parse_type("invalid")
+
+    result = qb.parse_type("custom")
+    assert result == {"match": {"playlist_type.keyword": "custom"}}

backend/playlist/urls.py

@@ -0,0 +1,27 @@
+"""all playlist API urls"""
+
+from django.urls import path
+from playlist import views
+
+urlpatterns = [
+    path(
+        "",
+        views.PlaylistApiListView.as_view(),
+        name="api-playlist-list",
+    ),
+    path(
+        "custom/",
+        views.PlaylistCustomApiListView.as_view(),
+        name="api-custom-playlist-list",
+    ),
+    path(
+        "custom/<slug:playlist_id>/",
+        views.PlaylistCustomApiView.as_view(),
+        name="api-custom-playlist",
+    ),
+    path(
+        "<slug:playlist_id>/",
+        views.PlaylistApiView.as_view(),
+        name="api-playlist",
+    ),
+]

backend/playlist/views.py

@@ -0,0 +1,273 @@
+"""all playlist API views"""
+
+import uuid
+
+from common.serializers import (
+    AsyncTaskResponseSerializer,
+    ErrorResponseSerializer,
+)
+from common.views_base import AdminWriteOnly, ApiBaseView
+from download.src.subscriptions import PlaylistSubscription
+from drf_spectacular.utils import OpenApiResponse, extend_schema
+from playlist.serializers import (
+    PlaylistBulkAddSerializer,
+    PlaylistCustomPostSerializer,
+    PlaylistDeleteQuerySerializer,
+    PlaylistListCustomPostSerializer,
+    PlaylistListQuerySerializer,
+    PlaylistListSerializer,
+    PlaylistSerializer,
+    PlaylistSingleUpdate,
+)
+from playlist.src.index import YoutubePlaylist
+from playlist.src.query_building import QueryBuilder
+from rest_framework.response import Response
+from task.tasks import subscribe_to
+from user.src.user_config import UserConfig
+
+
+class PlaylistApiListView(ApiBaseView):
+    """resolves to /api/playlist/
+    GET: returns list of indexed playlists
+    params:
+    - channel:str=<channel-id>
+    - subscribed: bool
+    - type:enum=regular|custom
+    POST: change subscribe state
+    """
+
+    search_base = "ta_playlist/_search/"
+    permission_classes = [AdminWriteOnly]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(PlaylistListSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        },
+        parameters=[PlaylistListQuerySerializer],
+    )
+    def get(self, request):
+        """get playlist list"""
+        query_serializer = PlaylistListQuerySerializer(
+            data=request.query_params
+        )
+        query_serializer.is_valid(raise_exception=True)
+        validated_query = query_serializer.validated_data
+        try:
+            data = QueryBuilder(**validated_query).build_data()
+        except ValueError as err:
+            error = ErrorResponseSerializer({"error": str(err)})
+            return Response(error.data, status=400)
+
+        self.data = data
+        self.get_document_list(request)
+
+        response_serializer = PlaylistListSerializer(self.response)
+
+        return Response(response_serializer.data)
+
+    @extend_schema(
+        request=PlaylistBulkAddSerializer(),
+        responses={
+            200: OpenApiResponse(AsyncTaskResponseSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        },
+    )
+    def post(self, request):
+        """async subscribe to list of playlists"""
+        data_serializer = PlaylistBulkAddSerializer(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+
+        pending = [i["playlist_id"] for i in validated_data["data"]]
+        if not pending:
+            error = ErrorResponseSerializer({"error": "nothing to subscribe"})
+            return Response(error.data, status=400)
+
+        url_str = " ".join(pending)
+        task = subscribe_to.delay(url_str, expected_type="playlist")
+
+        message = {
+            "message": "playlist subscribe task started",
+            "task_id": task.id,
+        }
+        serializer = AsyncTaskResponseSerializer(message)
+
+        return Response(serializer.data)
+
+
+class PlaylistCustomApiListView(ApiBaseView):
+    """resolves to /api/playlist/custom/
+    POST: Create new custom playlist
+    """
+
+    search_base = "ta_playlist/_search/"
+    permission_classes = [AdminWriteOnly]
+
+    @extend_schema(
+        request=PlaylistListCustomPostSerializer(),
+        responses={
+            200: OpenApiResponse(PlaylistSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="Bad request"
+            ),
+        },
+    )
+    def post(self, request):
+        """create new custom playlist"""
+        serializer = PlaylistListCustomPostSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        validated_data = serializer.validated_data
+
+        custom_name = validated_data["playlist_name"]
+        playlist_id = f"TA_playlist_{uuid.uuid4()}"
+        custom_playlist = YoutubePlaylist(playlist_id)
+        custom_playlist.create(custom_name)
+
+        response_serializer = PlaylistSerializer(custom_playlist.json_data)
+
+        return Response(response_serializer.data)
+
+
+class PlaylistCustomApiView(ApiBaseView):
+    """resolves to /api/playlist/custom/<playlist_id>/
+    POST: modify custom playlist
+    """
+
+    search_base = "ta_playlist/_doc/"
+    permission_classes = [AdminWriteOnly]
+
+    @extend_schema(
+        request=PlaylistCustomPostSerializer(),
+        responses={
+            200: OpenApiResponse(PlaylistSerializer()),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="bad request"
+            ),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="playlist not found"
+            ),
+        },
+    )
+    def post(self, request, playlist_id):
+        """modify custom playlist"""
+        data_serializer = PlaylistCustomPostSerializer(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+
+        self.get_document(playlist_id)
+        if not self.response:
+            error = ErrorResponseSerializer({"error": "playlist not found"})
+            return Response(error.data, status=404)
+
+        if not self.response["playlist_type"] == "custom":
+            error = ErrorResponseSerializer(
+                {"error": f"playlist with ID {playlist_id} is not custom"}
+            )
+            return Response(error.data, status=400)
+
+        action = validated_data.get("action")
+        video_id = validated_data.get("video_id")
+
+        playlist = YoutubePlaylist(playlist_id)
+        if action == "create":
+            try:
+                playlist.add_video_to_playlist(video_id)
+            except TypeError:
+                error = ErrorResponseSerializer(
+                    {"error": f"failed to add video {video_id} to playlist"}
+                )
+                return Response(error.data, status=400)
+        else:
+            hide = UserConfig(request.user.id).get_value("hide_watched")
+            playlist.move_video(video_id, action, hide_watched=hide)
+
+        response_serializer = PlaylistSerializer(playlist.json_data)
+
+        return Response(response_serializer.data)
+
+
+class PlaylistApiView(ApiBaseView):
+    """resolves to /api/playlist/<playlist_id>/
+    GET: returns metadata dict of playlist
+    """
+
+    search_base = "ta_playlist/_doc/"
+    permission_classes = [AdminWriteOnly]
+    valid_custom_actions = ["create", "remove", "up", "down", "top", "bottom"]
+
+    @extend_schema(
+        responses={
+            200: OpenApiResponse(PlaylistSerializer()),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="playlist not found"
+            ),
+        },
+    )
+    def get(self, request, playlist_id):
+        # pylint: disable=unused-argument
+        """get playlist"""
+        self.get_document(playlist_id)
+        if not self.response:
+            error = ErrorResponseSerializer({"error": "playlist not found"})
+            return Response(error.data, status=404)
+
+        response_serializer = PlaylistSerializer(self.response)
+
+        return Response(response_serializer.data)
+
+    @extend_schema(
+        request=PlaylistSingleUpdate(),
+        responses={
+            200: OpenApiResponse(PlaylistSerializer()),
+            404: OpenApiResponse(
+                ErrorResponseSerializer(), description="playlist not found"
+            ),
+        },
+    )
+    def post(self, request, playlist_id):
+        """update subscribed state of playlist"""
+        data_serializer = PlaylistSingleUpdate(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+
+        self.get_document(playlist_id)
+        if not self.response:
+            error = ErrorResponseSerializer({"error": "playlist not found"})
+            return Response(error.data, status=404)
+
+        subscribed = validated_data["playlist_subscribed"]
+        playlist_sub = PlaylistSubscription()
+        json_data = playlist_sub.change_subscribe(playlist_id, subscribed)
+
+        response_serializer = PlaylistSerializer(json_data)
+        return Response(response_serializer.data)
+
+    @extend_schema(
+        parameters=[PlaylistDeleteQuerySerializer],
+        responses={
+            204: OpenApiResponse(description="playlist deleted"),
+        },
+    )
+    def delete(self, request, playlist_id):
+        """delete playlist"""
+        print(f"{playlist_id}: delete playlist")
+
+        query_serializer = PlaylistDeleteQuerySerializer(
+            data=request.query_params
+        )
+        query_serializer.is_valid(raise_exception=True)
+        validated_query = query_serializer.validated_data
+
+        delete_videos = validated_query.get("delete_videos", False)
+
+        if delete_videos:
+            YoutubePlaylist(playlist_id).delete_videos_playlist()
+        else:
+            YoutubePlaylist(playlist_id).delete_metadata()
+
+        return Response(status=204)

backend/requirements-dev.txt

@@ -0,0 +1,10 @@
+-r requirements.txt
+ipython==9.2.0
+pre-commit==4.2.0
+pylint-django==2.6.1
+pylint==3.3.7
+pytest-django==4.11.1
+pytest==8.3.5
+python-dotenv==1.1.0
+requirementscheck==0.0.6
+types-requests==2.32.0.20250328

backend/requirements.txt

@@ -0,0 +1,15 @@
+apprise==1.9.3
+celery==5.5.2
+django-auth-ldap==5.2.0
+django-celery-beat==2.8.0
+django-cors-headers==4.7.0
+Django==5.2.1
+djangorestframework==3.16.0
+drf-spectacular==0.28.0
+Pillow==11.2.1
+redis==6.0.0
+requests==2.32.3
+ryd-client==0.0.6
+uvicorn==0.34.2
+whitenoise==6.9.0
+yt-dlp[default]==2025.4.30