bump TA_VERSION v0.3.6

Changed:
- Added auto tag to download queue item
- Add filesystem rescan before stream extract migration
- Add channel tags to search results
- Fix migration notification logic
- Fix filesystem rescan without task

.eslintrc.js

@@ -0,0 +1,17 @@
+'use strict';
+module.exports = {
+  extends: ['eslint:recommended', 'eslint-config-prettier'],
+  parserOptions: {
+    ecmaVersion: 2020,
+  },
+  env: {
+    browser: true,
+  },
+  rules: {
+    strict: ['error', 'global'],
+    'no-unused-vars': ['error', { vars: 'local' }],
+    eqeqeq: ['error', 'always', { null: 'ignore' }],
+    curly: ['error', 'multi-line'],
+    'no-var': 'error',
+  },
+};

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+github: bbilly1
+ko_fi: bbilly1
+custom: https://paypal.me/bbilly1

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -0,0 +1,55 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to help improve this project! Please read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide carefully before continuing.
+
+  - type: checkboxes
+    id: latest
+    attributes:
+      label: "I've read the documentation"
+      options:
+        - label: I'm running the latest version of Tube Archivist and have read the [release notes](https://github.com/tubearchivist/tubearchivist/releases/latest).
+          required: true
+        - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [bug report](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#bug-report) section.
+          required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: On what operating system are you running Tube Archivist on?
+      placeholder: ex. Linux, Unraid, Synology...
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: Your Bug Report
+      value: "## Describe the bug\n\n## Steps To Reproduce\n\n## Expected behavior"
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Anything else?
+      description: |
+        Links? References? Anything that will give us more context about the issue you are encountering!
+
+        Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -0,0 +1,34 @@
+name: Feature Request
+description: I have an idea for a great addition to this project
+title: "[Feature Request]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to help improve this project! This project is *very* selective with accepting new feature requests. Please read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide carefully before continuing.
+
+  - type: checkboxes
+    id: already
+    attributes:
+      label: "I've read the documentation"
+      options:
+        - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [feature request](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#feature-request) section.
+          required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Your Feature Request
+      value: "## Is your feature request related to a problem? Please describe.\n\n## Describe the solution you'd like\n\n## Additional context"
+      placeholder: Tell us what you see!
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: help
+    attributes:
+      label: Your help is needed!
+      description: This project is ambitious as it is, please contribute.
+      options:
+        - label: Yes I will work on this in the next few days or weeks.

.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml

@@ -0,0 +1,43 @@
+name: Installation Help
+description: There is an issue with installing or configuring Tube Archivist
+title: "[Installation]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This is most likely not the best place to ask for installation help. There is a helpful community on [Discord](https://github.com/tubearchivist/tubearchivist#connect).
+
+  - type: checkboxes
+    id: instructions
+    attributes:
+      label: Installation instructions
+      options:
+        - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [installation help](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#installation-help) section.
+          required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: On what operating system do you want to install Tube Archivist on?
+      placeholder: ex. Linux, Unraid, Synology...
+    validations:
+      required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Your installation or setup question
+      placeholder: Describe your issue
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+    validations:
+      required: true

.github/pull_request_template.md

@@ -0,0 +1,3 @@
+Thank you for taking the time to improve this project. Please take a look at the [How to make a Pull Request](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-make-a-pull-request) section to help get your contribution merged.
+
+You can delete this text before submitting.

.github/workflows/lint_js.yml

@@ -0,0 +1,16 @@
+name: lint_js
+
+on: [pull_request, push]
+
+jobs:
+  check:
+    name: lint_js
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      - run: npm ci
+      - run: npm run lint
+      - run: npm run format -- --check

.github/workflows/lint_python.yml

@@ -4,16 +4,21 @@ jobs:
   lint_python:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      # note: this logic is duplicated in the `validate` function in ./deploy.sh
+      # if you update this file, you should update that as well
       - run: pip install --upgrade pip wheel
       - run: pip install bandit black codespell flake8 flake8-bugbear
-                         flake8-comprehensions isort
-      - run: black --check --diff --line-length 79 .
-      - run: codespell
-      - run: flake8 . --count --max-complexity=10 --max-line-length=79
-                      --show-source --statistics
-      - run: isort --check-only --line-length 79 --profile black .
+                         flake8-comprehensions isort requests
+      - run: ./deploy.sh validate
+      # - run: black --check --diff --line-length 79 .
+      # - run: codespell --skip="./.git,./package.json,./package-lock.json,./node_modules"
+      # - run: flake8 . --count --max-complexity=10 --max-line-length=79
+      #                 --show-source --statistics
+      # - run: isort --check-only --line-length 79 --profile black .
       # - run: pip install -r tubearchivist/requirements.txt
       # - run: mkdir --parents --verbose .mypy_cache
       # - run: mypy --ignore-missing-imports --install-types --non-interactive .

.gitignore

@@ -5,4 +5,7 @@ __pycache__
 db.sqlite3
 
 # vscode custom conf
-.vscode
+.vscode
+
+# JavaScript stuff
+node_modules

ARCHITECTURE.md

@@ -0,0 +1,48 @@
+# The Inner Workings of Tube Archivist
+This is a high level overview of the architecture of Tube Archivist, intended for interested contributors to find your way around quickly.
+
+```
+                        Tube Archivist
+        ______________________|_____________________
+        |                     |                    |
+-------------------    ---------------    -------------------
+|                 |    |             |    |                 |
+|  DjangoProject  |    |  RedisJson  |    |  ElasticSearch  |
+|                 |    |             |    |                 |
+-------------------    ---------------    -------------------
+```
+
+## DjangoProject
+This is the main Python application. Django serves its data container internally with **Uwsgi** on port 8080, the interface is served with **Nginx** on the public port 8000.
+
+Users created static files like media files and artwork as well as application artwork like logos and fonts are served directly from Nginx, while the rest of the application uses uwsgi_pass to proxy the requests to uwsgi.
+
+Config files are located in the `docker_assets` folder. The script `run.sh` is the container `CMD` command and entry point, validating env vars, connection to ElasticSearch (ES) and will start the application.
+
+Compared to other Django projects, this application doesn't make use of the database models, due to a lack of integration with ES. This project has its own abstractions and integrations, treating ES as a REST API.
+
+Long running application tasks are handed off to **Celery** - using **Redis** as a broker - to run asynchronously from the main threads. 
+- All tasks are defined in the `home.tasks.py` module.
+
+There are three Django apps:
+- **config**: The root app, routing the main endpoints and the main `settings.py` file
+- **api**: The API app with its views and functionality
+- **home**: Most of the application logic, templates and views, will probably get split up further in the future.
+
+The *home* app is split up into packages in the `src` directory:
+- **download**: All download related classes, interact with yt-dlp, download artwork, handle the download queue and post processing tasks.
+- **es**: All index setup and validation classes, handles mapping validations and makes mapping changes, wrapper functions to simplify interactions with Elasticsearch, backup and restore.
+- **frontend**: All direct interactions with the frontend, like Django forms, searching, watched state changes, and legacy api_calls in the process of moving to the api app.
+- **index**: Contains all functionality for scraping and indexing videos, channels, playlists, comments, subtitles, etc...
+- **ta**: Loose collection of functions and classes, handle application config and contains redis wrapper classes.
+
+## RedisJson
+Holds the main application config json object that gets dynamically edited from the frontend, serves as a message broker for **Celery**. Redis serves as a temporary and thread safe link between Django and the frontend, storing progress messages and temporary queues for processing. Used to store locking keys for threads and execution details for tasks.
+
+- Wrapper classes to interact with Redis are located in the `home.src.ta.ta_redis.py` module.
+
+## ElasticSearch (ES)
+Is used to store and index all metadata, functions as an application database and makes it all searchable. The mapping defines which fields are indexed as searchable text fields and which fields are used for match filtering.
+
+- The index setup and validation is handled in the `home.src.es.index_setup.py` module.
+- Wrapper classes for making requests to ES are located in the `home.src.es.connect.py` module.

CONTRIBUTING.md

@@ -1,57 +1,81 @@
 ## Contributing to Tube Archivist
 
 Welcome, and thanks for showing interest in improving Tube Archivist!  
-If you haven't already, the best place to start is the README. This will give you an overview on what the project is all about.
 
-## Report a bug
+## Table of Content
+- [How to open an issue](#how-to-open-an-issue)
+  - [Bug Report](#bug-report)
+  - [Feature Request](#feature-request)
+  - [Installation Help](#installation-help)
+- [How to make a Pull Request](#how-to-make-a-pull-request)
+- [Improve to the Documentation](#improve-to-the-documentation)
+- [Development Environment](#development-environment)
+---
 
-If you notice something is not working as expected, check to see if it has been previously reported in the [open issues](https://github.com/tubearchivist/tubearchivist/issues).
-If it has not yet been disclosed, go ahead and create an issue.  
-If the issue doesn't move forward due to a lack of response, I assume it's solved and will close it after some time to keep the list fresh. 
+## How to open an issue
+Please read this carefully before opening any [issue](https://github.com/tubearchivist/tubearchivist/issues) on GitHub.
 
-## Wiki
+**Do**:
+- Do provide details and context, this matters a lot and makes it easier for people to help.
+- Do familiarize yourself with the project first, some questions answer themselves when using the project for some time. Familiarize yourself with the [Readme](https://github.com/tubearchivist/tubearchivist) and the [documentation](https://docs.tubearchivist.com/), this covers a lot of the common questions, particularly the [FAQ](https://docs.tubearchivist.com/faq/).
+- Do respond to questions within a day or two so issues can progress. If the issue doesn't move forward due to a lack of response, we'll assume it's solved and we'll close it after some time to keep the list fresh.
 
-The wiki is where all user functions are documented in detail. These pages are mirrored into the **docs** folder of the repo. This allows for pull requests and all other features like regular code. Make any changes there, and I'll sync them with the wiki tab.
+**Don't**:
+- Don't open *duplicates*, that includes open and closed issues.
+- Don't open an issue for something that's already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), this needs your help to implement it, not another issue.
+- Don't open an issue for something that's a [known limitation](https://github.com/tubearchivist/tubearchivist#known-limitations). These are *known* by definition and don't need another reminder. Some limitations may be solved in the future, maybe by you?
+- Don't overwrite the *issue template*, they are there for a reason. Overwriting that shows that you don't really care about this project. It shows that you have a misunderstanding how open source collaboration works and just want to push your ideas through. Overwriting the template may result in a ban.
 
-## Development Environment
+### Bug Report
+Bug reports are highly welcome! This project has improved a lot due to your help by providing feedback when something doesn't work as expected. The developers can't possibly cover all edge cases in an ever changing environment like YouTube and yt-dlp.
 
-I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment.  
+Please keep in mind:
+- Docker logs are the easiest way to understand what's happening when something goes wrong, *always* provide the logs upfront.
+- Set the environment variable `DJANGO_DEBUG=True` to Tube Archivist and reproduce the bug for a better log output. Don't forget to remove that variable again after.
+- A bug that can't be reproduced, is difficult or sometimes even impossible to fix. Provide very clear steps *how to reproduce*.
 
-This is my setup I have landed on, YMMV:
-- Clone the repo, work on it with your favorite code editor in your local filesystem. *testing* branch is the where all the changes are happening, might be unstable and is WIP.
-- Then I have a VM running standard Ubuntu Server LTS with docker installed. The VM keeps my projects separate and offers convenient snapshot functionality. The VM also offers ways to simulate lowend environments by limiting CPU cores and memory. You can use this [Ansible Docker Ubuntu](https://github.com/bbilly1/ansible-playbooks) playbook to get started quickly. But you could also just run docker on your host system.
-- The `Dockerfile` is structured in a way that the actual application code is in the last layer so rebuilding the image with only code changes utilizes the build cache for everything else and will just take a few seconds.
-- Take a look at the `deploy.sh` file. I have my local DNS resolve `tubearchivist.local` to the IP of the VM for convenience. To deploy the latest changes and rebuild the application to the testing VM run:
-```bash
-./deploy.sh test
-```
-- The command above will call the docker build command with `--build-arg INSTALL_DEBUG=1` to install additional useful debug tools.
-- The `test` argument takes another optional argument to build for a specific architecture valid options are: `amd64`, `arm64` and `multi`, default is `amd64`.
-- This `deploy.sh` script is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking. 
+### Feature Request
+This project needs your help to grow further. There is no shortage of ideas, see the open [issues on GH](https://github.com/tubearchivist/tubearchivist/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement) and the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), what this project lacks is contributors to implement these ideas.
 
-## Working with Elasticsearch
-Additionally to the required services as listed in the example docker-compose file, the **Dev Tools** of [Kibana](https://www.elastic.co/guide/en/kibana/current/docker.html) are invaluable for running and testing Elasticsearch queries.  
+Existing ideas are easily *multiple years* worth of development effort, at least at current speed. Best and fastest way to implement your feature is to do it yourself, that's why this project is open source after all. This project is *very* selective with accepting new feature requests at this point.  
 
-If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.
+Good feature requests usually fall into one or more of these categories:
+- You want to work on your own idea within the next few days or weeks.
+- Your idea is beneficial for a wide range of users, not just for you.
+- Your idea extends the current project by building on and improving existing functionality.
+- Your idea is quick and easy to implement, for an experienced as well as for a first time contributor.
 
-## Implementing a new feature
+Your request is likely going to be rejected if:
+- Your idea requires multiple days worth of development time and is unrealistic to be implemented any time soon.
+- There are already other ways to do what you are trying to do.
+- You are trying to do something that only applies to your platform, your specific workflow or your specific setup.
+- Your idea would fundamentally change how the project works or it wouldn't be able to be implemented with backwards compatibility.
+- Your idea is not a good fit for this project.
 
-Do you see anything on the roadmap that you would like to take a closer look at but you are not sure, what's the best way to tackle that? Or anything not on there yet you'd like to implement but are not sure how? Reach out on Discord and we'll look into it together.
+### Installation Help
+GitHub is most likely not the best place to ask for installation help. That's inherently individual and one on one.
+1. First step is always, help yourself. Start at the [Readme](https://github.com/tubearchivist/tubearchivist) or the additional platform specific installation pages in the [docs](https://docs.tubearchivist.com/).
+2. If that doesn't answer your question, open a `#support` thread on [Discord](https://www.tubearchivist.com/discord).
+3. Only if that is not an option, open an issue here.
 
-## Making changes
+IMPORTANT: When receiving help, contribute back to the community by improving the installation instructions with your newly gained knowledge.
 
-To fix a bug or implement a feature, fork the repository and make all changes to the testing branch. When ready, create a pull request.
+---
 
-## Releases
+## How to make a Pull Request
 
-There are three different docker tags:
-- **latest**: As the name implies is the latest multiarch release for regular usage.
-- **unstable**: Intermediate amd64 builds for quick testing and improved collaboration. Don't mix with a *latest* installation, for your testing environment only. This is untested and WIP and will have breaking changes between commits that might require a reset to resolve. 
-- **semantic versioning**: There will be a handful named version tags that will also have a matching release and tag on github.
+Thank you for contributing and helping improve this project. This is a quick checklist to help streamline the process:
 
-If you want to see what's in your container, checkout the matching release tag. A merge to **master** usually means a *latest* or *unstable* release. If you want to preview changes in your testing environment, pull the *unstable* tag or clone the repository and build the docker container with the Dockerfile from the **testing** branch.
+- For **code changes**, make your PR against the [testing branch](https://github.com/tubearchivist/tubearchivist/tree/testing). That's where all active development happens. This simplifies the later merging into *master*, minimizes any conflicts and usually allows for easy and convenient *fast-forward* merging.
+- For **documentation changes**, make your PR directly against the *master* branch.
+- Show off your progress, even if not yet complete, by creating a [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) PR first and switch it as *ready* when you are ready.
+- Make sure all your code is linted and formatted correctly, see below. The automatic GH action unfortunately needs to be triggered manually by a maintainer for first time contributors, but will trigger automatically for existing contributors.
 
-## Code formatting and linting
+### Making changes to the JavaScript
+
+The JavaScript does not require any build step; you just edit the files directly. However, there is config for eslint and prettier (a linter and formatter respectively); their use is recommended but not required. To use them, install `node`, run `npm i` from the root directory of this repository to install dependencies, then run `npm run lint` and `npm run format` to run eslint and prettier respectively.
+
+### Code formatting and linting
 
 To keep things clean and consistent for everybody, there is a github action setup to lint and check the changes. You can test your code locally first if you want. For example if you made changes in the **video** module, run
 
@@ -59,4 +83,67 @@ To keep things clean and consistent for everybody, there is a github action setu
 ./deploy.sh validate tubearchivist/home/src/index/video.py
 ```
 
-to validate your changes. If you omit the path, all the project files will get checked. This is subject to change as the codebase improves. 
+to validate your changes. If you omit the path, all the project files will get checked. This is subject to change as the codebase improves.
+
+---
+
+## Improve to the Documentation
+
+The documentation available at [docs.tubearchivist.com](https://docs.tubearchivist.com/) and is build from a separate repo [tubearchivist/docs](https://github.com/tubearchivist/docs). The Readme has additional instructions on how to make changes.
+
+---
+
+## Development Environment
+
+I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. You might be able to run the application directly, but this document assumes you're using docker.
+
+### Instructions
+
+Set up docker on your development machine.
+
+Clone this repository.
+
+Functional changes should be made against the unstable `testing` branch, so check that branch out, then make a new branch for your work.
+
+Edit the `docker-compose.yml` file and replace the [`image: bbilly1/tubearchivist` line](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L7) with `build: .`. Also make any other changes to the environment variables and so on necessary to run the application, just like you're launching the application as normal.
+
+Run `docker compose up --build`. This will bring up the application. Kill it with `ctrl-c` or by running `docker compose down` from a new terminal window in the same directory.
+
+Make your changes locally and re-run `docker compose up --build`. The `Dockerfile` is structured in a way that the actual application code is in the last layer so rebuilding the image with only code changes utilizes the build cache for everything else and will just take a few seconds.
+
+### Develop environment inside a VM
+
+You may find it nice to run everything inside of a VM, though this is not necessary. There's a `deploy.sh` script which has some helpers for this use case. YMMV, this is what one of the developers does:
+
+- Clone the repo, work on it with your favorite code editor in your local filesystem. *testing* branch is where all the changes are happening, might be unstable and is WIP.
+- Then I have a VM running standard Ubuntu Server LTS with docker installed. The VM keeps my projects separate and offers convenient snapshot functionality. The VM also offers ways to simulate low end environments by limiting CPU cores and memory. You can use this [Ansible Docker Ubuntu](https://github.com/bbilly1/ansible-playbooks) playbook to get started quickly. But you could also just run docker on your host system.
+- I have my local DNS resolve `tubearchivist.local` to the IP of the VM for convenience. To deploy the latest changes and rebuild the application to the testing VM run:
+```bash
+./deploy.sh test
+```
+- The command above will call the docker build command with `--build-arg INSTALL_DEBUG=1` to install additional useful debug tools.
+- The `test` argument takes another optional argument to build for a specific architecture valid options are: `amd64`, `arm64` and `multi`, default is `amd64`.
+- This `deploy.sh` script is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking.
+
+### Working with Elasticsearch
+Additionally to the required services as listed in the example docker-compose file, the **Dev Tools** of [Kibana](https://www.elastic.co/guide/en/kibana/current/docker.html) are invaluable for running and testing Elasticsearch queries.
+
+**Quick start**  
+Generate your access token in Elasitcsearch:
+```bash
+bin/elasticsearch-service-tokens create elastic/kibana kibana
+```
+
+Example docker compose, use same version as for Elasticsearch:
+```yml
+kibana:
+  image: docker.elastic.co/kibana/kibana:0.0.0
+  container_name: kibana
+  environment:
+	- "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
+	- "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
+  ports:
+	- "5601:5601"
+```
+
+If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.

Dockerfile

@@ -3,18 +3,18 @@
 
 
 # First stage to build python wheel
-FROM python:3.10.4-slim-bullseye AS builder
+FROM python:3.10.9-slim-bullseye AS builder
 ARG TARGETPLATFORM
 
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends build-essential gcc
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
 
 # install requirements
 COPY ./tubearchivist/requirements.txt /requirements.txt
 RUN pip install --user -r requirements.txt
 
 # build final image
-FROM python:3.10.4-slim-bullseye as tubearchivist
+FROM python:3.10.9-slim-bullseye as tubearchivist
 
 ARG TARGETPLATFORM
 ARG INSTALL_DEBUG
@@ -32,19 +32,23 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm
     curl \
     xz-utils && rm -rf /var/lib/apt/lists/*
 
-# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
-RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
+# install patched ffmpeg build, default to linux64
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
+    curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
+        | grep browser_download_url \
+        | grep ".*master.*linuxarm64.*tar.xz" \
+        | cut -d '"' -f 4 \
+        | xargs curl -L --output ffmpeg.tar.xz ; \
+    else \
     curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
         | grep browser_download_url \
         | grep ".*master.*linux64.*tar.xz" \
         | cut -d '"' -f 4 \
-        | xargs curl -L --output ffmpeg.tar.xz && \
-        tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
-        tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
-        rm ffmpeg.tar.xz \
-    ; elif [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
-        apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
-    ; fi
+        | xargs curl -L --output ffmpeg.tar.xz ; \
+    fi && \
+    tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
+    tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
+    rm ffmpeg.tar.xz
 
 # install debug tools for testing environment
 RUN if [ "$INSTALL_DEBUG" ] ; then \
@@ -54,12 +58,11 @@ RUN if [ "$INSTALL_DEBUG" ] ; then \
     ; fi
 
 # make folders
-RUN mkdir /cache
-RUN mkdir /youtube
-RUN mkdir /app
+RUN mkdir /cache /youtube /app
 
 # copy config files
 COPY docker_assets/nginx.conf /etc/nginx/sites-available/default
+RUN sed -i 's/^user www\-data\;$/user root\;/' /etc/nginx/nginx.conf
 
 # copy application into container
 COPY ./tubearchivist /app

README.md

@@ -1,158 +1,142 @@
 ![Tube Archivist](assets/tube-archivist-banner.jpg?raw=true "Tube Archivist Banner")  
 
-<center><h1>Your self hosted YouTube media server</h1></center>
+<h1 align="center">Your self hosted YouTube media server</h1>
+<div align="center">
+<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-docker.png" alt="tubearchivist-docker" title="Tube Archivist Docker Pulls" height="50" width="190"/></a>
+<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-star.png" alt="tubearchivist-github-star" title="Tube Archivist GitHub Stars" height="50" width="190"/></a>
+<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-forks.png" alt="tubearchivist-github-forks" title="Tube Archivist GitHub Forks" height="50" width="190"/></a>
+<a href="https://www.tubearchivist.com/discord" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-discord.png" alt="tubearchivist-discord" title="TA Discord Server Members" height="50" width="190"/></a>
+</div>
 
-Tube Archivist has a new home: https://github.com/tubearchivist/tubearchivist
+![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")
+[*more screenshots and video*](SHOWCASE.MD)
 
 ## Table of contents:
-* [Wiki](https://github.com/tubearchivist/tubearchivist/wiki) with [FAQ](https://github.com/tubearchivist/tubearchivist/wiki/FAQ)
+* [Docs](https://docs.tubearchivist.com/) with [FAQ](https://docs.tubearchivist.com/faq/), and API documentation
 * [Core functionality](#core-functionality)
-* [Screenshots](#screenshots)
-* [Problem Tube Archivist tries to solve](#problem-tube-archivist-tries-to-solve)
-* [Connect](#connect)
-* [Extended Universe](#extended-universe)
-* [Installing and updating](#installing-and-updating)
+* [Resources](#resources)
+* [Installing](#installing)
 * [Getting Started](#getting-started)
-* [Potential pitfalls](#potential-pitfalls)
-* [Roadmap](#roadmap)
 * [Known limitations](#known-limitations)
+* [Port Collisions](#port-collisions)
+* [Common Errors](#common-errors)
+* [Roadmap](#roadmap)
 * [Donate](#donate)
 
 ------------------------
 
 ## Core functionality
+Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. This includes:  
 * Subscribe to your favorite YouTube channels
 * Download Videos using **yt-dlp**
 * Index and make videos searchable
 * Play videos
 * Keep track of viewed and unviewed videos
-
-## Tube Archivist on YouTube
-[![ibracorp-youtube-video-thumb](assets/tube-archivist-ibracorp-O8H8Z01c0Ys.jpg)](https://www.youtube.com/watch?v=O8H8Z01c0Ys)
-
-## Screenshots
-![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")  
-*Home Page*
-
-![channels screenshot](assets/tube-archivist-screenshot-channels.png?raw=true "Tube Archivist Channels")  
-*All Channels*
-
-![single channel screenshot](assets/tube-archivist-screenshot-single-channel.png?raw=true "Tube Archivist Single Channel")  
-*Single Channel*
-
-![video page screenshot](assets/tube-archivist-screenshot-video.png?raw=true "Tube Archivist Video Page")  
-*Video Page*
-
-![video page screenshot](assets/tube-archivist-screenshot-download.png?raw=true "Tube Archivist Video Page")  
-*Downloads Page*
   
-## Problem Tube Archivist tries to solve
-Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface.
-
-## Connect
-- [Discord](https://discord.gg/AFwz8nE7BK): Connect with us on our Discord server.
+## Resources
+- [Discord](https://www.tubearchivist.com/discord): Connect with us on our Discord server.
 - [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit.
-
-## Extended Universe
 - [Browser Extension](https://github.com/tubearchivist/browser-extension) Tube Archivist Companion, for [Firefox](https://addons.mozilla.org/addon/tubearchivist-companion/) and [Chrome](https://chrome.google.com/webstore/detail/tubearchivist-companion/jjnkmicfnfojkkgobdfeieblocadmcie)
-- [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.
+- [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.  
 
-## Installing and updating
-Take a look at the example `docker-compose.yml` file provided. Use the *latest* or the named semantic version tag. The *unstable* tag is for intermediate testing and as the name implies, is **unstable** and not be used on your main installation but in a [testing environment](CONTRIBUTING.md).  
+## Installing
+For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. Minimal with dual core with 4 threads, better quad core plus. 
+This project requires docker. Ensure it is installed and running on your system.  
 
-Tube Archivist depends on three main components split up into separate docker containers:  
+The documentation has additional user provided instructions for [Unraid](https://docs.tubearchivist.com/installation/unraid/), [Synology](https://docs.tubearchivist.com/installation/synology/), [Podman](https://docs.tubearchivist.com/installation/podman/) and [True NAS](https://docs.tubearchivist.com/installation/truenas-scale/).
 
-### Tube Archivist
-The main Python application that displays and serves your video collection, built with Django.
-  - Serves the interface on port `8000`
-  - Needs a volume for the video archive at **/youtube**
-  - And another volume to save application data at **/cache**.
-  - The environment variables `ES_URL` and `REDIS_HOST` are needed to tell Tube Archivist where Elasticsearch and Redis respectively are located.
-  - The environment variables `HOST_UID` and `HOST_GID` allows Tube Archivist to `chown` the video files to the main host system user instead of the container user. Those two variables are optional, not setting them will disable that functionality. That might be needed if the underlying filesystem doesn't support `chown` like *NFS*. 
-  - Change the environment variables `TA_USERNAME` and `TA_PASSWORD` to create the initial credentials. 
-  - `ELASTIC_PASSWORD` is for the password for Elasticsearch. The environment variable `ELASTIC_USER` is optional, should you want to change the username from the default *elastic*.
-  - For the scheduler to know what time it is, set your timezone with the `TZ` environment variable, defaults to *UTC*.
+The instructions here should get you up and running quickly, for Docker beginners and full explanation about each environment variable, see the [docs](https://docs.tubearchivist.com/installation/docker-compose/).
 
-### Port collisions
+Take a look at the example [docker-compose.yml](https://github.com/tubearchivist/tubearchivist/blob/master/docker-compose.yml) and configure the required environment variables.
+
+**TubeArchivist**:  
+| Environment Var | Value |  |
+| ----------- | ----------- | ----------- |
+| TA_HOST | Server IP or hostname | Required |
+| TA_USERNAME | Initial username when logging into TA | Required |
+| TA_PASSWORD | Initial password when logging into TA | Required |
+| ELASTIC_PASSWORD | Password for ElasticSearch | Required |
+| REDIS_HOST | Hostname for Redis | Required |
+| TZ | Set your timezone for the scheduler | Required |
+| TA_PORT | Overwrite Nginx port | Optional |
+| TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
+| ES_URL | URL That ElasticSearch runs on | Optional |
+| HOST_GID | Allow TA to own the video files instead of container user | Optional |
+| HOST_UID | Allow TA to own the video files instead of container user | Optional |
+| ELASTIC_USER | Change the default ElasticSearch user | Optional |
+| REDIS_PORT | Port that Redis runs on | Optional |
+| TA_LDAP | Configure TA to use LDAP Authentication | [Read more](https://docs.tubearchivist.com/configuration/ldap/) |
+| ENABLE_CAST | Enable casting support | [Read more](https://docs.tubearchivist.com/configuration/cast/) |
+| DJANGO_DEBUG | Return additional error messages, for debug only |  |
+
+**ElasticSearch**  
+| Environment Var | Value | State |
+| ----------- | ----------- | ----------- |
+| ELASTIC_PASSWORD | Matching password `ELASTIC_PASSWORD` from TubeArchivist | Required |
+| http.port | Change the port ElasticSearch runs on | Optional |
+
+
+## Update
+Always use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tags are only for your testing environment, there might not be an update path for these testing builds. 
+
+You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. To take advantage of the latest fixes and improvements, make sure you are running the *latest and greatest*. After updating, check the footer to verify you are running the expected version.  
+
+  - This project is tested for updates between one or two releases maximum. Further updates back may or may not be supported and you might have to reset your index and configurations to update. Ideally apply new updates at least once per month.  
+  - There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.  
+  - All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version.   
+
+## Getting Started
+1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS and some other browsers a compatible format must be specified. For example:
+```
+bestvideo[vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4
+```
+2. Subscribe to some of your favorite YouTube channels on the **channels** page. 
+3. On the **downloads** page, click on *Rescan subscriptions* to add videos from the subscribed channels to your Download queue or click on *Add to download queue* to manually add Video IDs, links, channels or playlists.
+4. Click on *Start download* and let **Tube Archivist** to it's thing. 
+5. Enjoy your archived collection!
+
+
+### Port Collisions  
 If you have a collision on port `8000`, best solution is to use dockers *HOST_PORT* and *CONTAINER_PORT* distinction: To for example change the interface to port 9000 use `9000:8000` in your docker-compose file.  
 
-Should that not be an option, the Tube Archivist container takes these two additional environment variables:
-- **TA_PORT**: To actually change the port where nginx listens, make sure to also change the ports value in your docker-compose file.
-- **TA_UWSGI_PORT**: To change the default uwsgi port 8080 used for container internal networking between uwsgi serving the django application and nginx.  
+For more information on port collisions, check the docs.
 
-Changing any of these two environment variables will change the files *nginx.conf* and *uwsgi.ini* at startup using `sed` in your container.
+## Common Errors  
+Here is a list of common errors and their solutions.  
 
-### Elasticsearch
-**Note**: Newest Tube Archivist depends on Elasticsearch version 7.17 to provide an automatic updatepath in the future. 
-
-Use `bbilly1/tubearchivist-es` to automatically get the recommended version, or use the official image with the version tag in the docker-compose file.
-
-Stores video meta data and makes everything searchable. Also keeps track of the download queue.
-  - Needs to be accessible over the default port `9200`
-  - Needs a volume at **/usr/share/elasticsearch/data** to store data
-
-Follow the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html) for additional installation details.
-
-### Redis JSON
-Functions as a cache and temporary link between the application and the file system. Used to store and display messages and configuration variables.
-  - Needs to be accessible over the default port `6379`
-  - Needs a volume at **/data** to make your configuration changes permanent.
-
-### Redis on a custom port
-For some architectures it might be required to run Redis JSON on a nonstandard port. To for example change the Redis port to **6380**, set the following values:
-- Set the environment variable `REDIS_PORT=6380` to the *tubearchivist* service.
-- For the *archivist-redis* service, change the ports to `6380:6380`
-- Additionally set the following value to the *archivist-redis* service: `command: --port 6380 --loadmodule /usr/lib/redis/modules/rejson.so`
-
-### Updating Tube Archivist
-You will see the current version number of **Tube Archivist** in the footer of the interface so you can compare it with the latest release to make sure you are running the *latest and greatest*.  
-* There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.  
-* All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version.
-
-### Alternative installation instructions:
-- **arm64**: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, you can use `bbilly1/rejson`, an unofficial rebuild for arm64.
-- **Synology**: There is a [discussion thread](https://github.com/tubearchivist/tubearchivist/discussions/48) with Synology installation instructions.
-- **Unraid**: The three containers needed are all in the Community Applications. First install `TubeArchivist RedisJSON` followed by `TubeArchivist ES`, and finally you can install `TubeArchivist`. If you have unraid specific issues, report those to the [support thread](https://forums.unraid.net/topic/114073-support-crocs-tube-archivist/ "support thread").
-- **Helm Chart**: There is a Helm Chart available at https://github.com/insuusvenerati/helm-charts. Mostly self-explanatory but feel free to ask questions in the discord / subreddit.
-
-
-## Potential pitfalls
-### vm.max_map_count
+### `vm.max_map_count`
 **Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.
 
 To temporary set the value run:  
 ```
 sudo sysctl -w vm.max_map_count=262144
 ```  
-
 To apply the change permanently depends on your host operating system:  
-- For example on Ubuntu Server add `vm.max_map_count = 262144` to the file */etc/sysctl.conf*.
-- On Arch based systems create a file */etc/sysctl.d/max_map_count.conf* with the content `vm.max_map_count = 262144`. 
-- On any other platform look up in the documentation on how to pass kernel parameters.
+
+ - For example on Ubuntu Server add `vm.max_map_count = 262144` to the file `/etc/sysctl.conf`.
+ - On Arch based systems create a file `/etc/sysctl.d/max_map_count.conf` with the content `vm.max_map_count = 262144`. 
+ - On any other platform look up in the documentation on how to pass kernel parameters.  
+
 
 ### Permissions for elasticsearch
-If you see a message similar to `AccessDeniedException[/usr/share/elasticsearch/data/nodes]` when initially starting elasticsearch, that means the container is not allowed to write files to the volume.  
-That's most likely the case when you run `docker-compose` as an unprivileged user. To fix that issue, shutdown the container and on your host machine run:
+If you see a message similar to `Unable to access 'path.repo' (/usr/share/elasticsearch/data/snapshot)` or `failed to obtain node locks, tried [/usr/share/elasticsearch/data]` and `maybe these locations are not writable` when initially starting elasticsearch, that probably means the container is not allowed to write files to the volume.  
+To fix that issue, shutdown the container and on your host machine run:
 ```
-chown 1000:0 /path/to/mount/point
+chown 1000:0 -R /path/to/mount/point
 ```
-This will match the permissions with the **UID** and **GID** of elasticsearch within the container and should fix the issue.
+This will match the permissions with the **UID** and **GID** of elasticsearch process within the container and should fix the issue.  
+
 
 ### Disk usage
-The Elasticsearch index will turn to *read only* if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`, [link](https://github.com/tubearchivist/tubearchivist#disk-usage).  
+The Elasticsearch index will turn to ***read only*** if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`.  
 
 Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
 
-## Getting Started
-1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS and some other browsers a compatible format must be specified. For example:
-```
-bestvideo[VCODEC=avc1]+bestaudio[ACODEC=mp4a]/mp4
-```
-2. Subscribe to some of your favorite YouTube channels on the **channels** page. 
-3. On the **downloads** page, click on *Rescan subscriptions* to add videos from the subscribed channels to your Download queue or click on *Add to download queue* to manually add Video IDs, links, channels or playlists.
-4. Click on *Start download* and let **Tube Archivist** to it's thing. 
-5. Enjoy your archived collection!
-  
+## Known limitations
+- Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
+- Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
+- There is currently no flexibility in naming of the media files.
+
 ## Roadmap
 We have come far, nonetheless we are not short of ideas on how to improve and extend this project. Issues waiting for you to be tackled in no particular order:
 
@@ -162,19 +146,19 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex
 - [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/tubearchivist/tubearchivist/issues/97))
 - [ ] User created playlists, random and repeat controls ([#108](https://github.com/tubearchivist/tubearchivist/issues/108), [#220](https://github.com/tubearchivist/tubearchivist/issues/220))
 - [ ] Auto play or play next link ([#226](https://github.com/tubearchivist/tubearchivist/issues/226))
-- [ ] Show similar videos on video page
 - [ ] Multi language support
 - [ ] Show total video downloaded vs total videos available in channel
-- [ ] Make items in grid row configurable to use more of the screen
 - [ ] Add statistics of index
 - [ ] Download speed schedule ([#198](https://github.com/tubearchivist/tubearchivist/issues/198))
-- [ ] Implement complete offline media file import from json file ([#138](https://github.com/tubearchivist/tubearchivist/issues/138))
-- [ ] Filter and query in search form, search by url query ([#134](https://github.com/tubearchivist/tubearchivist/issues/134), [#139](https://github.com/tubearchivist/tubearchivist/issues/139))
-- [ ] Auto ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163))
+- [ ] Download or Ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163))
 - [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144))
-- [ ] Download video comments
 
 Implemented:
+- [X] Download video comments [2022-11-30]
+- [X] Show similar videos on video page [2022-11-30]
+- [X] Implement complete offline media file import from json file [2022-08-20]
+- [X] Filter and query in search form, search by url query [2022-07-23]
+- [X] Make items in grid row configurable to use more of the screen [2022-06-04]
 - [X] Add passing browser cookies to yt-dlp [2022-05-08]
 - [X] Add [SponsorBlock](https://sponsor.ajay.app/) integration [2022-04-16]
 - [X] Implement per channel settings [2022-03-26]
@@ -193,16 +177,32 @@ Implemented:
 - [X] Backup and restore [2021-09-22]
 - [X] Scan your file system to index already downloaded videos [2021-09-14]
 
-## Known limitations
-- Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
-- Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
-- For now this is meant to be run in a trusted network environment. Not everything is properly authenticated.
-- There is currently no flexibility in naming of the media files.
-
-
 ## Donate
 The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.  
 Second best way to support the development is to provide for caffeinated beverages:
+* [GitHub Sponsor](https://github.com/sponsors/bbilly1) become a sponsor here on GitHub
 * [Paypal.me](https://paypal.me/bbilly1) for a one time coffee
 * [Paypal Subscription](https://www.paypal.com/webapps/billing/plans/subscribe?plan_id=P-03770005GR991451KMFGVPMQ) for a monthly coffee
 * [ko-fi.com](https://ko-fi.com/bbilly1) for an alternative platform
+
+## Notable mentions
+This is a selection of places where this project has been featured on reddit, in the news, blogs or any other online media, newest on top.
+* **noted.lol**: Dev Debrief, An Interview With the Developer of Tube Archivist, [2023-03-30] [[link](https://noted.lol/dev-debrief-tube-archivist/)]
+* **console.substack.com**: Interview With Simon of Tube Archivist, [2023-01-29] [[link](https://console.substack.com/p/console-142#%C2%A7interview-with-simon-of-tube-archivist)]
+* **reddit.com**: Tube Archivist v0.3.0 - Now Archiving Comments, [2022-12-02] [[link](https://www.reddit.com/r/selfhosted/comments/zaonzp/tube_archivist_v030_now_archiving_comments/)]
+* **reddit.com**: Tube Archivist v0.2 - Now with Full Text Search, [2022-07-24] [[link](https://www.reddit.com/r/selfhosted/comments/w6jfa1/tube_archivist_v02_now_with_full_text_search/)]
+* **noted.lol**: How I Control What Media My Kids Watch Using Tube Archivist, [2022-03-27] [[link](https://noted.lol/how-i-control-what-media-my-kids-watch-using-tube-archivist/)]
+* **thehomelab.wiki**: Tube Archivist - A Youtube-DL Alternative on Steroids, [2022-01-27] [[link](https://thehomelab.wiki/books/news/page/tube-archivist-a-youtube-dl-alternative-on-steroids)]
+* **reddit.com**: Celebrating TubeArchivist v0.1, [2022-01-09] [[link](https://www.reddit.com/r/selfhosted/comments/rzh084/celebrating_tubearchivist_v01/)]
+* **linuxunplugged.com**: Pick: tubearchivist — Your self-hosted YouTube media server, [2021-09-11] [[link](https://linuxunplugged.com/425)] and [2021-10-05] [[link](https://linuxunplugged.com/426)]
+* **reddit.com**: Introducing Tube Archivist, your self hosted Youtube media server, [2021-09-12] [[link](https://www.reddit.com/r/selfhosted/comments/pmj07b/introducing_tube_archivist_your_self_hosted/)]
+
+
+## Sponsor
+Big thank you to [Digitalocean](https://www.digitalocean.com/) for generously donating credit for the tubearchivist.com VPS and buildserver. 
+<p>
+  <a href="https://www.digitalocean.com/">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/PoweredByDO/DO_Powered_by_Badge_blue.svg" width="201px">
+  </a>
+</p>
+

SHOWCASE.MD

@@ -0,0 +1,19 @@
+## Tube Archivist on YouTube
+[![ibracorp-youtube-video-thumb](assets/tube-archivist-ibracorp-O8H8Z01c0Ys.jpg)](https://www.youtube.com/watch?v=O8H8Z01c0Ys)
+Video featuring Tube Archivist generously created by [IBRACORP](https://www.youtube.com/@IBRACORP).
+
+## Screenshots
+![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")  
+*Home Page*: Your recent videos, continue watching incomplete videos.
+
+![channels screenshot](assets/tube-archivist-screenshot-channels.png?raw=true "Tube Archivist Channels")  
+*All Channels*: A list of all your indexed channels, filtered by subscribed only.
+
+![single channel screenshot](assets/tube-archivist-screenshot-single-channel.png?raw=true "Tube Archivist Single Channel")  
+*Single Channel*: Single channel page with additional metadata and sub pages.
+
+![video page screenshot](assets/tube-archivist-screenshot-video.png?raw=true "Tube Archivist Video Page")  
+*Video Page*: Stream your video directly from the interface.
+
+![video page screenshot](assets/tube-archivist-screenshot-download.png?raw=true "Tube Archivist Video Page")  
+*Downloads Page*: Add, control and monitor your download queue.

deploy.sh

@@ -18,11 +18,7 @@ set -e
 
 function sync_blackhole {
 
-    # docker commands need sudo, only build amd64
     host="blackhole.local"
-
-    read -sp 'Password: ' remote_pw
-    export PASS=$remote_pw
     
     rsync -a --progress --delete-after \
         --exclude ".git" \
@@ -30,10 +26,11 @@ function sync_blackhole {
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
         --exclude "db.sqlite3" \
+        --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
 
-    echo "$PASS" | ssh "$host" 'sudo -S docker buildx build --platform linux/amd64 -t bbilly1/tubearchivist:latest tubearchivist --load 2>/dev/null'
-    echo "$PASS" | ssh "$host" 'sudo -S docker compose up -d 2>/dev/null'
+    ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist'
+    ssh "$host" 'docker compose up -d'
 
 }
 
@@ -53,6 +50,7 @@ function sync_test {
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
         --exclude "db.sqlite3" \
+        --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
 
     # copy default docker-compose file if not exist
@@ -86,15 +84,17 @@ function validate {
 
     echo "run validate on $check_path"
 
+    # note: this logic is duplicated in the `./github/workflows/lint_python.yml` config
+    # if you update this file, you should update that as well
     echo "running black"
-    black --diff --color --check -l 79 "$check_path"
+    black --exclude "migrations/*" --diff --color --check -l 79 "$check_path"
     echo "running codespell"
-    codespell --skip="./.git" "$check_path"
+    codespell --skip="./.git,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
     echo "running flake8"
-    flake8 "$check_path" --count --max-complexity=10 --max-line-length=79 \
-        --show-source --statistics
+    flake8 "$check_path" --exclude "migrations" --count --max-complexity=10 \
+        --max-line-length=79 --show-source --statistics
     echo "running isort"
-    isort --check-only --diff --profile black -l 79 "$check_path"
+    isort --skip "migrations" --check-only --diff --profile black -l 79 "$check_path"
     printf "    \n> all validations passed\n"
 
 }
@@ -144,6 +144,7 @@ function sync_unstable {
 }
 
 
+# new function, sync only tag, build with build server
 function sync_docker {
 
     # check things
@@ -152,13 +153,40 @@ function sync_docker {
         return
     fi
 
+    echo "latest tags:"
+    git tag | tail -n 5 | sort -r
+
+    printf "\ncreate new version:\n"
+    read -r VERSION
+
+    echo "push new tag: $VERSION?"
+    read -rn 1
+
+    # create release tag
+    echo "commits since last version:"
+    git log "$(git describe --tags --abbrev=0)"..HEAD --oneline
+    git tag -a "$VERSION" -m "new release version $VERSION"
+    git push origin "$VERSION"
+
+}
+
+
+# old builder, sync tag, build and push locally
+function sync_docker_old {
+
+    # check things
+    if [[ $(git branch --show-current) != 'master' ]]; then
+        echo 'you are not on master, dummy!'
+        return
+    fi
+
     if [[ $(systemctl is-active docker) != 'active' ]]; then
         echo "starting docker"
         sudo systemctl start docker
     fi
 
     echo "latest tags:"
-    git tag | tail -n 10
+    git tag | tail -n 5 | sort -r
 
     printf "\ncreate new version:\n"
     read -r VERSION
@@ -169,14 +197,15 @@ function sync_docker {
     # start build
     sudo docker buildx build \
         --platform linux/amd64,linux/arm64 \
-        -t bbilly1/tubearchivist:latest \
+        -t bbilly1/tubearchivist \
+        -t bbilly1/tubearchivist:unstable \
         -t bbilly1/tubearchivist:"$VERSION" --push .
 
     # create release tag
     echo "commits since last version:"
     git log "$(git describe --tags --abbrev=0)"..HEAD --oneline
     git tag -a "$VERSION" -m "new release version $VERSION"
-    git push all "$VERSION"
+    git push origin "$VERSION"
 
 }
 
@@ -186,12 +215,9 @@ if [[ $1 == "blackhole" ]]; then
 elif [[ $1 == "test" ]]; then
     sync_test "$2"
 elif [[ $1 == "validate" ]]; then
-    # check package versions in requirements.txt for updates
-    python version_check.py
     validate "$2"
 elif [[ $1 == "docker" ]]; then
     sync_docker
-    sync_unstable
 elif [[ $1 == "unstable" ]]; then
     sync_unstable
 elif [[ $1 == "es" ]]; then

docker-compose.yml

@@ -3,8 +3,8 @@ version: '3.3'
 services:
   tubearchivist:
     container_name: tubearchivist
-    restart: always
-    image: bbilly1/tubearchivist:latest
+    restart: unless-stopped
+    image: bbilly1/tubearchivist
     ports:
       - 8000:8000
     volumes:
@@ -15,6 +15,7 @@ services:
       - REDIS_HOST=archivist-redis          # don't add protocol
       - HOST_UID=1000
       - HOST_GID=1000
+      - TA_HOST=tubearchivist.local         # set your host name
       - TA_USERNAME=tubearchivist           # your initial TA credentials
       - TA_PASSWORD=verysecret              # your initial TA credentials
       - ELASTIC_PASSWORD=verysecret         # set password for Elasticsearch
@@ -23,9 +24,9 @@ services:
       - archivist-es
       - archivist-redis
   archivist-redis:
-    image: redislabs/rejson:latest          # for arm64 use bbilly1/rejson
+    image: redis/redis-stack-server
     container_name: archivist-redis
-    restart: always
+    restart: unless-stopped
     expose:
       - "6379"
     volumes:
@@ -33,14 +34,15 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 7.17.3
+    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.7.0
     container_name: archivist-es
-    restart: always
+    restart: unless-stopped
     environment:
-      - "xpack.security.enabled=true"
       - "ELASTIC_PASSWORD=verysecret"       # matching Elasticsearch password
-      - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "xpack.security.enabled=true"
+      - "discovery.type=single-node"
+      - "path.repo=/usr/share/elasticsearch/data/snapshot"
     ulimits:
       memlock:
         soft: -1

docker_assets/nginx.conf

@@ -3,18 +3,22 @@ server {
     listen 8000;
     
     location /cache/videos/ {
+        auth_request /api/ping/;
         alias /cache/videos/;
     }
     
     location /cache/channels/ {
+        auth_request /api/ping/;
         alias /cache/channels/;
     }
     
     location /cache/playlists/ {
+        auth_request /api/ping/;
         alias /cache/playlists/;
     }
     
     location /media/ {
+        auth_request /api/ping/;
         alias /youtube/;
         types {
             text/vtt vtt;

docker_assets/run.sh

@@ -1,50 +1,23 @@
 #!/bin/bash
 # startup script inside the container for tubearchivist
 
-if [[ -z "$ELASTIC_USER" ]]; then
-    export ELASTIC_USER=elastic
-fi
+set -e
 
-ENV_VARS=("TA_USERNAME" "TA_PASSWORD" "ELASTIC_PASSWORD" "ELASTIC_USER")
-for each in "${ENV_VARS[@]}"; do
-    if ! [[ -v $each ]]; then
-        echo "missing environment variable $each"
-        exit 1
-    fi
-done
-
-# ugly nginx and uwsgi port overwrite with env vars
-if [[ -n "$TA_PORT" ]]; then
-    sed -i "s/8000/$TA_PORT/g" /etc/nginx/sites-available/default
-fi
-
-if [[ -n "$TA_UWSGI_PORT" ]]; then
-    sed -i "s/8080/$TA_UWSGI_PORT/g" /etc/nginx/sites-available/default
-    sed -i "s/8080/$TA_UWSGI_PORT/g" /app/uwsgi.ini
-fi
-
-# wait for elasticsearch
-counter=0
-until curl -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL" -fs; do
-    echo "waiting for elastic search to start"
-    counter=$((counter+1))
-    if [[ $counter -eq 12 ]]; then
-        # fail after 2 min
-        echo "failed to connect to elastic search, exiting..."
-        exit 1
-    fi
-    sleep 10
-done
-
-# start python application
-python manage.py makemigrations
+# django setup
 python manage.py migrate
-export DJANGO_SUPERUSER_PASSWORD=$TA_PASSWORD && \
-    python manage.py createsuperuser --noinput --name "$TA_USERNAME"
 
-python manage.py collectstatic --noinput -c
+if [[ -z "$DJANGO_DEBUG" ]]; then
+    python manage.py collectstatic --noinput -c
+fi
+
+# ta setup
+python manage.py ta_envcheck
+python manage.py ta_connection
+python manage.py ta_startup
+
+# start all tasks
 nginx &
 celery -A home.tasks worker --loglevel=INFO &
 celery -A home beat --loglevel=INFO \
-    -s "${BEAT_SCHEDULE_PATH:-/cache/celerybeat-schedule}" &
+    -s "${BEAT_SCHEDULE_PATH:-${cachedir}/celerybeat-schedule}" &
 uwsgi --ini uwsgi.ini

docker_assets/uwsgi.ini

@@ -5,4 +5,7 @@ pidfile = /tmp/project-master.pid
 vacuum = True
 max-requests = 5000
 socket = :8080
-buffer-size = 8192
+buffer-size = 8192
+log-5xx = true
+log-4xx = true
+disable-logging = true

docs/Channels.md

@@ -1,32 +1,3 @@
-# Channels Overview and Channel Detail Page
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-The channels are organized on two different levels, similar as the [playlists](Playlists):
-
-## Channels Overview
-Accessible at `/channel/` of your Tube Archivist, the **Overview Page** shows a list of all channels you have indexed. 
-- You can filter that list to show or hide subscribed channels with the toggle. Clicking on the channel banner or the channel name will direct you to the *Channel Detail Page*.
-- If you are subscribed to a channel a *Unsubscribe* button will show, if you aren't subscribed, a *Subscribe* button will show instead. 
-
-The **Subscribe to Channels** button <img src="assets/icon-add.png?raw=true" alt="add icon" width="20px" style="margin:0 5px;"> opens a text field to subscribe to a channel. You have a few options:
-- Enter the YouTube channel ID, a 25 character alphanumeric string. For example *UCBa659QWEk1AI4Tg--mrJ2A*
-- Enter the URL to the channel page on YouTube. For example *https://www.youtube.com/channel/UCBa659QWEk1AI4Tg--mrJ2A*
-- Enter the channel name for example: *https://www.youtube.com/c/TomScottGo*.
-- Enter the video URL for any video and let Tube Archivist extract the channel ID for you. For example *https://www.youtube.com/watch?v=2tdiKTSdE9Y*
-- Add one per line.
-
-You can search your indexed channels by clicking on the search icon <img src="assets/icon-search.png?raw=true" alt="search icon" width="20px" style="margin:0 5px;">. This will open a dedicated page.
-
-## Channel Detail
-Each channel will get a dedicated channel detail page accessible at `/channel/<channel-id>/` of your Tube Archivist. This page shows all the videos you have downloaded from this channel plus additional metadata. 
-- If you are subscribed to the channel, an *Unsubscribe* button will show, else the *Subscribe* button will show.
-- You can *Show* the channel description, that matches with the *About* tab on YouTube.
-- The **Mark as Watched** button will mark all videos of this channel as watched.
-- The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata additionally this will also delete playlists metadata belonging to that channel.
-- The button **Show Playlists** will go to the [playlists](Playlists) page and filter the list to only show playlists from this channel.
-
-### Channel Customize
-Clicking on the *Configure* button will open a form with options to configure settings on a per channel basis. Any configurations here will overwrite your settings from the [settings](Settings) page.
-- **Download Format**: Overwrite the download qualities for videos from this channel.
-- **Auto Delete**: Automatically delete watched videos from this channel after selected days.
-- **Index Playlists**: Automatically add all Playlists with at least a video downloaded to your index. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.
-- **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. Customize per channel: You can *disable* or *enable* SponsorBlock for certain channels only to overwrite the behavior set on the [Settings](settings) page. Selecting *unset* will remove the overwrite and your setting will fall back to the default on the settings page.
+Equivalent Channels page is: [here](https://docs.tubearchivist.com/channels/).

docs/Downloads.md

@@ -1,41 +1,3 @@
-# Downloads Page
-Accessible at `/downloads/` of your Tube Archivist, this page handles all the download functionality.
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-
-## Rescan Subscriptions
-The **Rescan Subscriptions** icon <img src="assets/icon-rescan.png?raw=true" alt="rescan icon" width="20px" style="margin:0 5px;"> will start a background task to look for new videos from the channels and playlists you are subscribed to. You can define the channel and playlist page size on the [settings page](Settings#subscriptions). With the default page size, expect this process to take around 2-3 seconds for each channel or playlist you are subscribed to. A status message will show the progress.
-
-Then for every video found, **Tube Archivist** will skip the video if it has already been downloaded or if you added it to the *ignored* list before. All the other videos will get added to the download queue. Expect this to take around 2 seconds for each video as **Tube Archivist** needs to grab some additional metadata. New videos will get added at the bottom of the download queue.
-
-## Download Queue
-The **Start Download** icon <img src="assets/icon-download.png?raw=true" alt="download icon" width="20px" style="margin:0 5px;"> will start the download process starting from the top of the queue. Take a look at the relevant settings on the [Settings Page](Settings#downloads). Once the process started, a progress message will show with additional details and controls: 
-- The stop icon <img src="assets/icon-stop.png?raw=true" alt="stop icon" width="20px" style="margin:0 5px;"> will gracefully stop the download process, once the current video has been finished successfully.
-- The cancel icon <img src="assets/icon-close-red.png?raw=true" alt="close icon" width="20px" style="margin:0 5px;"> is equivalent to killing the process and will stop the download immediately. Any leftover files will get deleted, the canceled video will still be available in the download queue.
-
-After downloading, Tube Archivist tries to add new videos to already indexed playlists.
-
-## Add to Download Queue
-The **Add to Download Queue** icon <img src="assets/icon-add.png?raw=true" alt="add icon" width="20px" style="margin:0 5px;"> opens a text field to manually add videos to the download queue. You have a few options:
-- Add a link to a YouTube video. For example *https://www.youtube.com/watch?v=2tdiKTSdE9Y*.
-- Add a YouTube video ID. For example *2tdiKTSdE9Y*.
-- Add a link to a YouTube video by providing the shortened URL, for example *https://youtu.be/2tdiKTSdE9Y*.
-- Add a Channel ID or Channel URL to add every available video to the download queue. This will ignore the channel page size as described before and is meant for an initial download of the whole channel. You can still ignore selected videos before starting the download.
-- Add a channel name like for example *https://www.youtube.com/c/TomScottGo*.
-- Add a playlist ID or URL to add every available video in the list to the download queue, for example *https://www.youtube.com/playlist?list=PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha* or *PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha*. 
-  - Note: When adding a playlist to the queue, this playlist will automatically get [indexed](Playlists#playlist-detail).
-  - Note: When you add a link to a video in a playlist, Tube Archivist assumes you want to download only the specific video and not the whole playlist, for example *https://www.youtube.com/watch?v=CINVwWHlzTY&list=PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha* will only add one video *CINVwWHlzTY* to the queue.
-- Add one link per line.
-
-## The Download Queue
-Below the three buttons you find the download queue. New items will get added at the bottom of the queue, the next video to download once you click on **Start Download** will be the first in the list.
-
-Every video in the download queue has two buttons:
-- **Ignore**: This will remove that video from the download queue and this video will not get added again, even when you **Rescan Subscriptions**.
-- **Download now**: This will give priority to this video. If the download process is already running, the prioritized video will get downloaded as soon as the current video is finished. If there is no download process running, this will start downloading this single video and stop after that.  
-
-You can flip the view by activating **Show Only Ignored Videos**. This will show all videos you have previously *ignored*.  
-Every video in the ignored list has two buttons:
-- **Forget**: This will delete the item form the ignored list.
-- **Add to Queue**: This will add the ignored video back to the download queue.  
-
-You can delete your download queue from the [Settings](Settings#actions) page.
+Equivalent Downloads page is: [here](https://docs.tubearchivist.com/downloads/).

docs/FAQ.md

@@ -1,34 +1,3 @@
-# Frequently Asked Questions
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-## 1. Scope of this project
-Tube Archivist is *Your self hosted YouTube media server*, which also defines the primary scope of what this project tries to do:
-- **Self hosted**: This assumes you have full control over the underlying operating system and hardware and can configure things to work properly with Docker, it's volumes and networks as well as whatever disk storage and filesystem you choose to use.
-- **YouTube**: Downloading, indexing and playing videos from YouTube, there are currently no plans to expand this to any additional platforms.
-- **Media server**: This project tries to be a stand alone media server in it's own web interface.
-
-Additionally to that, progress is also happening on:
-- **API**: Endpoints for additional integrations.
-- **Browser Extension**: To integrate between youtube.com and Tube Archivist.
-
-Defining the scope is important for the success of any project:
-- A scope too broad will result in development effort spreading too thin and will run into danger that his project tries to do too many things and none of them well.
-- A too narrow scope will make this project uninteresting and will exclude audiences that could also benefit from this project.
-- Not defining a scope will easily lead to misunderstandings and false hopes of where this project tries to go.
-
-Of course this is subject to change, as this project continues to grow and more people contribute.
-
-## 2. Emby/Plex/Jellyfin/Kodi integrations
-Although there are similarities between these excellent projects and Tube Archivist, they have a very different use case. Trying to fit the metadata relations and database structure of a YouTube archival project into these media servers that specialize in Movies and TV shows is always going to be limiting.
-
-Part of the scope is to be its own media server, so that's where the focus and effort of this project is. That being said, the nature of self hosted and open source software gives you all the possible freedom to use your media as you wish.
-
-## 3. To Docker or not to Docker
-This project is a classical docker application: There are multiple moving parts that need to be able to interact with each other and need to be compatible with multiple architectures and operating systems. Additionally Docker also drastically reduces development complexity which is highly appreciated.  
-
-So Docker is the only supported installation method. If you don't have any experience with Docker, consider investing the time to learn this very useful technology.
-
-## 4. Finetuning Elasticsearch
-A minimal configuration of Elasticsearch (ES) is provided in the example docker-compose.yml file. ES is highly configurable and very interesting to learn more about. Refer to the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) if you want to get into it.
-
-## 5. Advanced Authentication
-If you like to use things like SSO, LDAP or 2FA to login, consider using something like Authelia as a reverse proxy so this project can focus on the core task. Tube Archivist has a *remember me* checkbox at login to extend your sessions lifetime in your browser. 
+Equivalent FAQ page is: [here](https://docs.tubearchivist.com/faq/).

docs/Home.md

@@ -1,31 +1 @@
-# Tube Archivist Wiki
-Welcome to the official Tube Archivist Wiki. This is an up-to-date documentation of user functionality.
-
-Table of contents:
-* [FAQ](FAQ): Frequently asked questions what this project is and tries to do
-* [Channels](Channels): Browse your channels, handle channel subscriptions
-* [Playlists](Playlists): Browse your indexed playlists, handle playlist subscriptions
-* [Downloads](Downloads): Scanning subscriptions, handle download queue
-* [Settings](Settings): All the configuration options
-* [Video](Video): All details of a single video and playlist navigation.
-* [Users](Users): User management admin interface
-* [Installation](Installation): WIP - detailed installation instructions for various platforms.
-
-## Getting Started
-1. [Subscribe](Channels#channels-overview) to some of your favourite YouTube channels.
-2. [Scan](Downloads#rescan-subscriptions) subscriptions to add the latest videos to the download queue.
-3. [Add](Downloads#add-to-download-queue) additional videos, channels or playlist - ignore the ones you don't want to download.
-4. [Download](Downloads#download-queue) and let **Tube Archivist** do it's thing.
-5. Sit back and enjoy your archived and indexed collection!
-
-## General Navigation
-* Clicking on the channel name or the channel icon brings you to the dedicated channel page to show videos from that channel.
-* Clicking on a video title brings you to the dedicated video page and shows additional details.
-* Clicking on a video thumbnail opens the video player and starts streaming the selected video.
-* Clicking on the search icon <img src="assets/icon-search.png?raw=true" alt="gridview icon" width="20px" style="margin:0 5px;"> will open a dedicated search page to search over your complete index.
-* The pagination - if available - builds links for up to 10'000 results, use the search, sort or filter functionality to find what you are looking for.
-
-
-An empty checkbox icon <img src="assets/icon-unseen.png?raw=true" alt="unseen icon" width="20px" style="margin:0 5px;"> will show for videos you haven't marked as watched. Click on it and the icon will change to a filled checkbox <img src="assets/icon-seen.png?raw=true" alt="seen icon" width="20px" style="margin:0 5px;"> indicating it as watched - click again to revert.
-
-When available the <img src="assets/icon-gridview.png?raw=true" alt="gridview icon" width="20px" style="margin:0 5px;"> gridview icon will display the list in a grid. A grid row holds 3 items by default, use the <img src="assets/icon-add.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> icon to add more or the <img src="assets/icon-substract.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> icon to remove items per row, depending on your screen size. The <img src="assets/icon-listview.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> listview icon will arrange the items in a list. The sort icon <img src="assets/icon-sort.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> will open additional sort options.
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.

docs/Installation.md

@@ -1,69 +1,3 @@
-# Detailed Installation Instructions for Various Platforms
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-## Unraid
-
-Tube Archivist, and all if it's dependencies are located in the [community applications](https://forums.unraid.net/topic/38582-plug-in-community-applications/) store. The three containers you will need are as follows:
-
--   **TubeArchivist-RedisJSON**: This container acts as a cache and temporary link between the application and the file system. Used to store and display messages and configuration variables.
--   **TubeArchivist-ES**: ElasticSearch stores video meta data and makes everything searchable. Also keeps track of the download queue.
--   **TubeArchivist**: Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface.
-
-### Step 1: Install  `TubeArchivist-RedisJSON`
-
-![enter image description here](https://i.imgur.com/ycAqFRU.png)
-This is the easiest container to setup of the thee, just make sure that you do not have any port conflicts, and that your `/data` is mounted to the correct path. The other containers will map to the same directory.
-
-If you need to install `TubeArchivist-RedisJSON`on a different port, you'll have to follow [these steps](https://github.com/tubearchivist/tubearchivist#redis-on-a-custom-port) later on when installing the `TubeArchivist` container
-
-
-### Step 2: Install  `TubeArchivist-ES`
-![enter image description here](https://i.imgur.com/o6tsTdt.png)
-ElasticSeach is also pretty easy to setup. Again, make sure you have no port conflicts, make sure that you mapped `/usr/share/elasticsearch/data` to the same directory as `RedisJSON`, and make sure to change the default password to something more secure. 
-
-There is three additional settings in the "show more settings" area, but leave those as they are.
-
-
-### Step 3: Install  `TubeArchivist`
-
-![enter image description here](https://i.imgur.com/dwSCfgO.png)
-It's finally time to set up TubeArchivist!
-
- - `Port:`Again, make sure that you have no port conflicts on 8000.
-   
- - `Youtube Media Path:` is where you'll download all of your videos to.
-   Make sure that this is an empty directory to not cause confusion when
-   starting the application. If you have existing videos that you'd like
-   to import into Tube Archivist, please checkout the [settings
-   wiki.](https://github.com/tubearchivist/tubearchivist/wiki/Settings#manual-media-files-import)
-   
-   
-- `Appdata:` This should be the same base path as the other two containers.
-   
- - `TA Username:`This will be your username for TubeArchivist.
-   
- - `TA Password:`This will be your password for TubeArchivist.
-   
- - `Redis` This will be JUST the ip address of your redis container
-
- - `ElasticSearch Password:`This is the password you defined in the `TubeArchivist-ES` container.
- - `ElasticSearch:` This seems to cause some confusion, but it's a pretty simple step, just replace the IP and Port to match you `TubeArchivist-ES` container.
-
- (example: if your IP is 192.168.1.15, the value should be http://192.168.1.15:9200)
-
- - `Time Zone:` This is an important step for your scheduler, to find your timezone, use a site like [TimeZoneConverter](http://www.timezoneconverter.com/cgi-bin/findzone.tzc) 
-
-### From there, you should be able to start up your containers and you're good to go!
-If you're still having trouble, join us on [discord](https://discord.gg/AFwz8nE7BK) and come to the #unraid channel.
-
-<br />
-<br />
-
-## Truenas Scale 
-
-Truenas Scale can be a bit confusing, with its k3s kubernetes implementation. 
-
-However, there is a step by step guide available for it's users here:
-
-https://heavysetup.info/applications/tube-archivist/dataset/
-
-- Ensure you are navigating the columns under `Tube Archivist` on the left hand side of the screen
+Equivalent pages are located under *installation* on the left.

docs/Playlists.md

@@ -1,23 +1,3 @@
-# Playlist Overview and Playlist Detail Page
-The playlists are organized in two different levels, similar as the [channels](Channels):
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-## Playlist Overview
-Accessible at `/playlist/` of your Tube Archivist, this **Overview Page** shows a list of all playlists you have indexed over all your channels.
-- You can filter that list to show only subscribed to playlists with the toggle.
-
-You can index playlists of a channel from the channel detail page as described [here](Channels#channel-detail).
-
-The **Subscribe to Playlist** button <img src="assets/icon-add.png?raw=true" alt="add icon" width="20px" style="margin:0 5px;"> opens a text field to subscribe to playlists. You have a few options:
-- Enter the YouTube playlist id, for example: *PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha*
-- Enter the Youtube dedicated playlist url, for example: *https://www.youtube.com/playlist?list=PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha*
-- Add one per line.
-- NOTE: It doesn't make sense to subscribe to a playlist if you are already subscribed the corresponding channel as this will slow down the **Rescan Subscriptions** [task](Downloads#rescan-subscriptions).
-
-You can search your indexed playlists by clicking on the search icon <img src="assets/icon-search.png?raw=true" alt="search icon" width="20px" style="margin:0 5px;">. This will open a dedicated page.
-
-## Playlist Detail
-Each playlist will get a dedicated playlist detail page accessible at `/playlist/<playlist-id>/` of your Tube Archivist. This page shows all the videos you have downloaded from this playlist.
-
-- If you are subscribed to the playlist, an Unsubscribe button will show, else the Subscribe button will show.
-- The Mark as Watched button will mark all videos of this playlist as watched.
-- The **Delete Playlist** button will give you the option to delete just the *metadata* which won't delete any media files or *delete all* which will delete metadata plus all videos belonging to this playlist.
+Equivalent Playlist page is: [here](https://docs.tubearchivist.com/playlists/).

docs/Search.md

@@ -0,0 +1,3 @@
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
+
+Equivalent Search page is: [here](https://docs.tubearchivist.com/search/).

docs/Settings.md

@@ -1,162 +1,3 @@
-# Settings Page
-Accessible at `/settings/` of your **Tube Archivist**, this page holds all the configurations and additional functionality related to the database.
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-Click on **Update Settings** at the bottom of the form to apply your configurations.
-
-## Color scheme
-Switch between the easy on the eyes dark theme and the burning bright theme.
-
-## Archive View
-- **Page Size**: Defines how many results get displayed on a given page. Same value goes for all archive views.
-
-## Subscriptions
-Settings related to the channel management.
-- **Channel Page Size**: Defines how many pages will get analyzed by **Tube Archivist** each time you click on *Rescan Subscriptions*. The default page size used by yt-dlp is **50**, that's also the recommended value to set here. Any value higher will slow down the rescan process, for example if you set the value to 51, that means yt-dlp will have to go through 2 pages of results instead of 1 and by that doubling the time that process takes.
-
-## Downloads
-Settings related to the download process.
-- **Download Limit**: Stop the download process after downloading the set quantity of videos.
-- **Download Speed Limit**: Set your download speed limit in KB/s. This will pass the option `--limit-rate` to yt-dlp.
-- **Throttled Rate Limit**: Restart download if the download speed drops below this value in KB/s. This will pass the option `--throttled-rate` to yt-dlp. Using this option might have a negative effect if you have an unstable or slow internet connection.
-- **Sleep Interval**: Time in seconds to sleep between requests to YouTube. It's a good idea to set this to **3** seconds. Might be necessary to avoid throttling.
-- **Auto Delete Watched Videos**: Automatically delete videos marked as watched after selected days. If activated, checks your videos after download task is finished.
-
-## Download Format
-Additional settings passed to yt-dlp.
-- **Format**: This controls which streams get downloaded and is equivalent to passing `--format` to yt-dlp. Use one of the recommended one or look at the documentation of [yt-dlp](https://github.com/yt-dlp/yt-dlp#format-selection). Please note: The option `--merge-output-format mp4` is automatically passed to yt-dlp to guarantee browser compatibility. Similar to that, `--check-formats` is passed as well to check that the selected formats are actually downloadable.
-- **Embed Metadata**: This saves the available tags directly into the media file by passing `--embed-metadata` to yt-dlp.
-- **Embed Thumbnail**: This will save the thumbnail into the media file by passing `--embed-thumbnail` to yt-dlp.
-
-## Subtitles
-- **Download Setting**: Select the subtitle language you like to download. Add a comma separated list for multiple languages.
-- **Source Settings**: User created subtitles are provided from the uploader and are usually the video script. Auto generated is from YouTube, quality varies, particularly for auto translated tracks.
-- **Index Settings**: Enabling subtitle indexing will add the lines to Elasticsearch and will make subtitles searchable. This will increase the index size and is not recommended on low-end hardware.
-
-## Cookie
-Importing your YouTube Cookie into Tube Archivist allows yt-dlp to bypass age restrictions, gives access to private videos and your *watch later* or *liked videos*.
-
-### Security concerns
-Cookies are used to store your session and contain your access token to your google account, this file can be used to take over your account. Treat that file with utmost care as you would any other password or credential. *Tube Archivist* stores your cookie in Redis and will automatically append it to yt-dlp for every request.
-
-### Export your cookie
-- Install **Cookies.txt** addon for [chrome](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid) or [firefox](https://addons.mozilla.org/firefox/addon/cookies-txt).
-- Visit YouTube and login with whichever YouTube account you wish to use to generate the cookies.
-- Click on the extension icon in the toolbar - it will drop down showing the active cookies for YT.
-- Click Export to export the cookies, filename is by default *cookies.google.txt*.
-
-### Import your cookie
-Place the file *cookies.google.txt* into the *cache/import* folder of Tube Archivist and enable the cookie import. Once you click on *Update Application Configurations* to save your changes, your cookie will get imported and stored internally.
-
-Once imported, a **Validate Cookie File** button will show, where you can confirm if your cookie is working or not.
-
-### Use your cookie
-Once imported, additionally to the advantages above, your [Watch Later](https://www.youtube.com/playlist?list=WL) and [Liked Videos](https://www.youtube.com/playlist?list=LL) become a regular playlist you can download and subscribe to as any other [playlist](Playlists).
-
-### Limitation
-There is only one cookie per Tube Archivist instance, this will be shared between all users.
-
-## Integrations
-All third party integrations of TubeArchivist will **always** be *opt in*.
-- **API**: Your access token for the Tube Archivist API.
-- **returnyoutubedislike.com**: This will get return dislikes and average ratings for each video by integrating with the API from [returnyoutubedislike.com](https://www.returnyoutubedislike.com/).
-- **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. If a video doesn't have timestamps, or has unlocked timestamps, use the browser addon to contribute to this excellent project. Can also be activated and deactivated as a per [channel overwrite](Settings#channel-customize).
-- **Cast**: Enabling the cast integration in the settings page will load an additional JS library from **Google**.
-    * Requirements
-        - HTTPS
-            * To use the cast integration HTTPS needs to be enabled, which can be done using a reverse proxy. This is a requirement by Google as communication to the cast device is required to be encrypted, but the content itself is not.
-        - Supported Browser
-            * A supported browser is required for this integration such as Google Chrome. Other browsers, especially Chromium-based browsers, may support casting by enabling it in the settings.
-        - Subtitles
-            * Subtitles are supported however they do not work out of the box and require additional configuration. Due to requirements by Google, to use subtitles you need additional headers which will need to be configured in your reverse proxy. See this [page](https://developers.google.com/cast/docs/web_sender/advanced#cors_requirements) for the specific requirements.
-               > You need the following headers: Content-Type, Accept-Encoding, and Range. Note that the last two headers, Accept-Encoding and Range, are additional headers that you may not have needed previously.
-               > Wildcards "*" cannot be used for the Access-Control-Allow-Origin header. If the page has protected media content, it must use a domain instead of a wildcard.
-
-
-# Scheduler Setup
-Schedule settings expect a cron like format, where the first value is minute, second is hour and third is day of the week. Day 0 is Sunday, day 1 is Monday etc.
-
-Examples:
-- **0 15 \***: Run task every day at 15:00 in the afternoon.
-- **30 8 \*/2**: Run task every second day of the week (Sun, Tue, Thu, Sat) at 08:30 in the morning.
-- **0 \*/3,8-17 \***: Execute every hour divisible by 3, and every hour during office hours (8 in the morning - 5 in the afternoon).
-- **0 8,16 \***: Execute every day at 8 in the morning and at 4 in the afternoon.
-- **auto**: Sensible default.
-- **0**: (zero), deactivate that task.
-
-NOTE:
-- Changes in the scheduler settings require a container restart to take effect.
-- Cron format as *number*/*number* are none standard cron and are not supported by the scheduler, for example **0 0/12 \*** is invalid, use **0 \*/12 \*** instead.
-- Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason * or wildcards for minutes are not supported.
-
-## Rescan Subscriptions
-That's the equivalent task as run from the downloads page looking through your channel and playlist and add missing videos to the download queue.
-
-## Start download
-Start downloading all videos currently in the download queue.
-
-## Refresh Metadata
-Rescan videos, channels and playlists on youtube and update metadata periodically. This will also deactivate an item and exclude it from future refreshes if the link on YouTube is no longer available. This task is meant to be run once per day, set your schedule accordingly.
-
-The field **Refresh older than x days** takes a number where TubeArchivist will consider an item as *outdated*. This value is used to calculate how many items need to be refreshed today based on the total indexed. This will spread out the requests to YouTube. Sensible value here is **90** days.
-
-## Thumbnail check
-This will check if all expected thumbnails are there and will delete any artwork without matching video.
-
-## Index backup
-Create a zip file of the metadata and select **Max auto backups to keep** to automatically delete old backups created from this task.
-
-
-# Actions
-Additional database functionality.
-
-## Delete download queue
-The button **Delete all queued** will delete all pending videos from the download queue. The button **Delete all ignored** will delete all videos you have previously ignored.
-
-## Manual Media Files Import
-So far this depends on the video you are trying to import to be still available on YouTube to get the metadata. Add the files you'd like to import to the */cache/import* folder. Then start the process from the settings page *Manual Media Files Import*. Make sure to follow one of the two methods below.
-
-### Method 1:
-Add a matching *.json* file with the media file. Both files need to have the same base name, for example:
-- For the media file: \<base-name>.mp4
-- For the JSON file: \<base-name>.info.json
-- Alternate JSON file: \<base-name>.json
-
-**Tube Archivist** then looks for the 'id' key within the JSON file to identify the video.
-
-### Method 2:
-Detect the YouTube ID from filename, this accepts the default yt-dlp naming convention for file names like:
-- \<base-name>[\<youtube-id>].mp4
-- The YouTube ID in square brackets at the end of the filename is the crucial part.
-
-### Some notes:
-- This will **consume** the files you put into the import folder: Files will get converted to mp4 if needed (this might take a long time...) and moved to the archive, *.json* files will get deleted upon completion to avoid having duplicates on the next run.
-- For best file transcoding quality, convert your media files with desired settings first before importing (#138).
-- There should be no subdirectories added to */cache/import*, only video files. If your existing video library has video files inside subdirectories, you can get all the files into one directory by running `find ./ -mindepth 2 -type f -exec mv '{}' . \;` from the top-level directory of your existing video library. You can also delete any remaining empty subdirectories with `find ./ -mindepth 1 -type d -delete`.
-- Maybe start with a subset of your files to import to make sure everything goes well...
-- Follow the logs to monitor progress and errors: `docker-compose logs -f tubearchivist`.
-
-## Embed thumbnails into media file
-This will write or overwrite all thumbnails in the media file using the downloaded thumbnail. This is only necessary if you didn't download the files with the option *Embed Thumbnail* enabled or want to make sure all media files get the newest thumbnail. Follow the docker-compose logs to monitor progress.
-
-## Backup Database
-This will backup your metadata into a zip file. The file will get stored at *cache/backup* and will contain the necessary files to restore the Elasticsearch index formatted **nd-json** files plus a complete export of the index in a set of conventional **json** files.  
-
-BE AWARE: This will **not** backup any media files, just the metadata from the Elasticsearch.
-
-## Restore From Backup
-The restore functionality will expect the same zip file in *cache/backup* as created from the **Backup database** function. This will recreate the index from the snapshot. There will be a list of all available backup to choose from. The *source* tag can have these different values:
-- **manual**: For backups manually created from here on the settings page.
-- **auto**: For backups automatically created via a sceduled task.
-- **update**: For backups created after a Tube Archivist update due to changes in the index.
-- **False**: Undefined.
-
-BE AWARE: This will **replace** your current index with the one from the backup file. This won't restore any media files.
-
-## Rescan Filesystem
-This function will go through all your media files and looks at the whole index to try to find any issues:
-- Should the filename not match with the indexed media url, this will rename the video files correctly and update the index with the new link.
-- When you delete media files from the filesystem outside of the Tube Archivist interface, this will delete leftover metadata from the index.
-- When you have media files that are not indexed yet, this will grab the metadata from YouTube like it was a newly downloaded video. This can be useful when restoring from an older backup file with missing metadata but already downloaded mediafiles. NOTE: This only works if the media files are named in the same convention as Tube Archivist does, particularly the YouTube ID needs to be at the same index in the filename, alternatively see above for *Manual Media Files Import*.
--This will also check all of your thumbnails and download any that are missing.
-
-BE AWARE: There is no undo.
+Equivalent Settings page is: [here](https://docs.tubearchivist.com/settings/).

docs/Users.md

@@ -1,20 +1,3 @@
-# User Management
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-For now, **Tube Archivist** is a single user application. You can create multiple users with different names and passwords, they will share the same videos and permissions but some interface configurations are on a per user basis. *More is on the roadmap*.
-
-## Superuser
-The first user gets created with the environment variables **TA_USERNAME** and **TA_PASSWORD** from your docker-compose file. That first user will automatically have *superuser* privileges. 
-
-## Admin Interface
-When logged in from your *superuser* account, you are able to access the admin interface from the settings page or at `/admin/`. This interface holds all functionality for user management.
-
-## Create additional users
-From the admin interface when you click on *Accounts* you will get a list of all users. From there you can create additional users by clicking on *Add Account*, provide a name and confirm password and click on *Save* to create the user.
-
-## Changing users
-You can delete or change permissions and password of a user by clicking on the username from the *Accounts* list page and follow the interface from there. Changing the password of the *superuser* here will overwrite the password originally set with the environment variables.
-
-## Reset
-Delete all user configurations by deleting the file `cache/db.sqlite3` and restart the container. This will create the superuser again from the environment variables.
-
-NOTE: Future improvements here will most likely require such a reset.
+Equivalent Users page is: [here](https://docs.tubearchivist.com/users/).

docs/Video.md

@@ -1,11 +1,3 @@
-# Video Page
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-Every video downloaded gets a dedicated page accessible at `/video/<video-id>` of your Tube Archivist.
-
-Clicking on the channel name or the channel icon will bring you to the dedicated channel detail [page](Channels#channel-detail).
-
-The button **Delete Video** will delete that video including the media file.
-
-When available, a playlist navigation will show at the bottom. Clicking on the playlist name will bring you to the dedicated [Playlist Detail](Playlists#playlist-detail) page showing all videos downloaded from that playlist. The number in square brackets indicates the position of the current video in that playlist.
-
-Clicking on the next or previous video name or thumbnail will bring you to that dedicated video page.
+Equivalent Video page is: [here](https://docs.tubearchivist.com/video/).

package.json

@@ -0,0 +1,17 @@
+{
+  "private": true,
+  "scripts": {
+    "lint": "eslint 'tubearchivist/static/**/*.js'",
+    "format": "prettier --write 'tubearchivist/static/**/*.js'"
+  },
+  "devDependencies": {
+    "eslint": "^8.26.0",
+    "prettier": "^2.7.1",
+    "eslint-config-prettier": "^8.5.0"
+  },
+  "prettier": {
+    "singleQuote": true,
+    "arrowParens": "avoid",
+    "printWidth": 100
+  }
+}

tubearchivist/api/README.md

@@ -1,233 +1,3 @@
 # TubeArchivist API
-Documentation of available API endpoints.  
-**Note: This is very early alpha and will change!**
 
-## Authentication
-API token will get automatically created, accessible on the settings page. Token needs to be passed as an authorization header with every request. Additionally session based authentication is enabled too: When you are logged into your TubeArchivist instance, you'll have access to the api in the browser for testing.
-
-Curl example:
-```shell
-curl -v /api/video/<video-id>/ \
-    -H "Authorization: Token xxxxxxxxxx"
-```
-
-Python requests example:
-```python
-import requests
-
-url = "/api/video/<video-id>/"
-headers = {"Authorization": "Token xxxxxxxxxx"}
-response = requests.get(url, headers=headers)
-```
-
-## Pagination
-The list views return a paginate object with the following keys:
-- page_size: int current page size set in config
-- page_from: int first result idx
-- prev_pages: array of ints of previous pages, if available
-- current_page: int current page from query
-- max_hits: reached: bool if max of 10k results is reached
-- last_page: int of last page link
-- next_pages: array of ints of next pages
-- total_hits: int total results
-
-Pass page number as a query parameter: `page=2`. Defaults to *0*, `page=1` is redundant and falls back to *0*. If a page query doesn't return any results, you'll get `HTTP 404 Not Found`.
-
-## Login View
-Return token and user ID for username and password:  
-POST /api/login
-```json
-{
-    "username": "tubearchivist",
-    "password": "verysecret"
-}
-```
-
-after successful login returns 
-```json
-{
-    "token": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
-    "user_id": 1
-}
-```
-
-## Video List View
-/api/video/
-
-## Video Item View
-/api/video/\<video_id>/
-
-## Video Progress View
-/api/video/\<video_id>/progress  
-
-Progress is stored for each user.
-
-### Get last player position of a video
-GET /api/video/\<video_id>/progress
-```json
-{
-    "youtube_id": "<video_id>",
-    "user_id": 1,
-    "position": 100
-}
-```
-
-### Post player position of video
-POST /api/video/\<video_id>/progress
-```json
-{
-    "position": 100
-}
-```
-
-### Delete player position of video
-DELETE /api/video/\<video_id>/progress  
-
-
-## Sponsor Block View
-/api/video/\<video_id>/sponsor/
-
-Integrate with sponsorblock
-
-### Get list of segments
-GET /api/video/\<video_id>/sponsor/
-
-
-### Vote on existing segment
-**This only simulates the request**  
-POST /api/video/\<video_id>/sponsor/
-```json
-{
-    "vote": {
-        "uuid": "<uuid>",
-        "yourVote": 1
-    }
-}
-```
-yourVote needs to be *int*: 0 for downvote, 1 for upvote, 20 to undo vote
-
-### Create new segment
-**This only simulates the request**  
-POST /api/video/\<video_id>/sponsor/
-```json
-{
-    "segment": {
-        "startTime": 5,
-        "endTime": 10
-    }
-}
-```
-Timestamps either *int* or *float*, end time can't be before start time.
-
-
-## Channel List View
-/api/channel/
-
-### Subscribe to a list of channels
-POST /api/channel/
-```json
-{
-    "data": [
-        {"channel_id": "UC9-y-6csu5WGm29I7JiwpnA", "channel_subscribed": true}
-    ]
-}
-```
-
-## Channel Item View
-/api/channel/\<channel_id>/
-
-## Channel Videos View
-/api/channel/\<channel_id>/video/
-
-## Playlist List View
-/api/playlist/
-
-## Playlists Item View
-/api/playlist/\<playlist_id>/
-
-## Playlist Videos View
-/api/playlist/\<playlist_id>/video/
-
-## Download Queue List View
-GET /api/download/
-
-Parameter:
-- filter: pending, ignore
-
-### Add list of videos to download queue
-POST /api/download/
-```json
-{
-    "data": [
-        {"youtube_id": "NYj3DnI81AQ", "status": "pending"}
-    ]
-}
-```
-
-### Delete download queue items by filter
-DELETE /api/download/?filter=ignore  
-DELETE /api/download/?filter=pending
-
-## Download Queue Item View
-GET /api/download/\<video_id>/  
-POST /api/download/\<video_id>/
-
-Ignore video in download queue:
-```json
-{
-    "status": "ignore"
-}
-```
-
-Add to queue previously ignored video:
-```json
-{
-    "status": "pending"
-}
-```
-
-DELETE /api/download/\<video_id>/  
-Forget or delete from download queue
-
-## Ping View
-Validate your connection with the API  
-GET /api/ping
-
-When valid returns message with user id: 
-```json
-{
-    "response": "pong",
-    "user": 1
-}
-```
-
-## Task View
-Start a background task
-POST /api/task/
-```json
-{
-    "run": "task_name"
-}
-```
-
-List of valid task names:
-- **download_pending**: Start the download queue
-- **rescan_pending**: Rescan your subscriptions
-
-
-## Cookie View
-Check your youtube cookie settings  
-GET /api/cookie/
-```json
-{
-    "cookie_enabled": true
-}
-```
-
-POST /api/cookie/  
-Send empty post request to validate cookie.
-```json
-{
-    "cookie_validated": true
-}
-```
+All API documentation has moved to [docs.tubearchivist.com](https://docs.tubearchivist.com/).

tubearchivist/api/src/search_processor.py

@@ -22,7 +22,7 @@ class SearchProcess:
         self.processed = False
 
     def process(self):
-        """dedect type and process"""
+        """detect type and process"""
         if "_source" in self.response.keys():
             # single
             self.processed = self._process_result(self.response)
@@ -37,7 +37,7 @@ class SearchProcess:
         return self.processed
 
     def _process_result(self, result):
-        """dedect which type of data to process"""
+        """detect which type of data to process"""
         index = result["_index"]
         processed = False
         if index == "ta_video":
@@ -48,6 +48,8 @@ class SearchProcess:
             processed = self._process_playlist(result["_source"])
         if index == "ta_download":
             processed = self._process_download(result["_source"])
+        if index == "ta_comment":
+            processed = self._process_comment(result["_source"])
 
         return processed
 
@@ -62,7 +64,7 @@ class SearchProcess:
                 "channel_last_refresh": date_str,
                 "channel_banner_url": f"{art_base}_banner.jpg",
                 "channel_thumb_url": f"{art_base}_thumb.jpg",
-                "channel_tvart_url": False,
+                "channel_tvart_url": f"{art_base}_tvart.jpg",
             }
         )
 
@@ -74,7 +76,7 @@ class SearchProcess:
         media_url = urllib.parse.quote(video_dict["media_url"])
         vid_last_refresh = date_praser(video_dict["vid_last_refresh"])
         published = date_praser(video_dict["published"])
-        vid_thumb_url = ThumbManager().vid_thumb_path(video_id)
+        vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
         channel = self._process_channel(video_dict["channel"])
 
         if "subtitles" in video_dict:
@@ -113,7 +115,7 @@ class SearchProcess:
     def _process_download(self, download_dict):
         """run on single download item"""
         video_id = download_dict["youtube_id"]
-        vid_thumb_url = ThumbManager().vid_thumb_path(video_id)
+        vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
         published = date_praser(download_dict["published"])
 
         download_dict.update(
@@ -123,3 +125,17 @@ class SearchProcess:
             }
         )
         return dict(sorted(download_dict.items()))
+
+    def _process_comment(self, comment_dict):
+        """run on all comments, create reply thread"""
+        all_comments = comment_dict["comment_comments"]
+        processed_comments = []
+
+        for comment in all_comments:
+            if comment["comment_parent"] == "root":
+                comment.update({"comment_replies": []})
+                processed_comments.append(comment)
+            else:
+                processed_comments[-1]["comment_replies"].append(comment)
+
+        return processed_comments

tubearchivist/api/src/task_processor.py

@@ -1,54 +0,0 @@
-"""
-Functionality:
-- process tasks from API
-- validate
-- handover to celery
-"""
-
-from home.src.ta.ta_redis import RedisArchivist
-from home.tasks import download_pending, update_subscribed
-
-
-class TaskHandler:
-    """handle tasks from api"""
-
-    def __init__(self, data):
-        self.data = data
-
-    def run_task(self):
-        """map data and run"""
-        task_name = self.data["run"]
-        try:
-            to_run = self.exec_map(task_name)
-        except KeyError as err:
-            print(f"invalid task name {task_name}")
-            raise ValueError from err
-
-        response = to_run()
-        response.update({"task": task_name})
-        return response
-
-    def exec_map(self, task_name):
-        """map dict key and return function to execute"""
-        exec_map = {
-            "download_pending": self._download_pending,
-            "rescan_pending": self._rescan_pending,
-        }
-
-        return exec_map[task_name]
-
-    @staticmethod
-    def _rescan_pending():
-        """look for new items in subscribed channels"""
-        print("rescan subscribed channels")
-        update_subscribed.delay()
-        return {"success": True}
-
-    @staticmethod
-    def _download_pending():
-        """start the download queue"""
-        print("download pending")
-        running = download_pending.delay()
-        print("set task id: " + running.id)
-        RedisArchivist().set_message("dl_queue_id", running.id, expire=False)
-        return {"success": True}

tubearchivist/api/urls.py

@@ -1,96 +1,134 @@
 """all api urls"""
 
-from api.views import (
-    ChannelApiListView,
-    ChannelApiVideoView,
-    ChannelApiView,
-    CookieView,
-    DownloadApiListView,
-    DownloadApiView,
-    LoginApiView,
-    PingView,
-    PlaylistApiListView,
-    PlaylistApiVideoView,
-    PlaylistApiView,
-    TaskApiView,
-    VideoApiListView,
-    VideoApiView,
-    VideoProgressView,
-    VideoSponsorView,
-)
+from api import views
 from django.urls import path
 
 urlpatterns = [
-    path("ping/", PingView.as_view(), name="ping"),
-    path("login/", LoginApiView.as_view(), name="api-login"),
+    path("ping/", views.PingView.as_view(), name="ping"),
+    path("login/", views.LoginApiView.as_view(), name="api-login"),
     path(
         "video/",
-        VideoApiListView.as_view(),
+        views.VideoApiListView.as_view(),
         name="api-video-list",
     ),
     path(
         "video/<slug:video_id>/",
-        VideoApiView.as_view(),
+        views.VideoApiView.as_view(),
         name="api-video",
     ),
     path(
         "video/<slug:video_id>/progress/",
-        VideoProgressView.as_view(),
+        views.VideoProgressView.as_view(),
         name="api-video-progress",
     ),
+    path(
+        "video/<slug:video_id>/comment/",
+        views.VideoCommentView.as_view(),
+        name="api-video-comment",
+    ),
+    path(
+        "video/<slug:video_id>/similar/",
+        views.VideoSimilarView.as_view(),
+        name="api-video-similar",
+    ),
     path(
         "video/<slug:video_id>/sponsor/",
-        VideoSponsorView.as_view(),
+        views.VideoSponsorView.as_view(),
         name="api-video-sponsor",
     ),
     path(
         "channel/",
-        ChannelApiListView.as_view(),
+        views.ChannelApiListView.as_view(),
         name="api-channel-list",
     ),
     path(
         "channel/<slug:channel_id>/",
-        ChannelApiView.as_view(),
+        views.ChannelApiView.as_view(),
         name="api-channel",
     ),
     path(
         "channel/<slug:channel_id>/video/",
-        ChannelApiVideoView.as_view(),
+        views.ChannelApiVideoView.as_view(),
         name="api-channel-video",
     ),
     path(
         "playlist/",
-        PlaylistApiListView.as_view(),
+        views.PlaylistApiListView.as_view(),
         name="api-playlist-list",
     ),
     path(
         "playlist/<slug:playlist_id>/",
-        PlaylistApiView.as_view(),
+        views.PlaylistApiView.as_view(),
         name="api-playlist",
     ),
     path(
         "playlist/<slug:playlist_id>/video/",
-        PlaylistApiVideoView.as_view(),
+        views.PlaylistApiVideoView.as_view(),
         name="api-playlist-video",
     ),
     path(
         "download/",
-        DownloadApiListView.as_view(),
+        views.DownloadApiListView.as_view(),
         name="api-download-list",
     ),
     path(
         "download/<slug:video_id>/",
-        DownloadApiView.as_view(),
+        views.DownloadApiView.as_view(),
         name="api-download",
     ),
     path(
-        "task/",
-        TaskApiView.as_view(),
-        name="api-task",
+        "refresh/",
+        views.RefreshView.as_view(),
+        name="api-refresh",
+    ),
+    path(
+        "snapshot/",
+        views.SnapshotApiListView.as_view(),
+        name="api-snapshot-list",
+    ),
+    path(
+        "snapshot/<slug:snapshot_id>/",
+        views.SnapshotApiView.as_view(),
+        name="api-snapshot",
+    ),
+    path(
+        "task-name/",
+        views.TaskListView.as_view(),
+        name="api-task-list",
+    ),
+    path(
+        "task-name/<slug:task_name>/",
+        views.TaskNameListView.as_view(),
+        name="api-task-name-list",
+    ),
+    path(
+        "task-id/<slug:task_id>/",
+        views.TaskIDView.as_view(),
+        name="api-task-id",
     ),
     path(
         "cookie/",
-        CookieView.as_view(),
+        views.CookieView.as_view(),
         name="api-cookie",
     ),
+    path(
+        "watched/",
+        views.WatchedView.as_view(),
+        name="api-watched",
+    ),
+    path(
+        "search/",
+        views.SearchView.as_view(),
+        name="api-search",
+    ),
+    path(
+        "token/",
+        views.TokenView.as_view(),
+        name="api-token",
+    ),
+    path(
+        "notification/",
+        views.NotificationView.as_view(),
+        name="api-notification",
+    ),
 ]

tubearchivist/api/views.py

@@ -1,16 +1,27 @@
 """all API views"""
 
 from api.src.search_processor import SearchProcess
-from api.src.task_processor import TaskHandler
 from home.src.download.queue import PendingInteract
 from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.connect import ElasticWrap
+from home.src.es.snapshot import ElasticSnapshot
+from home.src.frontend.searching import SearchForm
+from home.src.frontend.watched import WatchState
+from home.src.index.channel import YoutubeChannel
 from home.src.index.generic import Pagination
-from home.src.index.video import SponsorBlock
+from home.src.index.reindex import ReindexProgress
+from home.src.index.video import SponsorBlock, YoutubeVideo
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import UrlListParser
-from home.src.ta.ta_redis import RedisArchivist, RedisQueue
-from home.tasks import extrac_dl, subscribe_to
+from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.task_manager import TaskCommand, TaskManager
+from home.src.ta.urlparser import Parser
+from home.tasks import (
+    BaseTask,
+    check_reindex,
+    download_pending,
+    extrac_dl,
+    subscribe_to,
+)
 from rest_framework.authentication import (
     SessionAuthentication,
     TokenAuthentication,
@@ -27,8 +38,8 @@ class ApiBaseView(APIView):
 
     authentication_classes = [SessionAuthentication, TokenAuthentication]
     permission_classes = [IsAuthenticated]
-    search_base = False
-    data = False
+    search_base = ""
+    data = ""
 
     def __init__(self):
         super().__init__()
@@ -41,7 +52,6 @@ class ApiBaseView(APIView):
     def get_document(self, document_id):
         """get single document from es"""
         path = f"{self.search_base}{document_id}"
-        print(path)
         response, status_code = ElasticWrap(path).get()
         try:
             self.response["data"] = SearchProcess(response).process()
@@ -52,9 +62,7 @@ class ApiBaseView(APIView):
 
     def initiate_pagination(self, request):
         """set initial pagination values"""
-        user_id = request.user.id
-        page_get = int(request.GET.get("page", 0))
-        self.pagination_handler = Pagination(page_get, user_id)
+        self.pagination_handler = Pagination(request)
         self.data.update(
             {
                 "size": self.pagination_handler.pagination["page_size"],
@@ -62,10 +70,11 @@ class ApiBaseView(APIView):
             }
         )
 
-    def get_document_list(self, request):
+    def get_document_list(self, request, pagination=True):
         """get a list of results"""
-        print(self.search_base)
-        self.initiate_pagination(request)
+        if pagination:
+            self.initiate_pagination(request)
+
         es_handler = ElasticWrap(self.search_base)
         response, status_code = es_handler.get(data=self.data)
         self.response["data"] = SearchProcess(response).process()
@@ -74,8 +83,11 @@ class ApiBaseView(APIView):
         else:
             self.status_code = 404
 
-        self.pagination_handler.validate(response["hits"]["total"]["value"])
-        self.response["paginate"] = self.pagination_handler.pagination
+        if pagination:
+            self.pagination_handler.validate(
+                response["hits"]["total"]["value"]
+            )
+            self.response["paginate"] = self.pagination_handler.pagination
 
 
 class VideoApiView(ApiBaseView):
@@ -91,6 +103,20 @@ class VideoApiView(ApiBaseView):
         self.get_document(video_id)
         return Response(self.response, status=self.status_code)
 
+    def delete(self, request, video_id):
+        # pylint: disable=unused-argument
+        """delete single video"""
+        message = {"video": video_id}
+        try:
+            YoutubeVideo(video_id).delete_media_file()
+            status_code = 200
+            message.update({"state": "delete"})
+        except FileNotFoundError:
+            status_code = 404
+            message.update({"state": "not found"})
+
+        return Response(message, status=status_code)
+
 
 class VideoApiListView(ApiBaseView):
     """resolves to /api/video/
@@ -131,7 +157,7 @@ class VideoProgressView(ApiBaseView):
         position = request.data.get("position", 0)
         key = f"{request.user.id}:progress:{video_id}"
         message = {"position": position, "youtube_id": video_id}
-        RedisArchivist().set_message(key, message, expire=False)
+        RedisArchivist().set_message(key, message)
         self.response = request.data
 
         return Response(self.response)
@@ -145,6 +171,46 @@ class VideoProgressView(ApiBaseView):
         return Response(self.response)
 
 
+class VideoCommentView(ApiBaseView):
+    """resolves to /api/video/<video_id>/comment/
+    handle video comments
+    GET: return all comments from video with reply threads
+    """
+
+    search_base = "ta_comment/_doc/"
+
+    def get(self, request, video_id):
+        """get video comments"""
+        # pylint: disable=unused-argument
+        self.get_document(video_id)
+
+        return Response(self.response, status=self.status_code)
+
+
+class VideoSimilarView(ApiBaseView):
+    """resolves to /api/video/<video-id>/similar/
+    GET: return max 3 videos similar to this
+    """
+
+    search_base = "ta_video/_search/"
+
+    def get(self, request, video_id):
+        """get similar videos"""
+        self.data = {
+            "size": 6,
+            "query": {
+                "more_like_this": {
+                    "fields": ["tags", "title"],
+                    "like": {"_id": video_id},
+                    "min_term_freq": 1,
+                    "max_query_terms": 25,
+                }
+            },
+        }
+        self.get_document_list(request, pagination=False)
+        return Response(self.response, status=self.status_code)
+
+
 class VideoSponsorView(ApiBaseView):
     """resolves to /api/video/<video_id>/sponsor/
     handle sponsor block integration
@@ -207,6 +273,20 @@ class ChannelApiView(ApiBaseView):
         self.get_document(channel_id)
         return Response(self.response, status=self.status_code)
 
+    def delete(self, request, channel_id):
+        # pylint: disable=unused-argument
+        """delete channel"""
+        message = {"channel": channel_id}
+        try:
+            YoutubeChannel(channel_id).delete_channel()
+            status_code = 200
+            message.update({"state": "delete"})
+        except FileNotFoundError:
+            status_code = 404
+            message.update({"state": "not found"})
+
+        return Response(message, status=status_code)
+
 
 class ChannelApiListView(ApiBaseView):
     """resolves to /api/channel/
@@ -215,14 +295,27 @@ class ChannelApiListView(ApiBaseView):
     """
 
     search_base = "ta_channel/_search/"
+    valid_filter = ["subscribed"]
 
     def get(self, request):
         """get request"""
-        self.get_document_list(request)
         self.data.update(
             {"sort": [{"channel_name.keyword": {"order": "asc"}}]}
         )
 
+        query_filter = request.GET.get("filter", False)
+        must_list = []
+        if query_filter:
+            if query_filter not in self.valid_filter:
+                message = f"invalid url query filder: {query_filter}"
+                print(message)
+                return Response({"message": message}, status=400)
+
+            must_list.append({"term": {"channel_subscribed": {"value": True}}})
+
+        self.data["query"] = {"bool": {"must": must_list}}
+        self.get_document_list(request)
+
         return Response(self.response)
 
     @staticmethod
@@ -321,7 +414,7 @@ class DownloadApiView(ApiBaseView):
     """
 
     search_base = "ta_download/_doc/"
-    valid_status = ["pending", "ignore"]
+    valid_status = ["pending", "ignore", "priority"]
 
     def get(self, request, video_id):
         # pylint: disable=unused-argument
@@ -331,15 +424,21 @@ class DownloadApiView(ApiBaseView):
 
     def post(self, request, video_id):
         """post to video to change status"""
-        item_status = request.data["status"]
+        item_status = request.data.get("status")
         if item_status not in self.valid_status:
             message = f"{video_id}: invalid status {item_status}"
             print(message)
             return Response({"message": message}, status=400)
 
+        _, status_code = PendingInteract(video_id).get_item()
+        if status_code == 404:
+            message = f"{video_id}: item not found {status_code}"
+            return Response({"message": message}, status=404)
+
         print(f"{video_id}: change status to {item_status}")
-        PendingInteract(video_id=video_id, status=item_status).update_status()
-        RedisQueue().clear_item(video_id)
+        PendingInteract(video_id, item_status).update_status()
+        if item_status == "priority":
+            download_pending.delay(auto_only=True)
 
         return Response(request.data)
 
@@ -348,7 +447,7 @@ class DownloadApiView(ApiBaseView):
         # pylint: disable=unused-argument
         """delete single video from queue"""
         print(f"{video_id}: delete from queue")
-        PendingInteract(video_id=video_id).delete_item()
+        PendingInteract(video_id).delete_item()
 
         return Response({"success": True})
 
@@ -367,13 +466,23 @@ class DownloadApiListView(ApiBaseView):
         """get request"""
         query_filter = request.GET.get("filter", False)
         self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
+
+        must_list = []
         if query_filter:
             if query_filter not in self.valid_filter:
                 message = f"invalid url query filder: {query_filter}"
                 print(message)
                 return Response({"message": message}, status=400)
 
-            self.data["query"] = {"term": {"status": {"value": query_filter}}}
+            must_list.append({"term": {"status": {"value": query_filter}}})
+
+        filter_channel = request.GET.get("channel", False)
+        if filter_channel:
+            must_list.append(
+                {"term": {"channel_id": {"value": filter_channel}}}
+            )
+
+        self.data["query"] = {"bool": {"must": must_list}}
 
         self.get_document_list(request)
         return Response(self.response)
@@ -381,8 +490,8 @@ class DownloadApiListView(ApiBaseView):
     @staticmethod
     def post(request):
         """add list of videos to download queue"""
-        print(f"request meta data: {request.META}")
         data = request.data
+        auto_start = bool(request.GET.get("autostart"))
         try:
             to_add = data["data"]
         except KeyError:
@@ -393,13 +502,13 @@ class DownloadApiListView(ApiBaseView):
         pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
         url_str = " ".join(pending)
         try:
-            youtube_ids = UrlListParser(url_str).process_list()
+            youtube_ids = Parser(url_str).parse()
         except ValueError:
             message = f"failed to parse: {url_str}"
             print(message)
             return Response({"message": message}, status=400)
 
-        extrac_dl.delay(youtube_ids)
+        extrac_dl.delay(youtube_ids, auto_start=auto_start)
 
         return Response(data)
 
@@ -450,25 +559,209 @@ class LoginApiView(ObtainAuthToken):
         return Response({"token": token.key, "user_id": user.pk})
 
 
-class TaskApiView(ApiBaseView):
-    """resolves to /api/task/
-    POST: start a new background task
+class SnapshotApiListView(ApiBaseView):
+    """resolves to /api/snapshot/
+    GET: returns snashot config plus list of existing snapshots
+    POST: take snapshot now
     """
 
+    @staticmethod
+    def get(request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        snapshots = ElasticSnapshot().get_snapshot_stats()
+
+        return Response(snapshots)
+
+    @staticmethod
+    def post(request):
+        """take snapshot now with post request"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().take_snapshot_now()
+
+        return Response(response)
+
+
+class SnapshotApiView(ApiBaseView):
+    """resolves to /api/snapshot/<snapshot-id>/
+    GET: return a single snapshot
+    POST: restore snapshot
+    DELETE: delete a snapshot
+    """
+
+    @staticmethod
+    def get(request, snapshot_id):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
+
+        if not snapshot:
+            return Response({"message": "snapshot not found"}, status=404)
+
+        return Response(snapshot)
+
+    @staticmethod
+    def post(request, snapshot_id):
+        """restore snapshot with post request"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().restore_all(snapshot_id)
+        if not response:
+            message = {"message": "failed to restore snapshot"}
+            return Response(message, status=400)
+
+        return Response(response)
+
+    @staticmethod
+    def delete(request, snapshot_id):
+        """delete snapshot from index"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
+        if not response:
+            message = {"message": "failed to delete snapshot"}
+            return Response(message, status=400)
+
+        return Response(response)
+
+
+class TaskListView(ApiBaseView):
+    """resolves to /api/task-name/
+    GET: return a list of all stored task results
+    """
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        all_results = TaskManager().get_all_results()
+
+        return Response(all_results)
+
+
+class TaskNameListView(ApiBaseView):
+    """resolves to /api/task-name/<task-name>/
+    GET: return a list of stored results of task
+    POST: start new background process
+    """
+
+    def get(self, request, task_name):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        if task_name not in BaseTask.TASK_CONFIG:
+            message = {"message": "invalid task name"}
+            return Response(message, status=404)
+
+        all_results = TaskManager().get_tasks_by_name(task_name)
+
+        return Response(all_results)
+
+    def post(self, request, task_name):
+        """
+        handle post request
+        404 for invalid task_name
+        400 if task can't be started here without argument
+        """
+        # pylint: disable=unused-argument
+        task_config = BaseTask.TASK_CONFIG.get(task_name)
+        if not task_config:
+            message = {"message": "invalid task name"}
+            return Response(message, status=404)
+
+        if not task_config.get("api-start"):
+            message = {"message": "can not start task through this endpoint"}
+            return Response(message, status=400)
+
+        message = TaskCommand().start(task_name)
+
+        return Response({"message": message})
+
+
+class TaskIDView(ApiBaseView):
+    """resolves to /api/task-id/<task-id>/
+    GET: return details of task id
+    """
+
+    valid_commands = ["stop", "kill"]
+
+    def get(self, request, task_id):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        task_result = TaskManager().get_task(task_id)
+        if not task_result:
+            message = {"message": "task id not found"}
+            return Response(message, status=404)
+
+        return Response(task_result)
+
+    def post(self, request, task_id):
+        """post command to task"""
+        command = request.data.get("command")
+        if not command or command not in self.valid_commands:
+            message = {"message": "no valid command found"}
+            return Response(message, status=400)
+
+        task_result = TaskManager().get_task(task_id)
+        if not task_result:
+            message = {"message": "task id not found"}
+            return Response(message, status=404)
+
+        task_conf = BaseTask.TASK_CONFIG.get(task_result.get("name"))
+        if command == "stop":
+            if not task_conf.get("api-stop"):
+                message = {"message": "task can not be stopped"}
+                return Response(message, status=400)
+
+            message_key = self._build_message_key(task_conf, task_id)
+            TaskCommand().stop(task_id, message_key)
+        if command == "kill":
+            if not task_conf.get("api-stop"):
+                message = {"message": "task can not be killed"}
+                return Response(message, status=400)
+
+            TaskCommand().kill(task_id)
+
+        return Response({"message": "command sent"})
+
+    def _build_message_key(self, task_conf, task_id):
+        """build message key to forward command to notification"""
+        return f"message:{task_conf.get('group')}:{task_id.split('-')[0]}"
+
+
+class RefreshView(ApiBaseView):
+    """resolves to /api/refresh/
+    GET: get refresh progress
+    POST: start a manual refresh task
+    """
+
+    def get(self, request):
+        """handle get request"""
+        request_type = request.GET.get("type")
+        request_id = request.GET.get("id")
+
+        if request_id and not request_type:
+            return Response({"status": "Bad Request"}, status=400)
+
+        try:
+            progress = ReindexProgress(
+                request_type=request_type, request_id=request_id
+            ).get_progress()
+        except ValueError:
+            return Response({"status": "Bad Request"}, status=400)
+
+        return Response(progress)
+
     def post(self, request):
         """handle post request"""
-
         data = request.data
-        print(data)
-        response = TaskHandler(data).run_task()
+        extract_videos = bool(request.GET.get("extract_videos", False))
+        check_reindex.delay(data=data, extract_videos=extract_videos)
 
-        return Response(response)
+        return Response(data)
 
 
 class CookieView(ApiBaseView):
     """resolves to /api/cookie/
     GET: check if cookie is enabled
     POST: verify validity of cookie
+    PUT: import cookie
     """
 
     @staticmethod
@@ -476,9 +769,11 @@ class CookieView(ApiBaseView):
         """handle get request"""
         # pylint: disable=unused-argument
         config = AppConfig().config
-        cookie_enabled = config["downloads"]["cookie_import"]
+        valid = RedisArchivist().get_message("cookie:valid")
+        response = {"cookie_enabled": config["downloads"]["cookie_import"]}
+        response.update(valid)
 
-        return Response({"cookie_enabled": cookie_enabled})
+        return Response(response)
 
     @staticmethod
     def post(request):
@@ -488,3 +783,94 @@ class CookieView(ApiBaseView):
         validated = CookieHandler(config).validate()
 
         return Response({"cookie_validated": validated})
+
+    @staticmethod
+    def put(request):
+        """handle put request"""
+        # pylint: disable=unused-argument
+        config = AppConfig().config
+        cookie = request.data.get("cookie")
+        if not cookie:
+            message = "missing cookie key in request data"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        print(f"cookie preview:\n\n{cookie[:300]}")
+        handler = CookieHandler(config)
+        handler.set_cookie(cookie)
+        validated = handler.validate()
+        if not validated:
+            handler.revoke()
+            message = {"cookie_import": "fail", "cookie_validated": validated}
+            print(f"cookie: {message}")
+            return Response({"message": message}, status=400)
+
+        message = {"cookie_import": "done", "cookie_validated": validated}
+        return Response(message)
+
+
+class WatchedView(ApiBaseView):
+    """resolves to /api/watched/
+    POST: change watched state of video, channel or playlist
+    """
+
+    def post(self, request):
+        """change watched state"""
+        youtube_id = request.data.get("id")
+        is_watched = request.data.get("is_watched")
+
+        if not youtube_id or is_watched is None:
+            message = {"message": "missing id or is_watched"}
+            return Response(message, status=400)
+
+        WatchState(youtube_id, is_watched).change()
+        return Response({"message": "success"}, status=200)
+
+
+class SearchView(ApiBaseView):
+    """resolves to /api/search/
+    GET: run a search with the string in the ?query parameter
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request
+        search through all indexes"""
+        search_query = request.GET.get("query", None)
+        if search_query is None:
+            return Response(
+                {"message": "no search query specified"}, status=400
+            )
+
+        search_results = SearchForm().multi_search(search_query)
+        return Response(search_results)
+
+
+class TokenView(ApiBaseView):
+    """resolves to /api/token/
+    DELETE: revoke the token
+    """
+
+    @staticmethod
+    def delete(request):
+        print("revoke API token")
+        request.user.auth_token.delete()
+        return Response({"success": True})
+
+
+class NotificationView(ApiBaseView):
+    """resolves to /api/notification/
+    GET: returns a list of notifications
+    filter query to filter messages by group
+    """
+
+    valid_filters = ["download", "settings", "channel"]
+
+    def get(self, request):
+        """get all notifications"""
+        query = "message"
+        filter_by = request.GET.get("filter", None)
+        if filter_by in self.valid_filters:
+            query = f"{query}:{filter_by}"
+
+        return Response(RedisArchivist().list_items(query))

tubearchivist/config/management/commands/ta_connection.py

@@ -0,0 +1,144 @@
+"""
+Functionality:
+- check that all connections are working
+"""
+
+from time import sleep
+
+import requests
+from django.core.management.base import BaseCommand, CommandError
+from home.src.es.connect import ElasticWrap
+from home.src.ta.ta_redis import RedisArchivist
+
+TOPIC = """
+
+#######################
+#  Connection check   #
+#######################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    TIMEOUT = 120
+    MIN_MAJOR, MAX_MAJOR = 8, 8
+    MIN_MINOR = 0
+
+    # pylint: disable=no-member
+    help = "Check connections"
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(TOPIC)
+        self._redis_connection_check()
+        self._redis_config_set()
+        self._es_connection_check()
+        self._es_version_check()
+        self._es_path_check()
+
+    def _redis_connection_check(self):
+        """check ir redis connection is established"""
+        self.stdout.write("[1] connect to Redis")
+        redis_conn = RedisArchivist().conn
+        for _ in range(5):
+            try:
+                pong = redis_conn.execute_command("PING")
+                if pong:
+                    self.stdout.write(
+                        self.style.SUCCESS("    ✓ Redis connection verified")
+                    )
+                    return
+
+            except Exception:  # pylint: disable=broad-except
+                self.stdout.write("    ... retry Redis connection")
+                sleep(2)
+
+        message = "    🗙 Redis connection failed"
+        self.stdout.write(self.style.ERROR(f"{message}"))
+        RedisArchivist().exec("PING")
+        sleep(60)
+        raise CommandError(message)
+
+    def _redis_config_set(self):
+        """set config for redis if not set already"""
+        self.stdout.write("[2] set Redis config")
+        redis_conn = RedisArchivist().conn
+        timeout_is = int(redis_conn.config_get("timeout").get("timeout"))
+        if not timeout_is:
+            redis_conn.config_set("timeout", 3600)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ Redis config set"))
+
+    def _es_connection_check(self):
+        """wait for elasticsearch connection"""
+        self.stdout.write("[3] connect to Elastic Search")
+        total = self.TIMEOUT // 5
+        for i in range(total):
+            self.stdout.write(f"    ... waiting for ES [{i}/{total}]")
+            try:
+                _, status_code = ElasticWrap("/").get(
+                    timeout=1, print_error=False
+                )
+            except requests.exceptions.ConnectionError:
+                sleep(5)
+                continue
+
+            if status_code and status_code == 200:
+                self.stdout.write(
+                    self.style.SUCCESS("    ✓ ES connection established")
+                )
+                return
+
+        response, status_code = ElasticWrap("/").get(
+            timeout=1, print_error=False
+        )
+
+        message = "    🗙 ES connection failed"
+        self.stdout.write(self.style.ERROR(f"{message}"))
+        self.stdout.write(f"    error message: {response}")
+        self.stdout.write(f"    status code: {status_code}")
+        sleep(60)
+        raise CommandError(message)
+
+    def _es_version_check(self):
+        """check for minimal elasticsearch version"""
+        self.stdout.write("[4] Elastic Search version check")
+        response, _ = ElasticWrap("/").get()
+        version = response["version"]["number"]
+        major = int(version.split(".")[0])
+
+        if self.MIN_MAJOR <= major <= self.MAX_MAJOR:
+            self.stdout.write(
+                self.style.SUCCESS("    ✓ ES version check passed")
+            )
+            return
+
+        message = (
+            "    🗙 ES connection failed. "
+            + f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}"
+        )
+        self.stdout.write(self.style.ERROR(f"{message}"))
+        sleep(60)
+        raise CommandError(message)
+
+    def _es_path_check(self):
+        """check that path.repo var is set"""
+        self.stdout.write("[5] check ES path.repo env var")
+        response, _ = ElasticWrap("_nodes/_all/settings").get()
+        for node in response["nodes"].values():
+            if node["settings"]["path"].get("repo"):
+                self.stdout.write(
+                    self.style.SUCCESS("    ✓ path.repo env var is set")
+                )
+                return
+
+            message = (
+                "    🗙 path.repo env var not found. "
+                + "set the following env var to the ES container:\n"
+                + "    path.repo=/usr/share/elasticsearch/data/snapshot"
+            )
+            self.stdout.write(self.style.ERROR(f"{message}"))
+            sleep(60)
+            raise CommandError(message)

tubearchivist/config/management/commands/ta_envcheck.py

@@ -0,0 +1,196 @@
+"""
+Functionality:
+- Check environment at startup
+- Process config file overwrites from env var
+- Stop startup on error
+- python management.py ta_envcheck
+"""
+
+import os
+import re
+
+from django.core.management.base import BaseCommand, CommandError
+from home.models import Account
+
+LOGO = """
+
+                         ....  .....
+                  ...'',;:cc,. .;::;;,'...
+               ..,;:cccllclc,  .:ccllllcc;,..
+            ..,:cllcc:;,'.',.  ....'',;ccllc:,..
+          ..;cllc:,'..                ...,:cccc:'.
+         .;cccc;..                        ..,:ccc:'.
+       .ckkkOkxollllllllllllc.      .,:::;.  .,cclc;
+      .:0MMMMMMMMMMMMMMMMMMMX:     .cNMMMWx.   .;clc:
+     .;lOXK0000KNMMMMX00000KO;     ;KMMMMMNl.   .;ccl:,.
+     .;:c:'.....kMMMNo........    'OMMMWMMMK:    '::;;'.
+   .......     .xMMMNl           .dWMMXdOMMMO'   ........
+   .:cc:;.     .xMMMNc          .lNMMNo.:XMMWx.    .:cl:.
+   .:llc,.     .:xxxd,          ;KMMMk. .oWMMNl.   .:llc'
+   .cll:.     .;:;;:::,.       'OMMMK:';''kWMMK:   .;llc,
+   .cll:.     .,;;;;;;,.     .,xWMMNl.:l:.;KMMMO'  .;llc'
+   .:llc.      .cOOOk;      .lKNMMWx..:l:..lNMMWx. .:llc'
+   .;lcc,.     .xMMMNc      :KMMMM0, .:lc. .xWMMNl.'ccl:.
+    .cllc.     .xMMMNc     'OMMMMXc...:lc...,0MMMKl:lcc,.
+    .,ccl:.    .xMMMNc    .xWMMMWo.,;;:lc;;;.cXMMMXdcc;.
+     .,clc:.   .xMMMNc   .lNMMMWk. .':clc:,. .dWMMW0o;.
+      .,clcc,. .ckkkx;   .okkkOx,    .';,.    'kKKK0l.
+       .':lcc:'.....      .  ..            ..,;cllc,.
+         .,cclc,....                     ....;clc;..
+          ..,:,..,c:'..              ...';:,..,:,.
+            ....:lcccc:;,'''.....'',;;:clllc,....
+               .'',;:cllllllccccclllllcc:,'..
+                   ...'',,;;;;;;;;;,''...
+                            .....
+
+"""
+
+TOPIC = """
+#######################
+#  Environment Setup  #
+#######################
+
+"""
+
+EXPECTED_ENV_VARS = [
+    "TA_USERNAME",
+    "TA_PASSWORD",
+    "ELASTIC_PASSWORD",
+    "ES_URL",
+    "TA_HOST",
+]
+INST = "https://github.com/tubearchivist/tubearchivist#installing-and-updating"
+NGINX = "/etc/nginx/sites-available/default"
+UWSGI = "/app/uwsgi.ini"
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+    help = "Check environment before startup"
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(LOGO)
+        self.stdout.write(TOPIC)
+        self._expected_vars()
+        self._elastic_user_overwrite()
+        self._ta_port_overwrite()
+        self._ta_uwsgi_overwrite()
+        self._enable_cast_overwrite()
+        self._create_superuser()
+
+    def _expected_vars(self):
+        """check if expected env vars are set"""
+        self.stdout.write("[1] checking expected env vars")
+        env = os.environ
+        for var in EXPECTED_ENV_VARS:
+            if not env.get(var):
+                message = f"    🗙 expected env var {var} not set\n    {INST}"
+                self.stdout.write(self.style.ERROR(message))
+                raise CommandError(message)
+
+        message = "    ✓ all expected env vars are set"
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _elastic_user_overwrite(self):
+        """check for ELASTIC_USER overwrite"""
+        self.stdout.write("[2] set default ES user")
+        if not os.environ.get("ELASTIC_USER"):
+            os.environ.setdefault("ELASTIC_USER", "elastic")
+
+        env = os.environ.get("ELASTIC_USER")
+
+        self.stdout.write(self.style.SUCCESS(f"    ✓ ES user is set to {env}"))
+
+    def _ta_port_overwrite(self):
+        """set TA_PORT overwrite for nginx"""
+        self.stdout.write("[3] check TA_PORT overwrite")
+        overwrite = os.environ.get("TA_PORT")
+        if not overwrite:
+            self.stdout.write(self.style.SUCCESS("    TA_PORT is not set"))
+            return
+
+        regex = re.compile(r"listen [0-9]{1,5}")
+        to_overwrite = f"listen {overwrite}"
+        changed = file_overwrite(NGINX, regex, to_overwrite)
+        if changed:
+            message = f"    ✓ TA_PORT changed to {overwrite}"
+        else:
+            message = f"    ✓ TA_PORT already set to {overwrite}"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _ta_uwsgi_overwrite(self):
+        """set TA_UWSGI_PORT overwrite"""
+        self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
+        overwrite = os.environ.get("TA_UWSGI_PORT")
+        if not overwrite:
+            message = "    TA_UWSGI_PORT is not set"
+            self.stdout.write(self.style.SUCCESS(message))
+            return
+
+        # nginx
+        regex = re.compile(r"uwsgi_pass localhost:[0-9]{1,5}")
+        to_overwrite = f"uwsgi_pass localhost:{overwrite}"
+        changed = file_overwrite(NGINX, regex, to_overwrite)
+
+        # uwsgi
+        regex = re.compile(r"socket = :[0-9]{1,5}")
+        to_overwrite = f"socket = :{overwrite}"
+        changed = file_overwrite(UWSGI, regex, to_overwrite)
+
+        if changed:
+            message = f"    ✓ TA_UWSGI_PORT changed to {overwrite}"
+        else:
+            message = f"    ✓ TA_UWSGI_PORT already set to {overwrite}"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _enable_cast_overwrite(self):
+        """cast workaround, remove auth for static files in nginx"""
+        self.stdout.write("[5] check ENABLE_CAST overwrite")
+        overwrite = os.environ.get("ENABLE_CAST")
+        if not overwrite:
+            self.stdout.write(self.style.SUCCESS("    ENABLE_CAST is not set"))
+            return
+
+        regex = re.compile(r"[^\S\r\n]*auth_request /api/ping/;\n")
+        changed = file_overwrite(NGINX, regex, "")
+        if changed:
+            message = "    ✓ process nginx to enable Cast"
+        else:
+            message = "    ✓ Cast is already enabled in nginx"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _create_superuser(self):
+        """create superuser if not exist"""
+        self.stdout.write("[6] create superuser")
+        is_created = Account.objects.filter(is_superuser=True)
+        if is_created:
+            message = "    superuser already created"
+            self.stdout.write(self.style.SUCCESS(message))
+            return
+
+        name = os.environ.get("TA_USERNAME")
+        password = os.environ.get("TA_PASSWORD")
+        Account.objects.create_superuser(name, password)
+        message = f"    ✓ new superuser with name {name} created"
+        self.stdout.write(self.style.SUCCESS(message))
+
+
+def file_overwrite(file_path, regex, overwrite):
+    """change file content from old to overwrite, return true when changed"""
+    with open(file_path, "r", encoding="utf-8") as f:
+        file_content = f.read()
+
+    changed = re.sub(regex, overwrite, file_content)
+    if changed == file_content:
+        return False
+
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(changed)
+
+    return True

tubearchivist/config/management/commands/ta_startup.py

@@ -0,0 +1,223 @@
+"""
+Functionality:
+- Application startup
+- Apply migrations
+"""
+
+import os
+from time import sleep
+
+from django.core.management.base import BaseCommand, CommandError
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.es.index_setup import ElasitIndexWrap
+from home.src.es.snapshot import ElasticSnapshot
+from home.src.index.filesystem import Filesystem
+from home.src.index.video_streams import MediaStreamExtractor
+from home.src.ta.config import AppConfig, ReleaseVersion
+from home.src.ta.helper import clear_dl_cache
+from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.task_manager import TaskManager
+
+TOPIC = """
+
+#######################
+#  Application Start  #
+#######################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(TOPIC)
+        self._sync_redis_state()
+        self._make_folders()
+        self._release_locks()
+        self._clear_tasks()
+        self._clear_dl_cache()
+        self._version_check()
+        self._mig_index_setup()
+        self._mig_snapshot_check()
+        self._mig_set_streams()
+        self._mig_set_autostart()
+
+    def _sync_redis_state(self):
+        """make sure redis gets new config.json values"""
+        self.stdout.write("[1] set new config.json values")
+        needs_update = AppConfig().load_new_defaults()
+        if needs_update:
+            self.stdout.write(
+                self.style.SUCCESS("    ✓ new config values set")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no new config values"))
+
+    def _make_folders(self):
+        """make expected cache folders"""
+        self.stdout.write("[2] create expected cache folders")
+        folders = [
+            "backup",
+            "channels",
+            "download",
+            "import",
+            "playlists",
+            "videos",
+        ]
+        cache_dir = AppConfig().config["application"]["cache_dir"]
+        for folder in folders:
+            folder_path = os.path.join(cache_dir, folder)
+            os.makedirs(folder_path, exist_ok=True)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ expected folders created"))
+
+    def _release_locks(self):
+        """make sure there are no leftover locks set in redis"""
+        self.stdout.write("[3] clear leftover locks in redis")
+        all_locks = [
+            "dl_queue_id",
+            "dl_queue",
+            "downloading",
+            "manual_import",
+            "reindex",
+            "rescan",
+            "run_backup",
+            "startup_check",
+        ]
+
+        redis_con = RedisArchivist()
+        has_changed = False
+        for lock in all_locks:
+            if redis_con.del_message(lock):
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ cleared lock {lock}")
+                )
+                has_changed = True
+
+        if not has_changed:
+            self.stdout.write(self.style.SUCCESS("    no locks found"))
+
+    def _clear_tasks(self):
+        """clear tasks and messages"""
+        self.stdout.write("[4] clear task leftovers")
+        TaskManager().fail_pending()
+        redis_con = RedisArchivist()
+        to_delete = redis_con.list_keys("message:")
+        if to_delete:
+            for key in to_delete:
+                redis_con.del_message(key)
+
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {len(to_delete)} messages")
+            )
+
+    def _clear_dl_cache(self):
+        """clear leftover files from dl cache"""
+        self.stdout.write("[5] clear leftover files from dl cache")
+        config = AppConfig().config
+        leftover_files = clear_dl_cache(config)
+        if leftover_files:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {leftover_files} files")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no files found"))
+
+    def _version_check(self):
+        """remove new release key if updated now"""
+        self.stdout.write("[6] check for first run after update")
+        new_version = ReleaseVersion().is_updated()
+        if new_version:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ update to {new_version} completed")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no new update found"))
+
+    def _mig_index_setup(self):
+        """migration: validate index mappings"""
+        self.stdout.write("[MIGRATION] validate index mappings")
+        ElasitIndexWrap().setup()
+
+    def _mig_snapshot_check(self):
+        """migration setup snapshots"""
+        self.stdout.write("[MIGRATION] setup snapshots")
+        ElasticSnapshot().setup()
+
+    def _mig_set_streams(self):
+        """migration: update from 0.3.5 to 0.3.6, set streams and media_size"""
+        self.stdout.write("[MIGRATION] index streams and media size")
+        videos = AppConfig().config["application"]["videos"]
+        data = {
+            "query": {
+                "bool": {"must_not": [{"exists": {"field": "streams"}}]}
+            },
+            "_source": ["media_url", "youtube_id"],
+        }
+        all_missing = IndexPaginate("ta_video", data).get_results()
+        if not all_missing:
+            self.stdout.write("    no videos need updating")
+            return
+
+        self.stdout.write("    start filesystem rescan")
+        Filesystem().process()
+        total = len(all_missing)
+        for idx, missing in enumerate(all_missing):
+            media_url = missing["media_url"]
+            youtube_id = missing["youtube_id"]
+            media_path = os.path.join(videos, media_url)
+            if not os.path.exists(media_path):
+                self.stdout.write(f"    file not found: {media_path}")
+                continue
+
+            media = MediaStreamExtractor(media_path)
+            vid_data = {
+                "doc": {
+                    "streams": media.extract_metadata(),
+                    "media_size": media.get_file_size(),
+                }
+            }
+            path = f"ta_video/_update/{youtube_id}"
+            response, status_code = ElasticWrap(path).post(data=vid_data)
+            if not status_code == 200:
+                self.stdout.errors(
+                    f"    update failed: {path}, {response}, {status_code}"
+                )
+
+            if idx % 100 == 0:
+                self.stdout.write(f"    progress {idx}/{total}")
+
+    def _mig_set_autostart(self):
+        """migration: update from 0.3.5 to 0.3.6 set auto_start to false"""
+        self.stdout.write("[MIGRATION] set default download auto_start")
+        data = {
+            "query": {
+                "bool": {"must_not": [{"exists": {"field": "auto_start"}}]}
+            },
+            "script": {"source": "ctx._source['auto_start'] = false"},
+        }
+        path = "ta_download/_update_by_query"
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code == 200:
+            updated = response.get("updated", 0)
+            if updated:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"    ✓ {updated} videos updated in ta_download"
+                    )
+                )
+            else:
+                self.stdout.write(
+                    "    no videos needed updating in ta_download"
+                )
+            return
+
+        message = "    🗙 ta_download auto_start update failed"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)

tubearchivist/config/settings.py

@@ -14,8 +14,11 @@ import hashlib
 from os import environ, path
 from pathlib import Path
 
+import ldap
 from corsheaders.defaults import default_headers
+from django_auth_ldap.config import LDAPSearch
 from home.src.ta.config import AppConfig
+from home.src.ta.helper import ta_host_parser
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -24,14 +27,13 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
 
-PW_HASH = hashlib.sha256(environ.get("TA_PASSWORD").encode())
+PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode())
 SECRET_KEY = PW_HASH.hexdigest()
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = bool(environ.get("DJANGO_DEBUG"))
 
-ALLOWED_HOSTS = ["*"]
-
+ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(environ["TA_HOST"])
 
 # Application definition
 
@@ -49,6 +51,7 @@ INSTALLED_APPS = [
     "rest_framework",
     "rest_framework.authtoken",
     "api",
+    "config",
 ]
 
 MIDDLEWARE = [
@@ -83,6 +86,96 @@ TEMPLATES = [
 
 WSGI_APPLICATION = "config.wsgi.application"
 
+if bool(environ.get("TA_LDAP")):
+    # pylint: disable=global-at-module-level
+    global AUTH_LDAP_SERVER_URI
+    AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
+
+    global AUTH_LDAP_BIND_DN
+    AUTH_LDAP_BIND_DN = environ.get("TA_LDAP_BIND_DN")
+
+    global AUTH_LDAP_BIND_PASSWORD
+    AUTH_LDAP_BIND_PASSWORD = environ.get("TA_LDAP_BIND_PASSWORD")
+
+    """
+    Since these are new environment variables, taking the opporunity to use
+    more accurate env names.
+    Given Names are *_technically_* different from Personal names, as people
+    who change their names have different given names and personal names,
+    and they go by personal names. Additionally, "LastName" is actually
+    incorrect for many cultures, such as Korea, where the
+    family name comes first, and the personal name comes last.
+
+    But we all know people are going to try to guess at these, so still want
+    to include names that people will guess, hence using first/last as well.
+    """
+
+    # Attribute mapping options
+
+    global AUTH_LDAP_USER_ATTR_MAP_USERNAME
+    AUTH_LDAP_USER_ATTR_MAP_USERNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_USERNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_UID")
+        or "uid"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME
+    AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_PERSONALNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_FIRSTNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_GIVENNAME")
+        or "givenName"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_SURNAME
+    AUTH_LDAP_USER_ATTR_MAP_SURNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_SURNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_LASTNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_FAMILYNAME")
+        or "sn"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_EMAIL
+    AUTH_LDAP_USER_ATTR_MAP_EMAIL = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_EMAIL")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_MAIL")
+        or "mail"
+    )
+
+    global AUTH_LDAP_USER_BASE
+    AUTH_LDAP_USER_BASE = environ.get("TA_LDAP_USER_BASE")
+
+    global AUTH_LDAP_USER_FILTER
+    AUTH_LDAP_USER_FILTER = environ.get("TA_LDAP_USER_FILTER")
+
+    global AUTH_LDAP_USER_SEARCH
+    # pylint: disable=no-member
+    AUTH_LDAP_USER_SEARCH = LDAPSearch(
+        AUTH_LDAP_USER_BASE,
+        ldap.SCOPE_SUBTREE,
+        "(&("
+        + AUTH_LDAP_USER_ATTR_MAP_USERNAME
+        + "=%(user)s)"
+        + AUTH_LDAP_USER_FILTER
+        + ")",
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP
+    AUTH_LDAP_USER_ATTR_MAP = {
+        "username": AUTH_LDAP_USER_ATTR_MAP_USERNAME,
+        "first_name": AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME,
+        "last_name": AUTH_LDAP_USER_ATTR_MAP_SURNAME,
+        "email": AUTH_LDAP_USER_ATTR_MAP_EMAIL,
+    }
+
+    if bool(environ.get("TA_LDAP_DISABLE_CERT_CHECK")):
+        global AUTH_LDAP_GLOBAL_OPTIONS
+        AUTH_LDAP_GLOBAL_OPTIONS = {
+            ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
+        }
+
+    global AUTHENTICATION_BACKENDS
+    AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
 
 # Database
 # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
@@ -163,4 +256,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.1.6"
+TA_VERSION = "v0.3.6"

tubearchivist/home/apps.py

@@ -1,108 +1,6 @@
 """handle custom startup functions"""
 
-import os
-import sys
-
 from django.apps import AppConfig
-from home.src.es.connect import ElasticWrap
-from home.src.es.index_setup import index_check
-from home.src.ta.config import AppConfig as ArchivistConfig
-from home.src.ta.ta_redis import RedisArchivist
-
-
-class StartupCheck:
-    """checks to run at application startup"""
-
-    MIN_MAJOR, MAX_MAJOR = 7, 7
-    MIN_MINOR = 17
-
-    def __init__(self):
-        self.config_handler = ArchivistConfig()
-        self.redis_con = RedisArchivist()
-        self.has_run = self.get_has_run()
-
-    def run(self):
-        """run all startup checks"""
-        print("run startup checks")
-        self.es_version_check()
-        self.release_lock()
-        index_check()
-        self.sync_redis_state()
-        self.make_folders()
-        self.set_has_run()
-
-    def get_has_run(self):
-        """validate if check has already executed"""
-        return self.redis_con.get_message("startup_check")
-
-    def set_has_run(self):
-        """startup checks run"""
-        message = {"status": True}
-        self.redis_con.set_message("startup_check", message, expire=120)
-
-    def sync_redis_state(self):
-        """make sure redis gets new config.json values"""
-        print("sync redis")
-        self.config_handler.load_new_defaults()
-
-    def make_folders(self):
-        """make needed cache folders here so docker doesn't mess it up"""
-        folders = [
-            "download",
-            "channels",
-            "videos",
-            "playlists",
-            "import",
-            "backup",
-        ]
-        cache_dir = self.config_handler.config["application"]["cache_dir"]
-        for folder in folders:
-            folder_path = os.path.join(cache_dir, folder)
-            try:
-                os.makedirs(folder_path)
-            except FileExistsError:
-                continue
-
-    def release_lock(self):
-        """make sure there are no leftover locks set in redis"""
-        all_locks = [
-            "startup_check",
-            "manual_import",
-            "downloading",
-            "dl_queue",
-            "dl_queue_id",
-            "rescan",
-        ]
-        for lock in all_locks:
-            response = self.redis_con.del_message(lock)
-            if response:
-                print("deleted leftover key from redis: " + lock)
-
-    def is_invalid(self, version):
-        """return true if es version is invalid, false if ok"""
-        major, minor = [int(i) for i in version.split(".")[:2]]
-        if not self.MIN_MAJOR <= major <= self.MAX_MAJOR:
-            return True
-
-        if minor >= self.MIN_MINOR:
-            return False
-
-        return True
-
-    def es_version_check(self):
-        """check for minimal elasticsearch version"""
-        response, _ = ElasticWrap("/").get()
-        version = response["version"]["number"]
-        invalid = self.is_invalid(version)
-
-        if invalid:
-            print(
-                "required elasticsearch version: "
-                + f"{self.MIN_MAJOR}.{self.MIN_MINOR}"
-            )
-            sys.exit(1)
-
-        print("elasticsearch version check passed")
 
 
 class HomeConfig(AppConfig):
@@ -110,11 +8,3 @@ class HomeConfig(AppConfig):
 
     default_auto_field = "django.db.models.BigAutoField"
     name = "home"
-
-    def ready(self):
-        startup = StartupCheck()
-        if startup.has_run["status"]:
-            print("startup checks run in other thread")
-            return
-
-        startup.run()

tubearchivist/home/config.json

@@ -12,23 +12,28 @@
         "grid_items": 3
     },
     "subscriptions": {
-        "auto_search": false,
         "auto_download": false,
-        "channel_size": 50
+        "channel_size": 50,
+        "live_channel_size": 50,
+        "shorts_channel_size": 50,
+        "auto_start": false
     },
     "downloads": {
-        "limit_count": false,
         "limit_speed": false,
         "sleep_interval": 3,
         "autodelete_days": false,
         "format": false,
+        "format_sort": false,
         "add_metadata": false,
         "add_thumbnail": false,
         "subtitle": false,
         "subtitle_source": false,
         "subtitle_index": false,
+        "comment_max": false,
+        "comment_sort": "top",
         "cookie_import": false,
         "throttledratelimit": false,
+        "extractor_lang": false,
         "integrate_ryd": false,
         "integrate_sponsorblock": false
     },
@@ -36,9 +41,9 @@
         "app_root": "/app",
         "cache_dir": "/cache",
         "videos": "/youtube",
-        "file_template": "%(id)s_%(title)s.mp4",
         "colors": "dark",
-        "enable_cast": false
+        "enable_cast": false,
+        "enable_snapshot": true
     },
     "scheduler": {
         "update_subscribed": false,
@@ -46,7 +51,8 @@
         "check_reindex":  {"minute": "0", "hour": "12", "day_of_week": "*"},
         "check_reindex_days": 90,
         "thumbnail_check": {"minute": "0", "hour": "17", "day_of_week": "*"},
-        "run_backup": {"minute": "0", "hour": "8", "day_of_week": "0"},
-        "run_backup_rotate": 5
+        "run_backup": false,
+        "run_backup_rotate": 5,
+        "version_check": "rand-d"
     }
 }

tubearchivist/home/migrations/0001_initial.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.1.5 on 2023-02-02 06:49
+
+from django.db import migrations, models
+import home.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('auth', '0012_alter_user_first_name_max_length'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Account',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('password', models.CharField(max_length=128, verbose_name='password')),
+                ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')),
+                ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')),
+                ('name', models.CharField(max_length=150, unique=True)),
+                ('is_staff', models.BooleanField(default=False)),
+                ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')),
+                ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')),
+            ],
+            options={
+                'abstract': False,
+            },
+            managers=[
+                ('objects', home.models.AccountManager()),
+            ],
+        ),
+    ]

tubearchivist/home/src/download/queue.py

@@ -15,9 +15,10 @@ from home.src.download.thumbnails import ThumbManager
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.playlist import YoutubePlaylist
+from home.src.index.video_constants import VideoTypeEnum
+from home.src.index.video_streams import DurationConverter
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import DurationConverter
-from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.helper import is_shorts
 
 
 class PendingIndex:
@@ -95,13 +96,13 @@ class PendingIndex:
 class PendingInteract:
     """interact with items in download queue"""
 
-    def __init__(self, video_id=False, status=False):
-        self.video_id = video_id
+    def __init__(self, youtube_id=False, status=False):
+        self.youtube_id = youtube_id
         self.status = status
 
     def delete_item(self):
         """delete single item from pending"""
-        path = f"ta_download/_doc/{self.video_id}"
+        path = f"ta_download/_doc/{self.youtube_id}"
         _, _ = ElasticWrap(path).delete(refresh=True)
 
     def delete_by_status(self):
@@ -111,29 +112,57 @@ class PendingInteract:
         _, _ = ElasticWrap(path).post(data=data)
 
     def update_status(self):
-        """update status field of pending item"""
-        data = {"doc": {"status": self.status}}
-        path = f"ta_download/_update/{self.video_id}"
+        """update status of pending item"""
+        if self.status == "priority":
+            data = {"doc": {"status": "pending", "auto_start": True}}
+        else:
+            data = {"doc": {"status": self.status}}
+
+        path = f"ta_download/_update/{self.youtube_id}/?refresh=true"
         _, _ = ElasticWrap(path).post(data=data)
 
+    def get_item(self):
+        """return pending item dict"""
+        path = f"ta_download/_doc/{self.youtube_id}"
+        response, status_code = ElasticWrap(path).get()
+        return response["_source"], status_code
+
+    def get_channel(self):
+        """
+        get channel metadata from queue to not depend on channel to be indexed
+        """
+        data = {
+            "size": 1,
+            "query": {"term": {"channel_id": {"value": self.youtube_id}}},
+        }
+        response, _ = ElasticWrap("ta_download/_search").get(data=data)
+        hits = response["hits"]["hits"]
+        if not hits:
+            channel_name = "NA"
+        else:
+            channel_name = hits[0]["_source"].get("channel_name", "NA")
+
+        return {
+            "channel_id": self.youtube_id,
+            "channel_name": channel_name,
+        }
+
 
 class PendingList(PendingIndex):
     """manage the pending videos list"""
 
     yt_obs = {
-        "default_search": "ytsearch",
-        "quiet": True,
-        "check_formats": "selected",
         "noplaylist": True,
         "writethumbnail": True,
         "simulate": True,
-        "socket_timeout": 3,
+        "check_formats": None,
     }
 
-    def __init__(self, youtube_ids=False):
+    def __init__(self, youtube_ids=False, task=False):
         super().__init__()
         self.config = AppConfig().config
         self.youtube_ids = youtube_ids
+        self.task = task
         self.to_skip = False
         self.missing_videos = False
 
@@ -142,47 +171,53 @@ class PendingList(PendingIndex):
         self.missing_videos = []
         self.get_download()
         self.get_indexed()
-        for entry in self.youtube_ids:
-            # notify
-            mess_dict = {
-                "status": "message:add",
-                "level": "info",
-                "title": "Adding to download queue.",
-                "message": "Extracting lists",
-            }
-            RedisArchivist().set_message("message:add", mess_dict)
+        total = len(self.youtube_ids)
+        for idx, entry in enumerate(self.youtube_ids):
             self._process_entry(entry)
+            if not self.task:
+                continue
+
+            self.task.send_progress(
+                message_lines=[f"Extracting items {idx + 1}/{total}"],
+                progress=(idx + 1) / total,
+            )
 
     def _process_entry(self, entry):
         """process single entry from url list"""
+        vid_type = self._get_vid_type(entry)
         if entry["type"] == "video":
-            self._add_video(entry["url"])
+            self._add_video(entry["url"], vid_type)
         elif entry["type"] == "channel":
-            self._parse_channel(entry["url"])
+            self._parse_channel(entry["url"], vid_type)
         elif entry["type"] == "playlist":
             self._parse_playlist(entry["url"])
-            new_thumbs = PlaylistSubscription().process_url_str(
-                [entry], subscribed=False
-            )
-            ThumbManager().download_playlist(new_thumbs)
+            PlaylistSubscription().process_url_str([entry], subscribed=False)
         else:
             raise ValueError(f"invalid url_type: {entry}")
 
-    def _add_video(self, url):
+    @staticmethod
+    def _get_vid_type(entry):
+        """add vid type enum if available"""
+        vid_type_str = entry.get("vid_type")
+        if not vid_type_str:
+            return VideoTypeEnum.UNKNOWN
+
+        return VideoTypeEnum(vid_type_str)
+
+    def _add_video(self, url, vid_type):
         """add video to list"""
         if url not in self.missing_videos and url not in self.to_skip:
-            self.missing_videos.append(url)
+            self.missing_videos.append((url, vid_type))
         else:
             print(f"{url}: skipped adding already indexed video to download.")
 
-    def _parse_channel(self, url):
+    def _parse_channel(self, url, vid_type):
         """add all videos of channel to list"""
         video_results = ChannelSubscription().get_last_youtube_videos(
-            url, limit=False
+            url, limit=False, query_filter=vid_type
         )
-        youtube_ids = [i[0] for i in video_results]
-        for video_id in youtube_ids:
-            self._add_video(video_id)
+        for video_id, _, vid_type in video_results:
+            self._add_video(video_id, vid_type)
 
     def _parse_playlist(self, url):
         """add all videos of playlist to list"""
@@ -191,52 +226,69 @@ class PendingList(PendingIndex):
         video_results = playlist.json_data.get("playlist_entries")
         youtube_ids = [i["youtube_id"] for i in video_results]
         for video_id in youtube_ids:
-            self._add_video(video_id)
+            # match vid_type later
+            self._add_video(video_id, VideoTypeEnum.UNKNOWN)
 
-    def add_to_pending(self, status="pending"):
+    def add_to_pending(self, status="pending", auto_start=False):
         """add missing videos to pending list"""
         self.get_channels()
         bulk_list = []
 
-        thumb_handler = ThumbManager()
-        for idx, youtube_id in enumerate(self.missing_videos):
-            video_details = self.get_youtube_details(youtube_id)
+        total = len(self.missing_videos)
+        for idx, (youtube_id, vid_type) in enumerate(self.missing_videos):
+            if self.task and self.task.is_stopped():
+                break
+
+            print(f"{youtube_id}: [{idx + 1}/{total}]: add to queue")
+            self._notify_add(idx, total)
+            video_details = self.get_youtube_details(youtube_id, vid_type)
             if not video_details:
                 continue
 
-            video_details["status"] = status
+            video_details.update(
+                {
+                    "status": status,
+                    "auto_start": auto_start,
+                }
+            )
+
             action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
             bulk_list.append(json.dumps(action))
             bulk_list.append(json.dumps(video_details))
 
-            thumb_needed = [(youtube_id, video_details["vid_thumb_url"])]
-            thumb_handler.download_vid(thumb_needed)
-            self._notify_add(idx)
+            url = video_details["vid_thumb_url"]
+            ThumbManager(youtube_id).download_video_thumb(url)
 
-        if bulk_list:
-            # add last newline
-            bulk_list.append("\n")
-            query_str = "\n".join(bulk_list)
-            _, _ = ElasticWrap("_bulk").post(query_str, ndjson=True)
+            if len(bulk_list) >= 20:
+                self._ingest_bulk(bulk_list)
+                bulk_list = []
 
-    def _notify_add(self, idx):
+        self._ingest_bulk(bulk_list)
+
+    def _ingest_bulk(self, bulk_list):
+        """add items to queue in bulk"""
+        if not bulk_list:
+            return
+
+        # add last newline
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+        _, _ = ElasticWrap("_bulk?refresh=true").post(query_str, ndjson=True)
+
+    def _notify_add(self, idx, total):
         """send notification for adding videos to download queue"""
-        progress = f"{idx + 1}/{len(self.missing_videos)}"
-        mess_dict = {
-            "status": "message:add",
-            "level": "info",
-            "title": "Adding new videos to download queue.",
-            "message": "Progress: " + progress,
-        }
-        if idx + 1 == len(self.missing_videos):
-            RedisArchivist().set_message("message:add", mess_dict, expire=4)
-        else:
-            RedisArchivist().set_message("message:add", mess_dict)
+        if not self.task:
+            return
 
-        if idx + 1 % 25 == 0:
-            print("adding to queue progress: " + progress)
+        self.task.send_progress(
+            message_lines=[
+                "Adding new videos to download queue.",
+                f"Extracting items {idx + 1}/{total}",
+            ],
+            progress=(idx + 1) / total,
+        )
 
-    def get_youtube_details(self, youtube_id):
+    def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEOS):
         """get details from youtubedl for single pending video"""
         vid = YtWrap(self.yt_obs, self.config).extract(youtube_id)
         if not vid:
@@ -247,12 +299,34 @@ class PendingList(PendingIndex):
             print(f"{youtube_id}: skipping premium video, id not matching")
             return False
         # stop if video is streaming live now
-        if vid["is_live"]:
+        if vid["live_status"] in ["is_upcoming", "is_live"]:
+            print(f"{youtube_id}: skip is_upcoming or is_live")
             return False
 
-        return self._parse_youtube_details(vid)
+        if vid["live_status"] == "was_live":
+            vid_type = VideoTypeEnum.STREAMS
+        else:
+            if self._check_shorts(vid):
+                vid_type = VideoTypeEnum.SHORTS
+            else:
+                vid_type = VideoTypeEnum.VIDEOS
 
-    def _parse_youtube_details(self, vid):
+        return self._parse_youtube_details(vid, vid_type)
+
+    @staticmethod
+    def _check_shorts(vid):
+        """check if vid is shorts video"""
+        if vid["width"] > vid["height"]:
+            return False
+
+        duration = vid.get("duration")
+        if duration and isinstance(duration, int):
+            if duration > 60:
+                return False
+
+        return is_shorts(vid["id"])
+
+    def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS):
         """parse response"""
         vid_id = vid.get("id")
         duration_str = DurationConverter.get_str(vid["duration"])
@@ -271,7 +345,9 @@ class PendingList(PendingIndex):
             "channel_id": vid["channel_id"],
             "duration": duration_str,
             "published": published,
-            "timestamp": int(datetime.now().strftime("%s")),
+            "timestamp": int(datetime.now().timestamp()),
+            # Pulling enum value out so it is serializable
+            "vid_type": vid_type.value,
         }
         if self.all_channels:
             youtube_details.update(

tubearchivist/home/src/download/subscriptions.py

@@ -5,19 +5,23 @@ Functionality:
 """
 
 from home.src.download import queue  # partial import
+from home.src.download.thumbnails import ThumbManager
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import IndexPaginate
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
+from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
 from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.urlparser import Parser
 
 
 class ChannelSubscription:
     """manage the list of channels subscribed"""
 
-    def __init__(self):
+    def __init__(self, task=False):
         self.config = AppConfig().config
+        self.task = task
 
     @staticmethod
     def get_channels(subscribed_only=True):
@@ -34,53 +38,103 @@ class ChannelSubscription:
 
         return all_channels
 
-    def get_last_youtube_videos(self, channel_id, limit=True):
+    def get_last_youtube_videos(
+        self, channel_id, limit=True, query_filter=VideoTypeEnum.UNKNOWN
+    ):
         """get a list of last videos from channel"""
-        obs = {
-            "skip_download": True,
-            "extract_flat": True,
-        }
-        if limit:
-            obs["playlistend"] = self.config["subscriptions"]["channel_size"]
+        queries = self._build_queries(query_filter, limit)
 
-        url = f"https://www.youtube.com/channel/{channel_id}/videos"
-        channel = YtWrap(obs, self.config).extract(url)
-        if not channel:
-            return False
+        last_videos = []
+
+        for vid_type_enum, limit_amount in queries:
+            obs = {
+                "skip_download": True,
+                "extract_flat": True,
+            }
+            if limit:
+                obs["playlistend"] = limit_amount
+
+            vid_type = vid_type_enum.value
+            channel = YtWrap(obs, self.config).extract(
+                f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
+            )
+            if not channel:
+                continue
+            last_videos.extend(
+                [(i["id"], i["title"], vid_type) for i in channel["entries"]]
+            )
 
-        last_videos = [(i["id"], i["title"]) for i in channel["entries"]]
         return last_videos
 
+    def _build_queries(self, query_filter, limit):
+        """build query list for vid_type"""
+        limit_map = {
+            "videos": self.config["subscriptions"]["channel_size"],
+            "streams": self.config["subscriptions"]["live_channel_size"],
+            "shorts": self.config["subscriptions"]["shorts_channel_size"],
+        }
+
+        queries = []
+
+        if query_filter and query_filter.value != "unknown":
+            if limit:
+                query_limit = limit_map.get(query_filter.value)
+            else:
+                query_limit = False
+
+            queries.append((query_filter, query_limit))
+
+            return queries
+
+        for query_item, default_limit in limit_map.items():
+            if not default_limit:
+                # is deactivated in config
+                continue
+
+            if limit:
+                query_limit = default_limit
+            else:
+                query_limit = False
+
+            queries.append((VideoTypeEnum(query_item), query_limit))
+
+        return queries
+
     def find_missing(self):
         """add missing videos from subscribed channels to pending"""
         all_channels = self.get_channels()
+        if not all_channels:
+            return False
+
         pending = queue.PendingList()
         pending.get_download()
         pending.get_indexed()
 
         missing_videos = []
 
+        total = len(all_channels)
         for idx, channel in enumerate(all_channels):
             channel_id = channel["channel_id"]
+            print(f"{channel_id}: find missing videos.")
             last_videos = self.get_last_youtube_videos(channel_id)
 
             if last_videos:
-                for video in last_videos:
-                    if video[0] not in pending.to_skip:
-                        missing_videos.append(video[0])
-            # notify
-            message = {
-                "status": "message:rescan",
-                "level": "info",
-                "title": "Scanning channels: Looking for new videos.",
-                "message": f"Progress: {idx + 1}/{len(all_channels)}",
-            }
-            if idx + 1 == len(all_channels):
-                RedisArchivist().set_message(
-                    "message:rescan", message=message, expire=4
+                for video_id, _, vid_type in last_videos:
+                    if video_id not in pending.to_skip:
+                        missing_videos.append((video_id, vid_type))
+
+            if not self.task:
+                continue
+
+            if self.task:
+                if self.task.is_stopped():
+                    self.task.send_progress(["Received Stop signal."])
+                    break
+
+                self.task.send_progress(
+                    message_lines=[f"Scanning Channel {idx + 1}/{total}"],
+                    progress=(idx + 1) / total,
                 )
-            else:
-                RedisArchivist().set_message("message:rescan", message=message)
 
         return missing_videos
 
@@ -97,8 +151,9 @@ class ChannelSubscription:
 class PlaylistSubscription:
     """manage the playlist download functionality"""
 
-    def __init__(self):
+    def __init__(self, task=False):
         self.config = AppConfig().config
+        self.task = task
 
     @staticmethod
     def get_playlists(subscribed_only=True):
@@ -120,18 +175,13 @@ class PlaylistSubscription:
 
     def process_url_str(self, new_playlists, subscribed=True):
         """process playlist subscribe form url_str"""
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"published": {"order": "desc"}}],
-        }
+        data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
         all_indexed = IndexPaginate("ta_video", data).get_results()
         all_youtube_ids = [i["youtube_id"] for i in all_indexed]
 
-        new_thumbs = []
         for idx, playlist in enumerate(new_playlists):
-            url_type = playlist["type"]
             playlist_id = playlist["url"]
-            if not url_type == "playlist":
+            if not playlist["type"] == "playlist":
                 print(f"{playlist_id} not a playlist, skipping...")
                 continue
 
@@ -142,8 +192,11 @@ class PlaylistSubscription:
             playlist_h.upload_to_es()
             playlist_h.add_vids_to_playlist()
             self.channel_validate(playlist_h.json_data["playlist_channel_id"])
-            thumb = playlist_h.json_data["playlist_thumbnail"]
-            new_thumbs.append((playlist_id, thumb))
+
+            url = playlist_h.json_data["playlist_thumbnail"]
+            thumb = ThumbManager(playlist_id, item_type="playlist")
+            thumb.download_playlist_thumb(url)
+
             # notify
             message = {
                 "status": "message:subplaylist",
@@ -152,11 +205,9 @@ class PlaylistSubscription:
                 "message": f"Processing {idx + 1} of {len(new_playlists)}",
             }
             RedisArchivist().set_message(
-                "message:subplaylist", message=message
+                "message:subplaylist", message=message, expire=True
             )
 
-        return new_thumbs
-
     @staticmethod
     def channel_validate(channel_id):
         """make sure channel of playlist is there"""
@@ -183,14 +234,18 @@ class PlaylistSubscription:
     def find_missing(self):
         """find videos in subscribed playlists not downloaded yet"""
         all_playlists = [i["playlist_id"] for i in self.get_playlists()]
+        if not all_playlists:
+            return False
+
         to_ignore = self.get_to_ignore()
 
         missing_videos = []
+        total = len(all_playlists)
         for idx, playlist_id in enumerate(all_playlists):
             size_limit = self.config["subscriptions"]["channel_size"]
             playlist = YoutubePlaylist(playlist_id)
-            playlist.update_playlist()
-            if not playlist:
+            is_active = playlist.update_playlist()
+            if not is_active:
                 playlist.deactivate()
                 continue
 
@@ -200,17 +255,124 @@ class PlaylistSubscription:
 
             all_missing = [i for i in playlist_entries if not i["downloaded"]]
 
-            message = {
-                "status": "message:rescan",
-                "level": "info",
-                "title": "Scanning playlists: Looking for new videos.",
-                "message": f"Progress: {idx + 1}/{len(all_playlists)}",
-            }
-            RedisArchivist().set_message("message:rescan", message=message)
-
             for video in all_missing:
                 youtube_id = video["youtube_id"]
                 if youtube_id not in to_ignore:
                     missing_videos.append(youtube_id)
 
+            if not self.task:
+                continue
+
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
+                    progress=(idx + 1) / total,
+                )
+                if self.task.is_stopped():
+                    self.task.send_progress(["Received Stop signal."])
+                    break
+
         return missing_videos
+
+
+class SubscriptionScanner:
+    """add missing videos to queue"""
+
+    def __init__(self, task=False):
+        self.task = task
+        self.missing_videos = False
+        self.auto_start = AppConfig().config["subscriptions"].get("auto_start")
+
+    def scan(self):
+        """scan channels and playlists"""
+        if self.task:
+            self.task.send_progress(["Rescanning channels and playlists."])
+
+        self.missing_videos = []
+        self.scan_channels()
+        if self.task and not self.task.is_stopped():
+            self.scan_playlists()
+
+        return self.missing_videos
+
+    def scan_channels(self):
+        """get missing from channels"""
+        channel_handler = ChannelSubscription(task=self.task)
+        missing = channel_handler.find_missing()
+        if not missing:
+            return
+
+        for vid_id, vid_type in missing:
+            self.missing_videos.append(
+                {"type": "video", "vid_type": vid_type, "url": vid_id}
+            )
+
+    def scan_playlists(self):
+        """get missing from playlists"""
+        playlist_handler = PlaylistSubscription(task=self.task)
+        missing = playlist_handler.find_missing()
+        if not missing:
+            return
+
+        for i in missing:
+            self.missing_videos.append(
+                {
+                    "type": "video",
+                    "vid_type": VideoTypeEnum.VIDEOS.value,
+                    "url": i,
+                }
+            )
+
+
+class SubscriptionHandler:
+    """subscribe to channels and playlists from url_str"""
+
+    def __init__(self, url_str, task=False):
+        self.url_str = url_str
+        self.task = task
+        self.to_subscribe = False
+
+    def subscribe(self):
+        """subscribe to url_str items"""
+        if self.task:
+            self.task.send_progress(["Processing form content."])
+        self.to_subscribe = Parser(self.url_str).parse()
+
+        total = len(self.to_subscribe)
+        for idx, item in enumerate(self.to_subscribe):
+            if self.task:
+                self._notify(idx, item, total)
+
+            self.subscribe_type(item)
+
+    def subscribe_type(self, item):
+        """process single item"""
+        if item["type"] == "playlist":
+            PlaylistSubscription().process_url_str([item])
+            return
+
+        if item["type"] == "video":
+            # extract channel id from video
+            vid = queue.PendingList().get_youtube_details(item["url"])
+            channel_id = vid["channel_id"]
+        elif item["type"] == "channel":
+            channel_id = item["url"]
+        else:
+            raise ValueError("failed to subscribe to: " + item["url"])
+
+        self._subscribe(channel_id)
+
+    def _subscribe(self, channel_id):
+        """subscribe to channel"""
+        ChannelSubscription().change_subscribe(
+            channel_id, channel_subscribed=True
+        )
+
+    def _notify(self, idx, item, total):
+        """send notification message to redis"""
+        subscribe_type = item["type"].title()
+        message_lines = [
+            f"Subscribe to {subscribe_type}",
+            f"Progress: {idx + 1}/{total}",
+        ]
+        self.task.send_progress(message_lines, progress=(idx + 1) / total)

tubearchivist/home/src/download/thumbnails.py

@@ -6,249 +6,228 @@ functionality:
 
 import base64
 import os
-from collections import Counter
 from io import BytesIO
 from time import sleep
 
 import requests
-from home.src.download import queue  # partial import
-from home.src.download import subscriptions  # partial import
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import ignore_filelist
-from home.src.ta.ta_redis import RedisArchivist
 from mutagen.mp4 import MP4, MP4Cover
-from PIL import Image, ImageFile, ImageFilter
+from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
 
 ImageFile.LOAD_TRUNCATED_IMAGES = True
 
 
-class ThumbManager:
-    """handle thumbnails related functions"""
+class ThumbManagerBase:
+    """base class for thumbnail management"""
 
     CONFIG = AppConfig().config
-    MEDIA_DIR = CONFIG["application"]["videos"]
     CACHE_DIR = CONFIG["application"]["cache_dir"]
     VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
     CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
     PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
 
-    def get_all_thumbs(self):
-        """get all video artwork already downloaded"""
-        all_thumb_folders = ignore_filelist(os.listdir(self.VIDEO_DIR))
-        all_thumbs = []
-        for folder in all_thumb_folders:
-            folder_path = os.path.join(self.VIDEO_DIR, folder)
-            if os.path.isfile(folder_path):
-                self.update_path(folder)
-                all_thumbs.append(folder_path)
-                continue
-                # raise exemption here in a future version
-                # raise FileExistsError("video cache dir has files inside")
+    def __init__(self, item_id, item_type, fallback=False):
+        self.item_id = item_id
+        self.item_type = item_type
+        self.fallback = fallback
 
-            all_folder_thumbs = ignore_filelist(os.listdir(folder_path))
-            all_thumbs.extend(all_folder_thumbs)
+    def download_raw(self, url):
+        """download thumbnail for video"""
+        if not url:
+            return self.get_fallback()
 
-        return all_thumbs
+        for i in range(3):
+            try:
+                response = requests.get(url, stream=True, timeout=5)
+                if response.ok:
+                    try:
+                        img = Image.open(response.raw)
+                        if isinstance(img, Image.Image):
+                            return img
+                        return self.get_fallback()
 
-    def update_path(self, file_name):
-        """reorganize thumbnails into folders as update path from v0.0.5"""
-        folder_name = file_name[0].lower()
-        folder_path = os.path.join(self.VIDEO_DIR, folder_name)
-        old_file = os.path.join(self.VIDEO_DIR, file_name)
-        new_file = os.path.join(folder_path, file_name)
-        os.makedirs(folder_path, exist_ok=True)
-        os.rename(old_file, new_file)
+                    except (UnidentifiedImageError, OSError):
+                        print(f"failed to open thumbnail: {url}")
+                        return self.get_fallback()
 
-    def get_needed_thumbs(self, missing_only=False):
-        """get a list of all missing thumbnails"""
-        all_thumbs = self.get_all_thumbs()
+                if response.status_code == 404:
+                    return self.get_fallback()
 
-        pending = queue.PendingList()
-        pending.get_download()
-        pending.get_indexed()
+            except requests.exceptions.RequestException:
+                print(f"{self.item_id}: retry thumbnail download {url}")
+                sleep((i + 1) ** i)
 
-        needed_thumbs = []
-        for video in pending.all_videos:
-            youtube_id = video["youtube_id"]
-            thumb_url = video["vid_thumb_url"]
-            if missing_only:
-                if youtube_id + ".jpg" not in all_thumbs:
-                    needed_thumbs.append((youtube_id, thumb_url))
-            else:
-                needed_thumbs.append((youtube_id, thumb_url))
+        return False
 
-        for video in pending.all_pending + pending.all_ignored:
-            youtube_id = video["youtube_id"]
-            thumb_url = video["vid_thumb_url"]
-            if missing_only:
-                if youtube_id + ".jpg" not in all_thumbs:
-                    needed_thumbs.append((youtube_id, thumb_url))
-            else:
-                needed_thumbs.append((youtube_id, thumb_url))
+    def get_fallback(self):
+        """get fallback thumbnail if not available"""
+        print(f"{self.item_id}: failed to extract thumbnail, use fallback")
+        if self.fallback:
+            img_raw = Image.open(self.fallback)
+            return img_raw
 
-        return needed_thumbs
-
-    def get_missing_channels(self):
-        """get all channel artwork"""
-        all_channel_art = os.listdir(self.CHANNEL_DIR)
-        files = [i[0:24] for i in all_channel_art]
-        cached_channel_ids = [k for (k, v) in Counter(files).items() if v > 1]
-        channel_sub = subscriptions.ChannelSubscription()
-        channels = channel_sub.get_channels(subscribed_only=False)
-
-        missing_channels = []
-        for channel in channels:
-            channel_id = channel["channel_id"]
-            if channel_id not in cached_channel_ids:
-                channel_banner = channel["channel_banner_url"]
-                channel_thumb = channel["channel_thumb_url"]
-                missing_channels.append(
-                    (channel_id, channel_thumb, channel_banner)
-                )
-
-        return missing_channels
-
-    def get_missing_playlists(self):
-        """get all missing playlist artwork"""
-        all_downloaded = ignore_filelist(os.listdir(self.PLAYLIST_DIR))
-        all_ids_downloaded = [i.replace(".jpg", "") for i in all_downloaded]
-        playlist_sub = subscriptions.PlaylistSubscription()
-        playlists = playlist_sub.get_playlists(subscribed_only=False)
-
-        missing_playlists = []
-        for playlist in playlists:
-            playlist_id = playlist["playlist_id"]
-            if playlist_id not in all_ids_downloaded:
-                playlist_thumb = playlist["playlist_thumbnail"]
-                missing_playlists.append((playlist_id, playlist_thumb))
-
-        return missing_playlists
-
-    def get_raw_img(self, img_url, thumb_type):
-        """get raw image from youtube and handle 404"""
-        try:
-            app_root = self.CONFIG["application"]["app_root"]
-        except KeyError:
-            # lazy keyerror fix to not have to deal with a strange startup
-            # racing contition between the threads in HomeConfig.ready()
-            app_root = "/app"
+        app_root = self.CONFIG["application"]["app_root"]
         default_map = {
             "video": os.path.join(
                 app_root, "static/img/default-video-thumb.jpg"
             ),
+            "playlist": os.path.join(
+                app_root, "static/img/default-video-thumb.jpg"
+            ),
             "icon": os.path.join(
                 app_root, "static/img/default-channel-icon.jpg"
             ),
             "banner": os.path.join(
                 app_root, "static/img/default-channel-banner.jpg"
             ),
+            "tvart": os.path.join(
+                app_root, "static/img/default-channel-art.jpg"
+            ),
         }
-        if img_url:
-            try:
-                response = requests.get(img_url, stream=True)
-            except ConnectionError:
-                sleep(5)
-                response = requests.get(img_url, stream=True)
-            if not response.ok and not response.status_code == 404:
-                print("retry thumbnail download for " + img_url)
-                sleep(5)
-                response = requests.get(img_url, stream=True)
-        else:
-            response = False
-        if not response or response.status_code == 404:
-            # use default
-            img_raw = Image.open(default_map[thumb_type])
-        else:
-            # use response
-            img_obj = response.raw
-            img_raw = Image.open(img_obj)
+
+        img_raw = Image.open(default_map[self.item_type])
 
         return img_raw
 
-    def download_vid(self, missing_thumbs, notify=True):
-        """download all missing thumbnails from list"""
-        print(f"downloading {len(missing_thumbs)} thumbnails")
-        for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs):
-            folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower())
-            thumb_path = os.path.join(
-                self.CACHE_DIR, self.vid_thumb_path(youtube_id)
-            )
 
+class ThumbManager(ThumbManagerBase):
+    """handle thumbnails related functions"""
+
+    def __init__(self, item_id, item_type="video", fallback=False):
+        super().__init__(item_id, item_type, fallback=fallback)
+
+    def download(self, url):
+        """download thumbnail"""
+        print(f"{self.item_id}: download {self.item_type} thumbnail")
+        if self.item_type == "video":
+            self.download_video_thumb(url)
+        elif self.item_type == "channel":
+            self.download_channel_art(url)
+        elif self.item_type == "playlist":
+            self.download_playlist_thumb(url)
+
+    def delete(self):
+        """delete thumbnail file"""
+        print(f"{self.item_id}: delete {self.item_type} thumbnail")
+        if self.item_type == "video":
+            self.delete_video_thumb()
+        elif self.item_type == "channel":
+            self.delete_channel_thumb()
+        elif self.item_type == "playlist":
+            self.delete_playlist_thumb()
+
+    def download_video_thumb(self, url, skip_existing=False):
+        """pass url for video thumbnail"""
+        folder_path = os.path.join(self.VIDEO_DIR, self.item_id[0].lower())
+        thumb_path = self.vid_thumb_path(absolute=True)
+
+        if skip_existing and os.path.exists(thumb_path):
+            return
+
+        os.makedirs(folder_path, exist_ok=True)
+        img_raw = self.download_raw(url)
+        width, height = img_raw.size
+
+        if not width / height == 16 / 9:
+            new_height = width / 16 * 9
+            offset = (height - new_height) / 2
+            img_raw = img_raw.crop((0, offset, width, height - offset))
+
+        img_raw.convert("RGB").save(thumb_path)
+
+    def vid_thumb_path(self, absolute=False, create_folder=False):
+        """build expected path for video thumbnail from youtube_id"""
+        folder_name = self.item_id[0].lower()
+        folder_path = os.path.join("videos", folder_name)
+        thumb_path = os.path.join(folder_path, f"{self.item_id}.jpg")
+        if absolute:
+            thumb_path = os.path.join(self.CACHE_DIR, thumb_path)
+
+        if create_folder:
+            folder_path = os.path.join(self.CACHE_DIR, folder_path)
             os.makedirs(folder_path, exist_ok=True)
-            img_raw = self.get_raw_img(thumb_url, "video")
 
-            width, height = img_raw.size
-            if not width / height == 16 / 9:
-                new_height = width / 16 * 9
-                offset = (height - new_height) / 2
-                img_raw = img_raw.crop((0, offset, width, height - offset))
-            img_raw.convert("RGB").save(thumb_path)
+        return thumb_path
 
-            progress = f"{idx + 1}/{len(missing_thumbs)}"
-            if notify:
-                mess_dict = {
-                    "status": "message:add",
-                    "level": "info",
-                    "title": "Processing Videos",
-                    "message": "Downloading Thumbnails, Progress: " + progress,
-                }
-                if idx + 1 == len(missing_thumbs):
-                    RedisArchivist().set_message(
-                        "message:add", mess_dict, expire=4
-                    )
-                else:
-                    RedisArchivist().set_message("message:add", mess_dict)
+    def download_channel_art(self, urls, skip_existing=False):
+        """pass tuple of channel thumbnails"""
+        channel_thumb, channel_banner, channel_tv = urls
+        self._download_channel_thumb(channel_thumb, skip_existing)
+        self._download_channel_banner(channel_banner, skip_existing)
+        self._download_channel_tv(channel_tv, skip_existing)
 
-            if idx + 1 % 25 == 0:
-                print("thumbnail progress: " + progress)
+    def _download_channel_thumb(self, channel_thumb, skip_existing):
+        """download channel thumbnail"""
 
-    def download_chan(self, missing_channels):
-        """download needed artwork for channels"""
-        print(f"downloading {len(missing_channels)} channel artwork")
-        for channel in missing_channels:
-            channel_id, channel_thumb, channel_banner = channel
+        thumb_path = os.path.join(
+            self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg"
+        )
+        self.item_type = "icon"
 
-            thumb_path = os.path.join(
-                self.CHANNEL_DIR, channel_id + "_thumb.jpg"
-            )
-            img_raw = self.get_raw_img(channel_thumb, "icon")
-            img_raw.convert("RGB").save(thumb_path)
+        if skip_existing and os.path.exists(thumb_path):
+            return
 
-            banner_path = os.path.join(
-                self.CHANNEL_DIR, channel_id + "_banner.jpg"
-            )
-            img_raw = self.get_raw_img(channel_banner, "banner")
-            img_raw.convert("RGB").save(banner_path)
+        img_raw = self.download_raw(channel_thumb)
+        img_raw.convert("RGB").save(thumb_path)
 
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Processing Channels",
-                "message": "Downloading Channel Art.",
-            }
-            RedisArchivist().set_message("message:download", mess_dict)
+    def _download_channel_banner(self, channel_banner, skip_existing):
+        """download channel banner"""
 
-    def download_playlist(self, missing_playlists):
-        """download needed artwork for playlists"""
-        print(f"downloading {len(missing_playlists)} playlist artwork")
-        for playlist in missing_playlists:
-            playlist_id, playlist_thumb_url = playlist
-            thumb_path = os.path.join(self.PLAYLIST_DIR, playlist_id + ".jpg")
-            img_raw = self.get_raw_img(playlist_thumb_url, "video")
-            img_raw.convert("RGB").save(thumb_path)
+        banner_path = os.path.join(
+            self.CHANNEL_DIR, self.item_id + "_banner.jpg"
+        )
+        self.item_type = "banner"
+        if skip_existing and os.path.exists(banner_path):
+            return
 
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Processing Playlists",
-                "message": "Downloading Playlist Art.",
-            }
-            RedisArchivist().set_message("message:download", mess_dict)
+        img_raw = self.download_raw(channel_banner)
+        img_raw.convert("RGB").save(banner_path)
 
-    def get_base64_blur(self, youtube_id):
+    def _download_channel_tv(self, channel_tv, skip_existing):
+        """download channel tv art"""
+        art_path = os.path.join(self.CHANNEL_DIR, self.item_id + "_tvart.jpg")
+        self.item_type = "tvart"
+        if skip_existing and os.path.exists(art_path):
+            return
+
+        img_raw = self.download_raw(channel_tv)
+        img_raw.convert("RGB").save(art_path)
+
+    def download_playlist_thumb(self, url, skip_existing=False):
+        """pass thumbnail url"""
+        thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
+        if skip_existing and os.path.exists(thumb_path):
+            return
+
+        img_raw = self.download_raw(url)
+        img_raw.convert("RGB").save(thumb_path)
+
+    def delete_video_thumb(self):
+        """delete video thumbnail if exists"""
+        thumb_path = self.vid_thumb_path()
+        to_delete = os.path.join(self.CACHE_DIR, thumb_path)
+        if os.path.exists(to_delete):
+            os.remove(to_delete)
+
+    def delete_channel_thumb(self):
+        """delete all artwork of channel"""
+        thumb = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg")
+        banner = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_banner.jpg")
+        if os.path.exists(thumb):
+            os.remove(thumb)
+        if os.path.exists(banner):
+            os.remove(banner)
+
+    def delete_playlist_thumb(self):
+        """delete playlist thumbnail"""
+        thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
+        if os.path.exists(thumb_path):
+            os.remove(thumb_path)
+
+    def get_vid_base64_blur(self):
         """return base64 encoded placeholder"""
-        img_path = self.vid_thumb_path(youtube_id)
-        file_path = os.path.join(self.CACHE_DIR, img_path)
+        file_path = os.path.join(self.CACHE_DIR, self.vid_thumb_path())
         img_raw = Image.open(file_path)
         img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20))
         img_blur = img_raw.filter(ImageFilter.BLUR)
@@ -260,91 +239,169 @@ class ThumbManager:
 
         return data_url
 
+
+class ValidatorCallback:
+    """handle callback validate thumbnails page by page"""
+
+    def __init__(self, source, index_name):
+        self.source = source
+        self.index_name = index_name
+
+    def run(self):
+        """run the task for page"""
+        print(f"{self.index_name}: validate artwork")
+        if self.index_name == "ta_video":
+            self._validate_videos()
+        elif self.index_name == "ta_channel":
+            self._validate_channels()
+        elif self.index_name == "ta_playlist":
+            self._validate_playlists()
+
+    def _validate_videos(self):
+        """check if video thumbnails are correct"""
+        for video in self.source:
+            url = video["_source"]["vid_thumb_url"]
+            handler = ThumbManager(video["_source"]["youtube_id"])
+            handler.download_video_thumb(url, skip_existing=True)
+
+    def _validate_channels(self):
+        """check if all channel artwork is there"""
+        for channel in self.source:
+            urls = (
+                channel["_source"]["channel_thumb_url"],
+                channel["_source"]["channel_banner_url"],
+                channel["_source"]["channel_tvart_url"],
+            )
+            handler = ThumbManager(channel["_source"]["channel_id"])
+            handler.download_channel_art(urls, skip_existing=True)
+
+    def _validate_playlists(self):
+        """check if all playlist artwork is there"""
+        for playlist in self.source:
+            url = playlist["_source"]["playlist_thumbnail"]
+            handler = ThumbManager(playlist["_source"]["playlist_id"])
+            handler.download_playlist_thumb(url, skip_existing=True)
+
+
+class ThumbValidator:
+    """validate thumbnails"""
+
+    INDEX = [
+        {
+            "data": {
+                "query": {"term": {"active": {"value": True}}},
+                "_source": ["vid_thumb_url", "youtube_id"],
+            },
+            "name": "ta_video",
+        },
+        {
+            "data": {
+                "query": {"term": {"channel_active": {"value": True}}},
+                "_source": {
+                    "excludes": ["channel_description", "channel_overwrites"]
+                },
+            },
+            "name": "ta_channel",
+        },
+        {
+            "data": {
+                "query": {"term": {"playlist_active": {"value": True}}},
+                "_source": ["playlist_id", "playlist_thumbnail"],
+            },
+            "name": "ta_playlist",
+        },
+    ]
+
+    def __init__(self, task):
+        self.task = task
+
+    def validate(self):
+        """validate all indexes"""
+        for index in self.INDEX:
+            total = self._get_total(index["name"])
+            if not total:
+                continue
+
+            paginate = IndexPaginate(
+                index_name=index["name"],
+                data=index["data"],
+                size=1000,
+                callback=ValidatorCallback,
+                task=self.task,
+                total=total,
+            )
+            _ = paginate.get_results()
+
     @staticmethod
-    def vid_thumb_path(youtube_id):
-        """build expected path for video thumbnail from youtube_id"""
-        folder_name = youtube_id[0].lower()
-        folder_path = os.path.join("videos", folder_name)
-        thumb_path = os.path.join(folder_path, youtube_id + ".jpg")
-        return thumb_path
+    def _get_total(index_name):
+        """get total documents in index"""
+        path = f"{index_name}/_count"
+        response, _ = ElasticWrap(path).get()
 
-    def delete_vid_thumb(self, youtube_id):
-        """delete video thumbnail if exists"""
-        thumb_path = self.vid_thumb_path(youtube_id)
-        to_delete = os.path.join(self.CACHE_DIR, thumb_path)
-        if os.path.exists(to_delete):
-            os.remove(to_delete)
+        return response.get("count")
 
-    def delete_chan_thumb(self, channel_id):
-        """delete all artwork of channel"""
-        thumb = os.path.join(self.CHANNEL_DIR, channel_id + "_thumb.jpg")
-        banner = os.path.join(self.CHANNEL_DIR, channel_id + "_banner.jpg")
-        if os.path.exists(thumb):
-            os.remove(thumb)
-        if os.path.exists(banner):
-            os.remove(banner)
 
-    def cleanup_downloaded(self):
-        """find downloaded thumbnails without video indexed"""
-        all_thumbs = self.get_all_thumbs()
-        all_indexed = self.get_needed_thumbs()
-        all_needed_thumbs = [i[0] + ".jpg" for i in all_indexed]
-        for thumb in all_thumbs:
-            if thumb not in all_needed_thumbs:
-                # cleanup
-                youtube_id = thumb.rstrip(".jpg")
-                self.delete_vid_thumb(youtube_id)
+class ThumbFilesystem:
+    """sync thumbnail files to media files"""
 
-    def get_thumb_list(self):
-        """get list of mediafiles and matching thumbnails"""
-        pending = queue.PendingList()
-        pending.get_download()
-        pending.get_indexed()
+    INDEX_NAME = "ta_video"
 
-        video_list = []
-        for video in pending.all_videos:
-            youtube_id = video["youtube_id"]
-            media_url = os.path.join(self.MEDIA_DIR, video["media_url"])
+    def __init__(self, task=False):
+        self.task = task
+
+    def embed(self):
+        """entry point"""
+        data = {
+            "query": {"match_all": {}},
+            "_source": ["media_url", "youtube_id"],
+        }
+        paginate = IndexPaginate(
+            index_name=self.INDEX_NAME,
+            data=data,
+            size=200,
+            callback=EmbedCallback,
+            task=self.task,
+            total=self._get_total(),
+        )
+        _ = paginate.get_results()
+
+    def _get_total(self):
+        """get total documents in index"""
+        path = f"{self.INDEX_NAME}/_count"
+        response, _ = ElasticWrap(path).get()
+
+        return response.get("count")
+
+
+class EmbedCallback:
+    """callback class to embed thumbnails"""
+
+    CONFIG = AppConfig().config
+    CACHE_DIR = CONFIG["application"]["cache_dir"]
+    MEDIA_DIR = CONFIG["application"]["videos"]
+    FORMAT = MP4Cover.FORMAT_JPEG
+
+    def __init__(self, source, index_name):
+        self.source = source
+        self.index_name = index_name
+
+    def run(self):
+        """run embed"""
+        for video in self.source:
+            video_id = video["_source"]["youtube_id"]
+            media_url = os.path.join(
+                self.MEDIA_DIR, video["_source"]["media_url"]
+            )
             thumb_path = os.path.join(
-                self.CACHE_DIR, self.vid_thumb_path(youtube_id)
-            )
-            video_list.append(
-                {
-                    "media_url": media_url,
-                    "thumb_path": thumb_path,
-                }
+                self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path()
             )
+            if os.path.exists(thumb_path):
+                self.embed(media_url, thumb_path)
 
-        return video_list
+    def embed(self, media_url, thumb_path):
+        """embed thumb in single media file"""
+        video = MP4(media_url)
+        with open(thumb_path, "rb") as f:
+            video["covr"] = [MP4Cover(f.read(), imageformat=self.FORMAT)]
 
-    @staticmethod
-    def write_all_thumbs(video_list):
-        """rewrite the thumbnail into media file"""
-
-        counter = 1
-        for video in video_list:
-            # loop through all videos
-            media_url = video["media_url"]
-            thumb_path = video["thumb_path"]
-
-            mutagen_vid = MP4(media_url)
-            with open(thumb_path, "rb") as f:
-                mutagen_vid["covr"] = [
-                    MP4Cover(f.read(), imageformat=MP4Cover.FORMAT_JPEG)
-                ]
-            mutagen_vid.save()
-            if counter % 50 == 0:
-                print(f"thumbnail write progress {counter}/{len(video_list)}")
-            counter = counter + 1
-
-
-def validate_thumbnails():
-    """check if all thumbnails are there and organized correctly"""
-    handler = ThumbManager()
-    thumbs_to_download = handler.get_needed_thumbs(missing_only=True)
-    handler.download_vid(thumbs_to_download)
-    missing_channels = handler.get_missing_channels()
-    handler.download_chan(missing_channels)
-    missing_playlists = handler.get_missing_playlists()
-    handler.download_playlist(missing_playlists)
-    handler.cleanup_downloaded()
+        video.save()

tubearchivist/home/src/download/yt_dlp_base.py

@@ -5,6 +5,8 @@ functionality:
 """
 
 import os
+from datetime import datetime
+from http import cookiejar
 from io import StringIO
 
 import yt_dlp
@@ -18,7 +20,9 @@ class YtWrap:
         "default_search": "ytsearch",
         "quiet": True,
         "check_formats": "selected",
-        "socket_timeout": 2,
+        "socket_timeout": 10,
+        "extractor_retries": 3,
+        "retries": 10,
     }
 
     def __init__(self, obs_request, config=False):
@@ -54,9 +58,12 @@ class YtWrap:
         """make extract request"""
         try:
             response = yt_dlp.YoutubeDL(self.obs).extract_info(url)
+        except cookiejar.LoadError:
+            print("cookie file is invalid")
+            return False
         except (yt_dlp.utils.ExtractorError, yt_dlp.utils.DownloadError):
             print(f"{url}: failed to get info from youtube")
-            response = False
+            return False
 
         return response
 
@@ -78,25 +85,74 @@ class CookieHandler:
         """import cookie from file"""
         cache_path = self.config["application"]["cache_dir"]
         import_path = os.path.join(cache_path, "import", "cookies.google.txt")
-        with open(import_path, encoding="utf-8") as cookie_file:
-            cookie = cookie_file.read()
 
-        RedisArchivist().set_message("cookie", cookie, expire=False)
+        try:
+            with open(import_path, encoding="utf-8") as cookie_file:
+                cookie = cookie_file.read()
+        except FileNotFoundError as err:
+            print(f"cookie: {import_path} file not found")
+            raise err
+
+        self.set_cookie(cookie)
 
         os.remove(import_path)
         print("cookie: import successful")
 
+    def set_cookie(self, cookie):
+        """set cookie str and activate in cofig"""
+        RedisArchivist().set_message("cookie", cookie)
+        path = ".downloads.cookie_import"
+        RedisArchivist().set_message("config", True, path=path)
+        self.config["downloads"]["cookie_import"] = True
+        print("cookie: activated and stored in Redis")
+
     @staticmethod
     def revoke():
         """revoke cookie"""
         RedisArchivist().del_message("cookie")
+        RedisArchivist().del_message("cookie:valid")
+        RedisArchivist().set_message(
+            "config", False, path=".downloads.cookie_import"
+        )
         print("cookie: revoked")
 
     def validate(self):
         """validate cookie using the liked videos playlist"""
+        print("validating cookie")
         obs_request = {
             "skip_download": True,
             "extract_flat": True,
         }
-        response = YtWrap(obs_request, self.config).extract("LL")
-        return bool(response)
+        validator = YtWrap(obs_request, self.config)
+        response = bool(validator.extract("LL"))
+        self.store_validation(response)
+
+        # update in redis to avoid expiring
+        modified = validator.obs["cookiefile"].getvalue()
+        if modified:
+            RedisArchivist().set_message("cookie", modified)
+
+        if not response:
+            mess_dict = {
+                "status": "message:download",
+                "level": "error",
+                "title": "Cookie validation failed, exiting...",
+                "message": "",
+            }
+            RedisArchivist().set_message(
+                "message:download", mess_dict, expire=4
+            )
+            print("cookie validation failed, exiting...")
+
+        return response
+
+    @staticmethod
+    def store_validation(response):
+        """remember last validation"""
+        now = datetime.now()
+        message = {
+            "status": response,
+            "validated": int(now.timestamp()),
+            "validated_str": now.strftime("%Y-%m-%d %H:%M"),
+        }
+        RedisArchivist().set_message("cookie:valid", message)

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -15,11 +15,12 @@ from home.src.download.subscriptions import PlaylistSubscription
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
+from home.src.index.comments import CommentList
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo, index_new_video
+from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import clean_string, ignore_filelist
-from home.src.ta.ta_redis import RedisArchivist, RedisQueue
 
 
 class DownloadPostProcess:
@@ -27,7 +28,7 @@ class DownloadPostProcess:
 
     def __init__(self, download):
         self.download = download
-        self.now = int(datetime.now().strftime("%s"))
+        self.now = int(datetime.now().timestamp())
         self.pending = False
 
     def run(self):
@@ -39,6 +40,7 @@ class DownloadPostProcess:
         self.auto_delete_all()
         self.auto_delete_overwrites()
         self.validate_playlists()
+        self.get_comments()
 
     def auto_delete_all(self):
         """handle auto delete"""
@@ -92,7 +94,7 @@ class DownloadPostProcess:
     def validate_playlists(self):
         """look for playlist needing to update"""
         for id_c, channel_id in enumerate(self.download.channels):
-            channel = YoutubeChannel(channel_id)
+            channel = YoutubeChannel(channel_id, task=self.download.task)
             overwrites = self.pending.channel_overwrites.get(channel_id, False)
             if overwrites and overwrites.get("index_playlists"):
                 # validate from remote
@@ -100,7 +102,7 @@ class DownloadPostProcess:
                 continue
 
             # validate from local
-            playlists = channel.get_indexed_playlists()
+            playlists = channel.get_indexed_playlists(active_only=True)
             all_channel_playlist = [i["playlist_id"] for i in playlists]
             self._validate_channel_playlist(all_channel_playlist, id_c)
 
@@ -113,6 +115,7 @@ class DownloadPostProcess:
             playlist.build_json(scrape=True)
             if not playlist.json_data:
                 playlist.deactivate()
+                continue
 
             playlist.add_vids_to_playlist()
             playlist.upload_to_es()
@@ -120,23 +123,22 @@ class DownloadPostProcess:
 
     def _notify_playlist_progress(self, all_channel_playlist, id_c, id_p):
         """notify to UI"""
-        title = (
-            "Processing playlists for channels: "
-            + f"{id_c + 1}/{len(self.download.channels)}"
-        )
-        message = f"Progress: {id_p + 1}/{len(all_channel_playlist)}"
-        mess_dict = {
-            "status": "message:download",
-            "level": "info",
-            "title": title,
-            "message": message,
-        }
-        if id_p + 1 == len(all_channel_playlist):
-            RedisArchivist().set_message(
-                "message:download", mess_dict, expire=4
-            )
-        else:
-            RedisArchivist().set_message("message:download", mess_dict)
+        if not self.download.task:
+            return
+
+        total_channel = len(self.download.channels)
+        total_playlist = len(all_channel_playlist)
+
+        message = [
+            f"Post Processing Channels: {id_c}/{total_channel}",
+            f"Validate Playlists {id_p + 1}/{total_playlist}",
+        ]
+        progress = (id_c + 1) / total_channel
+        self.download.task.send_progress(message, progress=progress)
+
+    def get_comments(self):
+        """get comments from youtube"""
+        CommentList(self.download.videos, task=self.download.task).index()
 
 
 class VideoDownloader:
@@ -145,113 +147,106 @@ class VideoDownloader:
     if not initiated with list, take from queue
     """
 
-    def __init__(self, youtube_id_list=False):
+    def __init__(self, youtube_id_list=False, task=False):
         self.obs = False
         self.video_overwrites = False
         self.youtube_id_list = youtube_id_list
+        self.task = task
         self.config = AppConfig().config
         self._build_obs()
         self.channels = set()
+        self.videos = set()
 
-    def run_queue(self):
+    def run_queue(self, auto_only=False):
         """setup download queue in redis loop until no more items"""
-        pending = PendingList()
-        pending.get_download()
-        pending.get_channels()
-        self.video_overwrites = pending.video_overwrites
-
-        queue = RedisQueue()
-
-        limit_queue = self.config["downloads"]["limit_count"]
-        if limit_queue:
-            queue.trim(limit_queue - 1)
-
+        self._get_overwrites()
         while True:
-            youtube_id = queue.get_next()
-            if not youtube_id:
+            video_data = self._get_next(auto_only)
+            if self.task.is_stopped() or not video_data:
                 break
 
+            youtube_id = video_data.get("youtube_id")
+            print(f"{youtube_id}: Downloading video")
+            self._notify(video_data, "Validate download format")
+
             success = self._dl_single_vid(youtube_id)
             if not success:
                 continue
 
+            self._notify(video_data, "Add video metadata to index")
+
             vid_dict = index_new_video(
-                youtube_id, video_overwrites=self.video_overwrites
+                youtube_id,
+                video_overwrites=self.video_overwrites,
+                video_type=VideoTypeEnum(video_data["vid_type"]),
             )
             self.channels.add(vid_dict["channel"]["channel_id"])
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Moving....",
-                "message": "Moving downloaded file to storage folder",
-            }
-            RedisArchivist().set_message("message:download", mess_dict, False)
+            self.videos.add(vid_dict["youtube_id"])
 
+            self._notify(video_data, "Move downloaded file to archive")
             self.move_to_archive(vid_dict)
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Completed",
-                "message": "",
-            }
-            RedisArchivist().set_message("message:download", mess_dict, 10)
             self._delete_from_pending(youtube_id)
 
         # post processing
         self._add_subscribed_channels()
         DownloadPostProcess(self).run()
 
-    @staticmethod
-    def add_pending():
-        """add pending videos to download queue"""
-        mess_dict = {
-            "status": "message:download",
-            "level": "info",
-            "title": "Looking for videos to download",
-            "message": "Scanning your download queue.",
-        }
-        RedisArchivist().set_message("message:download", mess_dict)
-        pending = PendingList()
-        pending.get_download()
-        to_add = [i["youtube_id"] for i in pending.all_pending]
-        if not to_add:
-            # there is nothing pending
-            print("download queue is empty")
-            mess_dict = {
-                "status": "message:download",
-                "level": "error",
-                "title": "Download queue is empty",
-                "message": "Add some videos to the queue first.",
-            }
-            RedisArchivist().set_message("message:download", mess_dict)
+    def _notify(self, video_data, message):
+        """send progress notification to task"""
+        if not self.task:
             return
 
-        RedisQueue().add_list(to_add)
+        typ = VideoTypeEnum(video_data["vid_type"]).value.rstrip("s").title()
+        title = video_data.get("title")
+        self.task.send_progress([f"Processing {typ}: {title}", message])
 
-    @staticmethod
-    def _progress_hook(response):
+    def _get_next(self, auto_only):
+        """get next item in queue"""
+        must_list = [{"term": {"status": {"value": "pending"}}}]
+        if auto_only:
+            must_list.append({"term": {"auto_start": {"value": True}}})
+
+        data = {
+            "size": 1,
+            "query": {"bool": {"must": must_list}},
+            "sort": [
+                {"auto_start": {"order": "desc"}},
+                {"timestamp": {"order": "asc"}},
+            ],
+        }
+        path = "ta_download/_search"
+        response, _ = ElasticWrap(path).get(data=data)
+        if not response["hits"]["hits"]:
+            return False
+
+        return response["hits"]["hits"][0]["_source"]
+
+    def _get_overwrites(self):
+        """get channel overwrites"""
+        pending = PendingList()
+        pending.get_download()
+        pending.get_channels()
+        self.video_overwrites = pending.video_overwrites
+
+    def _progress_hook(self, response):
         """process the progress_hooks from yt_dlp"""
-        # title
-        path = os.path.split(response["filename"])[-1][12:]
-        filename = os.path.splitext(os.path.splitext(path)[0])[0]
-        filename_clean = filename.replace("_", " ")
-        title = "Downloading: " + filename_clean
-        # message
+        progress = False
         try:
+            size = response.get("_total_bytes_str")
+            if size.strip() == "N/A":
+                size = response.get("_total_bytes_estimate_str", "N/A")
+
             percent = response["_percent_str"]
-            size = response["_total_bytes_str"]
+            progress = float(percent.strip("%")) / 100
             speed = response["_speed_str"]
             eta = response["_eta_str"]
             message = f"{percent} of {size} at {speed} - time left: {eta}"
         except KeyError:
             message = "processing"
-        mess_dict = {
-            "status": "message:download",
-            "level": "info",
-            "title": title,
-            "message": message,
-        }
-        RedisArchivist().set_message("message:download", mess_dict)
+
+        if self.task:
+            title = response["info_dict"]["title"]
+            self.task.send_progress([title, message], progress=progress)
 
     def _build_obs(self):
         """collection to build all obs passed to yt-dlp"""
@@ -262,29 +257,26 @@ class VideoDownloader:
     def _build_obs_basic(self):
         """initial obs"""
         self.obs = {
-            "default_search": "ytsearch",
             "merge_output_format": "mp4",
-            "restrictfilenames": True,
             "outtmpl": (
                 self.config["application"]["cache_dir"]
-                + "/download/"
-                + self.config["application"]["file_template"]
+                + "/download/%(id)s.mp4"
             ),
             "progress_hooks": [self._progress_hook],
             "noprogress": True,
-            "quiet": True,
             "continuedl": True,
-            "retries": 3,
             "writethumbnail": False,
             "noplaylist": True,
-            "check_formats": "selected",
-            "socket_timeout": 3,
         }
 
     def _build_obs_user(self):
         """build user customized options"""
         if self.config["downloads"]["format"]:
             self.obs["format"] = self.config["downloads"]["format"]
+        if self.config["downloads"]["format_sort"]:
+            format_sort = self.config["downloads"]["format_sort"]
+            format_sort_list = [i.strip() for i in format_sort.split(",")]
+            self.obs["format_sort"] = format_sort_list
         if self.config["downloads"]["limit_speed"]:
             self.obs["ratelimit"] = (
                 self.config["downloads"]["limit_speed"] * 1024
@@ -377,25 +369,25 @@ class VideoDownloader:
         new_folder = os.path.join(videos, channel_name)
         if not os.path.exists(new_folder):
             os.makedirs(new_folder)
-            if host_uid and host_gid:
-                os.chown(new_folder, host_uid, host_gid)
+        if host_uid and host_gid:
+            os.chown(new_folder, host_uid, host_gid)
         # find real filename
         cache_dir = self.config["application"]["cache_dir"]
         all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))
         for file_str in all_cached:
             if vid_dict["youtube_id"] in file_str:
                 old_file = file_str
-        old_file_path = os.path.join(cache_dir, "download", old_file)
-        new_file_path = os.path.join(videos, vid_dict["media_url"])
+        old_path = os.path.join(cache_dir, "download", old_file)
+        new_path = os.path.join(videos, vid_dict["media_url"])
         # move media file and fix permission
-        shutil.move(old_file_path, new_file_path)
+        shutil.move(old_path, new_path, copy_function=shutil.copyfile)
         if host_uid and host_gid:
-            os.chown(new_file_path, host_uid, host_gid)
+            os.chown(new_path, host_uid, host_gid)
 
     @staticmethod
     def _delete_from_pending(youtube_id):
         """delete downloaded video from pending index if its there"""
-        path = f"ta_download/_doc/{youtube_id}"
+        path = f"ta_download/_doc/{youtube_id}?refresh=true"
         _, _ = ElasticWrap(path).delete()
 
     def _add_subscribed_channels(self):

tubearchivist/home/src/es/backup.py

@@ -0,0 +1,243 @@
+"""
+Functionality:
+- Handle json zip file based backup
+- create backup
+- restore backup
+"""
+
+import json
+import os
+import zipfile
+from datetime import datetime
+
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import get_mapping, ignore_filelist
+
+
+class ElasticBackup:
+    """dump index to nd-json files for later bulk import"""
+
+    def __init__(self, reason=False, task=False):
+        self.config = AppConfig().config
+        self.cache_dir = self.config["application"]["cache_dir"]
+        self.timestamp = datetime.now().strftime("%Y%m%d")
+        self.index_config = get_mapping()
+        self.reason = reason
+        self.task = task
+
+    def backup_all_indexes(self):
+        """backup all indexes, add reason to init"""
+        print("backup all indexes")
+        if not self.reason:
+            raise ValueError("missing backup reason in ElasticBackup")
+
+        if self.task:
+            self.task.send_progress(["Scanning your index."])
+        for index in self.index_config:
+            index_name = index["index_name"]
+            print(f"backup: export in progress for {index_name}")
+            if not self.index_exists(index_name):
+                print(f"skip backup for not yet existing index {index_name}")
+                continue
+
+            self.backup_index(index_name)
+
+        if self.task:
+            self.task.send_progress(["Compress files to zip archive."])
+        self.zip_it()
+        if self.reason == "auto":
+            self.rotate_backup()
+
+    def backup_index(self, index_name):
+        """export all documents of a single index"""
+        paginate = IndexPaginate(
+            f"ta_{index_name}",
+            data={"query": {"match_all": {}}},
+            keep_source=True,
+            callback=BackupCallback,
+            task=self.task,
+            total=self._get_total(index_name),
+        )
+        _ = paginate.get_results()
+
+    @staticmethod
+    def _get_total(index_name):
+        """get total documents in index"""
+        path = f"ta_{index_name}/_count"
+        response, _ = ElasticWrap(path).get()
+
+        return response.get("count")
+
+    def zip_it(self):
+        """pack it up into single zip file"""
+        file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
+        folder = os.path.join(self.cache_dir, "backup")
+
+        to_backup = []
+        for file in os.listdir(folder):
+            if file.endswith(".json"):
+                to_backup.append(os.path.join(folder, file))
+
+        backup_file = os.path.join(folder, file_name)
+
+        comp = zipfile.ZIP_DEFLATED
+        with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
+            for backup_file in to_backup:
+                zip_f.write(backup_file, os.path.basename(backup_file))
+
+        # cleanup
+        for backup_file in to_backup:
+            os.remove(backup_file)
+
+    def post_bulk_restore(self, file_name):
+        """send bulk to es"""
+        file_path = os.path.join(self.cache_dir, file_name)
+        with open(file_path, "r", encoding="utf-8") as f:
+            data = f.read()
+
+        if not data.strip():
+            return
+
+        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
+
+    def get_all_backup_files(self):
+        """build all available backup files for view"""
+        backup_dir = os.path.join(self.cache_dir, "backup")
+        backup_files = os.listdir(backup_dir)
+        all_backup_files = ignore_filelist(backup_files)
+        all_available_backups = [
+            i
+            for i in all_backup_files
+            if i.startswith("ta_") and i.endswith(".zip")
+        ]
+        all_available_backups.sort(reverse=True)
+
+        backup_dicts = []
+        for backup_file in all_available_backups:
+            file_split = backup_file.split("-")
+            if len(file_split) == 2:
+                timestamp = file_split[1].strip(".zip")
+                reason = False
+            elif len(file_split) == 3:
+                timestamp = file_split[1]
+                reason = file_split[2].strip(".zip")
+
+            to_add = {
+                "filename": backup_file,
+                "timestamp": timestamp,
+                "reason": reason,
+            }
+            backup_dicts.append(to_add)
+
+        return backup_dicts
+
+    def restore(self, filename):
+        """
+        restore from backup zip file
+        call reset from ElasitIndexWrap first to start blank
+        """
+        zip_content = self._unpack_zip_backup(filename)
+        self._restore_json_files(zip_content)
+
+    def _unpack_zip_backup(self, filename):
+        """extract backup zip and return filelist"""
+        backup_dir = os.path.join(self.cache_dir, "backup")
+        file_path = os.path.join(backup_dir, filename)
+
+        with zipfile.ZipFile(file_path, "r") as z:
+            zip_content = z.namelist()
+            z.extractall(backup_dir)
+
+        return zip_content
+
+    def _restore_json_files(self, zip_content):
+        """go through the unpacked files and restore"""
+        backup_dir = os.path.join(self.cache_dir, "backup")
+
+        for idx, json_f in enumerate(zip_content):
+            self._notify_restore(idx, json_f, len(zip_content))
+            file_name = os.path.join(backup_dir, json_f)
+
+            if not json_f.startswith("es_") or not json_f.endswith(".json"):
+                os.remove(file_name)
+                continue
+
+            print("restoring: " + json_f)
+            self.post_bulk_restore(file_name)
+            os.remove(file_name)
+
+    def _notify_restore(self, idx, json_f, total_files):
+        """notify restore progress"""
+        message = [f"Restore index from json backup file {json_f}."]
+        progress = (idx + 1) / total_files
+        self.task.send_progress(message_lines=message, progress=progress)
+
+    @staticmethod
+    def index_exists(index_name):
+        """check if index already exists to skip"""
+        _, status_code = ElasticWrap(f"ta_{index_name}").get()
+        exists = status_code == 200
+
+        return exists
+
+    def rotate_backup(self):
+        """delete old backups if needed"""
+        rotate = self.config["scheduler"]["run_backup_rotate"]
+        if not rotate:
+            return
+
+        all_backup_files = self.get_all_backup_files()
+        auto = [i for i in all_backup_files if i["reason"] == "auto"]
+
+        if len(auto) <= rotate:
+            print("no backup files to rotate")
+            return
+
+        backup_dir = os.path.join(self.cache_dir, "backup")
+
+        all_to_delete = auto[rotate:]
+        for to_delete in all_to_delete:
+            file_path = os.path.join(backup_dir, to_delete["filename"])
+            print(f"remove old backup file: {file_path}")
+            os.remove(file_path)
+
+
+class BackupCallback:
+    """handle backup ndjson writer as callback for IndexPaginate"""
+
+    def __init__(self, source, index_name):
+        self.source = source
+        self.index_name = index_name
+        self.timestamp = datetime.now().strftime("%Y%m%d")
+
+    def run(self):
+        """run the junk task"""
+        file_content = self._build_bulk()
+        self._write_es_json(file_content)
+
+    def _build_bulk(self):
+        """build bulk query data from all_results"""
+        bulk_list = []
+
+        for document in self.source:
+            document_id = document["_id"]
+            es_index = document["_index"]
+            action = {"index": {"_index": es_index, "_id": document_id}}
+            source = document["_source"]
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(source))
+
+        # add last newline
+        bulk_list.append("\n")
+        file_content = "\n".join(bulk_list)
+
+        return file_content
+
+    def _write_es_json(self, file_content):
+        """write nd-json file for es _bulk API to disk"""
+        cache_dir = AppConfig().config["application"]["cache_dir"]
+        file_name = f"es_{self.index_name.lstrip('ta_')}-{self.timestamp}.json"
+        file_path = os.path.join(cache_dir, "backup", file_name)
+        with open(file_path, "a+", encoding="utf-8") as f:
+            f.write(file_content)

tubearchivist/home/src/es/connect.py

@@ -3,6 +3,7 @@ functionality:
 - wrapper around requests to call elastic search
 - reusable search_after to extract total index
 """
+# pylint: disable=missing-timeout
 
 import json
 
@@ -31,13 +32,15 @@ class ElasticWrap:
         self.auth = self.config["application"]["es_auth"]
         self.url = f"{es_url}/{self.path}"
 
-    def get(self, data=False):
+    def get(self, data=False, timeout=10, print_error=True):
         """get data from es"""
         if data:
-            response = requests.get(self.url, json=data, auth=self.auth)
+            response = requests.get(
+                self.url, json=data, auth=self.auth, timeout=timeout
+            )
         else:
-            response = requests.get(self.url, auth=self.auth)
-        if not response.ok:
+            response = requests.get(self.url, auth=self.auth, timeout=timeout)
+        if print_error and not response.ok:
             print(response.text)
 
         return response.json(), response.status_code
@@ -91,19 +94,25 @@ class ElasticWrap:
 
 
 class IndexPaginate:
-    """use search_after to go through whole index"""
+    """use search_after to go through whole index
+    kwargs:
+    - size: int, overwrite DEFAULT_SIZE
+    - keep_source: bool, keep _source key from es results
+    - callback: obj, Class implementing run method callback for every loop
+    - task: task object to send notification
+    - total: int, total items in index for progress message
+    """
 
     DEFAULT_SIZE = 500
 
-    def __init__(self, index_name, data, size=False, keep_source=False):
+    def __init__(self, index_name, data, **kwargs):
         self.index_name = index_name
         self.data = data
         self.pit_id = False
-        self.size = size
-        self.keep_source = keep_source
+        self.kwargs = kwargs
 
     def get_results(self):
-        """get all results"""
+        """get all results, add task and total for notifications"""
         self.get_pit()
         self.validate_data()
         all_results = self.run_loop()
@@ -119,36 +128,49 @@ class IndexPaginate:
     def validate_data(self):
         """add pit and size to data"""
         if "sort" not in self.data.keys():
-            print(self.data)
-            raise ValueError("missing sort key in data")
+            self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
 
-        size = self.size or self.DEFAULT_SIZE
-
-        self.data["size"] = size
+        self.data["size"] = self.kwargs.get("size") or self.DEFAULT_SIZE
         self.data["pit"] = {"id": self.pit_id, "keep_alive": "10m"}
 
     def run_loop(self):
         """loop through results until last hit"""
         all_results = []
+        counter = 0
         while True:
             response, _ = ElasticWrap("_search").get(data=self.data)
             all_hits = response["hits"]["hits"]
-            if all_hits:
-                for hit in all_hits:
-                    if self.keep_source:
-                        source = hit
-                    else:
-                        source = hit["_source"]
-                    search_after = hit["sort"]
-                    all_results.append(source)
-                # update search_after with last hit data
-                self.data["search_after"] = search_after
-            else:
+            if not all_hits:
                 break
 
+            for hit in all_hits:
+                if self.kwargs.get("keep_source"):
+                    all_results.append(hit)
+                else:
+                    all_results.append(hit["_source"])
+
+            if self.kwargs.get("callback"):
+                self.kwargs.get("callback")(all_hits, self.index_name).run()
+
+            if self.kwargs.get("task"):
+                print(f"{self.index_name}: processing page {counter}")
+                self._notify(len(all_results))
+
+            counter += 1
+
+            # update search_after with last hit data
+            self.data["search_after"] = all_hits[-1]["sort"]
+
         return all_results
 
+    def _notify(self, processed):
+        """send notification on task"""
+        total = self.kwargs.get("total")
+        progress = processed / total
+        index_clean = self.index_name.lstrip("ta_").title()
+        message = [f"Processing {index_clean}s {processed}/{total}"]
+        self.kwargs.get("task").send_progress(message, progress=progress)
+
     def clean_pit(self):
         """delete pit from elastic search"""
-        data = {"id": self.pit_id}
-        ElasticWrap("_pit").delete(data=data)
+        ElasticWrap("_pit").delete(data={"id": self.pit_id})

tubearchivist/home/src/es/index_mapping.json

@@ -37,8 +37,17 @@
                     "type": "text"
                 },
                 "channel_last_refresh": {
-                    "type": "date",
-                    "format": "epoch_second"
+                    "type": "date"
+                },
+                "channel_tags": {
+                    "type": "text",
+                    "analyzer": "english",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
                 },
                 "channel_overwrites": {
                     "properties": {
@@ -120,8 +129,17 @@
                             "type": "text"
                         },
                         "channel_last_refresh": {
-                            "type": "date",
-                            "format": "epoch_second"
+                            "type": "date"
+                        },
+                        "channel_tags": {
+                            "type": "text",
+                            "analyzer": "english",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "ignore_above": 256
+                                }
+                            }
                         },
                         "channel_overwrites": {
                             "properties": {
@@ -148,6 +166,9 @@
                     "type": "keyword",
                     "index": false
                 },
+                "media_size": {
+                    "type": "long"
+                },
                 "tags": {
                     "type": "text",
                     "analyzer": "english",
@@ -180,6 +201,9 @@
                 "youtube_id": {
                     "type": "keyword"
                 },
+                "vid_type": {
+                    "type": "keyword"
+                },
                 "published": {
                     "type": "date"
                 },
@@ -193,6 +217,9 @@
                         }
                     }
                 },
+                "comment_count": {
+                    "type": "long"
+                },
                 "stats" : {
                     "properties" : {
                         "average_rating" : {
@@ -235,6 +262,30 @@
                         }
                     }
                 },
+                "streams": {
+                    "properties": {
+                        "type": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "index": {
+                            "type": "short",
+                            "index": false
+                        },
+                        "codec": {
+                            "type": "text"
+                        },
+                        "width": {
+                            "type": "short"
+                        },
+                        "height": {
+                            "type": "short"
+                        },
+                        "bitrate": {
+                            "type": "integer"
+                        }
+                    }
+                },
                 "sponsorblock": {
                     "properties": {
                         "last_refresh": {
@@ -323,6 +374,12 @@
                 },
                 "youtube_id": {
                     "type": "keyword"
+                },
+                "vid_type": {
+                    "type": "keyword"
+                },
+                "auto_start": {
+                    "type": "boolean"
                 }
             },
             "expected_set": {
@@ -460,6 +517,75 @@
                 },
                 "number_of_replicas": "0"
             }
+        },
+        {
+            "index_name": "comment",
+            "expected_map": {
+                "youtube_id": {
+                    "type": "keyword"
+                },
+                "comment_last_refresh": {
+                    "type": "date"
+                },
+                "comment_channel_id": {
+                    "type": "keyword"
+                },
+                "comment_comments": {
+                    "properties": {
+                        "comment_id": {
+                            "type": "keyword"
+                        },
+                        "comment_text": {
+                            "type" : "text"
+                        },
+                        "comment_timestamp": {
+                            "type": "date"
+                        },
+                        "comment_time_text": {
+                            "type" : "text"
+                        },
+                        "comment_likecount": {
+                            "type": "long"
+                        },
+                        "comment_is_favorited": {
+                            "type": "boolean"
+                        },
+                        "comment_author": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "ignore_above": 256,
+                                    "normalizer": "to_lower"
+                                }
+                            }
+                        },
+                        "comment_author_id": {
+                            "type": "keyword"
+                        },
+                        "comment_author_thumbnail": {
+                            "type": "keyword"
+                        },
+                        "comment_author_is_uploader": {
+                            "type": "boolean"
+                        },
+                        "comment_parent": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            },
+            "expected_set": {
+                "analysis": {
+                    "normalizer": {
+                        "to_lower": {
+                            "type": "custom",
+                            "filter": ["lowercase"]
+                        }
+                    }
+                },
+                "number_of_replicas": "0"
+            }
         }
     ]
 }

tubearchivist/home/src/es/index_setup.py

@@ -5,22 +5,17 @@ functionality:
 - backup and restore metadata
 """
 
-import json
-import os
-import zipfile
-from datetime import datetime
-
-from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.es.backup import ElasticBackup
+from home.src.es.connect import ElasticWrap
+from home.src.es.snapshot import ElasticSnapshot
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import ignore_filelist
+from home.src.ta.helper import get_mapping
 
 
 class ElasticIndex:
-    """
-    handle mapping and settings on elastic search for a given index
-    """
+    """interact with a single index"""
 
-    def __init__(self, index_name, expected_map, expected_set):
+    def __init__(self, index_name, expected_map=False, expected_set=False):
         self.index_name = index_name
         self.expected_map = expected_map
         self.expected_set = expected_set
@@ -61,23 +56,23 @@ class ElasticIndex:
             if list(value.keys()) == ["properties"]:
                 for key_n, value_n in value["properties"].items():
                     if key not in now_map:
-                        print(key_n, value_n)
+                        print(f"detected mapping change: {key_n}, {value_n}")
                         return True
                     if key_n not in now_map[key]["properties"].keys():
-                        print(key_n, value_n)
+                        print(f"detected mapping change: {key_n}, {value_n}")
                         return True
                     if not value_n == now_map[key]["properties"][key_n]:
-                        print(key_n, value_n)
+                        print(f"detected mapping change: {key_n}, {value_n}")
                         return True
 
                 continue
 
             # not nested
             if key not in now_map.keys():
-                print(key, value)
+                print(f"detected mapping change: {key}, {value}")
                 return True
             if not value == now_map[key]:
-                print(key, value)
+                print(f"detected mapping change: {key}, {value}")
                 return True
 
         return False
@@ -100,6 +95,8 @@ class ElasticIndex:
 
     def rebuild_index(self):
         """rebuild with new mapping"""
+        print(f"applying new mappings to index ta_{self.index_name}...")
+        self.create_blank(for_backup=True)
         self.reindex("backup")
         self.delete_index(backup=False)
         self.create_blank()
@@ -126,277 +123,88 @@ class ElasticIndex:
 
         _, _ = ElasticWrap(path).delete()
 
-    def create_blank(self):
+    def create_blank(self, for_backup=False):
         """apply new mapping and settings for blank new index"""
+        print(f"create new blank index with name ta_{self.index_name}...")
+        path = f"ta_{self.index_name}"
+        if for_backup:
+            path = f"{path}_backup"
+
         data = {}
         if self.expected_set:
             data.update({"settings": self.expected_set})
         if self.expected_map:
             data.update({"mappings": {"properties": self.expected_map}})
 
-        _, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
+        _, _ = ElasticWrap(path).put(data)
 
 
-class ElasticBackup:
-    """dump index to nd-json files for later bulk import"""
+class ElasitIndexWrap:
+    """interact with all index mapping and setup"""
 
-    def __init__(self, index_config, reason):
-        self.config = AppConfig().config
-        self.cache_dir = self.config["application"]["cache_dir"]
-        self.index_config = index_config
-        self.reason = reason
-        self.timestamp = datetime.now().strftime("%Y%m%d")
-        self.backup_files = []
+    def __init__(self):
+        self.index_config = get_mapping()
+        self.backup_run = False
 
-    @staticmethod
-    def get_all_documents(index_name):
-        """export all documents of a single index"""
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"_doc": {"order": "desc"}}],
-        }
-        paginate = IndexPaginate(f"ta_{index_name}", data, keep_source=True)
-        all_results = paginate.get_results()
-
-        return all_results
-
-    @staticmethod
-    def build_bulk(all_results):
-        """build bulk query data from all_results"""
-        bulk_list = []
-
-        for document in all_results:
-            document_id = document["_id"]
-            es_index = document["_index"]
-            action = {"index": {"_index": es_index, "_id": document_id}}
-            source = document["_source"]
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(source))
-
-        # add last newline
-        bulk_list.append("\n")
-        file_content = "\n".join(bulk_list)
-
-        return file_content
-
-    def write_es_json(self, file_content, index_name):
-        """write nd-json file for es _bulk API to disk"""
-        file_name = f"es_{index_name}-{self.timestamp}.json"
-        file_path = os.path.join(self.cache_dir, "backup", file_name)
-        with open(file_path, "w", encoding="utf-8") as f:
-            f.write(file_content)
-
-        self.backup_files.append(file_path)
-
-    def write_ta_json(self, all_results, index_name):
-        """write generic json file to disk"""
-        file_name = f"ta_{index_name}-{self.timestamp}.json"
-        file_path = os.path.join(self.cache_dir, "backup", file_name)
-        to_write = [i["_source"] for i in all_results]
-        file_content = json.dumps(to_write)
-        with open(file_path, "w", encoding="utf-8") as f:
-            f.write(file_content)
-
-        self.backup_files.append(file_path)
-
-    def zip_it(self):
-        """pack it up into single zip file"""
-        file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
-        backup_folder = os.path.join(self.cache_dir, "backup")
-        backup_file = os.path.join(backup_folder, file_name)
-
-        with zipfile.ZipFile(
-            backup_file, "w", compression=zipfile.ZIP_DEFLATED
-        ) as zip_f:
-            for backup_file in self.backup_files:
-                zip_f.write(backup_file, os.path.basename(backup_file))
-
-        # cleanup
-        for backup_file in self.backup_files:
-            os.remove(backup_file)
-
-    def post_bulk_restore(self, file_name):
-        """send bulk to es"""
-        file_path = os.path.join(self.cache_dir, file_name)
-        with open(file_path, "r", encoding="utf-8") as f:
-            data = f.read()
-
-        if not data.strip():
-            return
-
-        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
-
-    def get_all_backup_files(self):
-        """build all available backup files for view"""
-        backup_dir = os.path.join(self.cache_dir, "backup")
-        backup_files = os.listdir(backup_dir)
-        all_backup_files = ignore_filelist(backup_files)
-        all_available_backups = [
-            i
-            for i in all_backup_files
-            if i.startswith("ta_") and i.endswith(".zip")
-        ]
-        all_available_backups.sort(reverse=True)
-
-        backup_dicts = []
-        for backup_file in all_available_backups:
-            file_split = backup_file.split("-")
-            if len(file_split) == 2:
-                timestamp = file_split[1].strip(".zip")
-                reason = False
-            elif len(file_split) == 3:
-                timestamp = file_split[1]
-                reason = file_split[2].strip(".zip")
-
-            to_add = {
-                "filename": backup_file,
-                "timestamp": timestamp,
-                "reason": reason,
-            }
-            backup_dicts.append(to_add)
-
-        return backup_dicts
-
-    def unpack_zip_backup(self, filename):
-        """extract backup zip and return filelist"""
-        backup_dir = os.path.join(self.cache_dir, "backup")
-        file_path = os.path.join(backup_dir, filename)
-
-        with zipfile.ZipFile(file_path, "r") as z:
-            zip_content = z.namelist()
-            z.extractall(backup_dir)
-
-        return zip_content
-
-    def restore_json_files(self, zip_content):
-        """go through the unpacked files and restore"""
-        backup_dir = os.path.join(self.cache_dir, "backup")
-
-        for json_f in zip_content:
-
-            file_name = os.path.join(backup_dir, json_f)
-
-            if not json_f.startswith("es_") or not json_f.endswith(".json"):
-                os.remove(file_name)
+    def setup(self):
+        """setup elastic index, run at startup"""
+        for index in self.index_config:
+            index_name, expected_map, expected_set = self._config_split(index)
+            handler = ElasticIndex(index_name, expected_map, expected_set)
+            if not handler.exists:
+                handler.create_blank()
                 continue
 
-            print("restoring: " + json_f)
-            self.post_bulk_restore(file_name)
-            os.remove(file_name)
+            rebuild = handler.validate()
+            if rebuild:
+                self._check_backup()
+                handler.rebuild_index()
+                continue
+
+            # else all good
+            print(f"ta_{index_name} index is created and up to date...")
+
+    def reset(self):
+        """reset all indexes to blank"""
+        self.delete_all()
+        self.create_all_blank()
+
+    def delete_all(self):
+        """delete all indexes"""
+        print("reset elastic index")
+        for index in self.index_config:
+            index_name, _, _ = self._config_split(index)
+            handler = ElasticIndex(index_name)
+            handler.delete_index(backup=False)
+
+    def create_all_blank(self):
+        """create all blank indexes"""
+        print("create all new indexes in elastic from template")
+        for index in self.index_config:
+            index_name, expected_map, expected_set = self._config_split(index)
+            handler = ElasticIndex(index_name, expected_map, expected_set)
+            handler.create_blank()
 
     @staticmethod
-    def index_exists(index_name):
-        """check if index already exists to skip"""
-        _, status_code = ElasticWrap(f"ta_{index_name}").get()
-        exists = status_code == 200
-
-        return exists
-
-    def rotate_backup(self):
-        """delete old backups if needed"""
-        rotate = self.config["scheduler"]["run_backup_rotate"]
-        if not rotate:
-            return
-
-        all_backup_files = self.get_all_backup_files()
-        auto = [i for i in all_backup_files if i["reason"] == "auto"]
-
-        if len(auto) <= rotate:
-            print("no backup files to rotate")
-            return
-
-        backup_dir = os.path.join(self.cache_dir, "backup")
-
-        all_to_delete = auto[rotate:]
-        for to_delete in all_to_delete:
-            file_path = os.path.join(backup_dir, to_delete["filename"])
-            print(f"remove old backup file: {file_path}")
-            os.remove(file_path)
-
-
-def get_mapping():
-    """read index_mapping.json and get expected mapping and settings"""
-    with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
-        index_config = json.load(f).get("index_config")
-
-    return index_config
-
-
-def index_check(force_restore=False):
-    """check if all indexes are created and have correct mapping"""
-
-    backed_up = False
-    index_config = get_mapping()
-
-    for index in index_config:
+    def _config_split(index):
+        """split index config keys"""
         index_name = index["index_name"]
         expected_map = index["expected_map"]
         expected_set = index["expected_set"]
-        handler = ElasticIndex(index_name, expected_map, expected_set)
-        # force restore
-        if force_restore:
-            handler.delete_index(backup=False)
-            handler.create_blank()
-            continue
 
-        # create new
-        if not handler.exists:
-            print(f"create new blank index with name ta_{index_name}...")
-            handler.create_blank()
-            continue
+        return index_name, expected_map, expected_set
 
-        # validate index
-        rebuild = handler.validate()
-        if rebuild:
-            # make backup before rebuild
-            if not backed_up:
-                print("running backup first")
-                backup_all_indexes(reason="update")
-                backed_up = True
+    def _check_backup(self):
+        """create backup if needed"""
+        if self.backup_run:
+            return
 
-            print(f"applying new mappings to index ta_{index_name}...")
-            handler.rebuild_index()
-            continue
+        config = AppConfig().config
+        if config["application"]["enable_snapshot"]:
+            # take snapshot if enabled
+            ElasticSnapshot().take_snapshot_now(wait=True)
+        else:
+            # fallback to json backup
+            ElasticBackup(reason="update").backup_all_indexes()
 
-        # else all good
-        print(f"ta_{index_name} index is created and up to date...")
-
-
-def get_available_backups():
-    """return dict of available backups for settings view"""
-    index_config = get_mapping()
-    backup_handler = ElasticBackup(index_config, reason=False)
-    all_backup_files = backup_handler.get_all_backup_files()
-    return all_backup_files
-
-
-def backup_all_indexes(reason):
-    """backup all es indexes to disk"""
-    index_config = get_mapping()
-    backup_handler = ElasticBackup(index_config, reason)
-
-    for index in backup_handler.index_config:
-        index_name = index["index_name"]
-        print(f"backup: export in progress for {index_name}")
-        if not backup_handler.index_exists(index_name):
-            continue
-        all_results = backup_handler.get_all_documents(index_name)
-        file_content = backup_handler.build_bulk(all_results)
-        backup_handler.write_es_json(file_content, index_name)
-        backup_handler.write_ta_json(all_results, index_name)
-
-    backup_handler.zip_it()
-
-    if reason == "auto":
-        backup_handler.rotate_backup()
-
-
-def restore_from_backup(filename):
-    """restore indexes from backup file"""
-    # delete
-    index_check(force_restore=True)
-    # recreate
-    index_config = get_mapping()
-    backup_handler = ElasticBackup(index_config, reason=False)
-    zip_content = backup_handler.unpack_zip_backup(filename)
-    backup_handler.restore_json_files(zip_content)
+        self.backup_run = True

tubearchivist/home/src/es/snapshot.py

@@ -0,0 +1,286 @@
+"""
+functionality:
+- handle snapshots in ES
+"""
+
+from datetime import datetime
+from os import environ
+from time import sleep
+from zoneinfo import ZoneInfo
+
+from home.src.es.connect import ElasticWrap
+from home.src.ta.helper import get_mapping
+
+
+class ElasticSnapshot:
+    """interact with snapshots on ES"""
+
+    REPO = "ta_snapshot"
+    REPO_SETTINGS = {
+        "compress": "true",
+        "chunk_size": "1g",
+        "location": "/usr/share/elasticsearch/data/snapshot",
+    }
+    POLICY = "ta_daily"
+
+    def __init__(self):
+        self.all_indices = self._get_all_indices()
+
+    def _get_all_indices(self):
+        """return all indices names managed by TA"""
+        mapping = get_mapping()
+        all_indices = [f"ta_{i['index_name']}" for i in mapping]
+
+        return all_indices
+
+    def setup(self):
+        """setup the snapshot in ES, create or update if needed"""
+        print("snapshot: run setup")
+        repo_exists = self._check_repo_exists()
+        if not repo_exists:
+            self.create_repo()
+
+        policy_exists = self._check_policy_exists()
+        if not policy_exists:
+            self.create_policy()
+
+        is_outdated = self._needs_startup_snapshot()
+        if is_outdated:
+            _ = self.take_snapshot_now()
+
+    def _check_repo_exists(self):
+        """check if expected repo already exists"""
+        path = f"_snapshot/{self.REPO}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 200:
+            print(f"snapshot: repo {self.REPO} already created")
+            matching = response[self.REPO]["settings"] == self.REPO_SETTINGS
+            if not matching:
+                print(f"snapshot: update repo settings {self.REPO_SETTINGS}")
+
+            return matching
+
+        print(f"snapshot: setup repo {self.REPO} config {self.REPO_SETTINGS}")
+        return False
+
+    def create_repo(self):
+        """create filesystem repo"""
+        path = f"_snapshot/{self.REPO}"
+        data = {
+            "type": "fs",
+            "settings": self.REPO_SETTINGS,
+        }
+        response, statuscode = ElasticWrap(path).post(data=data)
+        if statuscode == 200:
+            print(f"snapshot: repo setup correctly: {response}")
+
+    def _check_policy_exists(self):
+        """check if snapshot policy is set correctly"""
+        policy = self._get_policy()
+        expected_policy = self._build_policy_data()
+        if not policy:
+            print(f"snapshot: create policy {self.POLICY} {expected_policy}")
+            return False
+
+        if policy["policy"] != expected_policy:
+            print(f"snapshot: update policy settings {expected_policy}")
+            return False
+
+        print("snapshot: policy is set.")
+        return True
+
+    def _get_policy(self):
+        """get policy from es"""
+        path = f"_slm/policy/{self.POLICY}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode != 200:
+            return False
+
+        return response[self.POLICY]
+
+    def create_policy(self):
+        """create snapshot lifetime policy"""
+        path = f"_slm/policy/{self.POLICY}"
+        data = self._build_policy_data()
+        response, statuscode = ElasticWrap(path).put(data)
+        if statuscode == 200:
+            print(f"snapshot: policy setup correctly: {response}")
+
+    def _build_policy_data(self):
+        """build policy dict from config"""
+        at_12 = datetime.now().replace(hour=12, minute=0, second=0)
+        hour = at_12.astimezone(ZoneInfo("UTC")).hour
+
+        return {
+            "schedule": f"0 0 {hour} * * ?",
+            "name": f"<{self.POLICY}_>",
+            "repository": self.REPO,
+            "config": {
+                "indices": self.all_indices,
+                "ignore_unavailable": True,
+                "include_global_state": True,
+            },
+            "retention": {
+                "expire_after": "30d",
+                "min_count": 5,
+                "max_count": 50,
+            },
+        }
+
+    def _needs_startup_snapshot(self):
+        """check if last snapshot is expired"""
+        snap_dicts = self._get_all_snapshots()
+        if not snap_dicts:
+            print("snapshot: create initial snapshot")
+            return True
+
+        last_stamp = snap_dicts[0]["end_stamp"]
+        now = int(datetime.now().timestamp())
+        outdated = (now - last_stamp) / 60 / 60 > 24
+        if outdated:
+            print("snapshot: is outdated, create new now")
+
+        print("snapshot: last snapshot is up-to-date")
+        return outdated
+
+    def take_snapshot_now(self, wait=False):
+        """execute daily snapshot now"""
+        path = f"_slm/policy/{self.POLICY}/_execute"
+        response, statuscode = ElasticWrap(path).post()
+        if statuscode == 200:
+            print(f"snapshot: executing now: {response}")
+
+        if wait:
+            self._wait_for_snapshot(response["snapshot_name"])
+
+        return response
+
+    def _wait_for_snapshot(self, snapshot_name):
+        """return after snapshot_name completes"""
+        path = f"_snapshot/{self.REPO}/{snapshot_name}"
+
+        while True:
+            # wait for task to be created
+            sleep(1)
+            _, statuscode = ElasticWrap(path).get()
+            if statuscode == 200:
+                break
+
+        while True:
+            # wait for snapshot success
+            response, statuscode = ElasticWrap(path).get()
+            snapshot_state = response["snapshots"][0]["state"]
+            if snapshot_state == "SUCCESS":
+                break
+
+            print(f"snapshot: {snapshot_name} in state {snapshot_state}")
+            print("snapshot: wait to complete")
+            sleep(5)
+
+        print(f"snapshot: completed - {response}")
+
+    def get_snapshot_stats(self):
+        """get snapshot info for frontend"""
+        snapshot_info = self._build_policy_details()
+        if snapshot_info:
+            snapshot_info.update({"snapshots": self._get_all_snapshots()})
+
+        return snapshot_info
+
+    def get_single_snapshot(self, snapshot_id):
+        """get single snapshot metadata"""
+        path = f"_snapshot/{self.REPO}/{snapshot_id}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 404:
+            print(f"snapshots: not found: {snapshot_id}")
+            return False
+
+        snapshot = response["snapshots"][0]
+        return self._parse_single_snapshot(snapshot)
+
+    def _get_all_snapshots(self):
+        """get a list of all registered snapshots"""
+        path = f"_snapshot/{self.REPO}/*?sort=start_time&order=desc"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 404:
+            print("snapshots: not configured")
+            return False
+
+        all_snapshots = response["snapshots"]
+        if not all_snapshots:
+            print("snapshots: no snapshots found")
+            return False
+
+        snap_dicts = []
+        for snapshot in all_snapshots:
+            snap_dict = self._parse_single_snapshot(snapshot)
+            snap_dicts.append(snap_dict)
+
+        return snap_dicts
+
+    def _parse_single_snapshot(self, snapshot):
+        """extract relevant metadata from single snapshot"""
+        snap_dict = {
+            "id": snapshot["snapshot"],
+            "state": snapshot["state"],
+            "es_version": snapshot["version"],
+            "start_date": self._date_converter(snapshot["start_time"]),
+            "end_date": self._date_converter(snapshot["end_time"]),
+            "end_stamp": snapshot["end_time_in_millis"] // 1000,
+            "duration_s": snapshot["duration_in_millis"] // 1000,
+        }
+        return snap_dict
+
+    def _build_policy_details(self):
+        """get additional policy details"""
+        policy = self._get_policy()
+        if not policy:
+            return False
+
+        next_exec = policy["next_execution_millis"] // 1000
+        next_exec_date = datetime.fromtimestamp(next_exec)
+        next_exec_str = next_exec_date.strftime("%Y-%m-%d %H:%M")
+        expire_after = policy["policy"]["retention"]["expire_after"]
+        policy_metadata = {
+            "next_exec": next_exec,
+            "next_exec_str": next_exec_str,
+            "expire_after": expire_after,
+        }
+        return policy_metadata
+
+    @staticmethod
+    def _date_converter(date_utc):
+        """convert datetime string"""
+        expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
+        date = datetime.strptime(date_utc, expected_format)
+        local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
+        converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ")))
+        converted_str = converted.strftime("%Y-%m-%d %H:%M")
+
+        return converted_str
+
+    def restore_all(self, snapshot_name):
+        """restore snapshot by name"""
+        for index in self.all_indices:
+            _, _ = ElasticWrap(index).delete()
+
+        path = f"_snapshot/{self.REPO}/{snapshot_name}/_restore"
+        data = {"indices": "*"}
+        response, statuscode = ElasticWrap(path).post(data=data)
+        if statuscode == 200:
+            print(f"snapshot: executing now: {response}")
+            return response
+
+        print(f"snapshot: failed to restore, {statuscode} {response}")
+        return False
+
+    def delete_single_snapshot(self, snapshot_id):
+        """delete single snapshot from index"""
+        path = f"_snapshot/{self.REPO}/{snapshot_id}"
+        response, statuscode = ElasticWrap(path).delete()
+        if statuscode == 200:
+            print(f"snapshot: deleting {snapshot_id} {response}")
+            return response
+
+        print(f"snapshot: failed to delete, {statuscode} {response}")
+        return False

tubearchivist/home/src/frontend/api_calls.py

@@ -4,31 +4,14 @@ Functionality:
 - called via user input
 """
 
-from home.src.download.queue import PendingInteract
 from home.src.download.subscriptions import (
     ChannelSubscription,
     PlaylistSubscription,
 )
-from home.src.frontend.searching import SearchForm
-from home.src.frontend.watched import WatchState
-from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
-from home.src.index.video import YoutubeVideo
-from home.src.ta.helper import UrlListParser
-from home.src.ta.ta_redis import RedisArchivist, RedisQueue
-from home.tasks import (
-    download_pending,
-    download_single,
-    index_channel_playlists,
-    kill_dl,
-    re_sync_thumbs,
-    rescan_filesystem,
-    run_backup,
-    run_manual_import,
-    run_restore_backup,
-    subscribe_to,
-    update_subscribed,
-)
+from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.urlparser import Parser
+from home.tasks import run_restore_backup, subscribe_to
 
 
 class PostData:
@@ -51,54 +34,26 @@ class PostData:
     def exec_map(self):
         """map dict key and return function to execute"""
         exec_map = {
-            "watched": self._watched,
-            "un_watched": self._un_watched,
             "change_view": self._change_view,
             "change_grid": self._change_grid,
-            "rescan_pending": self._rescan_pending,
-            "ignore": self._ignore,
-            "dl_pending": self._dl_pending,
-            "queue": self._queue_handler,
             "unsubscribe": self._unsubscribe,
             "subscribe": self._subscribe,
             "sort_order": self._sort_order,
             "hide_watched": self._hide_watched,
             "show_subed_only": self._show_subed_only,
-            "dlnow": self._dlnow,
             "show_ignored_only": self._show_ignored_only,
-            "forgetIgnore": self._forget_ignore,
-            "addSingle": self._add_single,
-            "deleteQueue": self._delete_queue,
-            "manual-import": self._manual_import,
-            "re-embed": self._re_embed,
-            "db-backup": self._db_backup,
             "db-restore": self._db_restore,
-            "fs-rescan": self._fs_rescan,
-            "multi_search": self._multi_search,
-            "delete-video": self._delete_video,
-            "delete-channel": self._delete_channel,
             "delete-playlist": self._delete_playlist,
-            "find-playlists": self._find_playlists,
         }
 
         return exec_map[self.to_exec]
 
-    def _watched(self):
-        """mark as watched"""
-        WatchState(self.exec_val).mark_as_watched()
-        return {"success": True}
-
-    def _un_watched(self):
-        """mark as unwatched"""
-        WatchState(self.exec_val).mark_as_unwatched()
-        return {"success": True}
-
     def _change_view(self):
         """process view changes in home, channel, and downloads"""
         origin, new_view = self.exec_val.split(":")
         key = f"{self.current_user}:view:{origin}"
         print(f"change view: {key} to {new_view}")
-        RedisArchivist().set_message(key, {"status": new_view}, expire=False)
+        RedisArchivist().set_message(key, {"status": new_view})
         return {"success": True}
 
     def _change_grid(self):
@@ -109,56 +64,14 @@ class PostData:
 
         key = f"{self.current_user}:grid_items"
         print(f"change grid items: {grid_items}")
-        RedisArchivist().set_message(key, {"status": grid_items}, expire=False)
-        return {"success": True}
-
-    @staticmethod
-    def _rescan_pending():
-        """look for new items in subscribed channels"""
-        print("rescan subscribed channels")
-        update_subscribed.delay()
-        return {"success": True}
-
-    def _ignore(self):
-        """ignore from download queue"""
-        video_id = self.exec_val
-        print(f"{video_id}: ignore video from download queue")
-        PendingInteract(video_id=video_id, status="ignore").update_status()
-        # also clear from redis queue
-        RedisQueue().clear_item(video_id)
-        return {"success": True}
-
-    @staticmethod
-    def _dl_pending():
-        """start the download queue"""
-        print("download pending")
-        running = download_pending.delay()
-        task_id = running.id
-        print(f"{task_id}: set task id")
-        RedisArchivist().set_message("dl_queue_id", task_id, expire=False)
-        return {"success": True}
-
-    def _queue_handler(self):
-        """queue controls from frontend"""
-        to_execute = self.exec_val
-        if to_execute == "stop":
-            print("stopping download queue")
-            RedisQueue().clear()
-        elif to_execute == "kill":
-            task_id = RedisArchivist().get_message("dl_queue_id")
-            if not isinstance(task_id, str):
-                task_id = False
-            else:
-                print("brutally killing " + task_id)
-            kill_dl(task_id)
-
+        RedisArchivist().set_message(key, {"status": grid_items})
         return {"success": True}
 
     def _unsubscribe(self):
         """unsubscribe from channels or playlists"""
         id_unsub = self.exec_val
         print(f"{id_unsub}: unsubscribe")
-        to_unsub_list = UrlListParser(id_unsub).process_list()
+        to_unsub_list = Parser(id_unsub).parse()
         for to_unsub in to_unsub_list:
             unsub_type = to_unsub["type"]
             unsub_id = to_unsub["url"]
@@ -187,11 +100,11 @@ class PostData:
         sort_order = {"status": self.exec_val}
         if self.exec_val in ["asc", "desc"]:
             RedisArchivist().set_message(
-                f"{self.current_user}:sort_order", sort_order, expire=False
+                f"{self.current_user}:sort_order", sort_order
             )
         else:
             RedisArchivist().set_message(
-                f"{self.current_user}:sort_by", sort_order, expire=False
+                f"{self.current_user}:sort_by", sort_order
             )
         return {"success": True}
 
@@ -200,7 +113,7 @@ class PostData:
         key = f"{self.current_user}:hide_watched"
         message = {"status": bool(int(self.exec_val))}
         print(f"toggle {key}: {message}")
-        RedisArchivist().set_message(key, message, expire=False)
+        RedisArchivist().set_message(key, message)
         return {"success": True}
 
     def _show_subed_only(self):
@@ -208,17 +121,7 @@ class PostData:
         key = f"{self.current_user}:show_subed_only"
         message = {"status": bool(int(self.exec_val))}
         print(f"toggle {key}: {message}")
-        RedisArchivist().set_message(key, message, expire=False)
-        return {"success": True}
-
-    def _dlnow(self):
-        """start downloading single vid now"""
-        youtube_id = self.exec_val
-        print(f"{youtube_id}: downloading now")
-        running = download_single.delay(youtube_id=youtube_id)
-        task_id = running.id
-        print("set task id: " + task_id)
-        RedisArchivist().set_message("dl_queue_id", task_id, expire=False)
+        RedisArchivist().set_message(key, message)
         return {"success": True}
 
     def _show_ignored_only(self):
@@ -227,49 +130,7 @@ class PostData:
         key = f"{self.current_user}:show_ignored_only"
         value = {"status": show_value}
         print(f"Filter download view ignored only: {show_value}")
-        RedisArchivist().set_message(key, value, expire=False)
-        return {"success": True}
-
-    def _forget_ignore(self):
-        """delete from ta_download index"""
-        video_id = self.exec_val
-        print(f"{video_id}: forget from download")
-        PendingInteract(video_id=video_id).delete_item()
-        return {"success": True}
-
-    def _add_single(self):
-        """add single youtube_id to download queue"""
-        video_id = self.exec_val
-        print(f"{video_id}: add single vid to download queue")
-        PendingInteract(video_id=video_id, status="pending").update_status()
-        return {"success": True}
-
-    def _delete_queue(self):
-        """delete download queue"""
-        status = self.exec_val
-        print("deleting from download queue: " + status)
-        PendingInteract(status=status).delete_by_status()
-        return {"success": True}
-
-    @staticmethod
-    def _manual_import():
-        """run manual import from settings page"""
-        print("starting manual import")
-        run_manual_import.delay()
-        return {"success": True}
-
-    @staticmethod
-    def _re_embed():
-        """rewrite thumbnails into media files"""
-        print("start video thumbnail embed process")
-        re_sync_thumbs.delay()
-        return {"success": True}
-
-    @staticmethod
-    def _db_backup():
-        """backup es to zip from settings page"""
-        print("backing up database")
-        run_backup.delay("manual")
+        RedisArchivist().set_message(key, value)
         return {"success": True}
 
     def _db_restore(self):
@@ -279,32 +140,6 @@ class PostData:
         run_restore_backup.delay(filename)
         return {"success": True}
 
-    @staticmethod
-    def _fs_rescan():
-        """start file system rescan task"""
-        print("start filesystem scan")
-        rescan_filesystem.delay()
-        return {"success": True}
-
-    def _multi_search(self):
-        """search through all indexes"""
-        search_query = self.exec_val
-        print("searching for: " + search_query)
-        search_results = SearchForm().multi_search(search_query)
-        return search_results
-
-    def _delete_video(self):
-        """delete media file, metadata and thumb"""
-        youtube_id = self.exec_val
-        YoutubeVideo(youtube_id).delete_media_file()
-        return {"success": True}
-
-    def _delete_channel(self):
-        """delete channel and all matching videos"""
-        channel_id = self.exec_val
-        YoutubeChannel(channel_id).delete_channel()
-        return {"success": True}
-
     def _delete_playlist(self):
         """delete playlist, only metadata or incl all videos"""
         playlist_dict = self.exec_val
@@ -317,9 +152,3 @@ class PostData:
             YoutubePlaylist(playlist_id).delete_videos_playlist()
 
         return {"success": True}
-
-    def _find_playlists(self):
-        """add all playlists of a channel"""
-        channel_id = self.exec_val
-        index_channel_playlists.delay(channel_id)
-        return {"success": True}

tubearchivist/home/src/frontend/forms.py

@@ -44,6 +44,12 @@ class UserSettingsForm(forms.Form):
 class ApplicationSettingsForm(forms.Form):
     """handle all application settings"""
 
+    AUTOSTART_CHOICES = [
+        ("", "-- change subscription autostart --"),
+        ("0", "disable auto start"),
+        ("1", "enable auto start"),
+    ]
+
     METADATA_CHOICES = [
         ("", "-- change metadata embed --"),
         ("0", "don't embed metadata"),
@@ -68,10 +74,10 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable sponsorblock integration"),
     ]
 
-    CAST_CHOICES = [
-        ("", "-- change Cast integration --"),
-        ("0", "disable Cast"),
-        ("1", "enable Cast"),
+    SNAPSHOT_CHOICES = [
+        ("", "-- change snapshot settings --"),
+        ("0", "disable system snapshots"),
+        ("1", "enable system snapshots"),
     ]
 
     SUBTITLE_SOURCE_CHOICES = [
@@ -86,19 +92,37 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable subtitle index"),
     ]
 
+    COMMENT_SORT_CHOICES = [
+        ("", "-- change comments sort settings --"),
+        ("top", "sort comments by top"),
+        ("new", "sort comments by new"),
+    ]
+
     COOKIE_IMPORT_CHOICES = [
         ("", "-- change cookie settings"),
         ("0", "disable cookie"),
         ("1", "enable cookie"),
     ]
 
-    subscriptions_channel_size = forms.IntegerField(required=False)
-    downloads_limit_count = forms.IntegerField(required=False)
+    subscriptions_channel_size = forms.IntegerField(
+        required=False, min_value=1
+    )
+    subscriptions_live_channel_size = forms.IntegerField(
+        required=False, min_value=0
+    )
+    subscriptions_shorts_channel_size = forms.IntegerField(
+        required=False, min_value=0
+    )
+    subscriptions_auto_start = forms.ChoiceField(
+        widget=forms.Select, choices=AUTOSTART_CHOICES, required=False
+    )
     downloads_limit_speed = forms.IntegerField(required=False)
     downloads_throttledratelimit = forms.IntegerField(required=False)
     downloads_sleep_interval = forms.IntegerField(required=False)
     downloads_autodelete_days = forms.IntegerField(required=False)
     downloads_format = forms.CharField(required=False)
+    downloads_format_sort = forms.CharField(required=False)
+    downloads_extractor_lang = forms.CharField(required=False)
     downloads_add_metadata = forms.ChoiceField(
         widget=forms.Select, choices=METADATA_CHOICES, required=False
     )
@@ -112,6 +136,10 @@ class ApplicationSettingsForm(forms.Form):
     downloads_subtitle_index = forms.ChoiceField(
         widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False
     )
+    downloads_comment_max = forms.CharField(required=False)
+    downloads_comment_sort = forms.ChoiceField(
+        widget=forms.Select, choices=COMMENT_SORT_CHOICES, required=False
+    )
     downloads_cookie_import = forms.ChoiceField(
         widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False
     )
@@ -121,8 +149,8 @@ class ApplicationSettingsForm(forms.Form):
     downloads_integrate_sponsorblock = forms.ChoiceField(
         widget=forms.Select, choices=SP_CHOICES, required=False
     )
-    application_enable_cast = forms.ChoiceField(
-        widget=forms.Select, choices=CAST_CHOICES, required=False
+    application_enable_snapshot = forms.ChoiceField(
+        widget=forms.Select, choices=SNAPSHOT_CHOICES, required=False
     )
 
 
@@ -159,12 +187,14 @@ class MultiSearchForm(forms.Form):
 class AddToQueueForm(forms.Form):
     """text area form to add to downloads"""
 
+    HELP_TEXT = "Enter at least one video, channel or playlist id/URL here..."
+
     vid_url = forms.CharField(
         label=False,
         widget=forms.Textarea(
             attrs={
                 "rows": 4,
-                "placeholder": "Enter Video Urls or IDs here...",
+                "placeholder": HELP_TEXT,
             }
         ),
     )

tubearchivist/home/src/frontend/searching.py

@@ -11,6 +11,7 @@ from datetime import datetime
 
 from home.src.download.thumbnails import ThumbManager
 from home.src.es.connect import ElasticWrap
+from home.src.index.video_streams import DurationConverter
 from home.src.ta.config import AppConfig
 
 
@@ -19,6 +20,7 @@ class SearchHandler:
 
     def __init__(self, path, config, data=False):
         self.max_hits = None
+        self.aggs = None
         self.path = path
         self.config = config
         self.data = data
@@ -34,62 +36,22 @@ class SearchHandler:
             # simulate list for single result to reuse rest of class
             return_value = [response]
 
-        # stop if empty
         if not return_value:
             return False
 
-        all_videos = []
-        all_channels = []
         for idx, hit in enumerate(return_value):
             return_value[idx] = self.hit_cleanup(hit)
-            if hit["_index"] == "ta_video":
-                video_dict, channel_dict = self.vid_cache_link(hit)
-                if video_dict not in all_videos:
-                    all_videos.append(video_dict)
-                if channel_dict not in all_channels:
-                    all_channels.append(channel_dict)
-            elif hit["_index"] == "ta_channel":
-                channel_dict = self.channel_cache_link(hit)
-                if channel_dict not in all_channels:
-                    all_channels.append(channel_dict)
+
+        if response.get("aggregations"):
+            self.aggs = response["aggregations"]
+            if "total_duration" in self.aggs:
+                duration_sec = self.aggs["total_duration"]["value"]
+                self.aggs["total_duration"].update(
+                    {"value_str": DurationConverter().get_str(duration_sec)}
+                )
 
         return return_value
 
-    @staticmethod
-    def vid_cache_link(hit):
-        """download thumbnails into cache"""
-        vid_thumb = hit["source"]["vid_thumb_url"]
-        youtube_id = hit["source"]["youtube_id"]
-        channel_id_hit = hit["source"]["channel"]["channel_id"]
-        chan_thumb = hit["source"]["channel"]["channel_thumb_url"]
-        try:
-            chan_banner = hit["source"]["channel"]["channel_banner_url"]
-        except KeyError:
-            chan_banner = False
-        video_dict = {"youtube_id": youtube_id, "vid_thumb": vid_thumb}
-        channel_dict = {
-            "channel_id": channel_id_hit,
-            "chan_thumb": chan_thumb,
-            "chan_banner": chan_banner,
-        }
-        return video_dict, channel_dict
-
-    @staticmethod
-    def channel_cache_link(hit):
-        """build channel thumb links"""
-        channel_id_hit = hit["source"]["channel_id"]
-        chan_thumb = hit["source"]["channel_thumb_url"]
-        try:
-            chan_banner = hit["source"]["channel_banner_url"]
-        except KeyError:
-            chan_banner = False
-        channel_dict = {
-            "channel_id": channel_id_hit,
-            "chan_thumb": chan_thumb,
-            "chan_banner": chan_banner,
-        }
-        return channel_dict
-
     @staticmethod
     def hit_cleanup(hit):
         """clean up and parse data from a single hit"""
@@ -119,8 +81,8 @@ class SearchHandler:
 
         if "vid_thumb_url" in hit_keys:
             youtube_id = hit["source"]["youtube_id"]
-            thumb_path = ThumbManager().vid_thumb_path(youtube_id)
-            hit["source"]["vid_thumb_url"] = thumb_path
+            thumb_path = ThumbManager(youtube_id).vid_thumb_path()
+            hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
 
         if "channel_last_refresh" in hit_keys:
             refreshed = hit["source"]["channel_last_refresh"]
@@ -136,6 +98,11 @@ class SearchHandler:
                 date_str = datetime.strftime(date_refresh, "%d %b, %Y")
                 hit["source"]["channel"]["channel_last_refresh"] = date_str
 
+        if "subtitle_fragment_id" in hit_keys:
+            youtube_id = hit["source"]["youtube_id"]
+            thumb_path = ThumbManager(youtube_id).vid_thumb_path()
+            hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
+
         return hit
 
 
@@ -146,38 +113,12 @@ class SearchForm:
 
     def multi_search(self, search_query):
         """searching through index"""
-        path = "ta_video,ta_channel,ta_playlist/_search"
-        data = {
-            "size": 30,
-            "query": {
-                "multi_match": {
-                    "query": search_query,
-                    "type": "bool_prefix",
-                    "operator": "and",
-                    "fuzziness": "auto",
-                    "fields": [
-                        "category",
-                        "channel_description",
-                        "channel_name._2gram",
-                        "channel_name._3gram",
-                        "channel_name.search_as_you_type",
-                        "playlist_description",
-                        "playlist_name._2gram",
-                        "playlist_name._3gram",
-                        "playlist_name.search_as_you_type",
-                        "tags",
-                        "title._2gram",
-                        "title._3gram",
-                        "title.search_as_you_type",
-                    ],
-                }
-            },
-        }
-        look_up = SearchHandler(path, config=self.CONFIG, data=data)
+        path, query, query_type = SearchParser(search_query).run()
+        look_up = SearchHandler(path, config=self.CONFIG, data=query)
         search_results = look_up.get_data()
         all_results = self.build_results(search_results)
 
-        return {"results": all_results}
+        return {"results": all_results, "queryType": query_type}
 
     @staticmethod
     def build_results(search_results):
@@ -185,6 +126,7 @@ class SearchForm:
         video_results = []
         channel_results = []
         playlist_results = []
+        fulltext_results = []
         if search_results:
             for result in search_results:
                 if result["_index"] == "ta_video":
@@ -193,11 +135,347 @@ class SearchForm:
                     channel_results.append(result)
                 elif result["_index"] == "ta_playlist":
                     playlist_results.append(result)
+                elif result["_index"] == "ta_subtitle":
+                    fulltext_results.append(result)
 
         all_results = {
             "video_results": video_results,
             "channel_results": channel_results,
             "playlist_results": playlist_results,
+            "fulltext_results": fulltext_results,
         }
 
         return all_results
+
+
+class SearchParser:
+    """handle structured searches"""
+
+    def __init__(self, search_query):
+        self.query_words = search_query.lower().split()
+        self.query_map = {"term": [], "fuzzy": []}
+        self.append_to = "term"
+
+    def run(self):
+        """collection, return path and query dict for es"""
+        print(f"query words: {self.query_words}")
+        query_type = self._find_map()
+        self._run_words()
+        self._delete_unset()
+        self._match_data_types()
+
+        path, query = QueryBuilder(self.query_map, query_type).run()
+
+        return path, query, query_type
+
+    def _find_map(self):
+        """find query in keyword map"""
+        first_word = self.query_words[0]
+        key_word_map = self._get_map()
+
+        if ":" in first_word:
+            index_match, query_string = first_word.split(":")
+            if index_match in key_word_map:
+                self.query_map.update(key_word_map.get(index_match))
+                self.query_words[0] = query_string
+                return index_match
+
+        self.query_map.update(key_word_map.get("simple"))
+        print(f"query_map: {self.query_map}")
+
+        return "simple"
+
+    @staticmethod
+    def _get_map():
+        """return map to build on"""
+        return {
+            "simple": {
+                "index": "ta_video,ta_channel,ta_playlist",
+            },
+            "video": {
+                "index": "ta_video",
+                "channel": [],
+                "active": [],
+            },
+            "channel": {
+                "index": "ta_channel",
+                "active": [],
+                "subscribed": [],
+            },
+            "playlist": {
+                "index": "ta_playlist",
+                "active": [],
+                "subscribed": [],
+            },
+            "full": {
+                "index": "ta_subtitle",
+                "lang": [],
+                "source": [],
+            },
+        }
+
+    def _run_words(self):
+        """append word by word"""
+        for word in self.query_words:
+            if ":" in word:
+                keyword, search_string = word.split(":")
+                if keyword in self.query_map:
+                    self.append_to = keyword
+                    word = search_string
+
+            if word:
+                self.query_map[self.append_to].append(word)
+
+    def _delete_unset(self):
+        """delete unset keys"""
+        new_query_map = {}
+        for key, value in self.query_map.items():
+            if value:
+                new_query_map.update({key: value})
+        self.query_map = new_query_map
+
+    def _match_data_types(self):
+        """match values with data types"""
+        for key, value in self.query_map.items():
+            if key in ["term", "channel"]:
+                self.query_map[key] = " ".join(self.query_map[key])
+            if key in ["active", "subscribed"]:
+                self.query_map[key] = "yes" in value
+
+
+class QueryBuilder:
+    """build query for ES from form data"""
+
+    def __init__(self, query_map, query_type):
+        self.query_map = query_map
+        self.query_type = query_type
+
+    def run(self):
+        """build query"""
+        path = self._build_path()
+        query = self.build_query()
+        print(f"es path: {path}")
+        print(f"query: {query}")
+
+        return path, query
+
+    def _build_path(self):
+        """build es index search path"""
+        return f"{self.query_map.get('index')}/_search"
+
+    def build_query(self):
+        """build query based on query_type"""
+
+        exec_map = {
+            "simple": self._build_simple,
+            "video": self._build_video,
+            "channel": self._build_channel,
+            "playlist": self._build_playlist,
+            "full": self._build_fulltext,
+        }
+
+        build_must_list = exec_map[self.query_type]
+
+        if self.query_type == "full":
+            query = build_must_list()
+        else:
+            query = {
+                "size": 30,
+                "query": {"bool": {"must": build_must_list()}},
+            }
+
+        return query
+
+    def _get_fuzzy(self):
+        """return fuziness valuee"""
+        fuzzy_value = self.query_map.get("fuzzy", ["auto"])[0]
+        if fuzzy_value == "no":
+            return 0
+
+        if not fuzzy_value.isdigit():
+            return "auto"
+
+        if int(fuzzy_value) > 2:
+            return "2"
+
+        return fuzzy_value
+
+    def _build_simple(self):
+        """build simple cross index query"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "channel_name._2gram",
+                            "channel_name._3gram",
+                            "channel_name.search_as_you_type",
+                            "playlist_name._2gram",
+                            "playlist_name._3gram",
+                            "playlist_name.search_as_you_type",
+                            "title._2gram",
+                            "title._3gram",
+                            "title.search_as_you_type",
+                        ],
+                    }
+                }
+            )
+
+        return must_list
+
+    def _build_video(self):
+        """build video query"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "title._2gram^2",
+                            "title._3gram^2",
+                            "title.search_as_you_type^2",
+                            "tags",
+                            "category",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"active": {"value": active}}})
+
+        if (channel := self.query_map.get("channel")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": channel,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "channel.channel_name._2gram",
+                            "channel.channel_name._3gram",
+                            "channel.channel_name.search_as_you_type",
+                        ],
+                    }
+                }
+            )
+
+        return must_list
+
+    def _build_channel(self):
+        """build query for channel"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "channel_description",
+                            "channel_name._2gram^2",
+                            "channel_name._3gram^2",
+                            "channel_name.search_as_you_type^2",
+                            "channel_tags",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"channel_active": {"value": active}}})
+
+        if (subscribed := self.query_map.get("subscribed")) is not None:
+            must_list.append(
+                {"term": {"channel_subscribed": {"value": subscribed}}}
+            )
+
+        return must_list
+
+    def _build_playlist(self):
+        """build query for playlist"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "playlist_description",
+                            "playlist_name._2gram^2",
+                            "playlist_name._3gram^2",
+                            "playlist_name.search_as_you_type^2",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"playlist_active": {"value": active}}})
+
+        if (subscribed := self.query_map.get("subscribed")) is not None:
+            must_list.append(
+                {"term": {"playlist_subscribed": {"value": subscribed}}}
+            )
+
+        return must_list
+
+    def _build_fulltext(self):
+        """build query for fulltext search"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "match": {
+                        "subtitle_line": {
+                            "query": term,
+                            "fuzziness": self._get_fuzzy(),
+                        }
+                    }
+                }
+            )
+
+        if (lang := self.query_map.get("lang")) is not None:
+            must_list.append({"term": {"subtitle_lang": {"value": lang[0]}}})
+
+        if (source := self.query_map.get("source")) is not None:
+            must_list.append(
+                {"term": {"subtitle_source": {"value": source[0]}}}
+            )
+
+        query = {
+            "size": 30,
+            "_source": {"excludes": "subtitle_line"},
+            "query": {"bool": {"must": must_list}},
+            "highlight": {
+                "fields": {
+                    "subtitle_line": {
+                        "number_of_fragments": 0,
+                        "pre_tags": ['<span class="settings-current">'],
+                        "post_tags": ["</span>"],
+                    }
+                }
+            },
+        }
+
+        return query

tubearchivist/home/src/frontend/watched.py

@@ -6,93 +6,100 @@ functionality:
 from datetime import datetime
 
 from home.src.es.connect import ElasticWrap
-from home.src.ta.helper import UrlListParser
+from home.src.ta.urlparser import Parser
 
 
 class WatchState:
     """handle watched checkbox for videos and channels"""
 
-    def __init__(self, youtube_id):
+    def __init__(self, youtube_id, is_watched):
         self.youtube_id = youtube_id
-        self.stamp = int(datetime.now().strftime("%s"))
+        self.is_watched = is_watched
+        self.stamp = int(datetime.now().timestamp())
+        self.pipeline = f"_ingest/pipeline/watch_{youtube_id}"
 
-    def mark_as_watched(self):
-        """update es with new watched value"""
-        url_type = self.dedect_type()
+    def change(self):
+        """change watched state of item(s)"""
+        print(f"{self.youtube_id}: change watched state to {self.is_watched}")
+        url_type = self._dedect_type()
         if url_type == "video":
-            self.mark_vid_watched()
-        elif url_type == "channel":
-            self.mark_channel_watched()
-        elif url_type == "playlist":
-            self.mark_playlist_watched()
+            self.change_vid_state()
+            return
 
-        print(f"{self.youtube_id}: marked as watched")
+        self._add_pipeline()
+        path = f"ta_video/_update_by_query?pipeline=watch_{self.youtube_id}"
+        data = self._build_update_data(url_type)
+        _, _ = ElasticWrap(path).post(data)
+        self._delete_pipeline()
 
-    def mark_as_unwatched(self):
-        """revert watched state to false"""
-        url_type = self.dedect_type()
-        if url_type == "video":
-            self.mark_vid_watched(revert=True)
-
-        print(f"{self.youtube_id}: revert as unwatched")
-
-    def dedect_type(self):
+    def _dedect_type(self):
         """find youtube id type"""
-        print(self.youtube_id)
-        url_process = UrlListParser(self.youtube_id).process_list()
+        url_process = Parser(self.youtube_id).parse()
         url_type = url_process[0]["type"]
         return url_type
 
-    def mark_vid_watched(self, revert=False):
-        """change watched status of single video"""
+    def change_vid_state(self):
+        """change watched state of video"""
         path = f"ta_video/_update/{self.youtube_id}"
         data = {
-            "doc": {"player": {"watched": True, "watched_date": self.stamp}}
+            "doc": {
+                "player": {
+                    "watched": self.is_watched,
+                    "watched_date": self.stamp,
+                }
+            }
         }
-        if revert:
-            data["doc"]["player"]["watched"] = False
-
         response, status_code = ElasticWrap(path).post(data=data)
         if status_code != 200:
             print(response)
             raise ValueError("failed to mark video as watched")
 
-    def mark_channel_watched(self):
-        """change watched status of every video in channel"""
-        path = "ta_video/_update_by_query"
-        must_list = [
-            {"term": {"channel.channel_id": {"value": self.youtube_id}}},
-            {"term": {"player.watched": {"value": False}}},
-        ]
-        data = {
-            "query": {"bool": {"must": must_list}},
-            "script": {
-                "source": "ctx._source.player['watched'] = true",
-                "lang": "painless",
-            },
+    def _build_update_data(self, url_type):
+        """build update by query data based on url_type"""
+        term_key_map = {
+            "channel": "channel.channel_id",
+            "playlist": "playlist.keyword",
+        }
+        term_key = term_key_map.get(url_type)
+
+        return {
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {term_key: {"value": self.youtube_id}}},
+                        {
+                            "term": {
+                                "player.watched": {
+                                    "value": not self.is_watched
+                                }
+                            }
+                        },
+                    ],
+                }
+            }
         }
 
-        response, status_code = ElasticWrap(path).post(data=data)
-        if status_code != 200:
-            print(response)
-            raise ValueError("failed mark channel as watched")
-
-    def mark_playlist_watched(self):
-        """change watched state of all videos in playlist"""
-        path = "ta_video/_update_by_query"
-        must_list = [
-            {"term": {"playlist.keyword": {"value": self.youtube_id}}},
-            {"term": {"player.watched": {"value": False}}},
-        ]
+    def _add_pipeline(self):
+        """add ingest pipeline"""
         data = {
-            "query": {"bool": {"must": must_list}},
-            "script": {
-                "source": "ctx._source.player['watched'] = true",
-                "lang": "painless",
-            },
+            "description": f"{self.youtube_id}: watched {self.is_watched}",
+            "processors": [
+                {
+                    "set": {
+                        "field": "player.watched",
+                        "value": self.is_watched,
+                    }
+                },
+                {
+                    "set": {
+                        "field": "player.watched_date",
+                        "value": self.stamp,
+                    }
+                },
+            ],
         }
+        _, _ = ElasticWrap(self.pipeline).put(data)
 
-        response, status_code = ElasticWrap(path).post(data=data)
-        if status_code != 200:
-            print(response)
-            raise ValueError("failed mark playlist as watched")
+    def _delete_pipeline(self):
+        """delete pipeline"""
+        ElasticWrap(self.pipeline).delete()

tubearchivist/home/src/index/channel.py

@@ -6,145 +6,15 @@ functionality:
 
 import json
 import os
-import re
 from datetime import datetime
 
-import requests
-from bs4 import BeautifulSoup
 from home.src.download import queue  # partial import
 from home.src.download.thumbnails import ThumbManager
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.generic import YouTubeItem
 from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.helper import clean_string, requests_headers
-from home.src.ta.ta_redis import RedisArchivist
-
-
-class ChannelScraper:
-    """custom scraper using bs4 to scrape channel about page
-    will be able to be integrated into yt-dlp
-    once #2237 and #2350 are merged upstream
-    """
-
-    def __init__(self, channel_id):
-        self.channel_id = channel_id
-        self.soup = False
-        self.yt_json = False
-        self.json_data = False
-
-    def get_json(self):
-        """main method to return channel dict"""
-        self.get_soup()
-        self._extract_yt_json()
-        self._parse_channel_main()
-        self._parse_channel_meta()
-        return self.json_data
-
-    def get_soup(self):
-        """return soup from youtube"""
-        print(f"{self.channel_id}: scrape channel data from youtube")
-        url = f"https://www.youtube.com/channel/{self.channel_id}/about?hl=en"
-        cookies = {"CONSENT": "YES+xxxxxxxxxxxxxxxxxxxxxxxxxxx"}
-        response = requests.get(
-            url, cookies=cookies, headers=requests_headers()
-        )
-        if response.ok:
-            channel_page = response.text
-        else:
-            print(f"{self.channel_id}: failed to extract channel info")
-            raise ConnectionError
-        self.soup = BeautifulSoup(channel_page, "html.parser")
-
-    def _extract_yt_json(self):
-        """parse soup and get ytInitialData json"""
-        all_scripts = self.soup.find("body").find_all("script")
-        for script in all_scripts:
-            if "var ytInitialData = " in str(script):
-                script_content = str(script)
-                break
-        # extract payload
-        script_content = script_content.split("var ytInitialData = ")[1]
-        json_raw = script_content.rstrip(";</script>")
-        self.yt_json = json.loads(json_raw)
-
-    def _parse_channel_main(self):
-        """extract maintab values from scraped channel json data"""
-        main_tab = self.yt_json["header"]["c4TabbedHeaderRenderer"]
-        # build and return dict
-        self.json_data = {
-            "channel_active": True,
-            "channel_last_refresh": int(datetime.now().strftime("%s")),
-            "channel_subs": self._get_channel_subs(main_tab),
-            "channel_name": main_tab["title"],
-            "channel_banner_url": self._get_thumbnails(main_tab, "banner"),
-            "channel_tvart_url": self._get_thumbnails(main_tab, "tvBanner"),
-            "channel_id": self.channel_id,
-            "channel_subscribed": False,
-        }
-
-    @staticmethod
-    def _get_thumbnails(main_tab, thumb_name):
-        """extract banner url from main_tab"""
-        try:
-            all_banners = main_tab[thumb_name]["thumbnails"]
-            banner = sorted(all_banners, key=lambda k: k["width"])[-1]["url"]
-        except KeyError:
-            banner = False
-
-        return banner
-
-    @staticmethod
-    def _get_channel_subs(main_tab):
-        """process main_tab to get channel subs as int"""
-        try:
-            sub_text_simple = main_tab["subscriberCountText"]["simpleText"]
-            sub_text = sub_text_simple.split(" ")[0]
-            if sub_text[-1] == "K":
-                channel_subs = int(float(sub_text.replace("K", "")) * 1000)
-            elif sub_text[-1] == "M":
-                channel_subs = int(float(sub_text.replace("M", "")) * 1000000)
-            elif int(sub_text) >= 0:
-                channel_subs = int(sub_text)
-            else:
-                message = f"{sub_text} not dealt with"
-                print(message)
-        except KeyError:
-            channel_subs = 0
-
-        return channel_subs
-
-    def _parse_channel_meta(self):
-        """extract meta tab values from channel payload"""
-        # meta tab
-        meta_tab = self.yt_json["metadata"]["channelMetadataRenderer"]
-        all_thumbs = meta_tab["avatar"]["thumbnails"]
-        thumb_url = sorted(all_thumbs, key=lambda k: k["width"])[-1]["url"]
-        # stats tab
-        renderer = "twoColumnBrowseResultsRenderer"
-        all_tabs = self.yt_json["contents"][renderer]["tabs"]
-        for tab in all_tabs:
-            if "tabRenderer" in tab.keys():
-                if tab["tabRenderer"]["title"] == "About":
-                    about_tab = tab["tabRenderer"]["content"][
-                        "sectionListRenderer"
-                    ]["contents"][0]["itemSectionRenderer"]["contents"][0][
-                        "channelAboutFullMetadataRenderer"
-                    ]
-                    break
-        try:
-            channel_views_text = about_tab["viewCountText"]["simpleText"]
-            channel_views = int(re.sub(r"\D", "", channel_views_text))
-        except KeyError:
-            channel_views = 0
-
-        self.json_data.update(
-            {
-                "channel_description": meta_tab["description"],
-                "channel_thumb_url": thumb_url,
-                "channel_views": channel_views,
-            }
-        )
+from home.src.ta.helper import clean_string
 
 
 class YoutubeChannel(YouTubeItem):
@@ -153,36 +23,142 @@ class YoutubeChannel(YouTubeItem):
     es_path = False
     index_name = "ta_channel"
     yt_base = "https://www.youtube.com/channel/"
+    yt_obs = {
+        "extract_flat": True,
+        "allow_playlist_files": True,
+    }
 
-    def __init__(self, youtube_id):
+    def __init__(self, youtube_id, task=False):
         super().__init__(youtube_id)
-        self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.all_playlists = False
+        self.task = task
 
-    def build_json(self, upload=False):
+    def build_yt_url(self):
+        """overwrite base to use channel about page"""
+        return f"{self.yt_base}{self.youtube_id}/about"
+
+    def build_json(self, upload=False, fallback=False):
         """get from es or from youtube"""
         self.get_from_es()
         if self.json_data:
             return
 
         self.get_from_youtube()
+        if not self.youtube_meta and fallback:
+            self._video_fallback(fallback)
+        else:
+            self.process_youtube_meta()
+            self.get_channel_art()
+
         if upload:
             self.upload_to_es()
-        return
 
-    def get_from_youtube(self):
-        """use bs4 to scrape channel about page"""
-        self.json_data = ChannelScraper(self.youtube_id).get_json()
-        self.get_channel_art()
+    def process_youtube_meta(self):
+        """extract relevant fields"""
+        self.youtube_meta["thumbnails"].reverse()
+        channel_subs = self.youtube_meta.get("channel_follower_count") or 0
+        self.json_data = {
+            "channel_active": True,
+            "channel_description": self.youtube_meta.get("description", False),
+            "channel_id": self.youtube_id,
+            "channel_last_refresh": int(datetime.now().timestamp()),
+            "channel_name": self.youtube_meta["uploader"],
+            "channel_subs": channel_subs,
+            "channel_subscribed": False,
+            "channel_tags": self._parse_tags(self.youtube_meta.get("tags")),
+            "channel_banner_url": self._get_banner_art(),
+            "channel_thumb_url": self._get_thumb_art(),
+            "channel_tvart_url": self._get_tv_art(),
+            "channel_views": self.youtube_meta.get("view_count", 0),
+        }
+
+    def _parse_tags(self, tags):
+        """parse channel tags"""
+        if not tags:
+            return False
+
+        joined = " ".join(tags)
+        return [i.strip() for i in joined.split('"') if i and not i == " "]
+
+    def _get_thumb_art(self):
+        """extract thumb art"""
+        for i in self.youtube_meta["thumbnails"]:
+            if not i.get("width"):
+                continue
+            if i.get("width") == i.get("height"):
+                return i["url"]
+
+        return False
+
+    def _get_tv_art(self):
+        """extract tv artwork"""
+        for i in self.youtube_meta["thumbnails"]:
+            if i.get("id") == "avatar_uncropped":
+                return i["url"]
+            if not i.get("width"):
+                continue
+            if i["width"] // i["height"] < 2 and not i["width"] == i["height"]:
+                return i["url"]
+
+        return False
+
+    def _get_banner_art(self):
+        """extract banner artwork"""
+        for i in self.youtube_meta["thumbnails"]:
+            if not i.get("width"):
+                continue
+            if i["width"] // i["height"] > 5:
+                return i["url"]
+
+        return False
+
+    def _video_fallback(self, fallback):
+        """use video metadata as fallback"""
+        print(f"{self.youtube_id}: fallback to video metadata")
+        self.json_data = {
+            "channel_active": False,
+            "channel_last_refresh": int(datetime.now().timestamp()),
+            "channel_subs": fallback.get("channel_follower_count", 0),
+            "channel_name": fallback["uploader"],
+            "channel_banner_url": False,
+            "channel_tvart_url": False,
+            "channel_id": self.youtube_id,
+            "channel_subscribed": False,
+            "channel_tags": False,
+            "channel_description": False,
+            "channel_thumb_url": False,
+            "channel_views": 0,
+        }
+        self._info_json_fallback()
+
+    def _info_json_fallback(self):
+        """read channel info.json for additional metadata"""
+        info_json = os.path.join(
+            self.config["application"]["cache_dir"],
+            "import",
+            f"{self.youtube_id}.info.json",
+        )
+        if os.path.exists(info_json):
+            print(f"{self.youtube_id}: read info.json file")
+            with open(info_json, "r", encoding="utf-8") as f:
+                content = json.loads(f.read())
+
+            self.json_data.update(
+                {
+                    "channel_subs": content.get("channel_follower_count", 0),
+                    "channel_description": content.get("description", False),
+                }
+            )
+            os.remove(info_json)
 
     def get_channel_art(self):
         """download channel art for new channels"""
-        channel_id = self.youtube_id
-        channel_thumb = self.json_data["channel_thumb_url"]
-        channel_banner = self.json_data["channel_banner_url"]
-        ThumbManager().download_chan(
-            [(channel_id, channel_thumb, channel_banner)]
+        urls = (
+            self.json_data["channel_thumb_url"],
+            self.json_data["channel_banner_url"],
+            self.json_data["channel_tvart_url"],
         )
+        ThumbManager(self.youtube_id, item_type="channel").download(urls)
 
     def sync_to_videos(self):
         """sync new channel_dict to all videos of channel"""
@@ -218,6 +194,15 @@ class YoutubeChannel(YouTubeItem):
         }
         _, _ = ElasticWrap("ta_video/_delete_by_query").post(data)
 
+    def delete_es_comments(self):
+        """delete all comments from this channel"""
+        data = {
+            "query": {
+                "term": {"comment_channel_id": {"value": self.youtube_id}}
+            }
+        }
+        _, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
+
     def delete_playlists(self):
         """delete all indexed playlist from es"""
         all_playlists = self.get_indexed_playlists()
@@ -229,6 +214,9 @@ class YoutubeChannel(YouTubeItem):
         """delete channel and all videos"""
         print(f"{self.youtube_id}: delete channel")
         self.get_from_es()
+        if not self.json_data:
+            raise FileNotFoundError
+
         folder_path = self.get_folder_path()
         print(f"{self.youtube_id}: delete all media files")
         try:
@@ -244,6 +232,7 @@ class YoutubeChannel(YouTubeItem):
         self.delete_playlists()
         print(f"{self.youtube_id}: delete indexed videos")
         self.delete_es_videos()
+        self.delete_es_comments()
         self.del_in_es()
 
     def index_channel_playlists(self):
@@ -251,34 +240,29 @@ class YoutubeChannel(YouTubeItem):
         print(f"{self.youtube_id}: index all playlists")
         self.get_from_es()
         channel_name = self.json_data["channel_name"]
-        mess_dict = {
-            "status": "message:playlistscan",
-            "level": "info",
-            "title": "Looking for playlists",
-            "message": f"{channel_name}: Scanning channel in progress",
-        }
-        RedisArchivist().set_message("message:playlistscan", mess_dict)
+        self.task.send_progress([f"{channel_name}: Looking for Playlists"])
         self.get_all_playlists()
         if not self.all_playlists:
             print(f"{self.youtube_id}: no playlists found.")
             return
 
         all_youtube_ids = self.get_all_video_ids()
+        total = len(self.all_playlists)
         for idx, playlist in enumerate(self.all_playlists):
-            self._notify_single_playlist(idx, playlist)
-            self._index_single_playlist(playlist, all_youtube_ids)
+            if self.task:
+                self._notify_single_playlist(idx, total)
 
-    def _notify_single_playlist(self, idx, playlist):
+            self._index_single_playlist(playlist, all_youtube_ids)
+            print("add playlist: " + playlist[1])
+
+    def _notify_single_playlist(self, idx, total):
         """send notification"""
         channel_name = self.json_data["channel_name"]
-        mess_dict = {
-            "status": "message:playlistscan",
-            "level": "info",
-            "title": f"{channel_name}: Scanning channel for playlists",
-            "message": f"Progress: {idx + 1}/{len(self.all_playlists)}",
-        }
-        RedisArchivist().set_message("message:playlistscan", mess_dict)
-        print("add playlist: " + playlist[1])
+        message = [
+            f"{channel_name}: Scanning channel for playlists",
+            f"Progress: {idx + 1}/{total}",
+        ]
+        self.task.send_progress(message, progress=(idx + 1) / total)
 
     @staticmethod
     def _index_single_playlist(playlist, all_youtube_ids):
@@ -308,6 +292,17 @@ class YoutubeChannel(YouTubeItem):
 
         return all_youtube_ids
 
+    def get_channel_videos(self):
+        """get all videos from channel"""
+        data = {
+            "query": {
+                "term": {"channel.channel_id": {"value": self.youtube_id}}
+            },
+            "_source": ["youtube_id", "vid_type"],
+        }
+        all_videos = IndexPaginate("ta_video", data).get_results()
+        return all_videos
+
     def get_all_playlists(self):
         """get all playlists owned by this channel"""
         url = (
@@ -319,14 +314,16 @@ class YoutubeChannel(YouTubeItem):
         all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
         self.all_playlists = all_entries
 
-    def get_indexed_playlists(self):
+    def get_indexed_playlists(self, active_only=False):
         """get all indexed playlists from channel"""
-        data = {
-            "query": {
-                "term": {"playlist_channel_id": {"value": self.youtube_id}}
-            },
-            "sort": [{"playlist_channel.keyword": {"order": "desc"}}],
-        }
+        must_list = [
+            {"term": {"playlist_channel_id": {"value": self.youtube_id}}}
+        ]
+        if active_only:
+            must_list.append({"term": {"playlist_active": {"value": True}}})
+
+        data = {"query": {"bool": {"must": must_list}}}
+
         all_playlists = IndexPaginate("ta_playlist", data).get_results()
         return all_playlists
 
@@ -351,7 +348,8 @@ class YoutubeChannel(YouTubeItem):
                 to_write[key] = False
                 continue
             if value in [0, "0"]:
-                del to_write[key]
+                if key in to_write:
+                    del to_write[key]
                 continue
             if value == "1":
                 to_write[key] = True

tubearchivist/home/src/index/comments.py

@@ -0,0 +1,209 @@
+"""
+Functionality:
+- Download comments
+- Index comments in ES
+- Retrieve comments from ES
+"""
+
+from datetime import datetime
+
+from home.src.download.yt_dlp_base import YtWrap
+from home.src.es.connect import ElasticWrap
+from home.src.ta.config import AppConfig
+
+
+class Comments:
+    """interact with comments per video"""
+
+    def __init__(self, youtube_id, config=False):
+        self.youtube_id = youtube_id
+        self.es_path = f"ta_comment/_doc/{youtube_id}"
+        self.json_data = False
+        self.config = config
+        self.is_activated = False
+        self.comments_format = False
+
+    def build_json(self):
+        """build json document for es"""
+        print(f"{self.youtube_id}: get comments")
+        self.check_config()
+        if not self.is_activated:
+            return
+
+        comments_raw, channel_id = self.get_yt_comments()
+        if not comments_raw and not channel_id:
+            return
+
+        self.format_comments(comments_raw)
+
+        self.json_data = {
+            "youtube_id": self.youtube_id,
+            "comment_last_refresh": int(datetime.now().timestamp()),
+            "comment_channel_id": channel_id,
+            "comment_comments": self.comments_format,
+        }
+
+    def check_config(self):
+        """read config if not attached"""
+        if not self.config:
+            self.config = AppConfig().config
+
+        self.is_activated = bool(self.config["downloads"]["comment_max"])
+
+    def build_yt_obs(self):
+        """
+        get extractor config
+        max-comments,max-parents,max-replies,max-replies-per-thread
+        """
+        max_comments = self.config["downloads"]["comment_max"]
+        max_comments_list = [i.strip() for i in max_comments.split(",")]
+        comment_sort = self.config["downloads"]["comment_sort"]
+
+        yt_obs = {
+            "check_formats": None,
+            "skip_download": True,
+            "getcomments": True,
+            "extractor_args": {
+                "youtube": {
+                    "max_comments": max_comments_list,
+                    "comment_sort": [comment_sort],
+                }
+            },
+        }
+
+        return yt_obs
+
+    def get_yt_comments(self):
+        """get comments from youtube"""
+        yt_obs = self.build_yt_obs()
+        info_json = YtWrap(yt_obs).extract(self.youtube_id)
+        if not info_json:
+            return False, False
+
+        comments_raw = info_json.get("comments")
+        channel_id = info_json.get("channel_id")
+        return comments_raw, channel_id
+
+    def format_comments(self, comments_raw):
+        """process comments to match format"""
+        comments = []
+
+        if comments_raw:
+            for comment in comments_raw:
+                cleaned_comment = self.clean_comment(comment)
+                if not cleaned_comment:
+                    continue
+
+                comments.append(cleaned_comment)
+
+        self.comments_format = comments
+
+    def clean_comment(self, comment):
+        """parse metadata from comment for indexing"""
+        if not comment.get("text"):
+            # comment text can be empty
+            print(f"{self.youtube_id}: Failed to extract text, {comment}")
+            return False
+
+        time_text_datetime = datetime.utcfromtimestamp(comment["timestamp"])
+
+        if time_text_datetime.hour == 0 and time_text_datetime.minute == 0:
+            format_string = "%Y-%m-%d"
+        else:
+            format_string = "%Y-%m-%d %H:%M"
+
+        time_text = time_text_datetime.strftime(format_string)
+
+        cleaned_comment = {
+            "comment_id": comment["id"],
+            "comment_text": comment["text"].replace("\xa0", ""),
+            "comment_timestamp": comment["timestamp"],
+            "comment_time_text": time_text,
+            "comment_likecount": comment["like_count"],
+            "comment_is_favorited": comment["is_favorited"],
+            "comment_author": comment["author"],
+            "comment_author_id": comment["author_id"],
+            "comment_author_thumbnail": comment["author_thumbnail"],
+            "comment_author_is_uploader": comment["author_is_uploader"],
+            "comment_parent": comment["parent"],
+        }
+
+        return cleaned_comment
+
+    def upload_comments(self):
+        """upload comments to es"""
+        if not self.is_activated:
+            return
+
+        print(f"{self.youtube_id}: upload comments")
+        _, _ = ElasticWrap(self.es_path).put(self.json_data)
+
+        vid_path = f"ta_video/_update/{self.youtube_id}"
+        data = {"doc": {"comment_count": len(self.comments_format)}}
+        _, _ = ElasticWrap(vid_path).post(data=data)
+
+    def delete_comments(self):
+        """delete comments from es"""
+        print(f"{self.youtube_id}: delete comments")
+        _, _ = ElasticWrap(self.es_path).delete(refresh=True)
+
+    def get_es_comments(self):
+        """get comments from ES"""
+        response, statuscode = ElasticWrap(self.es_path).get()
+        if statuscode == 404:
+            print(f"comments: not found {self.youtube_id}")
+            return False
+
+        return response.get("_source")
+
+    def reindex_comments(self):
+        """update comments from youtube"""
+        self.check_config()
+        if not self.is_activated:
+            return
+
+        self.build_json()
+        if not self.json_data:
+            return
+
+        es_comments = self.get_es_comments()
+
+        if not self.comments_format:
+            return
+
+        if not self.comments_format and es_comments["comment_comments"]:
+            # don't overwrite comments in es
+            return
+
+        self.delete_comments()
+        self.upload_comments()
+
+
+class CommentList:
+    """interact with comments in group"""
+
+    def __init__(self, video_ids, task=False):
+        self.video_ids = video_ids
+        self.task = task
+        self.config = AppConfig().config
+
+    def index(self):
+        """index comments for list, init with task object to notify"""
+        if not self.config["downloads"].get("comment_max"):
+            return
+
+        total_videos = len(self.video_ids)
+        for idx, youtube_id in enumerate(self.video_ids):
+            if self.task:
+                self.notify(idx, total_videos)
+
+            comment = Comments(youtube_id, config=self.config)
+            comment.build_json()
+            if comment.json_data:
+                comment.upload_comments()
+
+    def notify(self, idx, total_videos):
+        """send notification on task"""
+        message = [f"Add comments for new videos {idx + 1}/{total_videos}"]
+        progress = (idx + 1) / total_videos
+        self.task.send_progress(message, progress=progress)

tubearchivist/home/src/index/filesystem.py

@@ -7,35 +7,37 @@ Functionality:
 
 import json
 import os
-import re
-import shutil
-import subprocess
 
 from home.src.download.queue import PendingList
-from home.src.download.yt_dlp_handler import VideoDownloader
 from home.src.es.connect import ElasticWrap
-from home.src.index.reindex import Reindex
+from home.src.index.comments import CommentList
 from home.src.index.video import index_new_video
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import clean_string, ignore_filelist
-from home.src.ta.ta_redis import RedisArchivist
+from PIL import ImageFile
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
 
 
-class FilesystemScanner:
-    """handle scanning and fixing from filesystem"""
+class ScannerBase:
+    """scan the filesystem base class"""
 
     CONFIG = AppConfig().config
     VIDEOS = CONFIG["application"]["videos"]
 
     def __init__(self):
-        self.all_downloaded = self.get_all_downloaded()
-        self.all_indexed = self.get_all_indexed()
-        self.mismatch = None
-        self.to_rename = None
-        self.to_index = None
-        self.to_delete = None
+        self.to_index = False
+        self.to_delete = False
+        self.mismatch = False
+        self.to_rename = False
 
-    def get_all_downloaded(self):
+    def scan(self):
+        """entry point, scan and compare"""
+        all_downloaded = self._get_all_downloaded()
+        all_indexed = self._get_all_indexed()
+        self.list_comarison(all_downloaded, all_indexed)
+
+    def _get_all_downloaded(self):
         """get a list of all video files downloaded"""
         channels = os.listdir(self.VIDEOS)
         all_channels = ignore_filelist(channels)
@@ -53,7 +55,7 @@ class FilesystemScanner:
         return all_downloaded
 
     @staticmethod
-    def get_all_indexed():
+    def _get_all_indexed():
         """get a list of all indexed videos"""
         index_handler = PendingList()
         index_handler.get_download()
@@ -68,64 +70,79 @@ class FilesystemScanner:
             all_indexed.append((youtube_id, media_url, published, title))
         return all_indexed
 
-    def list_comarison(self):
+    def list_comarison(self, all_downloaded, all_indexed):
         """compare the lists to figure out what to do"""
-        self.find_unindexed()
-        self.find_missing()
-        self.find_bad_media_url()
+        self._find_unindexed(all_downloaded, all_indexed)
+        self._find_missing(all_downloaded, all_indexed)
+        self._find_bad_media_url(all_downloaded, all_indexed)
 
-    def find_unindexed(self):
+    def _find_unindexed(self, all_downloaded, all_indexed):
         """find video files without a matching document indexed"""
-        all_indexed_ids = [i[0] for i in self.all_indexed]
-        to_index = []
-        for downloaded in self.all_downloaded:
+        all_indexed_ids = [i[0] for i in all_indexed]
+        self.to_index = []
+        for downloaded in all_downloaded:
             if downloaded[2] not in all_indexed_ids:
-                to_index.append(downloaded)
+                self.to_index.append(downloaded)
 
-        self.to_index = to_index
-
-    def find_missing(self):
+    def _find_missing(self, all_downloaded, all_indexed):
         """find indexed videos without matching media file"""
-        all_downloaded_ids = [i[2] for i in self.all_downloaded]
-        to_delete = []
-        for video in self.all_indexed:
+        all_downloaded_ids = [i[2] for i in all_downloaded]
+        self.to_delete = []
+        for video in all_indexed:
             youtube_id = video[0]
             if youtube_id not in all_downloaded_ids:
-                to_delete.append(video)
+                self.to_delete.append(video)
 
-        self.to_delete = to_delete
-
-    def find_bad_media_url(self):
+    def _find_bad_media_url(self, all_downloaded, all_indexed):
         """rename media files not matching the indexed title"""
-        to_fix = []
-        to_rename = []
-        for downloaded in self.all_downloaded:
+        self.mismatch = []
+        self.to_rename = []
+
+        for downloaded in all_downloaded:
             channel, filename, downloaded_id = downloaded
             # find in indexed
-            for indexed in self.all_indexed:
+            for indexed in all_indexed:
                 indexed_id, media_url, published, title = indexed
                 if indexed_id == downloaded_id:
                     # found it
-                    title_c = clean_string(title)
                     pub = published.replace("-", "")
-                    expected_filename = f"{pub}_{indexed_id}_{title_c}.mp4"
-                    new_url = os.path.join(channel, expected_filename)
-                    if expected_filename != filename:
+                    expected = f"{pub}_{indexed_id}_{clean_string(title)}.mp4"
+                    new_url = os.path.join(channel, expected)
+                    if expected != filename:
                         # file to rename
-                        to_rename.append(
-                            (channel, filename, expected_filename)
-                        )
+                        self.to_rename.append((channel, filename, expected))
                     if media_url != new_url:
                         # media_url to update in es
-                        to_fix.append((indexed_id, new_url))
+                        self.mismatch.append((indexed_id, new_url))
 
                     break
 
-        self.mismatch = to_fix
-        self.to_rename = to_rename
+
+class Filesystem(ScannerBase):
+    """handle scanning and fixing from filesystem"""
+
+    def __init__(self, task=False):
+        super().__init__()
+        self.task = task
+
+    def process(self):
+        """entry point"""
+        if self.task:
+            self.task.send_progress(["Scanning your archive and index."])
+        self.scan()
+        self.rename_files()
+        self.send_mismatch_bulk()
+        self.delete_from_index()
+        self.add_missing()
 
     def rename_files(self):
         """rename media files as identified by find_bad_media_url"""
+        if not self.to_rename:
+            return
+
+        total = len(self.to_rename)
+        if self.task:
+            self.task.send_progress([f"Rename {total} media files."])
         for bad_filename in self.to_rename:
             channel, filename, expected_filename = bad_filename
             print(f"renaming [{filename}] to [{expected_filename}]")
@@ -135,6 +152,12 @@ class FilesystemScanner:
 
     def send_mismatch_bulk(self):
         """build bulk update"""
+        if not self.mismatch:
+            return
+
+        total = len(self.mismatch)
+        if self.task:
+            self.task.send_progress([f"Fix media urls for {total} files"])
         bulk_list = []
         for video_mismatch in self.mismatch:
             youtube_id, media_url = video_mismatch
@@ -150,164 +173,33 @@ class FilesystemScanner:
 
     def delete_from_index(self):
         """find indexed but deleted mediafile"""
+        if not self.to_delete:
+            return
+
+        total = len(self.to_delete)
+        if self.task:
+            self.task.send_progress([f"Clean up {total} items from index."])
         for indexed in self.to_delete:
             youtube_id = indexed[0]
             print(f"deleting {youtube_id} from index")
             path = f"ta_video/_doc/{youtube_id}"
             _, _ = ElasticWrap(path).delete()
 
+    def add_missing(self):
+        """add missing videos to index"""
+        video_ids = [i[2] for i in self.to_index]
+        if not video_ids:
+            return
 
-class ManualImport:
-    """import and indexing existing video files"""
-
-    CONFIG = AppConfig().config
-    CACHE_DIR = CONFIG["application"]["cache_dir"]
-    IMPORT_DIR = os.path.join(CACHE_DIR, "import")
-
-    def __init__(self):
-        self.identified = self.import_folder_parser()
-
-    def import_folder_parser(self):
-        """detect files in import folder"""
-        import_files = os.listdir(self.IMPORT_DIR)
-        to_import = ignore_filelist(import_files)
-        to_import.sort()
-        video_files = [i for i in to_import if not i.endswith(".json")]
-
-        identified = []
-
-        for file_path in video_files:
-
-            file_dict = {"video_file": file_path}
-            file_name, _ = os.path.splitext(file_path)
-
-            matching_json = [
-                i
-                for i in to_import
-                if i.startswith(file_name) and i.endswith(".json")
-            ]
-            if matching_json:
-                json_file = matching_json[0]
-                youtube_id = self.extract_id_from_json(json_file)
-                file_dict.update({"json_file": json_file})
-            else:
-                youtube_id = self.extract_id_from_filename(file_name)
-                file_dict.update({"json_file": False})
-
-            file_dict.update({"youtube_id": youtube_id})
-            identified.append(file_dict)
-
-        return identified
-
-    @staticmethod
-    def extract_id_from_filename(file_name):
-        """
-        look at the file name for the youtube id
-        expects filename ending in [<youtube_id>].<ext>
-        """
-        id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", file_name)
-        if id_search:
-            youtube_id = id_search.group(1)
-            return youtube_id
-
-        print("failed to extract youtube id for: " + file_name)
-        raise Exception
-
-    def extract_id_from_json(self, json_file):
-        """open json file and extract id"""
-        json_path = os.path.join(self.CACHE_DIR, "import", json_file)
-        with open(json_path, "r", encoding="utf-8") as f:
-            json_content = f.read()
-
-        youtube_id = json.loads(json_content)["id"]
-
-        return youtube_id
-
-    def process_import(self):
-        """go through identified media files"""
-
-        all_videos_added = []
-
-        for media_file in self.identified:
-            json_file = media_file["json_file"]
-            video_file = media_file["video_file"]
-            youtube_id = media_file["youtube_id"]
-
-            video_path = os.path.join(self.CACHE_DIR, "import", video_file)
-
-            self.move_to_cache(video_path, youtube_id)
-
-            # identify and archive
-            vid_dict = index_new_video(youtube_id)
-            VideoDownloader([youtube_id]).move_to_archive(vid_dict)
-            youtube_id = vid_dict["youtube_id"]
-            thumb_url = vid_dict["vid_thumb_url"]
-            all_videos_added.append((youtube_id, thumb_url))
-
-            # cleanup
-            if os.path.exists(video_path):
-                os.remove(video_path)
-            if json_file:
-                json_path = os.path.join(self.CACHE_DIR, "import", json_file)
-                os.remove(json_path)
-
-        return all_videos_added
-
-    def move_to_cache(self, video_path, youtube_id):
-        """move identified video file to cache, convert to mp4"""
-        file_name = os.path.split(video_path)[-1]
-        video_file, ext = os.path.splitext(file_name)
-
-        # make sure youtube_id is in filename
-        if youtube_id not in video_file:
-            video_file = f"{video_file}_{youtube_id}"
-
-        # move, convert if needed
-        if ext == ".mp4":
-            new_file = video_file + ext
-            dest_path = os.path.join(self.CACHE_DIR, "download", new_file)
-            shutil.move(video_path, dest_path)
-        else:
-            print(f"processing with ffmpeg: {video_file}")
-            new_file = video_file + ".mp4"
-            dest_path = os.path.join(self.CACHE_DIR, "download", new_file)
-            subprocess.run(
-                [
-                    "ffmpeg",
-                    "-i",
-                    video_path,
-                    dest_path,
-                    "-loglevel",
-                    "warning",
-                    "-stats",
-                ],
-                check=True,
-            )
-
-
-def scan_filesystem():
-    """grouped function to delete and update index"""
-    filesystem_handler = FilesystemScanner()
-    filesystem_handler.list_comarison()
-    if filesystem_handler.to_rename:
-        print("renaming files")
-        filesystem_handler.rename_files()
-    if filesystem_handler.mismatch:
-        print("fixing media urls in index")
-        filesystem_handler.send_mismatch_bulk()
-    if filesystem_handler.to_delete:
-        print("delete metadata from index")
-        filesystem_handler.delete_from_index()
-    if filesystem_handler.to_index:
-        print("index new videos")
-        for missing_vid in filesystem_handler.to_index:
-            youtube_id = missing_vid[2]
+        total = len(video_ids)
+        for idx, youtube_id in enumerate(video_ids):
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[
+                        f"Index missing video {youtube_id}, {idx}/{total}"
+                    ],
+                    progress=(idx + 1) / total,
+                )
             index_new_video(youtube_id)
 
-
-def reindex_old_documents():
-    """daily refresh of old documents"""
-    handler = Reindex()
-    handler.check_outdated()
-    handler.reindex()
-    RedisArchivist().set_message("last_reindex", handler.now, expire=False)
+        CommentList(video_ids, task=self.task).index()

tubearchivist/home/src/index/generic.py

@@ -15,8 +15,8 @@ class YouTubeItem:
     """base class for youtube"""
 
     es_path = False
-    index_name = False
-    yt_base = False
+    index_name = ""
+    yt_base = ""
     yt_obs = {
         "skip_download": True,
         "noplaylist": True,
@@ -24,18 +24,27 @@ class YouTubeItem:
 
     def __init__(self, youtube_id):
         self.youtube_id = youtube_id
+        self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.config = AppConfig().config
         self.app_conf = self.config["application"]
         self.youtube_meta = False
         self.json_data = False
 
+    def build_yt_url(self):
+        """build youtube url"""
+        return self.yt_base + self.youtube_id
+
     def get_from_youtube(self):
         """use yt-dlp to get meta data from youtube"""
         print(f"{self.youtube_id}: get metadata from youtube")
-        url = self.yt_base + self.youtube_id
-        response = YtWrap(self.yt_obs, self.config).extract(url)
+        obs_request = self.yt_obs.copy()
+        if self.config["downloads"]["extractor_lang"]:
+            langs = self.config["downloads"]["extractor_lang"]
+            langs_list = [i.strip() for i in langs.split(",")]
+            obs_request["extractor_args"] = {"youtube": {"lang": langs_list}}
 
-        self.youtube_meta = response
+        url = self.build_yt_url()
+        self.youtube_meta = YtWrap(obs_request, self.config).extract(url)
 
     def get_from_es(self):
         """get indexed data from elastic search"""
@@ -56,11 +65,11 @@ class YouTubeItem:
             "ta_channel": "channel_active",
             "ta_playlist": "playlist_active",
         }
-        update_path = f"{self.index_name}/_update/{self.youtube_id}"
+        path = f"{self.index_name}/_update/{self.youtube_id}?refresh=true"
         data = {
             "script": f"ctx._source.{key_match.get(self.index_name)} = false"
         }
-        _, _ = ElasticWrap(update_path).post(data)
+        _, _ = ElasticWrap(path).post(data)
 
     def del_in_es(self):
         """delete item from elastic search"""
@@ -73,16 +82,25 @@ class Pagination:
     figure out the pagination based on page size and total_hits
     """
 
-    def __init__(self, page_get, user_id, search_get=False):
-        self.user_id = user_id
+    def __init__(self, request):
+        self.request = request
+        self.page_get = False
+        self.params = False
+        self.get_params()
         self.page_size = self.get_page_size()
-        self.page_get = page_get
-        self.search_get = search_get
         self.pagination = self.first_guess()
 
+    def get_params(self):
+        """process url query parameters"""
+        query_dict = self.request.GET.copy()
+        self.page_get = int(query_dict.get("page", 0))
+
+        _ = query_dict.pop("page", False)
+        self.params = query_dict.urlencode()
+
     def get_page_size(self):
         """get default or user modified page_size"""
-        key = f"{self.user_id}:page_size"
+        key = f"{self.request.user.id}:page_size"
         page_size = RedisArchivist().get_message(key)["status"]
         if not page_size:
             config = AppConfig().config
@@ -108,9 +126,9 @@ class Pagination:
             "prev_pages": prev_pages,
             "current_page": page_get,
             "max_hits": False,
+            "params": self.params,
         }
-        if self.search_get:
-            pagination.update({"search_get": self.search_get})
+
         return pagination
 
     def validate(self, total_hits):

tubearchivist/home/src/index/manual.py

@@ -0,0 +1,477 @@
+"""
+Functionality:
+- Handle manual import task
+- Scan and identify media files in import folder
+- Process import media files
+"""
+
+import json
+import os
+import re
+import shutil
+import subprocess
+
+from home.src.download.thumbnails import ThumbManager
+from home.src.index.comments import CommentList
+from home.src.index.video import YoutubeVideo
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import ignore_filelist
+from PIL import Image
+from yt_dlp.utils import ISO639Utils
+
+
+class ImportFolderScanner:
+    """import and indexing existing video files
+    - identify all media files belonging to a video
+    - identify youtube id
+    - convert if needed
+    """
+
+    CONFIG = AppConfig().config
+    CACHE_DIR = CONFIG["application"]["cache_dir"]
+    IMPORT_DIR = os.path.join(CACHE_DIR, "import")
+
+    EXT_MAP = {
+        "media": [".mp4", ".mkv", ".webm"],
+        "metadata": [".json"],
+        "thumb": [".jpg", ".png", ".webp"],
+        "subtitle": [".vtt"],
+    }
+
+    def __init__(self, task=False):
+        self.task = task
+        self.to_import = False
+
+    def scan(self):
+        """scan and match media files"""
+        if self.task:
+            self.task.send_progress(["Scanning your import folder."])
+
+        all_files = self.get_all_files()
+        self.match_files(all_files)
+        self.process_videos()
+
+        return self.to_import
+
+    def get_all_files(self):
+        """get all files in /import"""
+        rel_paths = ignore_filelist(os.listdir(self.IMPORT_DIR))
+        all_files = [os.path.join(self.IMPORT_DIR, i) for i in rel_paths]
+        all_files.sort()
+
+        return all_files
+
+    @staticmethod
+    def _get_template():
+        """base dict for video"""
+        return {
+            "media": False,
+            "video_id": False,
+            "metadata": False,
+            "thumb": False,
+            "subtitle": [],
+        }
+
+    def match_files(self, all_files):
+        """loop through all files, join what matches"""
+        self.to_import = []
+
+        current_video = self._get_template()
+        last_base = False
+
+        for file_path in all_files:
+            base_name, ext = self._detect_base_name(file_path)
+            key, file_path = self._detect_type(file_path, ext)
+            if not key or not file_path:
+                continue
+
+            if base_name != last_base:
+                if last_base:
+                    print(f"manual import: {current_video}")
+                    self.to_import.append(current_video)
+
+                current_video = self._get_template()
+                last_base = base_name
+
+            if key == "subtitle":
+                current_video["subtitle"].append(file_path)
+            else:
+                current_video[key] = file_path
+
+        if current_video.get("media"):
+            print(f"manual import: {current_video}")
+            self.to_import.append(current_video)
+
+    def _detect_base_name(self, file_path):
+        """extract base_name and ext for matching"""
+        base_name_raw, ext = os.path.splitext(file_path)
+        base_name, ext2 = os.path.splitext(base_name_raw)
+
+        if ext2:
+            if ISO639Utils.short2long(ext2.strip(".")) or ext2 == ".info":
+                # valid secondary extension
+                return base_name, ext
+
+        return base_name_raw, ext
+
+    def _detect_type(self, file_path, ext):
+        """detect metadata type for file"""
+
+        for key, value in self.EXT_MAP.items():
+            if ext in value:
+                return key, file_path
+
+        return False, False
+
+    def process_videos(self):
+        """loop through all videos"""
+        for idx, current_video in enumerate(self.to_import):
+            if not current_video["media"]:
+                print(f"{current_video}: no matching media file found.")
+                raise ValueError
+
+            if self.task:
+                self._notify(idx, current_video)
+
+            self._detect_youtube_id(current_video)
+            self._dump_thumb(current_video)
+            self._convert_thumb(current_video)
+            self._get_subtitles(current_video)
+            self._convert_video(current_video)
+            print(f"manual import: {current_video}")
+
+            ManualImport(current_video, self.CONFIG).run()
+
+        video_ids = [i["video_id"] for i in self.to_import]
+        CommentList(video_ids, task=self.task).index()
+
+    def _notify(self, idx, current_video):
+        """send notification back to task"""
+        filename = os.path.split(current_video["media"])[-1]
+        if len(filename) > 50:
+            filename = filename[:50] + "..."
+
+        message = [
+            f"Import queue processing video {idx + 1}/{len(self.to_import)}",
+            filename,
+        ]
+        progress = (idx + 1) / len(self.to_import)
+        self.task.send_progress(message, progress=progress)
+
+    def _detect_youtube_id(self, current_video):
+        """find video id from filename or json"""
+        youtube_id = self._extract_id_from_filename(current_video["media"])
+        if youtube_id:
+            current_video["video_id"] = youtube_id
+            return
+
+        youtube_id = self._extract_id_from_json(current_video["metadata"])
+        if youtube_id:
+            current_video["video_id"] = youtube_id
+            return
+
+        raise ValueError("failed to find video id")
+
+    @staticmethod
+    def _extract_id_from_filename(file_name):
+        """
+        look at the file name for the youtube id
+        expects filename ending in [<youtube_id>].<ext>
+        """
+        base_name, _ = os.path.splitext(file_name)
+        id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
+        if id_search:
+            youtube_id = id_search.group(1)
+            return youtube_id
+
+        print(f"id extraction failed from filename: {file_name}")
+
+        return False
+
+    def _extract_id_from_json(self, json_file):
+        """open json file and extract id"""
+        json_path = os.path.join(self.CACHE_DIR, "import", json_file)
+        with open(json_path, "r", encoding="utf-8") as f:
+            json_content = f.read()
+
+        youtube_id = json.loads(json_content)["id"]
+
+        return youtube_id
+
+    def _dump_thumb(self, current_video):
+        """extract embedded thumb before converting"""
+        if current_video["thumb"]:
+            return
+
+        media_path = current_video["media"]
+        _, ext = os.path.splitext(media_path)
+
+        new_path = False
+        if ext == ".mkv":
+            idx, thumb_type = self._get_mkv_thumb_stream(media_path)
+            if idx is not None:
+                new_path = self.dump_mpv_thumb(media_path, idx, thumb_type)
+
+        elif ext == ".mp4":
+            thumb_type = self.get_mp4_thumb_type(media_path)
+            if thumb_type:
+                new_path = self.dump_mp4_thumb(media_path, thumb_type)
+
+        if new_path:
+            current_video["thumb"] = new_path
+
+    def _get_mkv_thumb_stream(self, media_path):
+        """get stream idx of thumbnail for mkv files"""
+        streams = self._get_streams(media_path)
+        attachments = [
+            i for i in streams["streams"] if i["codec_type"] == "attachment"
+        ]
+
+        for idx, stream in enumerate(attachments):
+            tags = stream["tags"]
+            if "mimetype" in tags and tags["filename"].startswith("cover"):
+                _, ext = os.path.splitext(tags["filename"])
+                return idx, ext
+
+        return None, None
+
+    @staticmethod
+    def dump_mpv_thumb(media_path, idx, thumb_type):
+        """write cover to disk for mkv"""
+        _, media_ext = os.path.splitext(media_path)
+        new_path = f"{media_path.rstrip(media_ext)}{thumb_type}"
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-v",
+                "quiet",
+                f"-dump_attachment:t:{idx}",
+                new_path,
+                "-i",
+                media_path,
+            ],
+            check=False,
+        )
+
+        return new_path
+
+    def get_mp4_thumb_type(self, media_path):
+        """detect filetype of embedded thumbnail"""
+        streams = self._get_streams(media_path)
+
+        for stream in streams["streams"]:
+            if stream["codec_name"] in ["png", "jpg"]:
+                return stream["codec_name"]
+
+        return False
+
+    def _convert_thumb(self, current_video):
+        """convert all thumbnails to jpg"""
+        if not current_video["thumb"]:
+            return
+
+        thumb_path = current_video["thumb"]
+
+        base_path, ext = os.path.splitext(thumb_path)
+        if ext == ".jpg":
+            return
+
+        new_path = f"{base_path}.jpg"
+        img_raw = Image.open(thumb_path)
+        img_raw.convert("RGB").save(new_path)
+
+        os.remove(thumb_path)
+        current_video["thumb"] = new_path
+
+    def _get_subtitles(self, current_video):
+        """find all subtitles in media file"""
+        if current_video["subtitle"]:
+            return
+
+        media_path = current_video["media"]
+        streams = self._get_streams(media_path)
+        base_path, ext = os.path.splitext(media_path)
+
+        if ext == ".webm":
+            print(f"{media_path}: subtitle extract from webm not supported")
+            return
+
+        for idx, stream in enumerate(streams["streams"]):
+            if stream["codec_type"] == "subtitle":
+                lang = ISO639Utils.long2short(stream["tags"]["language"])
+                sub_path = f"{base_path}.{lang}.vtt"
+                self._dump_subtitle(idx, media_path, sub_path)
+                current_video["subtitle"].append(sub_path)
+
+    @staticmethod
+    def _dump_subtitle(idx, media_path, sub_path):
+        """extract subtitle from media file"""
+        subprocess.run(
+            ["ffmpeg", "-i", media_path, "-map", f"0:{idx}", sub_path],
+            check=True,
+        )
+
+    @staticmethod
+    def _get_streams(media_path):
+        """return all streams from media_path"""
+        streams_raw = subprocess.run(
+            [
+                "ffprobe",
+                "-v",
+                "error",
+                "-show_streams",
+                "-print_format",
+                "json",
+                media_path,
+            ],
+            capture_output=True,
+            check=True,
+        )
+        streams = json.loads(streams_raw.stdout.decode())
+
+        return streams
+
+    @staticmethod
+    def dump_mp4_thumb(media_path, thumb_type):
+        """save cover to disk"""
+        _, ext = os.path.splitext(media_path)
+        new_path = f"{media_path.rstrip(ext)}.{thumb_type}"
+
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-i",
+                media_path,
+                "-map",
+                "0:v",
+                "-map",
+                "-0:V",
+                "-c",
+                "copy",
+                new_path,
+            ],
+            check=True,
+        )
+
+        return new_path
+
+    def _convert_video(self, current_video):
+        """convert if needed"""
+        current_path = current_video["media"]
+        base_path, ext = os.path.splitext(current_path)
+        if ext == ".mp4":
+            return
+
+        new_path = base_path + ".mp4"
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-i",
+                current_path,
+                new_path,
+                "-loglevel",
+                "warning",
+                "-stats",
+            ],
+            check=True,
+        )
+        current_video["media"] = new_path
+        os.remove(current_path)
+
+
+class ManualImport:
+    """import single identified video"""
+
+    def __init__(self, current_video, config):
+        self.current_video = current_video
+        self.config = config
+
+    def run(self):
+        """run all"""
+        json_data = self.index_metadata()
+        self._move_to_archive(json_data)
+        self._cleanup(json_data)
+
+    def index_metadata(self):
+        """get metadata from yt or json"""
+        video_id = self.current_video["video_id"]
+        video = YoutubeVideo(video_id)
+        video.build_json(
+            youtube_meta_overwrite=self._get_info_json(),
+            media_path=self.current_video["media"],
+        )
+        if not video.json_data:
+            print(f"{video_id}: manual import failed, and no metadata found.")
+            raise ValueError
+
+        video.check_subtitles(subtitle_files=self.current_video["subtitle"])
+        video.upload_to_es()
+
+        if video.offline_import and self.current_video["thumb"]:
+            old_path = self.current_video["thumb"]
+            thumbs = ThumbManager(video_id)
+            new_path = thumbs.vid_thumb_path(absolute=True, create_folder=True)
+            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+        else:
+            url = video.json_data["vid_thumb_url"]
+            ThumbManager(video_id).download_video_thumb(url)
+
+        return video.json_data
+
+    def _get_info_json(self):
+        """read info_json from file"""
+        if not self.current_video["metadata"]:
+            return False
+
+        with open(self.current_video["metadata"], "r", encoding="utf-8") as f:
+            info_json = json.loads(f.read())
+
+        return info_json
+
+    def _move_to_archive(self, json_data):
+        """move identified media file to archive"""
+        videos = self.config["application"]["videos"]
+        host_uid = self.config["application"]["HOST_UID"]
+        host_gid = self.config["application"]["HOST_GID"]
+
+        channel, file = os.path.split(json_data["media_url"])
+        channel_folder = os.path.join(videos, channel)
+        if not os.path.exists(channel_folder):
+            os.makedirs(channel_folder)
+
+        if host_uid and host_gid:
+            os.chown(channel_folder, host_uid, host_gid)
+
+        old_path = self.current_video["media"]
+        new_path = os.path.join(channel_folder, file)
+        shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+        if host_uid and host_gid:
+            os.chown(new_path, host_uid, host_gid)
+
+        base_name, _ = os.path.splitext(new_path)
+        for old_path in self.current_video["subtitle"]:
+            lang = old_path.split(".")[-2]
+            new_path = f"{base_name}.{lang}.vtt"
+            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+
+    def _cleanup(self, json_data):
+        """cleanup leftover files"""
+        meta_data = self.current_video["metadata"]
+        if meta_data and os.path.exists(meta_data):
+            os.remove(meta_data)
+
+        thumb = self.current_video["thumb"]
+        if thumb and os.path.exists(thumb):
+            os.remove(thumb)
+
+        for subtitle_file in self.current_video["subtitle"]:
+            if os.path.exists(subtitle_file):
+                os.remove(subtitle_file)
+
+        channel_info = os.path.join(
+            self.config["application"]["cache_dir"],
+            "import",
+            f"{json_data['channel']['channel_id']}.info.json",
+        )
+        if os.path.exists(channel_info):
+            os.remove(channel_info)

tubearchivist/home/src/index/playlist.py

@@ -18,12 +18,14 @@ class YoutubePlaylist(YouTubeItem):
 
     es_path = False
     index_name = "ta_playlist"
-    yt_obs = {"extract_flat": True}
+    yt_obs = {
+        "extract_flat": True,
+        "allow_playlist_files": True,
+    }
     yt_base = "https://www.youtube.com/playlist?list="
 
     def __init__(self, youtube_id):
         super().__init__(youtube_id)
-        self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.all_members = False
         self.nav = False
         self.all_youtube_ids = []
@@ -38,23 +40,32 @@ class YoutubePlaylist(YouTubeItem):
 
         if scrape or not self.json_data:
             self.get_from_youtube()
+            if not self.youtube_meta:
+                self.json_data = False
+                return
+
             self.process_youtube_meta()
             self.get_entries()
             self.json_data["playlist_entries"] = self.all_members
-            self.get_playlist_art()
             self.json_data["playlist_subscribed"] = subscribed
 
     def process_youtube_meta(self):
         """extract relevant fields from youtube"""
+        try:
+            playlist_thumbnail = self.youtube_meta["thumbnails"][-1]["url"]
+        except IndexError:
+            print(f"{self.youtube_id}: thumbnail extraction failed")
+            playlist_thumbnail = False
+
         self.json_data = {
             "playlist_id": self.youtube_id,
             "playlist_active": True,
             "playlist_name": self.youtube_meta["title"],
             "playlist_channel": self.youtube_meta["channel"],
             "playlist_channel_id": self.youtube_meta["channel_id"],
-            "playlist_thumbnail": self.youtube_meta["thumbnails"][-1]["url"],
+            "playlist_thumbnail": playlist_thumbnail,
             "playlist_description": self.youtube_meta["description"] or False,
-            "playlist_last_refresh": int(datetime.now().strftime("%s")),
+            "playlist_last_refresh": int(datetime.now().timestamp()),
         }
 
     def get_entries(self, playlistend=False):
@@ -68,12 +79,12 @@ class YoutubePlaylist(YouTubeItem):
                 downloaded = entry["id"] in self.all_youtube_ids
             else:
                 downloaded = False
-            if not entry["uploader"]:
+            if not entry["channel"]:
                 continue
             to_append = {
                 "youtube_id": entry["id"],
                 "title": entry["title"],
-                "uploader": entry["uploader"],
+                "uploader": entry["channel"],
                 "idx": idx,
                 "downloaded": downloaded,
             }
@@ -81,12 +92,10 @@ class YoutubePlaylist(YouTubeItem):
 
         self.all_members = all_members
 
-    @staticmethod
-    def get_playlist_art():
+    def get_playlist_art(self):
         """download artwork of playlist"""
-        thumbnails = ThumbManager()
-        missing_playlists = thumbnails.get_missing_playlists()
-        thumbnails.download_playlist(missing_playlists)
+        url = self.json_data["playlist_thumbnail"]
+        ThumbManager(self.youtube_id, item_type="playlist").download(url)
 
     def add_vids_to_playlist(self):
         """sync the playlist id to videos"""
@@ -145,17 +154,15 @@ class YoutubePlaylist(YouTubeItem):
             previous_item = False
         else:
             previous_item = all_entries[current_idx - 1]
-            prev_thumb = ThumbManager().vid_thumb_path(
-                previous_item["youtube_id"]
-            )
-            previous_item["vid_thumb"] = prev_thumb
+            prev_id = previous_item["youtube_id"]
+            previous_item["vid_thumb"] = ThumbManager(prev_id).vid_thumb_path()
 
         if current_idx == len(all_entries) - 1:
             next_item = False
         else:
             next_item = all_entries[current_idx + 1]
-            next_thumb = ThumbManager().vid_thumb_path(next_item["youtube_id"])
-            next_item["vid_thumb"] = next_thumb
+            next_id = next_item["youtube_id"]
+            next_item["vid_thumb"] = ThumbManager(next_id).vid_thumb_path()
 
         self.nav = {
             "playlist_meta": {

tubearchivist/home/src/index/reindex.py

@@ -4,156 +4,291 @@ functionality:
 - index and update in es
 """
 
+import json
 import os
 import shutil
 from datetime import datetime
-from math import ceil
 from time import sleep
 
 from home.src.download.queue import PendingList
+from home.src.download.subscriptions import ChannelSubscription
 from home.src.download.thumbnails import ThumbManager
+from home.src.download.yt_dlp_base import CookieHandler
 from home.src.download.yt_dlp_handler import VideoDownloader
-from home.src.es.connect import ElasticWrap
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
+from home.src.index.comments import Comments
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo
 from home.src.ta.config import AppConfig
+from home.src.ta.ta_redis import RedisQueue
 
 
-class Reindex:
-    """check for outdated documents and refresh data from youtube"""
+class ReindexBase:
+    """base config class for reindex task"""
 
-    MATCH_FIELD = {
-        "ta_video": "active",
-        "ta_channel": "channel_active",
-        "ta_playlist": "playlist_active",
+    REINDEX_CONFIG = {
+        "video": {
+            "index_name": "ta_video",
+            "queue_name": "reindex:ta_video",
+            "active_key": "active",
+            "refresh_key": "vid_last_refresh",
+        },
+        "channel": {
+            "index_name": "ta_channel",
+            "queue_name": "reindex:ta_channel",
+            "active_key": "channel_active",
+            "refresh_key": "channel_last_refresh",
+        },
+        "playlist": {
+            "index_name": "ta_playlist",
+            "queue_name": "reindex:ta_playlist",
+            "active_key": "playlist_active",
+            "refresh_key": "playlist_last_refresh",
+        },
     }
+
     MULTIPLY = 1.2
+    DAYS3 = 60 * 60 * 24 * 3
 
     def __init__(self):
-        # config
-        self.now = int(datetime.now().strftime("%s"))
         self.config = AppConfig().config
+        self.now = int(datetime.now().timestamp())
+
+    def populate(self, all_ids, reindex_config):
+        """add all to reindex ids to redis queue"""
+        if not all_ids:
+            return
+
+        RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
+
+
+class ReindexPopulate(ReindexBase):
+    """add outdated and recent documents to reindex queue"""
+
+    def __init__(self):
+        super().__init__()
         self.interval = self.config["scheduler"]["check_reindex_days"]
-        # scan
-        self.all_youtube_ids = False
-        self.all_channel_ids = False
-        self.all_playlist_ids = False
 
-    def _get_daily(self):
-        """get daily refresh values"""
-        total_videos = self._get_total_hits("ta_video")
-        video_daily = ceil(total_videos / self.interval * self.MULTIPLY)
-        if video_daily >= 10000:
-            video_daily = 9999
+    def add_recent(self):
+        """add recent videos to refresh"""
+        gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
+        must_list = [
+            {"term": {"active": {"value": True}}},
+            {"range": {"published": {"gte": gte}}},
+        ]
+        data = {
+            "size": 10000,
+            "query": {"bool": {"must": must_list}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+        hits = response["hits"]["hits"]
+        if not hits:
+            return
 
-        total_channels = self._get_total_hits("ta_channel")
-        channel_daily = ceil(total_channels / self.interval * self.MULTIPLY)
-        total_playlists = self._get_total_hits("ta_playlist")
-        playlist_daily = ceil(total_playlists / self.interval * self.MULTIPLY)
-        return (video_daily, channel_daily, playlist_daily)
+        all_ids = [i["_source"]["youtube_id"] for i in hits]
+        reindex_config = self.REINDEX_CONFIG.get("video")
+        self.populate(all_ids, reindex_config)
 
-    def _get_total_hits(self, index):
+    def add_outdated(self):
+        """add outdated documents"""
+        for reindex_config in self.REINDEX_CONFIG.values():
+            total_hits = self._get_total_hits(reindex_config)
+            daily_should = self._get_daily_should(total_hits)
+            all_ids = self._get_outdated_ids(reindex_config, daily_should)
+            self.populate(all_ids, reindex_config)
+
+    @staticmethod
+    def _get_total_hits(reindex_config):
         """get total hits from index"""
-        match_field = self.MATCH_FIELD[index]
-        path = f"{index}/_search?filter_path=hits.total"
-        data = {"query": {"match": {match_field: True}}}
+        index_name = reindex_config["index_name"]
+        active_key = reindex_config["active_key"]
+        path = f"{index_name}/_search?filter_path=hits.total"
+        data = {"query": {"match": {active_key: True}}}
         response, _ = ElasticWrap(path).post(data=data)
         total_hits = response["hits"]["total"]["value"]
         return total_hits
 
-    def _get_unrated_vids(self):
-        """get max 200 videos without rating if ryd integration is enabled"""
-        data = {
-            "size": 200,
-            "query": {
-                "bool": {
-                    "must_not": [{"exists": {"field": "stats.average_rating"}}]
-                }
-            },
-        }
-        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+    def _get_daily_should(self, total_hits):
+        """calc how many should reindex daily"""
+        daily_should = int((total_hits // self.interval + 1) * self.MULTIPLY)
+        if daily_should >= 10000:
+            daily_should = 9999
 
-        missing_rating = [i["_id"] for i in response["hits"]["hits"]]
-        self.all_youtube_ids = self.all_youtube_ids + missing_rating
+        return daily_should
 
-    def _get_outdated_vids(self, size):
-        """get daily videos to refresh"""
+    def _get_outdated_ids(self, reindex_config, daily_should):
+        """get outdated from index_name"""
+        index_name = reindex_config["index_name"]
+        refresh_key = reindex_config["refresh_key"]
         now_lte = self.now - self.interval * 24 * 60 * 60
         must_list = [
-            {"match": {"active": True}},
-            {"range": {"vid_last_refresh": {"lte": now_lte}}},
+            {"match": {reindex_config["active_key"]: True}},
+            {"range": {refresh_key: {"lte": now_lte}}},
         ]
         data = {
-            "size": size,
+            "size": daily_should,
             "query": {"bool": {"must": must_list}},
-            "sort": [{"vid_last_refresh": {"order": "asc"}}],
+            "sort": [{refresh_key: {"order": "asc"}}],
             "_source": False,
         }
-        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+        response, _ = ElasticWrap(f"{index_name}/_search").get(data=data)
 
-        all_youtube_ids = [i["_id"] for i in response["hits"]["hits"]]
-        return all_youtube_ids
+        all_ids = [i["_id"] for i in response["hits"]["hits"]]
+        return all_ids
 
-    def _get_outdated_channels(self, size):
-        """get daily channels to refresh"""
-        now_lte = self.now - self.interval * 24 * 60 * 60
-        must_list = [
-            {"match": {"channel_active": True}},
-            {"range": {"channel_last_refresh": {"lte": now_lte}}},
-        ]
+
+class ReindexManual(ReindexBase):
+    """
+    manually add ids to reindex queue from API
+    data_example = {
+        "video": ["video1", "video2", "video3"],
+        "channel": ["channel1", "channel2", "channel3"],
+        "playlist": ["playlist1", "playlist2"],
+    }
+    extract_videos to also reindex all videos of channel/playlist
+    """
+
+    def __init__(self, extract_videos=False):
+        super().__init__()
+        self.extract_videos = extract_videos
+        self.data = False
+
+    def extract_data(self, data):
+        """process data"""
+        self.data = data
+        for key, values in self.data.items():
+            reindex_config = self.REINDEX_CONFIG.get(key)
+            if not reindex_config:
+                print(f"reindex type {key} not valid")
+                raise ValueError
+
+            self.process_index(reindex_config, values)
+
+    def process_index(self, index_config, values):
+        """process values per index"""
+        index_name = index_config["index_name"]
+        if index_name == "ta_video":
+            self._add_videos(values)
+        elif index_name == "ta_channel":
+            self._add_channels(values)
+        elif index_name == "ta_playlist":
+            self._add_playlists(values)
+
+    def _add_videos(self, values):
+        """add list of videos to reindex queue"""
+        if not values:
+            return
+
+        RedisQueue("reindex:ta_video").add_list(values)
+
+    def _add_channels(self, values):
+        """add list of channels to reindex queue"""
+        RedisQueue("reindex:ta_channel").add_list(values)
+
+        if self.extract_videos:
+            for channel_id in values:
+                all_videos = self._get_channel_videos(channel_id)
+                self._add_videos(all_videos)
+
+    def _add_playlists(self, values):
+        """add list of playlists to reindex queue"""
+        RedisQueue("reindex:ta_playlist").add_list(values)
+
+        if self.extract_videos:
+            for playlist_id in values:
+                all_videos = self._get_playlist_videos(playlist_id)
+                self._add_videos(all_videos)
+
+    def _get_channel_videos(self, channel_id):
+        """get all videos from channel"""
         data = {
-            "size": size,
-            "query": {"bool": {"must": must_list}},
-            "sort": [{"channel_last_refresh": {"order": "asc"}}],
-            "_source": False,
+            "query": {"term": {"channel.channel_id": {"value": channel_id}}},
+            "_source": ["youtube_id"],
         }
-        response, _ = ElasticWrap("ta_channel/_search").get(data=data)
+        all_results = IndexPaginate("ta_video", data).get_results()
+        return [i["youtube_id"] for i in all_results]
 
-        all_channel_ids = [i["_id"] for i in response["hits"]["hits"]]
-        return all_channel_ids
-
-    def _get_outdated_playlists(self, size):
-        """get daily outdated playlists to refresh"""
-        now_lte = self.now - self.interval * 24 * 60 * 60
-        must_list = [
-            {"match": {"playlist_active": True}},
-            {"range": {"playlist_last_refresh": {"lte": now_lte}}},
-        ]
+    def _get_playlist_videos(self, playlist_id):
+        """get all videos from playlist"""
         data = {
-            "size": size,
-            "query": {"bool": {"must": must_list}},
-            "sort": [{"playlist_last_refresh": {"order": "asc"}}],
-            "_source": False,
+            "query": {"term": {"playlist.keyword": {"value": playlist_id}}},
+            "_source": ["youtube_id"],
         }
-        response, _ = ElasticWrap("ta_playlist/_search").get(data=data)
+        all_results = IndexPaginate("ta_video", data).get_results()
+        return [i["youtube_id"] for i in all_results]
 
-        all_playlist_ids = [i["_id"] for i in response["hits"]["hits"]]
-        return all_playlist_ids
 
-    def check_outdated(self):
-        """add missing vids and channels"""
-        video_daily, channel_daily, playlist_daily = self._get_daily()
-        self.all_youtube_ids = self._get_outdated_vids(video_daily)
-        self.all_channel_ids = self._get_outdated_channels(channel_daily)
-        self.all_playlist_ids = self._get_outdated_playlists(playlist_daily)
+class Reindex(ReindexBase):
+    """reindex all documents from redis queue"""
 
-        integrate_ryd = self.config["downloads"]["integrate_ryd"]
-        if integrate_ryd:
-            self._get_unrated_vids()
+    def __init__(self, task=False):
+        super().__init__()
+        self.task = task
+        self.all_indexed_ids = False
 
-    @staticmethod
-    def _reindex_single_video(youtube_id):
+    def reindex_all(self):
+        """reindex all in queue"""
+        if not self.cookie_is_valid():
+            print("[reindex] cookie invalid, exiting...")
+            return
+
+        for name, index_config in self.REINDEX_CONFIG.items():
+            if not RedisQueue(index_config["queue_name"]).has_item():
+                continue
+
+            total = RedisQueue(index_config["queue_name"]).length()
+            while True:
+                has_next = self.reindex_index(name, index_config, total)
+                if not has_next:
+                    break
+
+    def reindex_index(self, name, index_config, total):
+        """reindex all of a single index"""
+        reindex = self.get_reindex_map(index_config["index_name"])
+        youtube_id = RedisQueue(index_config["queue_name"]).get_next()
+        if youtube_id:
+            self._notify(name, index_config, total)
+            reindex(youtube_id)
+            sleep_interval = self.config["downloads"].get("sleep_interval", 0)
+            sleep(sleep_interval)
+
+        return bool(youtube_id)
+
+    def get_reindex_map(self, index_name):
+        """return def to run for index"""
+        def_map = {
+            "ta_video": self._reindex_single_video,
+            "ta_channel": self._reindex_single_channel,
+            "ta_playlist": self._reindex_single_playlist,
+        }
+
+        return def_map.get(index_name)
+
+    def _notify(self, name, index_config, total):
+        """send notification back to task"""
+        remaining = RedisQueue(index_config["queue_name"]).length()
+        idx = total - remaining
+        message = [f"Reindexing {name.title()}s {idx}/{total}"]
+        progress = idx / total
+        self.task.send_progress(message, progress=progress)
+
+    def _reindex_single_video(self, youtube_id):
+        """wrapper to handle channel name changes"""
+        try:
+            self._reindex_single_video_call(youtube_id)
+        except FileNotFoundError:
+            ChannelUrlFixer(youtube_id, self.config).run()
+            self._reindex_single_video_call(youtube_id)
+
+    def _reindex_single_video_call(self, youtube_id):
         """refresh data for single video"""
         video = YoutubeVideo(youtube_id)
 
         # read current state
         video.get_from_es()
-        player = video.json_data["player"]
-        date_downloaded = video.json_data["date_downloaded"]
-        channel_dict = video.json_data["channel"]
-        playlist = video.json_data.get("playlist")
-        subtitles = video.json_data.get("subtitles")
+        es_meta = video.json_data.copy()
 
         # get new
         video.build_json()
@@ -161,45 +296,73 @@ class Reindex:
             video.deactivate()
             return
 
-        video.delete_subtitles(subtitles=subtitles)
+        video.delete_subtitles(subtitles=es_meta.get("subtitles"))
         video.check_subtitles()
 
         # add back
-        video.json_data["player"] = player
-        video.json_data["date_downloaded"] = date_downloaded
-        video.json_data["channel"] = channel_dict
-        if playlist:
-            video.json_data["playlist"] = playlist
+        video.json_data["player"] = es_meta.get("player")
+        video.json_data["date_downloaded"] = es_meta.get("date_downloaded")
+        video.json_data["channel"] = es_meta.get("channel")
+        if es_meta.get("playlist"):
+            video.json_data["playlist"] = es_meta.get("playlist")
 
         video.upload_to_es()
+        if es_meta.get("media_url") != video.json_data["media_url"]:
+            self._rename_media_file(
+                es_meta.get("media_url"), video.json_data["media_url"]
+            )
+
+        thumb_handler = ThumbManager(youtube_id)
+        thumb_handler.delete_video_thumb()
+        thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
+
+        Comments(youtube_id, config=self.config).reindex_comments()
 
-        thumb_handler = ThumbManager()
-        thumb_handler.delete_vid_thumb(youtube_id)
-        to_download = (youtube_id, video.json_data["vid_thumb_url"])
-        thumb_handler.download_vid([to_download], notify=False)
         return
 
+    def _rename_media_file(self, media_url_is, media_url_should):
+        """handle title change"""
+        print(f"[reindex] fix media_url {media_url_is} to {media_url_should}")
+        videos = self.config["application"]["videos"]
+        old_path = os.path.join(videos, media_url_is)
+        new_path = os.path.join(videos, media_url_should)
+        os.rename(old_path, new_path)
+
     @staticmethod
     def _reindex_single_channel(channel_id):
         """refresh channel data and sync to videos"""
+        # read current state
         channel = YoutubeChannel(channel_id)
         channel.get_from_es()
-        subscribed = channel.json_data["channel_subscribed"]
-        overwrites = channel.json_data.get("channel_overwrites", False)
+        es_meta = channel.json_data.copy()
+
+        # get new
         channel.get_from_youtube()
-        channel.json_data["channel_subscribed"] = subscribed
+        if not channel.youtube_meta:
+            channel.deactivate()
+            channel.get_from_es()
+            channel.sync_to_videos()
+            return
+
+        channel.process_youtube_meta()
+        channel.get_channel_art()
+
+        # add back
+        channel.json_data["channel_subscribed"] = es_meta["channel_subscribed"]
+        overwrites = es_meta.get("channel_overwrites")
         if overwrites:
             channel.json_data["channel_overwrites"] = overwrites
-        channel.upload_to_es()
-        channel.sync_to_videos()
 
-    @staticmethod
-    def _reindex_single_playlist(playlist_id, all_indexed_ids):
+        channel.upload_to_es()
+        ChannelFullScan(channel_id).scan()
+
+    def _reindex_single_playlist(self, playlist_id):
         """refresh playlist data"""
+        self._get_all_videos()
         playlist = YoutubePlaylist(playlist_id)
         playlist.get_from_es()
         subscribed = playlist.json_data["playlist_subscribed"]
-        playlist.all_youtube_ids = all_indexed_ids
+        playlist.all_youtube_ids = self.all_indexed_ids
         playlist.build_json(scrape=True)
         if not playlist.json_data:
             playlist.deactivate()
@@ -209,37 +372,98 @@ class Reindex:
         playlist.upload_to_es()
         return
 
-    def reindex(self):
-        """reindex what's needed"""
-        sleep_interval = self.config["downloads"]["sleep_interval"]
-        # videos
-        print(f"reindexing {len(self.all_youtube_ids)} videos")
-        for youtube_id in self.all_youtube_ids:
-            try:
-                self._reindex_single_video(youtube_id)
-            except FileNotFoundError:
-                # handle channel name change here
-                ChannelUrlFixer(youtube_id, self.config).run()
-                self._reindex_single_video(youtube_id)
-            if sleep_interval:
-                sleep(sleep_interval)
-        # channels
-        print(f"reindexing {len(self.all_channel_ids)} channels")
-        for channel_id in self.all_channel_ids:
-            self._reindex_single_channel(channel_id)
-            if sleep_interval:
-                sleep(sleep_interval)
-        # playlist
-        print(f"reindexing {len(self.all_playlist_ids)} playlists")
-        if self.all_playlist_ids:
-            handler = PendingList()
-            handler.get_download()
-            handler.get_indexed()
-            all_indexed_ids = [i["youtube_id"] for i in handler.all_videos]
-            for playlist_id in self.all_playlist_ids:
-                self._reindex_single_playlist(playlist_id, all_indexed_ids)
-                if sleep_interval:
-                    sleep(sleep_interval)
+    def _get_all_videos(self):
+        """add all videos for playlist index validation"""
+        if self.all_indexed_ids:
+            return
+
+        handler = PendingList()
+        handler.get_download()
+        handler.get_indexed()
+        self.all_indexed_ids = [i["youtube_id"] for i in handler.all_videos]
+
+    def cookie_is_valid(self):
+        """return true if cookie is enabled and valid"""
+        if not self.config["downloads"]["cookie_import"]:
+            # is not activated, continue reindex
+            return True
+
+        valid = CookieHandler(self.config).validate()
+        return valid
+
+
+class ReindexProgress(ReindexBase):
+    """
+    get progress of reindex task
+    request_type: key of self.REINDEX_CONFIG
+    request_id: id of request_type
+    return = {
+        "state": "running" | "queued" | False
+        "total_queued": int
+        "in_queue_name": "queue_name"
+    }
+    """
+
+    def __init__(self, request_type=False, request_id=False):
+        super().__init__()
+        self.request_type = request_type
+        self.request_id = request_id
+
+    def get_progress(self):
+        """get progress from task"""
+        queue_name, request_type = self._get_queue_name()
+        total = self._get_total_in_queue(queue_name)
+
+        progress = {
+            "total_queued": total,
+            "type": request_type,
+        }
+        state = self._get_state(total, queue_name)
+        progress.update(state)
+
+        return progress
+
+    def _get_queue_name(self):
+        """return queue_name, queue_type, raise exception on error"""
+        if not self.request_type:
+            return "all", "all"
+
+        reindex_config = self.REINDEX_CONFIG.get(self.request_type)
+        if not reindex_config:
+            print(f"reindex_config not found: {self.request_type}")
+            raise ValueError
+
+        return reindex_config["queue_name"], self.request_type
+
+    def _get_total_in_queue(self, queue_name):
+        """get all items in queue"""
+        total = 0
+        if queue_name == "all":
+            queues = [i["queue_name"] for i in self.REINDEX_CONFIG.values()]
+            for queue in queues:
+                total += len(RedisQueue(queue).get_all())
+        else:
+            total += len(RedisQueue(queue_name).get_all())
+
+        return total
+
+    def _get_state(self, total, queue_name):
+        """get state based on request_id"""
+        state_dict = {}
+        if self.request_id:
+            state = RedisQueue(queue_name).in_queue(self.request_id)
+            state_dict.update({"id": self.request_id, "state": state})
+
+            return state_dict
+
+        if total:
+            state = "running"
+        else:
+            state = "empty"
+
+        state_dict.update({"state": state})
+
+        return state_dict
 
 
 class ChannelUrlFixer:
@@ -293,9 +517,80 @@ class ChannelUrlFixer:
         """fix filepath"""
         print(f"{self.youtube_id}: fixing channel rename.")
         cache_dir = self.config["application"]["cache_dir"]
-        new_file_path = os.path.join(
+        new_path = os.path.join(
             cache_dir, "download", self.youtube_id + ".mp4"
         )
-        shutil.move(video_path_is, new_file_path)
+        shutil.move(video_path_is, new_path, copy_function=shutil.copyfile)
         VideoDownloader().move_to_archive(self.video.json_data)
         self.video.update_media_url()
+
+
+class ChannelFullScan:
+    """
+    update from v0.3.0 to v0.3.1
+    full scan of channel to fix vid_type mismatch
+    """
+
+    def __init__(self, channel_id):
+        self.channel_id = channel_id
+        self.to_update = False
+
+    def scan(self):
+        """match local with remote"""
+        print(f"{self.channel_id}: start full scan")
+        all_local_videos = self._get_all_local()
+        all_remote_videos = self._get_all_remote()
+        self.to_update = []
+        for video in all_local_videos:
+            video_id = video["youtube_id"]
+            remote_match = [i for i in all_remote_videos if i[0] == video_id]
+            if not remote_match:
+                print(f"{video_id}: no remote match found")
+                continue
+
+            expected_type = remote_match[0][-1]
+            if video["vid_type"] != expected_type:
+                self.to_update.append(
+                    {
+                        "video_id": video_id,
+                        "vid_type": expected_type,
+                    }
+                )
+
+        self.update()
+
+    def _get_all_remote(self):
+        """get all channel videos"""
+        sub = ChannelSubscription()
+        all_remote_videos = sub.get_last_youtube_videos(
+            self.channel_id, limit=False
+        )
+
+        return all_remote_videos
+
+    def _get_all_local(self):
+        """get all local indexed channel_videos"""
+        channel = YoutubeChannel(self.channel_id)
+        all_local_videos = channel.get_channel_videos()
+
+        return all_local_videos
+
+    def update(self):
+        """build bulk query for updates"""
+        if not self.to_update:
+            print(f"{self.channel_id}: nothing to update")
+            return
+
+        print(f"{self.channel_id}: fixing {len(self.to_update)} videos")
+        bulk_list = []
+        for video in self.to_update:
+            action = {
+                "update": {"_id": video.get("video_id"), "_index": "ta_video"}
+            }
+            source = {"doc": {"vid_type": video.get("vid_type")}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(source))
+        # add last newline
+        bulk_list.append("\n")
+        data = "\n".join(bulk_list)
+        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)

tubearchivist/home/src/index/subtitle.py

@@ -0,0 +1,325 @@
+"""
+functionality:
+- download subtitles
+- parse subtitles into it's cues
+- index dubtitles
+"""
+
+import json
+import os
+from datetime import datetime
+
+import requests
+from home.src.es.connect import ElasticWrap
+from home.src.ta.helper import requests_headers
+
+
+class YoutubeSubtitle:
+    """handle video subtitle functionality"""
+
+    def __init__(self, video):
+        self.video = video
+        self.languages = False
+
+    def _sub_conf_parse(self):
+        """add additional conf values to self"""
+        languages_raw = self.video.config["downloads"]["subtitle"]
+        if languages_raw:
+            self.languages = [i.strip() for i in languages_raw.split(",")]
+
+    def get_subtitles(self):
+        """check what to do"""
+        self._sub_conf_parse()
+        if not self.languages:
+            # no subtitles
+            return False
+
+        relevant_subtitles = []
+        for lang in self.languages:
+            user_sub = self._get_user_subtitles(lang)
+            if user_sub:
+                relevant_subtitles.append(user_sub)
+                continue
+
+            if self.video.config["downloads"]["subtitle_source"] == "auto":
+                auto_cap = self._get_auto_caption(lang)
+                if auto_cap:
+                    relevant_subtitles.append(auto_cap)
+
+        return relevant_subtitles
+
+    def _get_auto_caption(self, lang):
+        """get auto_caption subtitles"""
+        print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
+        all_subtitles = self.video.youtube_meta.get("automatic_captions")
+
+        if not all_subtitles:
+            return False
+
+        video_media_url = self.video.json_data["media_url"]
+        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
+        all_formats = all_subtitles.get(lang)
+        if not all_formats:
+            return False
+
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
+        subtitle.update(
+            {"lang": lang, "source": "auto", "media_url": media_url}
+        )
+
+        return subtitle
+
+    def _normalize_lang(self):
+        """normalize country specific language keys"""
+        all_subtitles = self.video.youtube_meta.get("subtitles")
+        if not all_subtitles:
+            return False
+
+        all_keys = list(all_subtitles.keys())
+        for key in all_keys:
+            lang = key.split("-")[0]
+            old = all_subtitles.pop(key)
+            if lang == "live_chat":
+                continue
+            all_subtitles[lang] = old
+
+        return all_subtitles
+
+    def _get_user_subtitles(self, lang):
+        """get subtitles uploaded from channel owner"""
+        print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
+        all_subtitles = self._normalize_lang()
+        if not all_subtitles:
+            return False
+
+        video_media_url = self.video.json_data["media_url"]
+        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
+        all_formats = all_subtitles.get(lang)
+        if not all_formats:
+            # no user subtitles found
+            return False
+
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
+        subtitle.update(
+            {"lang": lang, "source": "user", "media_url": media_url}
+        )
+
+        return subtitle
+
+    def download_subtitles(self, relevant_subtitles):
+        """download subtitle files to archive"""
+        videos_base = self.video.config["application"]["videos"]
+        indexed = []
+        for subtitle in relevant_subtitles:
+            dest_path = os.path.join(videos_base, subtitle["media_url"])
+            source = subtitle["source"]
+            lang = subtitle.get("lang")
+            response = requests.get(
+                subtitle["url"], headers=requests_headers(), timeout=30
+            )
+            if not response.ok:
+                print(f"{self.video.youtube_id}: failed to download subtitle")
+                print(response.text)
+                continue
+
+            parser = SubtitleParser(response.text, lang, source)
+            parser.process()
+            if not parser.all_cues:
+                continue
+
+            subtitle_str = parser.get_subtitle_str()
+            self._write_subtitle_file(dest_path, subtitle_str)
+            if self.video.config["downloads"]["subtitle_index"]:
+                query_str = parser.create_bulk_import(self.video, source)
+                self._index_subtitle(query_str)
+
+            indexed.append(subtitle)
+
+        return indexed
+
+    def _write_subtitle_file(self, dest_path, subtitle_str):
+        """write subtitle file to disk"""
+        # create folder here for first video of channel
+        os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
+        with open(dest_path, "w", encoding="utf-8") as subfile:
+            subfile.write(subtitle_str)
+
+        host_uid = self.video.config["application"]["HOST_UID"]
+        host_gid = self.video.config["application"]["HOST_GID"]
+        if host_uid and host_gid:
+            os.chown(dest_path, host_uid, host_gid)
+
+    @staticmethod
+    def _index_subtitle(query_str):
+        """send subtitle to es for indexing"""
+        _, _ = ElasticWrap("_bulk").post(data=query_str, ndjson=True)
+
+    def delete(self, subtitles=False):
+        """delete subtitles from index and filesystem"""
+        youtube_id = self.video.youtube_id
+        videos_base = self.video.config["application"]["videos"]
+        # delete files
+        if subtitles:
+            files = [i["media_url"] for i in subtitles]
+        else:
+            if not self.video.json_data.get("subtitles"):
+                return
+
+            files = [i["media_url"] for i in self.video.json_data["subtitles"]]
+
+        for file_name in files:
+            file_path = os.path.join(videos_base, file_name)
+            try:
+                os.remove(file_path)
+            except FileNotFoundError:
+                print(f"{youtube_id}: {file_path} failed to delete")
+        # delete from index
+        path = "ta_subtitle/_delete_by_query?refresh=true"
+        data = {"query": {"term": {"youtube_id": {"value": youtube_id}}}}
+        _, _ = ElasticWrap(path).post(data=data)
+
+
+class SubtitleParser:
+    """parse subtitle str from youtube"""
+
+    def __init__(self, subtitle_str, lang, source):
+        self.subtitle_raw = json.loads(subtitle_str)
+        self.lang = lang
+        self.source = source
+        self.all_cues = False
+
+    def process(self):
+        """extract relevant que data"""
+        self.all_cues = []
+        all_events = self.subtitle_raw.get("events")
+
+        if not all_events:
+            return
+
+        if self.source == "auto":
+            all_events = self._flat_auto_caption(all_events)
+
+        for idx, event in enumerate(all_events):
+            if "dDurationMs" not in event or "segs" not in event:
+                # some events won't have a duration or segs
+                print(f"skipping subtitle event without content: {event}")
+                continue
+
+            cue = {
+                "start": self._ms_conv(event["tStartMs"]),
+                "end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
+                "text": "".join([i.get("utf8") for i in event["segs"]]),
+                "idx": idx + 1,
+            }
+            self.all_cues.append(cue)
+
+    @staticmethod
+    def _flat_auto_caption(all_events):
+        """flatten autocaption segments"""
+        flatten = []
+        for event in all_events:
+            if "segs" not in event.keys():
+                continue
+            text = "".join([i.get("utf8") for i in event.get("segs")])
+            if not text.strip():
+                continue
+
+            if flatten:
+                # fix overlapping retiming issue
+                last = flatten[-1]
+                if "dDurationMs" not in last or "segs" not in last:
+                    # some events won't have a duration or segs
+                    print(f"skipping subtitle event without content: {event}")
+                    continue
+
+                last_end = last["tStartMs"] + last["dDurationMs"]
+                if event["tStartMs"] < last_end:
+                    joined = last["segs"][0]["utf8"] + "\n" + text
+                    last["segs"][0]["utf8"] = joined
+                    continue
+
+            event.update({"segs": [{"utf8": text}]})
+            flatten.append(event)
+
+        return flatten
+
+    @staticmethod
+    def _ms_conv(ms):
+        """convert ms to timestamp"""
+        hours = str((ms // (1000 * 60 * 60)) % 24).zfill(2)
+        minutes = str((ms // (1000 * 60)) % 60).zfill(2)
+        secs = str((ms // 1000) % 60).zfill(2)
+        millis = str(ms % 1000).zfill(3)
+
+        return f"{hours}:{minutes}:{secs}.{millis}"
+
+    def get_subtitle_str(self):
+        """create vtt text str from cues"""
+        subtitle_str = f"WEBVTT\nKind: captions\nLanguage: {self.lang}"
+
+        for cue in self.all_cues:
+            stamp = f"{cue.get('start')} --> {cue.get('end')}"
+            cue_text = f"\n\n{cue.get('idx')}\n{stamp}\n{cue.get('text')}"
+            subtitle_str = subtitle_str + cue_text
+
+        return subtitle_str
+
+    def create_bulk_import(self, video, source):
+        """subtitle lines for es import"""
+        documents = self._create_documents(video, source)
+        bulk_list = []
+
+        for document in documents:
+            document_id = document.get("subtitle_fragment_id")
+            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(document))
+
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+
+        return query_str
+
+    def _create_documents(self, video, source):
+        """process documents"""
+        documents = self._chunk_list(video.youtube_id)
+        channel = video.json_data.get("channel")
+        meta_dict = {
+            "youtube_id": video.youtube_id,
+            "title": video.json_data.get("title"),
+            "subtitle_channel": channel.get("channel_name"),
+            "subtitle_channel_id": channel.get("channel_id"),
+            "subtitle_last_refresh": int(datetime.now().timestamp()),
+            "subtitle_lang": self.lang,
+            "subtitle_source": source,
+        }
+
+        _ = [i.update(meta_dict) for i in documents]
+
+        return documents
+
+    def _chunk_list(self, youtube_id):
+        """join cues for bulk import"""
+        chunk_list = []
+
+        chunk = {}
+        for cue in self.all_cues:
+            if chunk:
+                text = f"{chunk.get('subtitle_line')} {cue.get('text')}\n"
+                chunk["subtitle_line"] = text
+            else:
+                idx = len(chunk_list) + 1
+                chunk = {
+                    "subtitle_index": idx,
+                    "subtitle_line": cue.get("text"),
+                    "subtitle_start": cue.get("start"),
+                }
+
+            chunk["subtitle_fragment_id"] = f"{youtube_id}-{self.lang}-{idx}"
+
+            if cue["idx"] % 5 == 0:
+                chunk["subtitle_end"] = cue.get("end")
+                chunk_list.append(chunk)
+                chunk = {}
+
+        return chunk_list

tubearchivist/home/src/index/video.py

@@ -4,7 +4,6 @@ functionality:
 - index and update in es
 """
 
-import json
 import os
 from datetime import datetime
 
@@ -12,313 +11,20 @@ import requests
 from django.conf import settings
 from home.src.es.connect import ElasticWrap
 from home.src.index import channel as ta_channel
+from home.src.index import comments as ta_comments
 from home.src.index import playlist as ta_playlist
 from home.src.index.generic import YouTubeItem
-from home.src.ta.helper import (
+from home.src.index.subtitle import YoutubeSubtitle
+from home.src.index.video_constants import VideoTypeEnum
+from home.src.index.video_streams import (
     DurationConverter,
-    clean_string,
-    randomizor,
-    requests_headers,
+    MediaStreamExtractor,
 )
+from home.src.ta.helper import clean_string, randomizor
 from home.src.ta.ta_redis import RedisArchivist
 from ryd_client import ryd_client
 
 
-class YoutubeSubtitle:
-    """handle video subtitle functionality"""
-
-    def __init__(self, video):
-        self.video = video
-        self.languages = False
-
-    def _sub_conf_parse(self):
-        """add additional conf values to self"""
-        languages_raw = self.video.config["downloads"]["subtitle"]
-        if languages_raw:
-            self.languages = [i.strip() for i in languages_raw.split(",")]
-
-    def get_subtitles(self):
-        """check what to do"""
-        self._sub_conf_parse()
-        if not self.languages:
-            # no subtitles
-            return False
-
-        relevant_subtitles = []
-        for lang in self.languages:
-            user_sub = self._get_user_subtitles(lang)
-            if user_sub:
-                relevant_subtitles.append(user_sub)
-                continue
-
-            if self.video.config["downloads"]["subtitle_source"] == "auto":
-                auto_cap = self._get_auto_caption(lang)
-                if auto_cap:
-                    relevant_subtitles.append(auto_cap)
-
-        return relevant_subtitles
-
-    def _get_auto_caption(self, lang):
-        """get auto_caption subtitles"""
-        print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
-        all_subtitles = self.video.youtube_meta.get("automatic_captions")
-
-        if not all_subtitles:
-            return False
-
-        video_media_url = self.video.json_data["media_url"]
-        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
-        all_formats = all_subtitles.get(lang)
-        if not all_formats:
-            return False
-
-        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
-        subtitle.update(
-            {"lang": lang, "source": "auto", "media_url": media_url}
-        )
-
-        return subtitle
-
-    def _normalize_lang(self):
-        """normalize country specific language keys"""
-        all_subtitles = self.video.youtube_meta.get("subtitles")
-        if not all_subtitles:
-            return False
-
-        all_keys = list(all_subtitles.keys())
-        for key in all_keys:
-            lang = key.split("-")[0]
-            old = all_subtitles.pop(key)
-            if lang == "live_chat":
-                continue
-            all_subtitles[lang] = old
-
-        return all_subtitles
-
-    def _get_user_subtitles(self, lang):
-        """get subtitles uploaded from channel owner"""
-        print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
-        all_subtitles = self._normalize_lang()
-        if not all_subtitles:
-            return False
-
-        video_media_url = self.video.json_data["media_url"]
-        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
-        all_formats = all_subtitles.get(lang)
-        if not all_formats:
-            # no user subtitles found
-            return False
-
-        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
-        subtitle.update(
-            {"lang": lang, "source": "user", "media_url": media_url}
-        )
-
-        return subtitle
-
-    def download_subtitles(self, relevant_subtitles):
-        """download subtitle files to archive"""
-        videos_base = self.video.config["application"]["videos"]
-        for subtitle in relevant_subtitles:
-            dest_path = os.path.join(videos_base, subtitle["media_url"])
-            source = subtitle["source"]
-            lang = subtitle.get("lang")
-            response = requests.get(
-                subtitle["url"], headers=requests_headers()
-            )
-            if not response.ok:
-                print(f"{self.video.youtube_id}: failed to download subtitle")
-                print(response.text)
-                continue
-
-            parser = SubtitleParser(response.text, lang, source)
-            parser.process()
-            subtitle_str = parser.get_subtitle_str()
-            self._write_subtitle_file(dest_path, subtitle_str)
-            if self.video.config["downloads"]["subtitle_index"]:
-                query_str = parser.create_bulk_import(self.video, source)
-                self._index_subtitle(query_str)
-
-    @staticmethod
-    def _write_subtitle_file(dest_path, subtitle_str):
-        """write subtitle file to disk"""
-        # create folder here for first video of channel
-        os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
-        with open(dest_path, "w", encoding="utf-8") as subfile:
-            subfile.write(subtitle_str)
-
-    @staticmethod
-    def _index_subtitle(query_str):
-        """send subtitle to es for indexing"""
-        _, _ = ElasticWrap("_bulk").post(data=query_str, ndjson=True)
-
-    def delete(self, subtitles=False):
-        """delete subtitles from index and filesystem"""
-        youtube_id = self.video.youtube_id
-        videos_base = self.video.config["application"]["videos"]
-        # delete files
-        if subtitles:
-            files = [i["media_url"] for i in subtitles]
-        else:
-            if not self.video.json_data.get("subtitles"):
-                return
-
-            files = [i["media_url"] for i in self.video.json_data["subtitles"]]
-
-        for file_name in files:
-            file_path = os.path.join(videos_base, file_name)
-            try:
-                os.remove(file_path)
-            except FileNotFoundError:
-                print(f"{youtube_id}: {file_path} failed to delete")
-        # delete from index
-        path = "ta_subtitle/_delete_by_query?refresh=true"
-        data = {"query": {"term": {"youtube_id": {"value": youtube_id}}}}
-        _, _ = ElasticWrap(path).post(data=data)
-
-
-class SubtitleParser:
-    """parse subtitle str from youtube"""
-
-    def __init__(self, subtitle_str, lang, source):
-        self.subtitle_raw = json.loads(subtitle_str)
-        self.lang = lang
-        self.source = source
-        self.all_cues = False
-
-    def process(self):
-        """extract relevant que data"""
-        all_events = self.subtitle_raw.get("events")
-        if self.source == "auto":
-            all_events = self._flat_auto_caption(all_events)
-
-        self.all_cues = []
-        for idx, event in enumerate(all_events):
-            if "dDurationMs" not in event or "segs" not in event:
-                # some events won't have a duration or segs
-                print(f"skipping subtitle event without content: {event}")
-                continue
-
-            cue = {
-                "start": self._ms_conv(event["tStartMs"]),
-                "end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
-                "text": "".join([i.get("utf8") for i in event["segs"]]),
-                "idx": idx + 1,
-            }
-            self.all_cues.append(cue)
-
-    @staticmethod
-    def _flat_auto_caption(all_events):
-        """flatten autocaption segments"""
-        flatten = []
-        for event in all_events:
-            if "segs" not in event.keys():
-                continue
-            text = "".join([i.get("utf8") for i in event.get("segs")])
-            if not text.strip():
-                continue
-
-            if flatten:
-                # fix overlapping retiming issue
-                last = flatten[-1]
-                if "dDurationMs" not in last or "segs" not in last:
-                    # some events won't have a duration or segs
-                    print(f"skipping subtitle event without content: {event}")
-                    continue
-
-                last_end = last["tStartMs"] + last["dDurationMs"]
-                if event["tStartMs"] < last_end:
-                    joined = last["segs"][0]["utf8"] + "\n" + text
-                    last["segs"][0]["utf8"] = joined
-                    continue
-
-            event.update({"segs": [{"utf8": text}]})
-            flatten.append(event)
-
-        return flatten
-
-    @staticmethod
-    def _ms_conv(ms):
-        """convert ms to timestamp"""
-        hours = str((ms // (1000 * 60 * 60)) % 24).zfill(2)
-        minutes = str((ms // (1000 * 60)) % 60).zfill(2)
-        secs = str((ms // 1000) % 60).zfill(2)
-        millis = str(ms % 1000).zfill(3)
-
-        return f"{hours}:{minutes}:{secs}.{millis}"
-
-    def get_subtitle_str(self):
-        """create vtt text str from cues"""
-        subtitle_str = f"WEBVTT\nKind: captions\nLanguage: {self.lang}"
-
-        for cue in self.all_cues:
-            stamp = f"{cue.get('start')} --> {cue.get('end')}"
-            cue_text = f"\n\n{cue.get('idx')}\n{stamp}\n{cue.get('text')}"
-            subtitle_str = subtitle_str + cue_text
-
-        return subtitle_str
-
-    def create_bulk_import(self, video, source):
-        """subtitle lines for es import"""
-        documents = self._create_documents(video, source)
-        bulk_list = []
-
-        for document in documents:
-            document_id = document.get("subtitle_fragment_id")
-            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(document))
-
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-
-        return query_str
-
-    def _create_documents(self, video, source):
-        """process documents"""
-        documents = self._chunk_list(video.youtube_id)
-        channel = video.json_data.get("channel")
-        meta_dict = {
-            "youtube_id": video.youtube_id,
-            "title": video.json_data.get("title"),
-            "subtitle_channel": channel.get("channel_name"),
-            "subtitle_channel_id": channel.get("channel_id"),
-            "subtitle_last_refresh": int(datetime.now().strftime("%s")),
-            "subtitle_lang": self.lang,
-            "subtitle_source": source,
-        }
-
-        _ = [i.update(meta_dict) for i in documents]
-
-        return documents
-
-    def _chunk_list(self, youtube_id):
-        """join cues for bulk import"""
-        chunk_list = []
-
-        chunk = {}
-        for cue in self.all_cues:
-            if chunk:
-                text = f"{chunk.get('subtitle_line')} {cue.get('text')}\n"
-                chunk["subtitle_line"] = text
-            else:
-                idx = len(chunk_list) + 1
-                chunk = {
-                    "subtitle_index": idx,
-                    "subtitle_line": cue.get("text"),
-                    "subtitle_start": cue.get("start"),
-                }
-
-            chunk["subtitle_fragment_id"] = f"{youtube_id}-{self.lang}-{idx}"
-
-            if cue["idx"] % 5 == 0:
-                chunk["subtitle_end"] = cue.get("end")
-                chunk_list.append(chunk)
-                chunk = {}
-
-        return chunk_list
-
-
 class SponsorBlock:
     """handle sponsor block integration"""
 
@@ -327,7 +33,7 @@ class SponsorBlock:
     def __init__(self, user_id=False):
         self.user_id = user_id
         self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
-        self.last_refresh = int(datetime.now().strftime("%s"))
+        self.last_refresh = int(datetime.now().timestamp())
 
     def get_sb_id(self):
         """get sponsorblock userid or generate if needed"""
@@ -339,7 +45,7 @@ class SponsorBlock:
         sb_id = RedisArchivist().get_message(key)
         if not sb_id["status"]:
             sb_id = {"status": randomizor(32)}
-            RedisArchivist().set_message(key, sb_id, expire=False)
+            RedisArchivist().set_message(key, sb_id)
 
         return sb_id
 
@@ -348,9 +54,17 @@ class SponsorBlock:
         url = f"{self.API}/skipSegments?videoID={youtube_id}"
         headers = {"User-Agent": self.user_agent}
         print(f"{youtube_id}: get sponsorblock timestamps")
-        response = requests.get(url, headers=headers)
+        try:
+            response = requests.get(url, headers=headers, timeout=10)
+        except requests.ReadTimeout:
+            print(f"{youtube_id}: sponsorblock API timeout")
+            return False
+
         if not response.ok:
-            print(f"{youtube_id}: sponsorblock failed: {response.text}")
+            print(f"{youtube_id}: sponsorblock failed: {response.status_code}")
+            if response.status_code == 503:
+                return False
+
             sponsor_dict = {
                 "last_refresh": self.last_refresh,
                 "is_enabled": True,
@@ -364,20 +78,14 @@ class SponsorBlock:
 
     def _get_sponsor_dict(self, all_segments):
         """format and process response"""
-        has_unlocked = False
-        cleaned_segments = []
-        for segment in all_segments:
-            if not segment["locked"]:
-                has_unlocked = True
-            del segment["userID"]
-            del segment["description"]
-            cleaned_segments.append(segment)
+        _ = [i.pop("description", None) for i in all_segments]
+        has_unlocked = not any(i.get("locked") for i in all_segments)
 
         sponsor_dict = {
             "last_refresh": self.last_refresh,
             "has_unlocked": has_unlocked,
             "is_enabled": True,
-            "segments": cleaned_segments,
+            "segments": all_segments,
         }
         return sponsor_dict
 
@@ -420,23 +128,34 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
     index_name = "ta_video"
     yt_base = "https://www.youtube.com/watch?v="
 
-    def __init__(self, youtube_id, video_overwrites=False):
+    def __init__(
+        self,
+        youtube_id,
+        video_overwrites=False,
+        video_type=VideoTypeEnum.VIDEOS,
+    ):
         super().__init__(youtube_id)
         self.channel_id = False
         self.video_overwrites = video_overwrites
-        self.es_path = f"{self.index_name}/_doc/{youtube_id}"
+        self.video_type = video_type
+        self.offline_import = False
 
-    def build_json(self):
+    def build_json(self, youtube_meta_overwrite=False, media_path=False):
         """build json dict of video"""
         self.get_from_youtube()
-        if not self.youtube_meta:
+        if not self.youtube_meta and not youtube_meta_overwrite:
             return
 
+        if not self.youtube_meta:
+            self.youtube_meta = youtube_meta_overwrite
+            self.offline_import = True
+
         self._process_youtube_meta()
         self._add_channel()
         self._add_stats()
         self.add_file_path()
-        self.add_player()
+        self.add_player(media_path)
+        self.add_streams(media_path)
         if self.config["downloads"]["integrate_ryd"]:
             self._get_ryd_stats()
 
@@ -466,7 +185,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         upload_date = self.youtube_meta["upload_date"]
         upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
         published = upload_date_time.strftime("%Y-%m-%d")
-        last_refresh = int(datetime.now().strftime("%s"))
+        last_refresh = int(datetime.now().timestamp())
         # base64_blur = ThumbManager().get_base64_blur(self.youtube_id)
         base64_blur = False
         # build json_data basics
@@ -481,13 +200,15 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             "vid_last_refresh": last_refresh,
             "date_downloaded": last_refresh,
             "youtube_id": self.youtube_id,
+            # Using .value to make json encodable
+            "vid_type": self.video_type.value,
             "active": True,
         }
 
     def _add_channel(self):
         """add channel dict to video json_data"""
         channel = ta_channel.YoutubeChannel(self.channel_id)
-        channel.build_json(upload=True)
+        channel.build_json(upload=True, fallback=self.youtube_meta)
         self.json_data.update({"channel": channel.json_data})
 
     def _add_stats(self):
@@ -495,13 +216,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         # likes
         like_count = self.youtube_meta.get("like_count", 0)
         dislike_count = self.youtube_meta.get("dislike_count", 0)
+        average_rating = self.youtube_meta.get("average_rating", 0)
         self.json_data.update(
             {
                 "stats": {
                     "view_count": self.youtube_meta["view_count"],
                     "like_count": like_count,
                     "dislike_count": dislike_count,
-                    "average_rating": self.youtube_meta["average_rating"],
+                    "average_rating": average_rating,
                 }
             }
         )
@@ -518,8 +240,39 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
         raise FileNotFoundError
 
-    def add_player(self):
+    def add_player(self, media_path=False):
         """add player information for new videos"""
+        vid_path = self._get_vid_path(media_path)
+
+        duration_handler = DurationConverter()
+        duration = duration_handler.get_sec(vid_path)
+        duration_str = duration_handler.get_str(duration)
+        self.json_data.update(
+            {
+                "player": {
+                    "watched": False,
+                    "duration": duration,
+                    "duration_str": duration_str,
+                }
+            }
+        )
+
+    def add_streams(self, media_path=False):
+        """add stream metadata"""
+        vid_path = self._get_vid_path(media_path)
+        media = MediaStreamExtractor(vid_path)
+        self.json_data.update(
+            {
+                "streams": media.extract_metadata(),
+                "media_size": media.get_file_size(),
+            }
+        )
+
+    def _get_vid_path(self, media_path=False):
+        """get path of media file"""
+        if media_path:
+            return media_path
+
         try:
             # when indexing from download task
             vid_path = self.build_dl_cache_path()
@@ -535,18 +288,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             else:
                 raise FileNotFoundError("could not find video file") from err
 
-        duration_handler = DurationConverter()
-        duration = duration_handler.get_sec(vid_path)
-        duration_str = duration_handler.get_str(duration)
-        self.json_data.update(
-            {
-                "player": {
-                    "watched": False,
-                    "duration": duration,
-                    "duration_str": duration_str,
-                }
-            }
-        )
+        return vid_path
 
     def add_file_path(self):
         """build media_url for where file will be located"""
@@ -568,6 +310,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         """delete video file, meta data"""
         print(f"{self.youtube_id}: delete video")
         self.get_from_es()
+        if not self.json_data:
+            raise FileNotFoundError
+
         video_base = self.app_conf["videos"]
         media_url = self.json_data.get("media_url")
         file_path = os.path.join(video_base, media_url)
@@ -579,6 +324,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         self.del_in_playlists()
         self.del_in_es()
         self.delete_subtitles()
+        self.delete_comments()
 
     def del_in_playlists(self):
         """remove downloaded in playlist"""
@@ -603,39 +349,69 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         print(f"{self.youtube_id}: delete subtitles")
         YoutubeSubtitle(self).delete(subtitles=subtitles)
 
+    def delete_comments(self):
+        """delete comments from es"""
+        comments = ta_comments.Comments(self.youtube_id, config=self.config)
+        comments.check_config()
+        if comments.is_activated:
+            comments.delete_comments()
+
     def _get_ryd_stats(self):
         """get optional stats from returnyoutubedislikeapi.com"""
+        # pylint: disable=broad-except
         try:
             print(f"{self.youtube_id}: get ryd stats")
             result = ryd_client.get(self.youtube_id)
-        except requests.exceptions.ConnectionError:
-            print(f"{self.youtube_id}: failed to query ryd api, skipping")
-            return False
+        except Exception as err:
+            print(f"{self.youtube_id}: failed to query ryd api {err}")
+            return
 
         if result["status"] == 404:
-            return False
+            return
 
         dislikes = {
-            "dislike_count": result["dislikes"],
-            "average_rating": result["rating"],
+            "dislike_count": result.get("dislikes", 0),
+            "average_rating": result.get("rating", 0),
         }
         self.json_data["stats"].update(dislikes)
 
-        return True
-
     def _get_sponsorblock(self):
         """get optional sponsorblock timestamps from sponsor.ajay.app"""
         sponsorblock = SponsorBlock().get_timestamps(self.youtube_id)
         if sponsorblock:
             self.json_data["sponsorblock"] = sponsorblock
 
-    def check_subtitles(self):
+    def check_subtitles(self, subtitle_files=False):
         """optionally add subtitles"""
+        if self.offline_import and subtitle_files:
+            indexed = self._offline_subtitles(subtitle_files)
+            self.json_data["subtitles"] = indexed
+            return
+
         handler = YoutubeSubtitle(self)
         subtitles = handler.get_subtitles()
         if subtitles:
-            self.json_data["subtitles"] = subtitles
-            handler.download_subtitles(relevant_subtitles=subtitles)
+            indexed = handler.download_subtitles(relevant_subtitles=subtitles)
+            self.json_data["subtitles"] = indexed
+
+    def _offline_subtitles(self, subtitle_files):
+        """import offline subtitles"""
+        base_name, _ = os.path.splitext(self.json_data["media_url"])
+        subtitles = []
+        for subtitle in subtitle_files:
+            lang = subtitle.split(".")[-2]
+            subtitle_media_url = f"{base_name}.{lang}.vtt"
+            to_add = {
+                "ext": "vtt",
+                "url": False,
+                "name": lang,
+                "lang": lang,
+                "source": "file",
+                "media_url": subtitle_media_url,
+            }
+            subtitles.append(to_add)
+
+        return subtitles
 
     def update_media_url(self):
         """update only media_url in es for reindex channel rename"""
@@ -644,9 +420,13 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         _, _ = ElasticWrap(path).post(data=data)
 
 
-def index_new_video(youtube_id, video_overwrites=False):
+def index_new_video(
+    youtube_id, video_overwrites=False, video_type=VideoTypeEnum.VIDEOS
+):
     """combined classes to create new video in index"""
-    video = YoutubeVideo(youtube_id, video_overwrites=video_overwrites)
+    video = YoutubeVideo(
+        youtube_id, video_overwrites=video_overwrites, video_type=video_type
+    )
     video.build_json()
     if not video.json_data:
         raise ValueError("failed to get metadata for " + youtube_id)

tubearchivist/home/src/index/video_constants.py

@@ -0,0 +1,12 @@
+"""video constants"""
+
+import enum
+
+
+class VideoTypeEnum(enum.Enum):
+    """all vid_type fields"""
+
+    VIDEOS = "videos"
+    STREAMS = "streams"
+    SHORTS = "shorts"
+    UNKNOWN = "unknown"

tubearchivist/home/src/index/video_streams.py

@@ -0,0 +1,132 @@
+"""extract metadata from video streams"""
+
+import json
+import subprocess
+from os import stat
+
+
+class DurationConverter:
+    """
+    using ffmpeg to get and parse duration from filepath
+    """
+
+    @staticmethod
+    def get_sec(file_path):
+        """read duration from file"""
+        duration = subprocess.run(
+            [
+                "ffprobe",
+                "-v",
+                "error",
+                "-show_entries",
+                "format=duration",
+                "-of",
+                "default=noprint_wrappers=1:nokey=1",
+                file_path,
+            ],
+            capture_output=True,
+            check=True,
+        )
+        duration_raw = duration.stdout.decode().strip()
+        if duration_raw == "N/A":
+            return 0
+
+        duration_sec = int(float(duration_raw))
+        return duration_sec
+
+    @staticmethod
+    def get_str(duration_sec):
+        """takes duration in sec and returns clean string"""
+        if not duration_sec:
+            # failed to extract
+            return "NA"
+
+        hours = int(duration_sec // 3600)
+        minutes = int((duration_sec - (hours * 3600)) // 60)
+        secs = int(duration_sec - (hours * 3600) - (minutes * 60))
+
+        duration_str = str()
+        if hours:
+            duration_str = str(hours).zfill(2) + ":"
+        if minutes:
+            duration_str = duration_str + str(minutes).zfill(2) + ":"
+        else:
+            duration_str = duration_str + "00:"
+        duration_str = duration_str + str(secs).zfill(2)
+        return duration_str
+
+
+class MediaStreamExtractor:
+    """extract stream metadata"""
+
+    def __init__(self, media_path):
+        self.media_path = media_path
+        self.metadata = []
+
+    def extract_metadata(self):
+        """entry point to extract metadata"""
+
+        cmd = [
+            "ffprobe",
+            "-v",
+            "quiet",
+            "-print_format",
+            "json",
+            "-show_streams",
+            "-show_format",
+            self.media_path,
+        ]
+        result = subprocess.run(
+            cmd, capture_output=True, text=True, check=False
+        )
+
+        if result.returncode != 0:
+            return self.metadata
+
+        streams = json.loads(result.stdout).get("streams")
+        for stream in streams:
+            self.process_stream(stream)
+
+        return self.metadata
+
+    def process_stream(self, stream):
+        """parse stream to metadata"""
+        codec_type = stream.get("codec_type")
+        if codec_type == "video":
+            self._extract_video_metadata(stream)
+        elif codec_type == "audio":
+            self._extract_audio_metadata(stream)
+        else:
+            return
+
+    def _extract_video_metadata(self, stream):
+        """parse video metadata"""
+        if "bit_rate" not in stream:
+            # is probably thumbnail
+            return
+
+        self.metadata.append(
+            {
+                "type": "video",
+                "index": stream["index"],
+                "codec": stream["codec_name"],
+                "width": stream["width"],
+                "height": stream["height"],
+                "bitrate": int(stream["bit_rate"]),
+            }
+        )
+
+    def _extract_audio_metadata(self, stream):
+        """extract audio metadata"""
+        self.metadata.append(
+            {
+                "type": "audio",
+                "index": stream["index"],
+                "codec": stream["codec_name"],
+                "bitrate": int(stream["bit_rate"]),
+            }
+        )
+
+    def get_file_size(self):
+        """get filesize in bytes"""
+        return stat(self.media_path).st_size

tubearchivist/home/src/ta/config.py

@@ -7,8 +7,11 @@ Functionality:
 import json
 import os
 import re
+from random import randint
 
+import requests
 from celery.schedules import crontab
+from django.conf import settings
 from home.src.ta.ta_redis import RedisArchivist
 
 
@@ -47,18 +50,6 @@ class AppConfig:
     @staticmethod
     def get_config_env():
         """read environment application variables"""
-        host_uid_env = os.environ.get("HOST_UID")
-        if host_uid_env:
-            host_uid = int(host_uid_env)
-        else:
-            host_uid = False
-
-        host_gid_env = os.environ.get("HOST_GID")
-        if host_gid_env:
-            host_gid = int(host_gid_env)
-        else:
-            host_gid = False
-
         es_pass = os.environ.get("ELASTIC_PASSWORD")
         es_user = os.environ.get("ELASTIC_USER", default="elastic")
 
@@ -66,8 +57,9 @@ class AppConfig:
             "REDIS_HOST": os.environ.get("REDIS_HOST"),
             "es_url": os.environ.get("ES_URL"),
             "es_auth": (es_user, es_pass),
-            "HOST_UID": host_uid,
-            "HOST_GID": host_gid,
+            "HOST_UID": int(os.environ.get("HOST_UID", False)),
+            "HOST_GID": int(os.environ.get("HOST_GID", False)),
+            "enable_cast": bool(os.environ.get("ENABLE_CAST")),
         }
 
         return application
@@ -99,7 +91,7 @@ class AppConfig:
             self.config[config_dict][config_value] = to_write
             updated.append((config_value, to_write))
 
-        RedisArchivist().set_message("config", self.config, expire=False)
+        RedisArchivist().set_message("config", self.config)
         return updated
 
     @staticmethod
@@ -111,7 +103,7 @@ class AppConfig:
 
             message = {"status": value}
             redis_key = f"{user_id}:{key}"
-            RedisArchivist().set_message(redis_key, message, expire=False)
+            RedisArchivist().set_message(redis_key, message)
 
     def get_colors(self):
         """overwrite config if user has set custom values"""
@@ -126,6 +118,15 @@ class AppConfig:
         self.config["application"]["colors"] = colors
         return colors
 
+    @staticmethod
+    def _build_rand_daily():
+        """build random daily schedule per installation"""
+        return {
+            "minute": randint(0, 59),
+            "hour": randint(0, 23),
+            "day_of_week": "*",
+        }
+
     def load_new_defaults(self):
         """check config.json for missing defaults"""
         default_config = self.get_config_file()
@@ -133,7 +134,10 @@ class AppConfig:
 
         # check for customizations
         if not redis_config:
-            return
+            config = self.get_config()
+            config["scheduler"]["version_check"] = self._build_rand_daily()
+            RedisArchivist().set_message("config", config)
+            return False
 
         needs_update = False
 
@@ -147,11 +151,16 @@ class AppConfig:
             # missing nested values
             for sub_key, sub_value in value.items():
                 if sub_key not in redis_config[key].keys():
+                    if sub_value == "rand-d":
+                        sub_value = self._build_rand_daily()
+
                     redis_config[key].update({sub_key: sub_value})
                     needs_update = True
 
         if needs_update:
-            RedisArchivist().set_message("config", redis_config, expire=False)
+            RedisArchivist().set_message("config", redis_config)
+
+        return needs_update
 
 
 class ScheduleBuilder:
@@ -163,8 +172,10 @@ class ScheduleBuilder:
         "check_reindex": "0 12 *",
         "thumbnail_check": "0 17 *",
         "run_backup": "0 18 0",
+        "version_check": "0 11 *",
     }
     CONFIG = ["check_reindex_days", "run_backup_rotate"]
+    MSG = "message:setting"
 
     def __init__(self):
         self.config = AppConfig().config
@@ -180,25 +191,27 @@ class ScheduleBuilder:
                 except ValueError:
                     print(f"failed: {key} {value}")
                     mess_dict = {
-                        "status": "message:setting",
+                        "status": self.MSG,
                         "level": "error",
                         "title": "Scheduler update failed.",
                         "message": "Invalid schedule input",
                     }
-                    RedisArchivist().set_message("message:setting", mess_dict)
+                    RedisArchivist().set_message(
+                        self.MSG, mess_dict, expire=True
+                    )
                     return
 
                 redis_config["scheduler"][key] = to_write
             if key in self.CONFIG and value:
                 redis_config["scheduler"][key] = int(value)
-        RedisArchivist().set_message("config", redis_config, expire=False)
+        RedisArchivist().set_message("config", redis_config)
         mess_dict = {
-            "status": "message:setting",
+            "status": self.MSG,
             "level": "info",
             "title": "Scheduler changed.",
             "message": "Please restart container for changes to take effect",
         }
-        RedisArchivist().set_message("message:setting", mess_dict)
+        RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
 
     def value_builder(self, key, value):
         """validate single cron form entry and return cron dict"""
@@ -252,6 +265,8 @@ class ScheduleBuilder:
 
     def build_schedule(self):
         """build schedule dict as expected by app.conf.beat_schedule"""
+        AppConfig().load_new_defaults()
+        self.config = AppConfig().config
         schedule_dict = {}
 
         for schedule_item in self.SCHEDULES:
@@ -259,18 +274,90 @@ class ScheduleBuilder:
             if not item_conf:
                 continue
 
-            minute = item_conf["minute"]
-            hour = item_conf["hour"]
-            day_of_week = item_conf["day_of_week"]
-            schedule_name = f"schedule_{schedule_item}"
-            to_add = {
-                schedule_name: {
-                    "task": schedule_item,
-                    "schedule": crontab(
-                        minute=minute, hour=hour, day_of_week=day_of_week
-                    ),
+            schedule_dict.update(
+                {
+                    f"schedule_{schedule_item}": {
+                        "task": schedule_item,
+                        "schedule": crontab(
+                            minute=item_conf["minute"],
+                            hour=item_conf["hour"],
+                            day_of_week=item_conf["day_of_week"],
+                        ),
+                    }
                 }
-            }
-            schedule_dict.update(to_add)
+            )
 
         return schedule_dict
+
+
+class ReleaseVersion:
+    """compare local version with remote version"""
+
+    REMOTE_URL = "https://www.tubearchivist.com/api/release/latest/"
+    NEW_KEY = "versioncheck:new"
+
+    def __init__(self):
+        self.local_version = self._parse_version(settings.TA_VERSION)
+        self.is_unstable = settings.TA_VERSION.endswith("-unstable")
+        self.remote_version = False
+        self.is_breaking = False
+        self.response = False
+
+    def check(self):
+        """check version"""
+        print(f"[{self.local_version}]: look for updates")
+        self.get_remote_version()
+        new_version, is_breaking = self._has_update()
+        if new_version:
+            message = {
+                "status": True,
+                "version": new_version,
+                "is_breaking": is_breaking,
+            }
+            RedisArchivist().set_message(self.NEW_KEY, message)
+            print(f"[{self.local_version}]: found new version {new_version}")
+
+    def get_remote_version(self):
+        """read version from remote"""
+        self.response = requests.get(self.REMOTE_URL, timeout=20).json()
+        remote_version_str = self.response["release_version"]
+        self.remote_version = self._parse_version(remote_version_str)
+        self.is_breaking = self.response["breaking_changes"]
+
+    def _has_update(self):
+        """check if there is an update"""
+        for idx, number in enumerate(self.local_version):
+            is_newer = self.remote_version[idx] > number
+            if is_newer:
+                return self.response["release_version"], self.is_breaking
+
+        if self.is_unstable and self.local_version == self.remote_version:
+            return self.response["release_version"], self.is_breaking
+
+        return False, False
+
+    @staticmethod
+    def _parse_version(version):
+        """return version parts"""
+        clean = version.rstrip("-unstable").lstrip("v")
+        return tuple((int(i) for i in clean.split(".")))
+
+    def is_updated(self):
+        """check if update happened in the mean time"""
+        message = self.get_update()
+        if not message:
+            return False
+
+        if self._parse_version(message.get("version")) == self.local_version:
+            RedisArchivist().del_message(self.NEW_KEY)
+            return settings.TA_VERSION
+
+        return False
+
+    def get_update(self):
+        """return new version dict if available"""
+        message = RedisArchivist().get_message(self.NEW_KEY)
+        if not message.get("status"):
+            return False
+
+        return message

tubearchivist/home/src/ta/helper.py

@@ -3,31 +3,32 @@ Loose collection of helper functions
 - don't import AppConfig class here to avoid circular imports
 """
 
+import json
+import os
 import random
 import re
 import string
-import subprocess
 import unicodedata
 from datetime import datetime
-from urllib.parse import parse_qs, urlparse
+from urllib.parse import urlparse
 
-from home.src.download.yt_dlp_base import YtWrap
+import requests
 
 
-def clean_string(file_name):
+def clean_string(file_name: str) -> str:
     """clean string to only asci characters"""
     whitelist = "-_.() " + string.ascii_letters + string.digits
     normalized = unicodedata.normalize("NFKD", file_name)
     ascii_only = normalized.encode("ASCII", "ignore").decode().strip()
-    white_listed = "".join(c for c in ascii_only if c in whitelist)
-    cleaned = re.sub(r"[ ]{2,}", " ", white_listed)
+    white_listed: str = "".join(c for c in ascii_only if c in whitelist)
+    cleaned: str = re.sub(r"[ ]{2,}", " ", white_listed)
     return cleaned
 
 
-def ignore_filelist(filelist):
+def ignore_filelist(filelist: list[str]) -> list[str]:
     """ignore temp files for os.listdir sanitizer"""
     to_ignore = ["Icon\r\r", "Temporary Items", "Network Trash Folder"]
-    cleaned = []
+    cleaned: list[str] = []
     for file_name in filelist:
         if file_name.startswith(".") or file_name in to_ignore:
             continue
@@ -37,13 +38,13 @@ def ignore_filelist(filelist):
     return cleaned
 
 
-def randomizor(length):
+def randomizor(length: int) -> str:
     """generate random alpha numeric string"""
-    pool = string.digits + string.ascii_letters
+    pool: str = string.digits + string.ascii_letters
     return "".join(random.choice(pool) for i in range(length))
 
 
-def requests_headers():
+def requests_headers() -> dict[str, str]:
     """build header with random user agent for requests outside of yt-dlp"""
 
     chrome_versions = (
@@ -95,7 +96,7 @@ def requests_headers():
     return {"User-Agent": template}
 
 
-def date_praser(timestamp):
+def date_praser(timestamp: int | str) -> str:
     """return formatted date string"""
     if isinstance(timestamp, int):
         date_obj = datetime.fromtimestamp(timestamp)
@@ -105,146 +106,59 @@ def date_praser(timestamp):
     return datetime.strftime(date_obj, "%d %b, %Y")
 
 
-class UrlListParser:
-    """take a multi line string and detect valid youtube ids"""
+def time_parser(timestamp: str) -> float:
+    """return seconds from timestamp, false on empty"""
+    if not timestamp:
+        return False
 
-    def __init__(self, url_str):
-        self.url_list = [i.strip() for i in url_str.split()]
+    if timestamp.isnumeric():
+        return int(timestamp)
 
-    def process_list(self):
-        """loop through the list"""
-        youtube_ids = []
-        for url in self.url_list:
-            parsed = urlparse(url)
-            print(f"processing: {url}")
-            print(parsed)
-            if not parsed.netloc:
-                # is not a url
-                id_type = self.find_valid_id(url)
-                youtube_id = url
-            elif "youtube.com" not in url and "youtu.be" not in url:
-                raise ValueError(f"{url} is not a youtube link")
-            elif parsed.path:
-                # is a url
-                youtube_id, id_type = self.detect_from_url(parsed)
-            else:
-                # not detected
-                raise ValueError(f"failed to detect {url}")
-
-            youtube_ids.append({"url": youtube_id, "type": id_type})
-
-        return youtube_ids
-
-    def detect_from_url(self, parsed):
-        """detect from parsed url"""
-        if parsed.netloc == "youtu.be":
-            # shortened
-            youtube_id = parsed.path.strip("/")
-            _ = self.find_valid_id(youtube_id)
-            return youtube_id, "video"
-
-        if parsed.query:
-            # detect from query string
-            query_parsed = parse_qs(parsed.query)
-            if "v" in query_parsed.keys():
-                youtube_id = query_parsed["v"][0]
-                _ = self.find_valid_id(youtube_id)
-                return youtube_id, "video"
-
-            if "list" in query_parsed.keys():
-                youtube_id = query_parsed["list"][0]
-                return youtube_id, "playlist"
-
-        if parsed.path.startswith("/channel/"):
-            # channel id in url
-            youtube_id = parsed.path.split("/")[2]
-            _ = self.find_valid_id(youtube_id)
-            return youtube_id, "channel"
-
-        # dedect channel with yt_dlp
-        youtube_id = self.extract_channel_name(parsed.geturl())
-        return youtube_id, "channel"
-
-    @staticmethod
-    def find_valid_id(id_str):
-        """dedect valid id from length of string"""
-        str_len = len(id_str)
-        if str_len == 11:
-            id_type = "video"
-        elif str_len == 24:
-            id_type = "channel"
-        elif str_len in [34, 18] or id_str in ["LL", "WL"]:
-            id_type = "playlist"
-        else:
-            # unable to parse
-            raise ValueError("not a valid id_str: " + id_str)
-
-        return id_type
-
-    @staticmethod
-    def extract_channel_name(url):
-        """find channel id from channel name with yt-dlp help"""
-        obs_request = {
-            "skip_download": True,
-            "extract_flat": True,
-            "playlistend": 0,
-        }
-        url_info = YtWrap(obs_request).extract(url)
-        try:
-            channel_id = url_info["channel_id"]
-        except KeyError as error:
-            print(f"failed to extract channel id from {url}")
-            raise ValueError from error
-
-        return channel_id
+    hours, minutes, seconds = timestamp.split(":", maxsplit=3)
+    return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
 
 
-class DurationConverter:
-    """
-    using ffmpeg to get and parse duration from filepath
-    """
+def clear_dl_cache(config: dict) -> int:
+    """clear leftover files from dl cache"""
+    print("clear download cache")
+    cache_dir = os.path.join(config["application"]["cache_dir"], "download")
+    leftover_files = os.listdir(cache_dir)
+    for cached in leftover_files:
+        to_delete = os.path.join(cache_dir, cached)
+        os.remove(to_delete)
 
-    @staticmethod
-    def get_sec(file_path):
-        """read duration from file"""
-        duration = subprocess.run(
-            [
-                "ffprobe",
-                "-v",
-                "error",
-                "-show_entries",
-                "format=duration",
-                "-of",
-                "default=noprint_wrappers=1:nokey=1",
-                file_path,
-            ],
-            capture_output=True,
-            check=True,
-        )
-        duration_raw = duration.stdout.decode().strip()
-        if duration_raw == "N/A":
-            return 0
+    return len(leftover_files)
 
-        duration_sec = int(float(duration_raw))
-        return duration_sec
 
-    @staticmethod
-    def get_str(duration_sec):
-        """takes duration in sec and returns clean string"""
-        if not duration_sec:
-            # failed to extract
-            return "NA"
+def get_mapping() -> dict:
+    """read index_mapping.json and get expected mapping and settings"""
+    with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
+        index_config: dict = json.load(f).get("index_config")
 
-        hours = duration_sec // 3600
-        minutes = (duration_sec - (hours * 3600)) // 60
-        secs = duration_sec - (hours * 3600) - (minutes * 60)
+    return index_config
 
-        duration_str = str()
-        if hours:
-            duration_str = str(hours).zfill(2) + ":"
-        if minutes:
-            duration_str = duration_str + str(minutes).zfill(2) + ":"
-        else:
-            duration_str = duration_str + "00:"
-        duration_str = duration_str + str(secs).zfill(2)
-        return duration_str
+
+def is_shorts(youtube_id: str) -> bool:
+    """check if youtube_id is a shorts video, bot not it it's not a shorts"""
+    shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
+    response = requests.head(
+        shorts_url, headers=requests_headers(), timeout=10
+    )
+
+    return response.status_code == 200
+
+
+def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
+    """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
+    allowed_hosts: list[str] = []
+    csrf_trusted_origins: list[str] = []
+    for host in ta_host.split():
+        host_clean = host.strip()
+        if not host_clean.startswith("http"):
+            host_clean = f"http://{host_clean}"
+
+        parsed = urlparse(host_clean)
+        allowed_hosts.append(f"{parsed.hostname}")
+        csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}")
+
+    return allowed_hosts, csrf_trusted_origins

tubearchivist/home/src/ta/ta_redis.py

@@ -2,6 +2,7 @@
 functionality:
 - interact with redis
 - hold temporary download queue in redis
+- interact with celery tasks results
 """
 
 import json
@@ -13,9 +14,9 @@ import redis
 class RedisBase:
     """connection base for redis"""
 
-    REDIS_HOST = os.environ.get("REDIS_HOST")
-    REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
-    NAME_SPACE = "ta:"
+    REDIS_HOST: str = str(os.environ.get("REDIS_HOST"))
+    REDIS_PORT: int = int(os.environ.get("REDIS_PORT") or 6379)
+    NAME_SPACE: str = "ta:"
 
     def __init__(self):
         self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
@@ -24,7 +25,7 @@ class RedisBase:
 class RedisArchivist(RedisBase):
     """collection of methods to interact with redis"""
 
-    CHANNELS = [
+    CHANNELS: list[str] = [
         "download",
         "add",
         "rescan",
@@ -34,73 +35,63 @@ class RedisArchivist(RedisBase):
         "setting",
     ]
 
-    def set_message(self, key, message, expire=True):
+    def set_message(
+        self,
+        key: str,
+        message: dict,
+        path: str = ".",
+        expire: bool | int = False,
+    ) -> None:
         """write new message to redis"""
         self.conn.execute_command(
-            "JSON.SET", self.NAME_SPACE + key, ".", json.dumps(message)
+            "JSON.SET", self.NAME_SPACE + key, path, json.dumps(message)
         )
 
         if expire:
             if isinstance(expire, bool):
-                secs = 20
+                secs: int = 20
             else:
                 secs = expire
             self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
 
-    def get_message(self, key):
+    def get_message(self, key: str) -> dict:
         """get message dict from redis"""
         reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
         if reply:
-            json_str = json.loads(reply)
-        else:
-            json_str = {"status": False}
+            return json.loads(reply)
 
-        return json_str
+        return {"status": False}
 
-    def list_items(self, query):
-        """list all matches"""
+    def list_keys(self, query: str) -> list:
+        """return all key matches"""
         reply = self.conn.execute_command(
             "KEYS", self.NAME_SPACE + query + "*"
         )
-        all_matches = [i.decode().lstrip(self.NAME_SPACE) for i in reply]
-        all_results = []
-        for match in all_matches:
-            json_str = self.get_message(match)
-            all_results.append(json_str)
+        if not reply:
+            return []
 
-        return all_results
+        return [i.decode().lstrip(self.NAME_SPACE) for i in reply]
 
-    def del_message(self, key):
+    def list_items(self, query: str) -> list:
+        """list all matches"""
+        all_matches = self.list_keys(query)
+        if not all_matches:
+            return []
+
+        return [self.get_message(i) for i in all_matches]
+
+    def del_message(self, key: str) -> bool:
         """delete key from redis"""
         response = self.conn.execute_command("DEL", self.NAME_SPACE + key)
         return response
 
-    def get_lock(self, lock_key):
-        """handle lock for task management"""
-        redis_lock = self.conn.lock(self.NAME_SPACE + lock_key)
-        return redis_lock
-
-    def get_progress(self):
-        """get a list of all progress messages"""
-        all_messages = []
-        for channel in self.CHANNELS:
-            key = "message:" + channel
-            reply = self.conn.execute_command(
-                "JSON.GET", self.NAME_SPACE + key
-            )
-            if reply:
-                json_str = json.loads(reply)
-                all_messages.append(json_str)
-
-        return all_messages
-
 
 class RedisQueue(RedisBase):
-    """dynamically interact with the download queue in redis"""
+    """dynamically interact with queues in redis"""
 
-    def __init__(self):
+    def __init__(self, queue_name: str):
         super().__init__()
-        self.key = self.NAME_SPACE + "dl_queue"
+        self.key = f"{self.NAME_SPACE}{queue_name}"
 
     def get_all(self):
         """return all elements in list"""
@@ -108,16 +99,29 @@ class RedisQueue(RedisBase):
         all_elements = [i.decode() for i in result]
         return all_elements
 
+    def length(self) -> int:
+        """return total elements in list"""
+        return self.conn.execute_command("LLEN", self.key)
+
+    def in_queue(self, element) -> str | bool:
+        """check if element is in list"""
+        result = self.conn.execute_command("LPOS", self.key, element)
+        if result is not None:
+            return "in_queue"
+
+        return False
+
     def add_list(self, to_add):
         """add list to queue"""
         self.conn.execute_command("RPUSH", self.key, *to_add)
 
-    def add_priority(self, to_add):
+    def add_priority(self, to_add: str) -> None:
         """add single video to front of queue"""
-        self.clear_item(to_add)
-        self.conn.execute_command("LPUSH", self.key, to_add)
+        item: str = json.dumps(to_add)
+        self.clear_item(item)
+        self.conn.execute_command("LPUSH", self.key, item)
 
-    def get_next(self):
+    def get_next(self) -> str | bool:
         """return next element in the queue, False if none"""
         result = self.conn.execute_command("LPOP", self.key)
         if not result:
@@ -126,14 +130,74 @@ class RedisQueue(RedisBase):
         next_element = result.decode()
         return next_element
 
-    def clear(self):
+    def clear(self) -> None:
         """delete list from redis"""
         self.conn.execute_command("DEL", self.key)
 
-    def clear_item(self, to_clear):
+    def clear_item(self, to_clear: str) -> None:
         """remove single item from list if it's there"""
         self.conn.execute_command("LREM", self.key, 0, to_clear)
 
-    def trim(self, size):
+    def trim(self, size: int) -> None:
         """trim the queue based on settings amount"""
         self.conn.execute_command("LTRIM", self.key, 0, size)
+
+    def has_item(self) -> bool:
+        """check if queue as at least one pending item"""
+        result = self.conn.execute_command("LRANGE", self.key, 0, 0)
+        return bool(result)
+
+
+class TaskRedis(RedisBase):
+    """interact with redis tasks"""
+
+    BASE: str = "celery-task-meta-"
+    EXPIRE: int = 60 * 60 * 24
+    COMMANDS: list[str] = ["STOP", "KILL"]
+
+    def get_all(self) -> list:
+        """return all tasks"""
+        all_keys = self.conn.execute_command("KEYS", f"{self.BASE}*")
+        return [i.decode().replace(self.BASE, "") for i in all_keys]
+
+    def get_single(self, task_id: str) -> dict:
+        """return content of single task"""
+        result = self.conn.execute_command("GET", self.BASE + task_id)
+        if not result:
+            return {}
+
+        return json.loads(result.decode())
+
+    def set_key(
+        self, task_id: str, message: dict, expire: bool | int = False
+    ) -> None:
+        """set value for lock, initial or update"""
+        key: str = f"{self.BASE}{task_id}"
+        self.conn.execute_command("SET", key, json.dumps(message))
+
+        if expire:
+            self.conn.execute_command("EXPIRE", key, self.EXPIRE)
+
+    def set_command(self, task_id: str, command: str) -> None:
+        """set task command"""
+        if command not in self.COMMANDS:
+            print(f"{command} not in valid commands {self.COMMANDS}")
+            raise ValueError
+
+        message = self.get_single(task_id)
+        if not message:
+            print(f"{task_id} not found")
+            raise KeyError
+
+        message.update({"command": command})
+        self.set_key(task_id, message)
+
+    def del_task(self, task_id: str) -> None:
+        """delete task result by id"""
+        self.conn.execute_command("DEL", f"{self.BASE}{task_id}")
+
+    def del_all(self) -> None:
+        """delete all task results"""
+        all_tasks = self.get_all()
+        for task_id in all_tasks:
+            self.del_task(task_id)

tubearchivist/home/src/ta/task_manager.py

@@ -0,0 +1,116 @@
+"""
+functionality:
+- interact with in redis stored task results
+- handle threads and locks
+"""
+
+from home import tasks as ta_tasks
+from home.src.ta.ta_redis import RedisArchivist, TaskRedis
+
+
+class TaskManager:
+    """manage tasks"""
+
+    def get_all_results(self):
+        """return all task results"""
+        handler = TaskRedis()
+        all_keys = handler.get_all()
+        if not all_keys:
+            return False
+
+        return [handler.get_single(i) for i in all_keys]
+
+    def get_tasks_by_name(self, task_name):
+        """get all tasks by name"""
+        all_results = self.get_all_results()
+        if not all_results:
+            return False
+
+        return [i for i in all_results if i.get("name") == task_name]
+
+    def get_task(self, task_id):
+        """get single task"""
+        return TaskRedis().get_single(task_id)
+
+    def is_pending(self, task):
+        """check if task_name is pending, pass task object"""
+        tasks = self.get_tasks_by_name(task.name)
+        if not tasks:
+            return False
+
+        return bool([i for i in tasks if i.get("status") == "PENDING"])
+
+    def is_stopped(self, task_id):
+        """check if task_id has received STOP command"""
+        task = self.get_task(task_id)
+
+        return task.get("command") == "STOP"
+
+    def get_pending(self, task_name):
+        """get all pending tasks of task_name"""
+        tasks = self.get_tasks_by_name(task_name)
+        if not tasks:
+            return False
+
+        return [i for i in tasks if i.get("status") == "PENDING"]
+
+    def init(self, task):
+        """pass task object from bind task to set initial pending message"""
+        message = {
+            "status": "PENDING",
+            "result": None,
+            "traceback": None,
+            "date_done": False,
+            "name": task.name,
+            "task_id": task.request.id,
+        }
+        TaskRedis().set_key(task.request.id, message)
+
+    def fail_pending(self):
+        """
+        mark all pending as failed,
+        run at startup to recover from hard reset
+        """
+        all_results = self.get_all_results()
+        if not all_results:
+            return
+
+        for result in all_results:
+            if result.get("status") == "PENDING":
+                result["status"] = "FAILED"
+                TaskRedis().set_key(result["task_id"], result, expire=True)
+
+
+class TaskCommand:
+    """run commands on task"""
+
+    def start(self, task_name):
+        """start task by task_name, only pass task that don't take args"""
+        task = ta_tasks.app.tasks.get(task_name).delay()
+        message = {
+            "task_id": task.id,
+            "status": task.status,
+            "task_name": task.name,
+        }
+
+        return message
+
+    def stop(self, task_id, message_key):
+        """
+        send stop signal to task_id,
+        needs to be implemented in task to take effect
+        """
+        print(f"[task][{task_id}]: received STOP signal.")
+        handler = TaskRedis()
+
+        task = handler.get_single(task_id)
+        if not task["name"] in ta_tasks.BaseTask.TASK_CONFIG:
+            raise ValueError
+
+        handler.set_command(task_id, "STOP")
+        RedisArchivist().set_message(message_key, "STOP", path=".command")
+
+    def kill(self, task_id):
+        """send kill signal to task_id"""
+        print(f"[task][{task_id}]: received KILL signal.")
+        ta_tasks.app.control.revoke(task_id, terminate=True)

tubearchivist/home/src/ta/urlparser.py

@@ -0,0 +1,134 @@
+"""
+Functionality:
+- detect valid youtube ids and links from multi line string
+- identify vid_type if possible
+"""
+
+from urllib.parse import parse_qs, urlparse
+
+from home.src.download.yt_dlp_base import YtWrap
+from home.src.index.video_constants import VideoTypeEnum
+
+
+class Parser:
+    """take a multi line string and detect valid youtube ids"""
+
+    def __init__(self, url_str):
+        self.url_list = [i.strip() for i in url_str.split()]
+
+    def parse(self):
+        """parse the list"""
+        ids = []
+        for url in self.url_list:
+            parsed = urlparse(url)
+            if parsed.netloc:
+                # is url
+                identified = self.process_url(parsed)
+            else:
+                # is not url
+                identified = self._find_valid_id(url)
+
+            if "vid_type" not in identified:
+                identified.update(self._detect_vid_type(parsed.path))
+
+            ids.append(identified)
+
+        return ids
+
+    def process_url(self, parsed):
+        """process as url"""
+        if parsed.netloc == "youtu.be":
+            # shortened
+            youtube_id = parsed.path.strip("/")
+            return self._validate_expected(youtube_id, "video")
+
+        query_parsed = parse_qs(parsed.query)
+        if "v" in query_parsed:
+            # video from v query str
+            youtube_id = query_parsed["v"][0]
+            return self._validate_expected(youtube_id, "video")
+
+        if "list" in query_parsed:
+            # playlist from list query str
+            youtube_id = query_parsed["list"][0]
+            return self._validate_expected(youtube_id, "playlist")
+
+        all_paths = parsed.path.strip("/").split("/")
+        if all_paths[0] == "shorts":
+            # is shorts video
+            item = self._validate_expected(all_paths[1], "video")
+            item.update({"vid_type": VideoTypeEnum.SHORTS.value})
+            return item
+
+        if all_paths[0] == "channel":
+            return self._validate_expected(all_paths[1], "channel")
+
+        # detect channel
+        channel_id = self._extract_channel_name(parsed.geturl())
+        return {"type": "channel", "url": channel_id}
+
+    def _validate_expected(self, youtube_id, expected_type):
+        """raise value error if not matching"""
+        matched = self._find_valid_id(youtube_id)
+        if matched["type"] != expected_type:
+            raise ValueError(
+                f"{youtube_id} not of expected type {expected_type}"
+            )
+
+        return {"type": expected_type, "url": youtube_id}
+
+    def _find_valid_id(self, id_str):
+        """detect valid id from length of string"""
+        if id_str in ("LL", "WL"):
+            return {"type": "playlist", "url": id_str}
+
+        if id_str.startswith("@"):
+            url = f"https://www.youtube.com/{id_str}"
+            channel_id = self._extract_channel_name(url)
+            return {"type": "channel", "url": channel_id}
+
+        len_id_str = len(id_str)
+        if len_id_str == 11:
+            item_type = "video"
+        elif len_id_str == 24:
+            item_type = "channel"
+        elif len_id_str in (34, 26, 18):
+            item_type = "playlist"
+        else:
+            raise ValueError(f"not a valid id_str: {id_str}")
+
+        return {"type": item_type, "url": id_str}
+
+    @staticmethod
+    def _extract_channel_name(url):
+        """find channel id from channel name with yt-dlp help"""
+        obs_request = {
+            "check_formats": None,
+            "skip_download": True,
+            "extract_flat": True,
+            "playlistend": 0,
+        }
+        url_info = YtWrap(obs_request).extract(url)
+        channel_id = url_info.get("channel_id", False)
+        if channel_id:
+            return channel_id
+
+        url = url_info.get("url", False)
+        if url:
+            # handle old channel name redirect with url path split
+            channel_id = urlparse(url).path.strip("/").split("/")[1]
+
+            return channel_id
+
+        print(f"failed to extract channel id from {url}")
+        raise ValueError
+
+    def _detect_vid_type(self, path):
+        """try to match enum from path, needs to be serializable"""
+        last = path.strip("/").split("/")[-1]
+        try:
+            vid_type = VideoTypeEnum(last).value
+        except ValueError:
+            vid_type = VideoTypeEnum.UNKNOWN.value
+
+        return {"vid_type": vid_type}

tubearchivist/home/tasks.py

@@ -8,271 +8,336 @@ Functionality:
 
 import os
 
-from celery import Celery, shared_task
-from home.apps import StartupCheck
+from celery import Celery, Task, shared_task
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import (
-    ChannelSubscription,
-    PlaylistSubscription,
+    SubscriptionHandler,
+    SubscriptionScanner,
 )
-from home.src.download.thumbnails import ThumbManager, validate_thumbnails
+from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator
 from home.src.download.yt_dlp_handler import VideoDownloader
-from home.src.es.index_setup import backup_all_indexes, restore_from_backup
+from home.src.es.backup import ElasticBackup
+from home.src.es.index_setup import ElasitIndexWrap
 from home.src.index.channel import YoutubeChannel
-from home.src.index.filesystem import (
-    ManualImport,
-    reindex_old_documents,
-    scan_filesystem,
-)
-from home.src.ta.config import AppConfig, ScheduleBuilder
-from home.src.ta.helper import UrlListParser
-from home.src.ta.ta_redis import RedisArchivist, RedisQueue
+from home.src.index.filesystem import Filesystem
+from home.src.index.manual import ImportFolderScanner
+from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
+from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
+from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.task_manager import TaskManager
+from home.src.ta.urlparser import Parser
 
 CONFIG = AppConfig().config
 REDIS_HOST = os.environ.get("REDIS_HOST")
 REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
-app = Celery("tasks", broker=f"redis://{REDIS_HOST}:{REDIS_PORT}")
+app = Celery(
+    "tasks",
+    broker=f"redis://{REDIS_HOST}:{REDIS_PORT}",
+    backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
+    result_extended=True,
+)
 app.config_from_object("django.conf:settings", namespace="ta:")
 app.autodiscover_tasks()
 app.conf.timezone = os.environ.get("TZ") or "UTC"
 
 
-@shared_task(name="update_subscribed")
-def update_subscribed():
-    """look for missing videos and add to pending"""
-    message = {
-        "status": "message:rescan",
-        "level": "info",
-        "title": "Rescanning channels and playlists.",
-        "message": "Looking for new videos.",
+class BaseTask(Task):
+    """base class to inherit each class from"""
+
+    # pylint: disable=abstract-method
+
+    TASK_CONFIG = {
+        "update_subscribed": {
+            "title": "Rescan your Subscriptions",
+            "group": "download:scan",
+            "api-start": True,
+            "api-stop": True,
+        },
+        "download_pending": {
+            "title": "Downloading",
+            "group": "download:run",
+            "api-start": True,
+            "api-stop": True,
+        },
+        "extract_download": {
+            "title": "Add to download queue",
+            "group": "download:add",
+            "api-stop": True,
+        },
+        "check_reindex": {
+            "title": "Reindex Documents",
+            "group": "reindex:run",
+        },
+        "manual_import": {
+            "title": "Manual video import",
+            "group": "setting:import",
+            "api-start": True,
+        },
+        "run_backup": {
+            "title": "Index Backup",
+            "group": "setting:backup",
+            "api-start": True,
+        },
+        "restore_backup": {
+            "title": "Restore Backup",
+            "group": "setting:restore",
+        },
+        "rescan_filesystem": {
+            "title": "Rescan your Filesystem",
+            "group": "setting:filesystemscan",
+            "api-start": True,
+        },
+        "thumbnail_check": {
+            "title": "Check your Thumbnails",
+            "group": "setting:thumbnailcheck",
+            "api-start": True,
+        },
+        "resync_thumbs": {
+            "title": "Sync Thumbnails to Media Files",
+            "group": "setting:thumbnailsync",
+            "api-start": True,
+        },
+        "index_playlists": {
+            "title": "Index Channel Playlist",
+            "group": "channel:indexplaylist",
+        },
+        "subscribe_to": {
+            "title": "Add Subscription",
+            "group": "subscription:add",
+        },
     }
-    RedisArchivist().set_message("message:rescan", message)
 
-    have_lock = False
-    my_lock = RedisArchivist().get_lock("rescan")
+    def on_failure(self, exc, task_id, args, kwargs, einfo):
+        """callback for task failure"""
+        print(f"{task_id} Failed callback")
+        message, key = self._build_message(level="error")
+        message.update({"messages": ["Task failed"]})
+        RedisArchivist().set_message(key, message, expire=20)
 
-    try:
-        have_lock = my_lock.acquire(blocking=False)
-        if have_lock:
-            channel_handler = ChannelSubscription()
-            missing_from_channels = channel_handler.find_missing()
-            playlist_handler = PlaylistSubscription()
-            missing_from_playlists = playlist_handler.find_missing()
-            missing = missing_from_channels + missing_from_playlists
-            if missing:
-                youtube_ids = [{"type": "video", "url": i} for i in missing]
-                pending_handler = PendingList(youtube_ids=youtube_ids)
-                pending_handler.parse_url_list()
-                pending_handler.add_to_pending()
+    def on_success(self, retval, task_id, args, kwargs):
+        """callback task completed successfully"""
+        print(f"{task_id} success callback")
+        message, key = self._build_message()
+        message.update({"messages": ["Task completed successfully"]})
+        RedisArchivist().set_message(key, message, expire=5)
 
-        else:
-            print("Did not acquire rescan lock.")
+    def before_start(self, task_id, args, kwargs):
+        """callback before initiating task"""
+        print(f"{self.name} create callback")
+        message, key = self._build_message()
+        message.update({"messages": ["New task received."]})
+        RedisArchivist().set_message(key, message)
 
-    finally:
-        if have_lock:
-            my_lock.release()
-
-
-@shared_task(name="download_pending")
-def download_pending():
-    """download latest pending videos"""
-    have_lock = False
-    my_lock = RedisArchivist().get_lock("downloading")
-
-    try:
-        have_lock = my_lock.acquire(blocking=False)
-        if have_lock:
-            downloader = VideoDownloader()
-            downloader.add_pending()
-            downloader.run_queue()
-        else:
-            print("Did not acquire download lock.")
-
-    finally:
-        if have_lock:
-            my_lock.release()
-
-
-@shared_task
-def download_single(youtube_id):
-    """start download single video now"""
-    queue = RedisQueue()
-    queue.add_priority(youtube_id)
-    print("Added to queue with priority: " + youtube_id)
-    # start queue if needed
-    have_lock = False
-    my_lock = RedisArchivist().get_lock("downloading")
-
-    try:
-        have_lock = my_lock.acquire(blocking=False)
-        if have_lock:
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Download single video",
-                "message": "processing",
+    def send_progress(self, message_lines, progress=False, title=False):
+        """send progress message"""
+        message, key = self._build_message()
+        message.update(
+            {
+                "messages": message_lines,
+                "progress": progress,
             }
-            RedisArchivist().set_message("message:download", mess_dict)
-            VideoDownloader().run_queue()
-        else:
-            print("Download queue already running.")
+        )
+        if title:
+            message["title"] = title
 
-    finally:
-        # release if only single run
-        if have_lock and not queue.get_next():
-            my_lock.release()
+        RedisArchivist().set_message(key, message)
+
+    def _build_message(self, level="info"):
+        """build message dict"""
+        task_id = self.request.id
+        message = self.TASK_CONFIG.get(self.name).copy()
+        message.update({"level": level, "id": task_id})
+        task_result = TaskManager().get_task(task_id)
+        if task_result:
+            command = task_result.get("command", False)
+            message.update({"command": command})
+
+        key = f"message:{message.get('group')}:{task_id.split('-')[0]}"
+        return message, key
+
+    def is_stopped(self):
+        """check if task is stopped"""
+        return TaskManager().is_stopped(self.request.id)
 
 
-@shared_task
-def extrac_dl(youtube_ids):
+@shared_task(name="update_subscribed", bind=True, base=BaseTask)
+def update_subscribed(self):
+    """look for missing videos and add to pending"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] rescan already running")
+        self.send_progress("Rescan already in progress.")
+        return
+
+    manager.init(self)
+    handler = SubscriptionScanner(task=self)
+    missing_videos = handler.scan()
+    auto_start = handler.auto_start
+    if missing_videos:
+        print(missing_videos)
+        extrac_dl.delay(missing_videos, auto_start=auto_start)
+
+
+@shared_task(name="download_pending", bind=True, base=BaseTask)
+def download_pending(self, auto_only=False):
+    """download latest pending videos"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] download queue already running")
+        self.send_progress("Download Queue is already running.")
+        return
+
+    manager.init(self)
+    VideoDownloader(task=self).run_queue(auto_only=auto_only)
+
+
+@shared_task(name="extract_download", bind=True, base=BaseTask)
+def extrac_dl(self, youtube_ids, auto_start=False):
     """parse list passed and add to pending"""
-    pending_handler = PendingList(youtube_ids=youtube_ids)
+    TaskManager().init(self)
+    if isinstance(youtube_ids, str):
+        to_add = Parser(youtube_ids).parse()
+    else:
+        to_add = youtube_ids
+
+    pending_handler = PendingList(youtube_ids=to_add, task=self)
     pending_handler.parse_url_list()
-    pending_handler.add_to_pending()
+    pending_handler.add_to_pending(auto_start=auto_start)
+
+    if auto_start:
+        download_pending.delay(auto_only=True)
 
 
-@shared_task(name="check_reindex")
-def check_reindex():
+@shared_task(bind=True, name="check_reindex", base=BaseTask)
+def check_reindex(self, data=False, extract_videos=False):
     """run the reindex main command"""
-    reindex_old_documents()
+    if data:
+        # started from frontend through API
+        print(f"[task][{self.name}] reindex {data}")
+        self.send_progress("Add items to the reindex Queue.")
+        ReindexManual(extract_videos=extract_videos).extract_data(data)
+
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] reindex queue is already running")
+        self.send_progress("Reindex Queue is already running.")
+        return
+
+    manager.init(self)
+    if not data:
+        # started from scheduler
+        populate = ReindexPopulate()
+        print(f"[task][{self.name}] reindex outdated documents")
+        self.send_progress("Add recent documents to the reindex Queue.")
+        populate.add_recent()
+        self.send_progress("Add outdated documents to the reindex Queue.")
+        populate.add_outdated()
+
+    Reindex(task=self).reindex_all()
 
 
-@shared_task
-def run_manual_import():
+@shared_task(bind=True, name="manual_import", base=BaseTask)
+def run_manual_import(self):
     """called from settings page, to go through import folder"""
-    print("starting media file import")
-    have_lock = False
-    my_lock = RedisArchivist().get_lock("manual_import")
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] manual import is already running")
+        self.send_progress("Manual import is already running.")
+        return
 
-    try:
-        have_lock = my_lock.acquire(blocking=False)
-        if have_lock:
-            import_handler = ManualImport()
-            if import_handler.identified:
-                all_videos_added = import_handler.process_import()
-                ThumbManager().download_vid(all_videos_added)
-        else:
-            print("Did not acquire lock form import.")
-
-    finally:
-        if have_lock:
-            my_lock.release()
+    manager.init(self)
+    ImportFolderScanner(task=self).scan()
 
 
-@shared_task(name="run_backup")
-def run_backup(reason="auto"):
+@shared_task(bind=True, name="run_backup", base=BaseTask)
+def run_backup(self, reason="auto"):
     """called from settings page, dump backup to zip file"""
-    backup_all_indexes(reason)
-    print("backup finished")
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] backup is already running")
+        self.send_progress("Backup is already running.")
+        return
+
+    manager.init(self)
+    ElasticBackup(reason=reason, task=self).backup_all_indexes()
 
 
-@shared_task
-def run_restore_backup(filename):
+@shared_task(bind=True, name="restore_backup", base=BaseTask)
+def run_restore_backup(self, filename):
     """called from settings page, dump backup to zip file"""
-    restore_from_backup(filename)
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] restore is already running")
+        self.send_progress("Restore is already running.")
+        return
+
+    manager.init(self)
+    self.send_progress(["Reset your Index"])
+    ElasitIndexWrap().reset()
+    ElasticBackup(task=self).restore(filename)
     print("index restore finished")
 
 
-def kill_dl(task_id):
-    """kill download worker task by ID"""
-    if task_id:
-        app.control.revoke(task_id, terminate=True)
-
-    _ = RedisArchivist().del_message("dl_queue_id")
-    RedisQueue().clear()
-
-    # clear cache
-    cache_dir = os.path.join(CONFIG["application"]["cache_dir"], "download")
-    for cached in os.listdir(cache_dir):
-        to_delete = os.path.join(cache_dir, cached)
-        os.remove(to_delete)
-
-    # notify
-    mess_dict = {
-        "status": "message:download",
-        "level": "error",
-        "title": "Canceling download process",
-        "message": "Canceling download queue now.",
-    }
-    RedisArchivist().set_message("message:download", mess_dict)
-
-
-@shared_task
-def rescan_filesystem():
+@shared_task(bind=True, name="rescan_filesystem", base=BaseTask)
+def rescan_filesystem(self):
     """check the media folder for mismatches"""
-    scan_filesystem()
-    validate_thumbnails()
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] filesystem rescan already running")
+        self.send_progress("Filesystem Rescan is already running.")
+        return
+
+    manager.init(self)
+    Filesystem(task=self).process()
+    ThumbValidator(task=self).validate()
 
 
-@shared_task(name="thumbnail_check")
-def thumbnail_check():
+@shared_task(bind=True, name="thumbnail_check", base=BaseTask)
+def thumbnail_check(self):
     """validate thumbnails"""
-    validate_thumbnails()
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] thumbnail check is already running")
+        self.send_progress("Thumbnail check is already running.")
+        return
+
+    manager.init(self)
+    ThumbValidator(task=self).validate()
 
 
-@shared_task
-def re_sync_thumbs():
+@shared_task(bind=True, name="resync_thumbs", base=BaseTask)
+def re_sync_thumbs(self):
     """sync thumbnails to mediafiles"""
-    handler = ThumbManager()
-    video_list = handler.get_thumb_list()
-    handler.write_all_thumbs(video_list)
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] thumb re-embed is already running")
+        self.send_progress("Thumbnail re-embed is already running.")
+        return
+
+    manager.init(self)
+    ThumbFilesystem(task=self).embed()
 
 
-@shared_task
-def subscribe_to(url_str):
+@shared_task(bind=True, name="subscribe_to", base=BaseTask)
+def subscribe_to(self, url_str):
     """take a list of urls to subscribe to"""
-    to_subscribe_list = UrlListParser(url_str).process_list()
-    counter = 1
-    for item in to_subscribe_list:
-        to_sub_id = item["url"]
-        if item["type"] == "playlist":
-            new_thumbs = PlaylistSubscription().process_url_str([item])
-            if new_thumbs:
-                ThumbManager().download_playlist(new_thumbs)
-            continue
-
-        if item["type"] == "video":
-            vid_details = PendingList().get_youtube_details(to_sub_id)
-            channel_id_sub = vid_details["channel_id"]
-        elif item["type"] == "channel":
-            channel_id_sub = to_sub_id
-        else:
-            raise ValueError("failed to subscribe to: " + to_sub_id)
-
-        ChannelSubscription().change_subscribe(
-            channel_id_sub, channel_subscribed=True
-        )
-        # notify for channels
-        message = {
-            "status": "message:subchannel",
-            "level": "info",
-            "title": "Subscribing to Channels",
-            "message": f"Processing {counter} of {len(to_subscribe_list)}",
-        }
-        RedisArchivist().set_message("message:subchannel", message=message)
-        counter = counter + 1
+    SubscriptionHandler(url_str, task=self).subscribe()
 
 
-@shared_task
-def index_channel_playlists(channel_id):
+@shared_task(bind=True, name="index_playlists", base=BaseTask)
+def index_channel_playlists(self, channel_id):
     """add all playlists of channel to index"""
-    channel = YoutubeChannel(channel_id)
-    # notify
-    mess_dict = {
-        "status": "message:playlistscan",
-        "level": "info",
-        "title": "Looking for playlists",
-        "message": f'Scanning channel "{channel.youtube_id}" in progress',
-    }
-    RedisArchivist().set_message("message:playlistscan", mess_dict)
+    channel = YoutubeChannel(channel_id, task=self)
     channel.index_channel_playlists()
 
 
-try:
-    app.conf.beat_schedule = ScheduleBuilder().build_schedule()
-except KeyError:
-    # update path from v0.0.8 to v0.0.9 to load new defaults
-    StartupCheck().sync_redis_state()
-    app.conf.beat_schedule = ScheduleBuilder().build_schedule()
+@shared_task(name="version_check")
+def version_check():
+    """check for new updates"""
+    ReleaseVersion().check()
+
+
+# start schedule here
+app.conf.beat_schedule = ScheduleBuilder().build_schedule()

tubearchivist/home/templates/home/about.html

@@ -8,8 +8,8 @@
     <div class="about-section">
         <h2>Useful Links</h2>
         <p>This project is in active and constant development, take a look at the <a href="https://github.com/tubearchivist/tubearchivist#roadmap" target="_blank">roadmap</a> for a overview.</p>
-        <p>For any questions on what a button or a function does, You can find the up-to-date user documentation on <a href="https://github.com/tubearchivist/tubearchivist/wiki" target="_blank">Github</a>.</p>
-        <p>All contributions are welcome: Open an <a href="https://github.com/tubearchivist/tubearchivist/issues" target="_blank">issue</a> for any bugs and errors, join us on <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> to discuss details. The <a href="https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md" target="_blank">contributing</a> page is a good place to get started.</p>
+        <p>All functionality is documented in our up-to-date <a href="https://docs.tubearchivist.com" target="_blank">user guide</a>.</p>
+        <p>All contributions are welcome: Open an <a href="https://github.com/tubearchivist/tubearchivist/issues" target="_blank">issue</a> for any bugs and errors, join us on <a href="https://www.tubearchivist.com/discord" target="_blank">Discord</a> to discuss details. The <a href="https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md" target="_blank">contributing</a> page is a good place to get started.</p>
     </div>
     <div class="about-section">
         <h2>Donate</h2>

tubearchivist/home/templates/home/base.html

@@ -79,16 +79,16 @@
             <div class="pagination">
                 {% if pagination %}
                     {% if pagination.current_page > 1 %}
-                        {% if pagination.search_get %}
-                        <a class="pagination-item" href="{{ request.path }}?search={{ pagination.search_get }}">First</a>
+                        {% if pagination.params %}
+                        <a class="pagination-item" href="{{ request.path }}?{{ pagination.params }}">First</a>
                         {% else %}
                         <a class="pagination-item" href="{{ request.path }}">First</a>
                         {% endif %}
                     {% endif %}
                     {% if pagination.prev_pages %}
                         {% for page in pagination.prev_pages %}
-                            {% if pagination.search_get %}
-                            <a class="pagination-item" href="?page={{ page }}&search={{ pagination.search_get }}">{{ page }}</a>
+                            {% if pagination.params %}
+                            <a class="pagination-item" href="?page={{ page }}&{{ pagination.params }}">{{ page }}</a>
                             {% else %}
                             <a class="pagination-item" href="?page={{ page }}">{{ page }}</a>
                             {% endif %}
@@ -100,16 +100,16 @@
                     {% if pagination.next_pages %}
                         <span> ></span>
                         {% for page in pagination.next_pages %}
-                            {% if pagination.search_get %}
-                            <a class="pagination-item" href="?page={{ page }}&search={{ pagination.search_get }}">{{ page }}</a>
+                            {% if pagination.params %}
+                            <a class="pagination-item" href="?page={{ page }}&{{ pagination.params }}">{{ page }}</a>
                             {% else %}
                             <a class="pagination-item" href="?page={{ page }}">{{ page }}</a>
                             {% endif %}
                         {% endfor %}
                     {% endif %}
                     {% if pagination.last_page > 0 %}
-                        {% if pagination.search_get %}
-                        <a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">
+                        {% if pagination.params %}
+                        <a class="pagination-item" href="?page={{ pagination.last_page }}&{{ pagination.params }}">
                             {% if pagination.max_hits %}
                                 Max ({{ pagination.last_page }})
                             {% else %}
@@ -132,7 +132,14 @@
     </div>
     <div class="footer">
         <div class="boxed-content">
-            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist {{ version }} </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/tubearchivist/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
+            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script></span>
+            <span>TubeArchivist</span>
+            <span>{{ version }}</span>
+            {% if ta_update %}
+                <span class="danger-zone">{{ ta_update.version }} available{% if ta_update.is_breaking %}<span class="danger-zone">Breaking Changes!</span>{% endif %}</span>
+                <span><a href="https://github.com/tubearchivist/tubearchivist/releases/tag/{{ ta_update.version }}" target="_blank">Release Page</a> | </span>
+            {% endif %}
+            <span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/tubearchivist/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://www.tubearchivist.com/discord" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
         </div>
     </div>
 </body>

tubearchivist/home/templates/home/channel.html

@@ -18,7 +18,7 @@
             </div>
         </div>
     </div>
-    <div id="notifications" data="channel"></div>
+    <div id="notifications" data="subscription"></div>
     <div class="view-controls">
         <div class="toggle">
             <span>Show subscribed only:</span>
@@ -36,7 +36,7 @@
             <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="channel" data-value="list" alt="list view">
         </div>
     </div>
-    <h2>Total matching channels: {{ max_hits }}</h2>
+    <h2>Total channels: {{ max_hits }}</h2>
     <div class="channel-list {{ view_style }}">
         {% if results %}
             {% for channel in results %}
@@ -81,4 +81,4 @@
     </div>
 </div>
 <script type="text/javascript" src="{% static 'progress.js' %}"></script>
-{% endblock content %}
+{% endblock content %}

tubearchivist/home/templates/home/channel_id.html

@@ -6,8 +6,24 @@
     <div class="channel-banner">
         <a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
     </div>
-    <div id="notifications" data="channel_id"></div>
-    <div class="info-box info-box-3">
+    <div class="info-box-item channel-nav">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
+        {% if has_streams %}
+            <a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
+        {% endif %}
+        {% if has_shorts %}
+            <a href="{% url 'channel_id_shorts' channel_info.channel_id %}"><h3>Shorts</h3></a>
+        {% endif %}
+        {% if has_playlists %}
+            <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
+        {% endif %}
+        <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+        {% if has_pending %}
+            <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
+        {% endif %}
+    </div>
+    <div id="notifications" data="channel reindex"></div>
+    <div class="info-box info-box-2">
         <div class="info-box-item">
             <div class="round-img">
                 <a href="{% url 'channel_id' channel_info.channel_id %}">
@@ -29,92 +45,15 @@
             </div>
         </div>
         <div class="info-box-item">
-            <div>
-                <p>Last refreshed: {{ channel_info.channel_last_refresh }}</p>
-                {% if channel_info.channel_active %}
-                    <p>Youtube: <a href="https://www.youtube.com/channel/{{ channel_info.channel_id }}" target="_blank">Active</a></p>
-                {% else %}
-                    <p>Youtube: Deactivated</p>
-                {% endif %}
-                <button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
-                <div class="delete-confirm" id="delete-button">
-                    <span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
-                </div>
-            </div>
-        </div>
-        <div class="info-box-item">
-            <div>
-                {% if channel_info.channel_views >= 1000000 %}
-                    <p>Channel views: {{ channel_info.channel_views|intword }}</p>
-                {% elif channel_info.channel_views > 0 %}
-                    <p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
-                {% endif %}
-                {% if max_hits %}
-                    <p>Total Videos archived: {{ max_hits }}</p>
-                    <p>Watched: <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
-                {% endif %}
-                <button {% if channel_info.channel_overwrites %} class="danger-button"{% endif %} onclick="showOverwrite()" title="Overwrite settings for channel {{ channel_info.channel_name }}">Configure</button>
-                <a href="/playlist/?search={{ channel_info.channel_id }}" title="Show all playlists belonging to {{ channel_info.channel_name }}"><button>Show Playlists</button></a>
-            </div>
+            {% if aggs %}
+                <p>{{ aggs.total_items.value }} videos <span class="space-carrot">|</span> {{ aggs.total_duration.value_str }} playback <span class="space-carrot">|</span> Total size {{ aggs.total_size.value|filesizeformat }}</p>
+                <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button>
+            {% endif %}
         </div>
     </div>
-    <div id="overwrite-form" class="info-box{% if not channel_info.channel_overwrites %} hidden-overwrite{% endif %}">
-        <div class="info-box-item">
-            <form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/" method="POST">
-                {% csrf_token %}
-                <div class="overwrite-form-item">
-                    <p>Download format: <span class="settings-current">
-                        {% if channel_info.channel_overwrites.download_format %}
-                            {{ channel_info.channel_overwrites.download_format }}
-                        {% else %}
-                            False
-                        {% endif %}</span></p>
-                    {{ channel_overwrite_form.download_format }}<br>
-                </div>
-                <div class="overwrite-form-item">
-                    <p>Auto delete watched videos after x days: <span class="settings-current">
-                        {% if channel_info.channel_overwrites.autodelete_days %}
-                            {{ channel_info.channel_overwrites.autodelete_days }}
-                        {% else %}
-                            False
-                        {% endif %}</span></p>
-                    {{ channel_overwrite_form.autodelete_days }}<br>
-                </div>
-                <div class="overwrite-form-item">
-                    <p>Index playlists: <span class="settings-current">
-                        {% if channel_info.channel_overwrites.index_playlists %}
-                            {{ channel_info.channel_overwrites.index_playlists }}
-                        {% else %}
-                            False
-                        {% endif %}</span></p>
-                    {{ channel_overwrite_form.index_playlists }}<br>
-                </div>
-                <div class="overwrite-form-item">
-                    <p>Enable <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a>: <span class="settings-current">
-                        {% if channel_info.channel_overwrites.integrate_sponsorblock %}
-                            {{ channel_info.channel_overwrites.integrate_sponsorblock }}
-                        {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %}
-                            Disabled
-                        {% else %}
-                            False
-                        {% endif %}</span></p>
-                    {{ channel_overwrite_form.integrate_sponsorblock }}<br>
-                </div>
-                <button type="submit">Save Channel Overwrites</button>
-            </form>
-        </div>
-    </div>
-    {% if channel_info.channel_description %}
-        <div class="info-box-item description-box">
-            <p>Description: <button onclick="textReveal()" id="text-reveal-button">Show</button></p>
-            <div id="text-reveal" class="description-text">
-                {{ channel_info.channel_description|linebreaks }}
-            </div>
-        </div>
-    {% endif %}
 </div>
 <div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
-    <div class="view-controls">
+    <div class="view-controls three">
         <div class="toggle">
             <span>Hide watched videos:</span>
             <div class="toggleBox">
@@ -167,7 +106,7 @@
                     <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
                                     <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}

tubearchivist/home/templates/home/channel_id_about.html

@@ -0,0 +1,142 @@
+{% extends "home/base.html" %}
+{% block content %}
+{% load static %}
+{% load humanize %}
+<div class="boxed-content">
+    <div class="channel-banner">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
+    </div>
+    <div class="info-box-item channel-nav">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
+        {% if has_streams %}
+            <a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
+        {% endif %}
+        {% if has_shorts %}
+            <a href="{% url 'channel_id_shorts' channel_info.channel_id %}"><h3>Shorts</h3></a>
+        {% endif %}
+        {% if has_playlists %}
+            <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
+        {% endif %}
+        <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+        {% if has_pending %}
+            <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
+        {% endif %}
+    </div>
+    <div id="notifications" data="channel reindex"></div>
+    <div class="info-box info-box-3">
+        <div class="info-box-item">
+            <div class="round-img">
+                <a href="{% url 'channel_id' channel_info.channel_id %}">
+                    <img src="{{ channel_info.channel_thumb_url }}" alt="channel-thumb">
+                </a>
+            </div>
+            <div>
+                <h3><a href="{% url 'channel_id' channel_info.channel_id %}">{{ channel_info.channel_name }}</a></h3>
+                {% if channel_info.channel_subs >= 1000000 %}
+                    <p>Subscribers: {{ channel_info.channel_subs|intword }}</p>
+                {% else %}
+                    <p>Subscribers: {{ channel_info.channel_subs|intcomma }}</p>
+                {% endif %}
+            </div>
+        </div>
+        <div class="info-box-item">
+            <div>
+                <p>Last refreshed: {{ channel_info.channel_last_refresh }}</p>
+                {% if channel_info.channel_active %}
+                    <p>Youtube: <a href="https://www.youtube.com/channel/{{ channel_info.channel_id }}" target="_blank">Active</a></p>
+                {% else %}
+                    <p>Youtube: Deactivated</p>
+                {% endif %}
+            </div>
+        </div>
+        <div class="info-box-item">
+            <div>
+                {% if channel_info.channel_views >= 1000000 %}
+                    <p>Channel views: {{ channel_info.channel_views|intword }}</p>
+                {% elif channel_info.channel_views > 0 %}
+                    <p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
+                {% endif %}
+                <div class="button-box">
+                    <button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
+                    <div class="delete-confirm" id="delete-button">
+                        <span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
+                    </div>
+                </div>
+                {% if reindex %}
+                    <p>Reindex scheduled</p>
+                {% else %}
+                    <div id="reindex-button" class="button-box">
+                        <button data-id="{{ channel_info.channel_id }}" data-type="channel" onclick="reindex(this)" title="Reindex Channel {{ channel_info.channel_name }}">Reindex</button>
+                        <button data-id="{{ channel_info.channel_id }}" data-type="channel" data-extract-videos="true" onclick="reindex(this)" title="Reindex Videos of {{ channel_info.channel_name }}">Reindex Videos</button>
+                    </div>
+                {% endif %}
+            </div>
+        </div>
+    </div>
+    {% if channel_info.channel_description %}
+        <div class="description-box">
+            <p id="text-expand" class="description-text">
+                {{ channel_info.channel_description|linebreaksbr|urlizetrunc:50 }}
+            </p>
+            <button onclick="textExpand()" id="text-expand-button">Show more</button>
+        </div>
+    {% endif %}
+    {% if channel_info.channel_tags %}
+        <div class="description-box">
+            <div class="video-tag-box">
+                {% for tag in channel_info.channel_tags %}
+                    <span class="video-tag">{{ tag }}</span>
+                {% endfor %}
+            </div>
+        </div>
+    {% endif %}
+    <div id="overwrite-form" class="info-box">
+        <div class="info-box-item">
+            <h2>Customize {{ channel_info.channel_name }}</h2>
+            <form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/about/" method="POST">
+                {% csrf_token %}
+                <div class="overwrite-form-item">
+                    <p>Download format: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.download_format %}
+                            {{ channel_info.channel_overwrites.download_format }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.download_format }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Auto delete watched videos after x days: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.autodelete_days %}
+                            {{ channel_info.channel_overwrites.autodelete_days }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.autodelete_days }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Index playlists: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.index_playlists %}
+                            {{ channel_info.channel_overwrites.index_playlists }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.index_playlists }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Enable <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a>: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.integrate_sponsorblock %}
+                            {{ channel_info.channel_overwrites.integrate_sponsorblock }}
+                        {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %}
+                            Disabled
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.integrate_sponsorblock }}<br>
+                </div>
+                <button type="submit">Save Channel Overwrites</button>
+            </form>
+        </div>
+    </div>
+</div>
+<script type="text/javascript" src="{% static 'progress.js' %}"></script>
+{% endblock content %}

tubearchivist/home/templates/home/channel_id_playlist.html

@@ -0,0 +1,70 @@
+{% extends "home/base.html" %}
+{% block content %}
+{% load static %}
+{% load humanize %}
+<div class="boxed-content">
+    <div class="channel-banner">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
+    </div>
+    <div class="info-box-item channel-nav">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
+        {% if has_streams %}
+            <a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
+        {% endif %}
+        {% if has_shorts %}
+            <a href="{% url 'channel_id_shorts' channel_info.channel_id %}"><h3>Shorts</h3></a>
+        {% endif %}
+        {% if has_playlists %}
+            <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
+        {% endif %}
+        <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+        {% if has_pending %}
+            <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
+        {% endif %}
+    </div>
+    <div id="notifications" data="channel reindex"></div>
+    <div class="view-controls">
+        <div class="toggle">
+            <span>Show subscribed only:</span>
+            <div class="toggleBox">
+                <input id="show_subed_only" onclick="toggleCheckbox(this)" type="checkbox" {% if show_subed_only %}checked{% endif %}>
+                {% if not show_subed_only %}
+                    <label for="" class="ofbtn">Off</label>
+                {% else %}
+                    <label for="" class="onbtn">On</label>
+                {% endif %}
+            </div>
+        </div>
+        <div class="view-icons">
+            <img src="{% static 'img/icon-gridview.svg' %}" onclick="changeView(this)" data-origin="playlist" data-value="grid" alt="grid view">
+            <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="playlist" data-value="list" alt="list view">
+        </div>
+    </div>
+    <div class="playlist-list {{ view_style }}">
+        {% if results %}
+            {% for playlist in results %}
+            <div class="playlist-item {{ view_style }}">
+                <div class="playlist-thumbnail">
+                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}">
+                        <img src="/cache/playlists/{{ playlist.source.playlist_id }}.jpg" alt="{{ playlist.source.playlist_id }}-thumbnail">
+                    </a>
+                </div>
+                <div class="playlist-desc {{ view_style }}">
+                    <a href="{% url 'channel_id' playlist.source.playlist_channel_id %}"><h3>{{ playlist.source.playlist_channel }}</h3></a>
+                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
+                    <p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
+                    {% if playlist.source.playlist_subscribed %}
+                        <button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
+                    {% else %}
+                        <button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
+                    {% endif %}
+                </div>
+            </div>
+            {% endfor %}
+        {% else %}
+            <h2>No playlists found...</h2>
+        {% endif %}
+    </div>
+</div>
+<script type="text/javascript" src="{% static 'progress.js' %}"></script>
+{% endblock content %}

tubearchivist/home/templates/home/downloads.html

@@ -3,7 +3,7 @@
 {% block content %}
 <div class="boxed-content">
     <div class="title-bar">
-        <h1>Downloads</h1>
+        <h1>Downloads {% if channel_filter_id %} for {{ channel_filter_name }}{% endif %}</h1>
     </div>
     <div id="notifications" data="download"></div>
     <div id="downloadControl"></div>
@@ -20,15 +20,16 @@
             <img id="animate-icon" onclick="showForm()" src="{% static 'img/icon-add.svg' %}" alt="add-icon">
             <p>Add to download queue</p>
             <div class="show-form">
-                <form id='hidden-form' action="/downloads/" method="post">
+                <div id='hidden-form' novalidate>
                     {% csrf_token %}
                     {{ add_form }}
-                    <button type="submit">Add to download queue</button>
-                </form>
+                    <button onclick="addToQueue()">Add to queue</button>
+                    <button onclick="addToQueue(true)">Download now</button>
+                </div>
             </div>
         </div>
     </div>
-    <div class="view-controls">
+    <div class="view-controls three">
         <div class="toggle">
             <span>Show only ignored videos:</span>
             <div class="toggleBox">
@@ -41,38 +42,70 @@
             </div>
         </div>
         <div class="view-icons">
+            {% if channel_agg_list|length > 1 %}
+                <select name="channel_filter" id="channel_filter" onchange="channelFilterDownload(this.value)">
+                    <option value="all" {% if not channel_filter_id %}selected{% endif %}>all</option>
+                    {% for channel in channel_agg_list %}
+                        <option {% if channel_filter_id == channel.id %}selected{% endif %} value="{{ channel.id }}">{{ channel.name }} ({{channel.count}})</option>
+                    {% endfor %}
+                </select>
+            {% endif %}
+            {% if view_style == "grid" %}
+                <div class="grid-count">
+                    {% if grid_items < 7 %}
+                        <img src="{% static 'img/icon-add.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"1"}}" alt="grid plus row">
+                    {% endif %}
+                    {% if grid_items > 3 %}
+                        <img src="{% static 'img/icon-substract.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"-1"}}" alt="grid minus row">
+                    {% endif %}
+                </div>
+            {% endif %}
             <img src="{% static 'img/icon-gridview.svg' %}" onclick="changeView(this)" data-origin="downloads" data-value="grid" alt="grid view">
             <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="downloads" data-value="list" alt="list view">
         </div>
     </div>
-    <h3>Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %}</h3>
-    <div class="dl-list {{ view_style }}">
+    <h3>Total videos in queue: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %} {% if channel_filter_id %} - from channel <i>{{ channel_filter_name }}</i>{% endif %}</h3>
+</div>
+<div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
+    <div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
         {% if results %}
             {% for video in results %}
-                <div class="dl-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
-                    <div class="dl-thumb {{ view_style }}">
-                        <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video_thumb">
-                        {% if show_ignored_only %}
-                            <span>ignored</span>
-                        {% else %}
-                            <span>queued</span>
-                        {% endif %}
+                <div class="video-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
+                    <div class="video-thumb-wrap {{ view_style }}">
+                        <div class="video-thumb">
+                            <img src="{{ video.source.vid_thumb_url }}" alt="video_thumb">
+                            <div class="video-tags">
+                                {% if show_ignored_only %}
+                                    <span>ignored</span>
+                                {% else %}
+                                    <span>queued</span>
+                                {% endif %}
+                                <span>{{ video.source.vid_type }}</span>
+                                {% if video.source.auto_start %}
+                                    <span>auto</span>
+                                {% endif %}
+                            </div>
+                        </div>
                     </div>
-                    <div class="dl-desc {{ view_style }}">
-                        <h3>{{ video.source.title }}</h3>
-                        {% if video.source.channel_indexed %}
+                    <div class="video-desc {{ view_style }}">
+                        <div>
+                            {% if video.source.channel_indexed %}
                             <a href="{% url 'channel_id' video.source.channel_id %}">{{ video.source.channel_name }}</a>
-                        {% else %}
-                            <span>{{ video.source.channel_name }}</span>
-                        {% endif %}
+                            {% else %}
+                                <span>{{ video.source.channel_name }}</span>
+                            {% endif %}
+                            <a href="https://www.youtube.com/watch?v={{ video.source.youtube_id }}" target="_blank"><h3>{{ video.source.title }}</h3></a>
+                        </div>
                         <p>Published: {{ video.source.published }} | Duration: {{ video.source.duration }} | {{ video.source.youtube_id }}</p>
-                        {% if show_ignored_only %}
-                            <button data-id="{{ video.source.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>
-                            <button data-id="{{ video.source.youtube_id }}" onclick="addSingle(this)">Add to queue</button>
-                        {% else %}
-                            <button data-id="{{ video.source.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
-                            <button id="{{ video.source.youtube_id }}" data-id="{{ video.source.youtube_id }}" onclick="downloadNow(this)">Download now</button>
-                        {% endif %}
+                        <div>
+                            {% if show_ignored_only %}
+                                <button data-id="{{ video.source.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>
+                                <button data-id="{{ video.source.youtube_id }}" onclick="addSingle(this)">Add to queue</button>
+                            {% else %}
+                                <button data-id="{{ video.source.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
+                                <button id="{{ video.source.youtube_id }}" data-id="{{ video.source.youtube_id }}" onclick="downloadNow(this)">Download now</button>
+                            {% endif %}
+                        </div>
                     </div>
                 </div>
             {% endfor %}
@@ -80,4 +113,4 @@
     </div>
 </div>
 <script type="text/javascript" src="{% static 'progress.js' %}"></script>
-{% endblock content %}
+{% endblock content %}

tubearchivist/home/templates/home/home.html

@@ -12,7 +12,7 @@
                     <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
                                     <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}
@@ -45,7 +45,7 @@
     <div class="title-bar">
         <h1>Recent Videos</h1>
     </div>
-    <div class="view-controls">
+    <div class="view-controls three">
         <div class="toggle">
             <span>Hide watched:</span>
             <div class="toggleBox">
@@ -98,7 +98,7 @@
                     <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
                                     <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}