clarify TA_HOST

Changed:
- Changed TA_HOST parser, fixing #441
- Changed thumb resync page size
- Added video id to download processing notification
- Added more notifications for zip backup

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -6,7 +6,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thanks for taking the time to help improve this project!
+        Thanks for taking the time to help improve this project! Please read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide carefully before continuing.
 
   - type: checkboxes
     id: latest
@@ -15,7 +15,7 @@ body:
       options:
         - label: I'm running the latest version of Tube Archivist and have read the [release notes](https://github.com/tubearchivist/tubearchivist/releases/latest).
           required: true
-        - label: I have read through the [wiki](https://github.com/tubearchivist/tubearchivist/wiki) and the [readme](https://github.com/tubearchivist/tubearchivist#installing-and-updating), particularly the [common errors](https://github.com/tubearchivist/tubearchivist#common-errors) section.
+        - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [bug report](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#bug-report) section.
           required: true
 
   - type: input

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -1,21 +1,19 @@
 name: Feature Request
-description: I want to work on a new feature.
+description: I have an idea for a great addition to this project
 title: "[Feature Request]: "
 
 body:
   - type: markdown
     attributes:
       value: |
-        Thanks for taking the time to help improve this project! This project is currently only taking new feature requests from contributors who want to make a PR with their ideas. The backlog of planned features outlined in the roadmap or the open issues here will keep us busy for the foreseeable future.
+        Thanks for taking the time to help improve this project! This project is *very* selective with accepting new feature requests. Please read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide carefully before continuing.
 
   - type: checkboxes
     id: already
     attributes:
-      label: Already implemented?
+      label: "I've read the documentation"
       options:
-        - label: I have read through the [wiki](https://github.com/tubearchivist/tubearchivist/wiki).
-          required: true
-        - label: I understand the [scope](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) of this project and am aware of the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and my idea is not already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap).
+        - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [feature request](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#feature-request) section.
           required: true
 
   - type: textarea
@@ -34,4 +32,3 @@ body:
       description: This project is ambitious as it is, please contribute.
       options:
         - label: Yes I will work on this in the next few days or weeks.
-          required: true

.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml

@@ -13,9 +13,7 @@ body:
     attributes:
       label: Installation instructions
       options:
-        - label: I have read and understand the [installation instructions](https://github.com/tubearchivist/tubearchivist#installing-and-updating).
-          required: true
-        - label: My issue is not described in the [common errors](https://github.com/tubearchivist/tubearchivist#common-errors) section.
+        - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [installation help](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#installation-help) section.
           required: true
 
   - type: input

.github/pull_request_template.md

@@ -0,0 +1,3 @@
+Thank you for taking the time to improve this project. Please take a look at the [How to make a Pull Request](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-make-a-pull-request) section to help get your contribution merged.
+
+You can delete this text before submitting.

CONTRIBUTING.md

@@ -1,17 +1,97 @@
 ## Contributing to Tube Archivist
 
 Welcome, and thanks for showing interest in improving Tube Archivist!  
-If you haven't already, the best place to start is the README. This will give you an overview on what the project is all about.
 
-## Report a bug
+## Table of Content
+- [How to open an issue](#how-to-open-an-issue)
+  - [Bug Report](#bug-report)
+  - [Feature Request](#feature-request)
+  - [Installation Help](#installation-help)
+- [How to make a Pull Request](#how-to-make-a-pull-request)
+- [Improve to the Documentation](#improve-to-the-documentation)
+- [Development Environment](#development-environment)
+---
 
-If you notice something is not working as expected, check to see if it has been previously reported in the [open issues](https://github.com/tubearchivist/tubearchivist/issues).
-If it has not yet been disclosed, go ahead and create an issue.  
-If the issue doesn't move forward due to a lack of response, I assume it's solved and will close it after some time to keep the list fresh. 
+## How to open an issue
+Please read this carefully before opening any [issue](https://github.com/tubearchivist/tubearchivist/issues) on GitHub.
 
-## Wiki
+**Do**:
+- Do provide details and context, this matters a lot and makes it easier for people to help.
+- Do familiarize yourself with the project first, some questions answer themselves when using the project for some time. Familiarize yourself with the [Readme](https://github.com/tubearchivist/tubearchivist) and the [documentation](https://docs.tubearchivist.com/), this covers a lot of the common questions, particularly the [FAQ](https://docs.tubearchivist.com/faq/).
+- Do respond to questions within a day or two so issues can progress. If the issue doesn't move forward due to a lack of response, we'll assume it's solved and we'll close it after some time to keep the list fresh.
 
-The wiki is where all user functions are documented in detail. These pages are mirrored into the **docs** folder of the repo. This allows for pull requests and all other features like regular code. Make any changes there, and I'll sync them with the wiki tab.
+**Don't**:
+- Don't open *duplicates*, that includes open and closed issues.
+- Don't open an issue for something that's already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), this needs your help to implement it, not another issue.
+- Don't open an issue for something that's a [known limitation](https://github.com/tubearchivist/tubearchivist#known-limitations). These are *known* by definition and don't need another reminder. Some limitations may be solved in the future, maybe by you?
+- Don't overwrite the *issue template*, they are there for a reason. Overwriting that shows that you don't really care about this project. It shows that you have a misunderstanding how open source collaboration works and just want to push your ideas through. Overwriting the template may result in a ban.
+
+### Bug Report
+Bug reports are highly welcome! This project has improved a lot due to your help by providing feedback when something doesn't work as expected. The developers can't possibly cover all edge cases in an ever changing environment like YouTube and yt-dlp.
+
+Please keep in mind:
+- Docker logs are the easiest way to understand what's happening when something goes wrong, *always* provide the logs upfront.
+- Set the environment variable `DJANGO_DEBUG=True` to Tube Archivist and reproduce the bug for a better log output. Don't forget to remove that variable again after.
+- A bug that can't be reproduced, is difficult or sometimes even impossible to fix. Provide very clear steps *how to reproduce*.
+
+### Feature Request
+This project needs your help to grow further. There is no shortage of ideas, see the open [issues on GH](https://github.com/tubearchivist/tubearchivist/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement) and the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), what this project lacks is contributors to implement these ideas.
+
+Existing ideas are easily *multiple years* worth of development effort, at least at current speed. Best and fastest way to implement your feature is to do it yourself, that's why this project is open source after all. This project is *very* selective with accepting new feature requests at this point.  
+
+Good feature requests usually fall into one or more of these categories:
+- You want to work on your own idea within the next few days or weeks.
+- Your idea is beneficial for a wide range of users, not just for you.
+- Your idea extends the current project by building on and improving existing functionality.
+- Your idea is quick and easy to implement, for an experienced as well as for a first time contributor.
+
+Your request is likely going to be rejected if:
+- Your idea requires multiple days worth of development time and is unrealistic to be implemented any time soon.
+- There are already other ways to do what you are trying to do.
+- You are trying to do something that only applies to your platform, your specific workflow or your specific setup.
+- Your idea would fundamentally change how the project works or it wouldn't be able to be implemented with backwards compatibility.
+- Your idea is not a good fit for this project.
+
+### Installation Help
+GitHub is most likely not the best place to ask for installation help. That's inherently individual and one on one.
+1. First step is always, help yourself. Start at the [Readme](https://github.com/tubearchivist/tubearchivist) or the additional platform specific installation pages in the [docs](https://docs.tubearchivist.com/).
+2. If that doesn't answer your question, open a `#support` thread on [Discord](https://www.tubearchivist.com/discord).
+3. Only if that is not an option, open an issue here.
+
+IMPORTANT: When receiving help, contribute back to the community by improving the installation instructions with your newly gained knowledge.
+
+---
+
+## How to make a Pull Request
+
+Thank you for contributing and helping improve this project. This is a quick checklist to help streamline the process:
+
+- For **code changes**, make your PR against the [testing branch](https://github.com/tubearchivist/tubearchivist/tree/testing). That's where all active development happens. This simplifies the later merging into *master*, minimizes any conflicts and usually allows for easy and convenient *fast-forward* merging.
+- For **documentation changes**, make your PR directly against the *master* branch.
+- Show off your progress, even if not yet complete, by creating a [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) PR first and switch it as *ready* when you are ready.
+- Make sure all your code is linted and formatted correctly, see below. The automatic GH action unfortunately needs to be triggered manually by a maintainer for first time contributors, but will trigger automatically for existing contributors.
+
+### Making changes to the JavaScript
+
+The JavaScript does not require any build step; you just edit the files directly. However, there is config for eslint and prettier (a linter and formatter respectively); their use is recommended but not required. To use them, install `node`, run `npm i` from the root directory of this repository to install dependencies, then run `npm run lint` and `npm run format` to run eslint and prettier respectively.
+
+### Code formatting and linting
+
+To keep things clean and consistent for everybody, there is a github action setup to lint and check the changes. You can test your code locally first if you want. For example if you made changes in the **video** module, run
+
+```shell
+./deploy.sh validate tubearchivist/home/src/index/video.py
+```
+
+to validate your changes. If you omit the path, all the project files will get checked. This is subject to change as the codebase improves.
+
+---
+
+## Improve to the Documentation
+
+The documentation available at [docs.tubearchivist.com](https://docs.tubearchivist.com/) and is build from a separate repo [tubearchivist/docs](https://github.com/tubearchivist/docs). The Readme has additional instructions on how to make changes.
+
+---
 
 ## Development Environment
 
@@ -35,18 +115,18 @@ Make your changes locally and re-run `docker compose up --build`. The `Dockerfil
 
 You may find it nice to run everything inside of a VM, though this is not necessary. There's a `deploy.sh` script which has some helpers for this use case. YMMV, this is what one of the developers does:
 
-- Clone the repo, work on it with your favorite code editor in your local filesystem. *testing* branch is the where all the changes are happening, might be unstable and is WIP.
-- Then I have a VM running standard Ubuntu Server LTS with docker installed. The VM keeps my projects separate and offers convenient snapshot functionality. The VM also offers ways to simulate lowend environments by limiting CPU cores and memory. You can use this [Ansible Docker Ubuntu](https://github.com/bbilly1/ansible-playbooks) playbook to get started quickly. But you could also just run docker on your host system.
+- Clone the repo, work on it with your favorite code editor in your local filesystem. *testing* branch is where all the changes are happening, might be unstable and is WIP.
+- Then I have a VM running standard Ubuntu Server LTS with docker installed. The VM keeps my projects separate and offers convenient snapshot functionality. The VM also offers ways to simulate low end environments by limiting CPU cores and memory. You can use this [Ansible Docker Ubuntu](https://github.com/bbilly1/ansible-playbooks) playbook to get started quickly. But you could also just run docker on your host system.
 - I have my local DNS resolve `tubearchivist.local` to the IP of the VM for convenience. To deploy the latest changes and rebuild the application to the testing VM run:
 ```bash
 ./deploy.sh test
 ```
 - The command above will call the docker build command with `--build-arg INSTALL_DEBUG=1` to install additional useful debug tools.
 - The `test` argument takes another optional argument to build for a specific architecture valid options are: `amd64`, `arm64` and `multi`, default is `amd64`.
-- This `deploy.sh` script is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking. 
+- This `deploy.sh` script is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking.
 
-## Working with Elasticsearch
-Additionally to the required services as listed in the example docker-compose file, the **Dev Tools** of [Kibana](https://www.elastic.co/guide/en/kibana/current/docker.html) are invaluable for running and testing Elasticsearch queries. 
+### Working with Elasticsearch
+Additionally to the required services as listed in the example docker-compose file, the **Dev Tools** of [Kibana](https://www.elastic.co/guide/en/kibana/current/docker.html) are invaluable for running and testing Elasticsearch queries.
 
 **Quick start**  
 Generate your access token in Elasitcsearch:
@@ -60,41 +140,10 @@ kibana:
   image: docker.elastic.co/kibana/kibana:0.0.0
   container_name: kibana
   environment:
-    - "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
-    - "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
+	- "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
+	- "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
   ports:
-    - "5601:5601"
+	- "5601:5601"
 ```
 
 If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.
-
-## Implementing a new feature
-
-Do you see anything on the roadmap that you would like to take a closer look at but you are not sure, what's the best way to tackle that? Or anything not on there yet you'd like to implement but are not sure how? Reach out on Discord and we'll look into it together.
-
-## Making changes
-
-To fix a bug or implement a feature, fork the repository and make all changes to the testing branch. When ready, create a pull request.
-
-## Making changes to the JavaScript
-
-The JavaScript does not require any build step; you just edit the files directly. However, there is config for eslint and prettier (a linter and formatter respectively); their use is recommended but not required. To use them, install `node`, run `npm i` from the root directory of this repository to install dependencies, then run `npm run lint` and `npm run format` to run eslint and prettier respectively.
-
-## Releases
-
-There are three different docker tags:
-- **latest**: As the name implies is the latest multiarch release for regular usage.
-- **unstable**: Intermediate amd64 builds for quick testing and improved collaboration. Don't mix with a *latest* installation, for your testing environment only. This is untested and WIP and will have breaking changes between commits that might require a reset to resolve. 
-- **semantic versioning**: There will be a handful named version tags that will also have a matching release and tag on github.
-
-If you want to see what's in your container, checkout the matching release tag. A merge to **master** usually means a *latest* or *unstable* release. If you want to preview changes in your testing environment, pull the *unstable* tag or clone the repository and build the docker container with the Dockerfile from the **testing** branch.
-
-## Code formatting and linting
-
-To keep things clean and consistent for everybody, there is a github action setup to lint and check the changes. You can test your code locally first if you want. For example if you made changes in the **video** module, run
-
-```shell
-./deploy.sh validate tubearchivist/home/src/index/video.py
-```
-
-to validate your changes. If you omit the path, all the project files will get checked. This is subject to change as the codebase improves. 

README.md

@@ -8,212 +8,82 @@
 <a href="https://www.tubearchivist.com/discord" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-discord.png" alt="tubearchivist-discord" title="TA Discord Server Members" height="50" width="190"/></a>
 </div>
 
+![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")
+
 ## Table of contents:
-* [Wiki](https://github.com/tubearchivist/tubearchivist/wiki) with [FAQ](https://github.com/tubearchivist/tubearchivist/wiki/FAQ)
+* [Docs](https://docs.tubearchivist.com/) with [FAQ](https://docs.tubearchivist.com/faq/), and API documentation
 * [Core functionality](#core-functionality)
-* [Screenshots](#screenshots)
-* [Problem Tube Archivist tries to solve](#problem-tube-archivist-tries-to-solve)
-* [Connect](#connect)
-* [Extended Universe](#extended-universe)
-* [Installing and updating](#installing-and-updating)
+* [Resources](#resources)
+* [Installing](#installing)
 * [Getting Started](#getting-started)
+* [Known limitations](#known-limitations)
+* [Port Collisions](#port-collisions)
 * [Common Errors](#common-errors)
 * [Roadmap](#roadmap)
-* [Known limitations](#known-limitations)
 * [Donate](#donate)
 
 ------------------------
 
 ## Core functionality
+Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. THis includes:  
 * Subscribe to your favorite YouTube channels
 * Download Videos using **yt-dlp**
 * Index and make videos searchable
 * Play videos
 * Keep track of viewed and unviewed videos
-
-## Tube Archivist on YouTube
-[![ibracorp-youtube-video-thumb](assets/tube-archivist-ibracorp-O8H8Z01c0Ys.jpg)](https://www.youtube.com/watch?v=O8H8Z01c0Ys)
-
-## Screenshots
-![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")  
-*Home Page*
-
-![channels screenshot](assets/tube-archivist-screenshot-channels.png?raw=true "Tube Archivist Channels")  
-*All Channels*
-
-![single channel screenshot](assets/tube-archivist-screenshot-single-channel.png?raw=true "Tube Archivist Single Channel")  
-*Single Channel*
-
-![video page screenshot](assets/tube-archivist-screenshot-video.png?raw=true "Tube Archivist Video Page")  
-*Video Page*
-
-![video page screenshot](assets/tube-archivist-screenshot-download.png?raw=true "Tube Archivist Video Page")  
-*Downloads Page*
   
-## Problem Tube Archivist tries to solve
-Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface.
-
-## Connect
+## Resources
 - [Discord](https://www.tubearchivist.com/discord): Connect with us on our Discord server.
 - [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit.
-
-## Extended Universe
 - [Browser Extension](https://github.com/tubearchivist/browser-extension) Tube Archivist Companion, for [Firefox](https://addons.mozilla.org/addon/tubearchivist-companion/) and [Chrome](https://chrome.google.com/webstore/detail/tubearchivist-companion/jjnkmicfnfojkkgobdfeieblocadmcie)
-- [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.
+- [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.  
+- [Showcase](SHOWCASE.MD) To see more screenshots and Youtube videos showing off TubeArchvist.
 
-## Installing and updating
-There's dedicated user-contributed install steps under [docs/Installation.md](./docs/Installation.md) for podman, Unraid, Truenas and Synology which you can use instead of this section if you happen to be using one of those. Otherwise, continue on.
+## Installing
+For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. Minimal with dual core with 4 threads, better quad core plus. 
+This project requires docker. Ensure it is installed and running on your system.  
 
-For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. Minimal with dual core with 4 threads, better quad core plus.  
+The documentation has additional user provided instructions for [Unraid](https://docs.tubearchivist.com/installation/unraid/), [Synology](https://docs.tubearchivist.com/installation/synology/), [Podman](https://docs.tubearchivist.com/installation/podman/) and [True NAS](https://docs.tubearchivist.com/installation/truenas-scale/).
 
-This project requires docker. Ensure it is installed and running on your system.
+The instructions here should get you up and running quickly, for Docker beginners and full explanation about each environment variable, see the [docs](https://docs.tubearchivist.com/installation/docker-compose/).
 
-Note for **arm64**: Tube Archivist is a multi arch container, same for redis. For Elasitc Search use the official image for arm64 support. Other architectures are not supported.
+Take a look at the example [docker-compose.yml](https://github.com/tubearchivist/tubearchivist/blob/master/docker-compose.yml) and configure the required environment variables.
 
-Save the [docker-compose.yml](./docker-compose.yml) file from this reposity somewhere permanent on your system, keeping it named `docker-compose.yml`. You'll need to refer to it whenever starting this application.
+**TubeArchivist**:  
+| Environment Var | Value |  |
+| ----------- | ----------- | ----------- |
+| TA_HOST | Server IP or hostname | Required |
+| TA_USERNAME | Initial username when logging into TA | Required |
+| TA_PASSWORD | Initial password when logging into TA | Required |
+| ELASTIC_PASSWORD | Password for ElasticSearch | Required |
+| REDIS_HOST | Hostname for Redis | Required |
+| TZ | Set your timezone for the scheduler | Required |
+| TA_PORT | Overwrite Nginx port | Optional |
+| TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
+| ES_URL | URL That ElasticSearch runs on | Optional |
+| HOST_GID | Allow TA to own the video files instead of container user | Optional |
+| HOST_UID | Allow TA to own the video files instead of container user | Optional |
+| ELASTIC_USER | Change the default ElasticSearch user | Optional |
+| REDIS_PORT | Port that Redis runs on | Optional |
+| TA_LDAP | Configure TA to use LDAP Authentication | [Read more](https://docs.tubearchivist.com/configuration/ldap/) |
+| ENABLE_CAST | Enable casting support | [Read more](https://docs.tubearchivist.com/configuration/cast/) |
+| DJANGO_DEBUG | Return additional error messages, for debug only |  |
 
-Edit the following values from that file:
-- under `tubearchivist`->`environment`:
-  - `HOST_UID`: your UID, if you want TubeArchivist to create files with your UID. Remove if you are OK with files being owned by the the container user.
-  - `HOST_GID`: as above but GID.
-  - `TA_HOST`: change it to the address of the machine you're running this on. This can be an IP address or a domain name.
-  - `TA_PASSWORD`: pick a password to use when logging in.
-  - `ELASTIC_PASSWORD`: pick a password for the elastic service. You won't need to type this yourself.
-  - `TZ`: your time zone. If you don't know yours, you can look it up [here](https://www.timezoneconverter.com/cgi-bin/findzone/findzone).
-- under `archivist-es`->`environment`:
-  - `"ELASTIC_PASSWORD=verysecret"`: change `verysecret` to match the ELASTIC_PASSWORD you picked above.
+**ElasticSearch**  
+| Environment Var | Value | State |
+| ----------- | ----------- | ----------- |
+| ELASTIC_PASSWORD | Matching password `ELASTIC_PASSWORD` from TubeArchivist | Required |
+| http.port | Change the port ElasticSearch runs on | Optional |
 
-By default Docker will store all data, including downloaded data, in its own data-root directory (which you can find by running `docker info` and looking for the "Docker Root Dir"). If you want to use other locations, you can replace the `media:`, `cache:`, `redis:`, and `es:` volume names with absolute paths; if you do, remove them from the `volumes:` list at the bottom of the file.
 
-From a terminal, `cd` into the directory you saved the `docker-compose.yml` file in and run `docker compose up --detach`. The first time you do this it will download the appropriate images, which can take a minute.
+## Update
+Always use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tags are only for your testing environment, there might not be an update path for these testing builds. 
 
-You can follow the logs with `docker compose logs -f`. Once it's ready it will print something like `celery@1234567890ab ready`. At this point you should be able to go to `http://your-host:8000` and log in with the `TA_USER`/`TA_PASSWORD` credentials.
+You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. To take advantage of the latest fixes and improvements, make sure you are running the *latest and greatest*. After updating, check the footer to verify you are running the expected version.  
 
-You can bring the application down by running `docker compose down` in the same directory.
-
-Use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tag is for intermediate testing and as the name implies, is **unstable** and not be used on your main installation but in a [testing environment](CONTRIBUTING.md).  
-
-## Installation Details
-
-Tube Archivist depends on three main components split up into separate docker containers:  
-
-### Tube Archivist
-The main Python application that displays and serves your video collection, built with Django.
-  - Serves the interface on port `8000`
-  - Needs a volume for the video archive at **/youtube**
-  - And another volume to save application data at **/cache**.
-  - The environment variables `ES_URL` and `REDIS_HOST` are needed to tell Tube Archivist where Elasticsearch and Redis respectively are located.
-  - The environment variables `HOST_UID` and `HOST_GID` allows Tube Archivist to `chown` the video files to the main host system user instead of the container user. Those two variables are optional, not setting them will disable that functionality. That might be needed if the underlying filesystem doesn't support `chown` like *NFS*. 
-  - Set the environment variable `TA_HOST` to match with the system running Tube Archivist. This can be a domain like *example.com*, a subdomain like *ta.example.com* or an IP address like *192.168.1.20*, add without the protocol and without the port. You can add multiple hostnames separated with a space. Any wrong configurations here will result in a `Bad Request (400)` response.
-  - Change the environment variables `TA_USERNAME` and `TA_PASSWORD` to create the initial credentials. 
-  - `ELASTIC_PASSWORD` is for the password for Elasticsearch. The environment variable `ELASTIC_USER` is optional, should you want to change the username from the default *elastic*.
-  - For the scheduler to know what time it is, set your timezone with the `TZ` environment variable, defaults to *UTC*.
-  - Set the environment variable `ENABLE_CAST=True` to send videos to your cast device, [read more](#enable-cast).
-
-### Port collisions
-If you have a collision on port `8000`, best solution is to use dockers *HOST_PORT* and *CONTAINER_PORT* distinction: To for example change the interface to port 9000 use `9000:8000` in your docker-compose file.  
-
-Should that not be an option, the Tube Archivist container takes these two additional environment variables:
-- **TA_PORT**: To actually change the port where nginx listens, make sure to also change the ports value in your docker-compose file.
-- **TA_UWSGI_PORT**: To change the default uwsgi port 8080 used for container internal networking between uwsgi serving the django application and nginx.  
-
-Changing any of these two environment variables will change the files *nginx.conf* and *uwsgi.ini* at startup using `sed` in your container.
-
-### LDAP Authentication
-You can configure LDAP with the following environment variables:
-
- - `TA_LDAP` (ex: `true`) Set to anything besides empty string to use LDAP authentication **instead** of local user authentication.
- - `TA_LDAP_SERVER_URI` (ex: `ldap://ldap-server:389`) Set to the uri of your LDAP server.
- - `TA_LDAP_DISABLE_CERT_CHECK` (ex: `true`) Set to anything besides empty string to disable certificate checking when connecting over LDAPS.
- - `TA_LDAP_BIND_DN` (ex: `uid=search-user,ou=users,dc=your-server`) DN of the user that is able to perform searches on your LDAP account.
- - `TA_LDAP_BIND_PASSWORD` (ex: `yoursecretpassword`) Password for the search user.
- - `TA_LDAP_USER_ATTR_MAP_USERNAME` (default: `uid`) Bind attribute used to map LDAP user's username
- - `TA_LDAP_USER_ATTR_MAP_PERSONALNAME` (default: `givenName`) Bind attribute used to match LDAP user's First Name/Personal Name.
- - `TA_LDAP_USER_ATTR_MAP_SURNAME` (default: `sn`) Bind attribute used to match LDAP user's Last Name/Surname.
- - `TA_LDAP_USER_ATTR_MAP_EMAIL` (default: `mail`) Bind attribute used to match LDAP user's EMail address
- - `TA_LDAP_USER_BASE` (ex: `ou=users,dc=your-server`) Search base for user filter.
- - `TA_LDAP_USER_FILTER` (ex: `(objectClass=user)`) Filter for valid users. Login usernames are matched using the attribute specified in `TA_LDAP_USER_ATTR_MAP_USERNAME` and should not be specified in this filter.
-
-When LDAP authentication is enabled, django passwords (e.g. the password defined in TA_PASSWORD), will not allow you to login, only the LDAP server is used.
-
-### Enable Cast
-As Cast doesn't support authentication, enabling this functionality will make your static files like artwork and media files accessible by guessing the links. That's read only access, the application itself is still protected.
-
-Enabling this integration will embed an additional third party JS library from **Google**.  
-
-**Requirements**:
-- HTTPS: To use the cast integration HTTPS needs to be enabled, which can be done using a reverse proxy. This is a requirement by Google as communication to the cast device is required to be encrypted, but the content itself is not.
-- Supported Browser: A supported browser is required for this integration such as Google Chrome. Other browsers, especially Chromium-based browsers, may support casting by enabling it in the settings.
-- Subtitles: Subtitles are supported however they do not work out of the box and require additional configuration. Due to requirements by Google, to use subtitles you need additional headers which will need to be configured in your reverse proxy. See this [page](https://developers.google.com/cast/docs/web_sender/advanced#cors_requirements) for the specific requirements.  
-You need the following headers: Content-Type, Accept-Encoding, and Range. Note that the last two headers, Accept-Encoding and Range, are additional headers that you may not have needed previously.  
-Wildcards "*" can not be used for the Access-Control-Allow-Origin header. If the page has protected media content, it must use a domain instead of a wildcard.
-
-### Elasticsearch
-**Note**: Tube Archivist depends on Elasticsearch 8. 
-
-Use `bbilly1/tubearchivist-es` to automatically get the recommended version, or use the official image with the version tag in the docker-compose file.
-
-Use official Elastic Search for **arm64**.  
-
-Stores video meta data and makes everything searchable. Also keeps track of the download queue.
-  - Needs to be accessible over the default port `9200`
-  - Needs a volume at **/usr/share/elasticsearch/data** to store data
-
-Follow the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html) for additional installation details.
-
-### Elasticsearch on a custom port
-Should you need to change the port for Elasticsearch to for example *9500*, follow these steps:
-- Set the environment variable `http.port=9500` to the ES container
-- Change the *expose* value for the ES container to match your port number
-- For the Tube Archivist container, change the *ES_URL* environment variable, e.g. `ES_URL=http://archivist-es:9500`
-
-### Redis JSON
-Functions as a cache and temporary link between the application and the file system. Used to store and display messages and configuration variables.
-  - Needs to be accessible over the default port `6379`
-  - Needs a volume at **/data** to make your configuration changes permanent.
-
-### Redis on a custom port
-For some architectures it might be required to run Redis JSON on a nonstandard port. To for example change the Redis port to **6380**, set the following values:
-- Set the environment variable `REDIS_PORT=6380` to the *tubearchivist* service.
-- For the *archivist-redis* service, change the ports to `6380:6380`
-- Additionally set the following value to the *archivist-redis* service: `command: --port 6380 --loadmodule /usr/lib/redis/modules/rejson.so`
-
-### Updating Tube Archivist
-You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. To take advantage of the latest fixes and improvements, make sure you are running the *latest and greatest*.  
-* This project is tested for updates between one or two releases maximum. Further updates back may or may not be supported and you might have to reset your index and configurations to update. Ideally apply new updates at least once per month.
-* There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.  
-* All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version.
-
-### Helm charts
-
-There is a Helm Chart available at https://github.com/insuusvenerati/helm-charts. Mostly self-explanatory but feel free to ask questions in the discord / subreddit.
-
-## Common Errors
-### vm.max_map_count
-**Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.
-
-To temporary set the value run:  
-```
-sudo sysctl -w vm.max_map_count=262144
-```  
-
-To apply the change permanently depends on your host operating system:  
-- For example on Ubuntu Server add `vm.max_map_count = 262144` to the file */etc/sysctl.conf*.
-- On Arch based systems create a file */etc/sysctl.d/max_map_count.conf* with the content `vm.max_map_count = 262144`. 
-- On any other platform look up in the documentation on how to pass kernel parameters.
-
-### Permissions for elasticsearch
-If you see a message similar to `Unable to access 'path.repo' (/usr/share/elasticsearch/data/snapshot)` or `failed to obtain node locks, tried [/usr/share/elasticsearch/data]` and `maybe these locations are not writable` when initially starting elasticsearch, that probably means the container is not allowed to write files to the volume.  
-To fix that issue, shutdown the container and on your host machine run:
-```
-chown 1000:0 -R /path/to/mount/point
-```
-This will match the permissions with the **UID** and **GID** of elasticsearch process within the container and should fix the issue.
-
-### Disk usage
-The Elasticsearch index will turn to *read only* if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`, [link](https://github.com/tubearchivist/tubearchivist#disk-usage).  
-
-Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
+  - This project is tested for updates between one or two releases maximum. Further updates back may or may not be supported and you might have to reset your index and configurations to update. Ideally apply new updates at least once per month.  
+  - There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.  
+  - All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version.   
 
 ## Getting Started
 1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS and some other browsers a compatible format must be specified. For example:
@@ -224,7 +94,49 @@ bestvideo[vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4
 3. On the **downloads** page, click on *Rescan subscriptions* to add videos from the subscribed channels to your Download queue or click on *Add to download queue* to manually add Video IDs, links, channels or playlists.
 4. Click on *Start download* and let **Tube Archivist** to it's thing. 
 5. Enjoy your archived collection!
-  
+
+
+### Port Collisions  
+If you have a collision on port `8000`, best solution is to use dockers *HOST_PORT* and *CONTAINER_PORT* distinction: To for example change the interface to port 9000 use `9000:8000` in your docker-compose file.  
+
+For more information on port collisions, check the docs.
+
+## Common Errors  
+Here is a list of common errors and their solutions.  
+
+### `vm.max_map_count`
+**Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.
+
+To temporary set the value run:  
+```
+sudo sysctl -w vm.max_map_count=262144
+```  
+To apply the change permanently depends on your host operating system:  
+
+ - For example on Ubuntu Server add `vm.max_map_count = 262144` to the file `/etc/sysctl.conf`.
+ - On Arch based systems create a file `/etc/sysctl.d/max_map_count.conf` with the content `vm.max_map_count = 262144`. 
+ - On any other platform look up in the documentation on how to pass kernel parameters.  
+
+
+### Permissions for elasticsearch
+If you see a message similar to `Unable to access 'path.repo' (/usr/share/elasticsearch/data/snapshot)` or `failed to obtain node locks, tried [/usr/share/elasticsearch/data]` and `maybe these locations are not writable` when initially starting elasticsearch, that probably means the container is not allowed to write files to the volume.  
+To fix that issue, shutdown the container and on your host machine run:
+```
+chown 1000:0 -R /path/to/mount/point
+```
+This will match the permissions with the **UID** and **GID** of elasticsearch process within the container and should fix the issue.  
+
+
+### Disk usage
+The Elasticsearch index will turn to ***read only*** if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`.  
+
+Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
+
+## Known limitations
+- Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
+- Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
+- There is currently no flexibility in naming of the media files.
+
 ## Roadmap
 We have come far, nonetheless we are not short of ideas on how to improve and extend this project. Issues waiting for you to be tackled in no particular order:
 
@@ -265,12 +177,6 @@ Implemented:
 - [X] Backup and restore [2021-09-22]
 - [X] Scan your file system to index already downloaded videos [2021-09-14]
 
-## Known limitations
-- Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
-- Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
-- There is currently no flexibility in naming of the media files.
-
-
 ## Donate
 The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.  
 Second best way to support the development is to provide for caffeinated beverages:
@@ -279,9 +185,9 @@ Second best way to support the development is to provide for caffeinated beverag
 * [Paypal Subscription](https://www.paypal.com/webapps/billing/plans/subscribe?plan_id=P-03770005GR991451KMFGVPMQ) for a monthly coffee
 * [ko-fi.com](https://ko-fi.com/bbilly1) for an alternative platform
 
-
 ## Notable mentions
-This is a selection of places where this project has been featured on reddit, in the news, blogs or any other online media. Sorted by newest on the top.
+This is a selection of places where this project has been featured on reddit, in the news, blogs or any other online media, newest on top.
+* **noted.lol**: Dev Debrief, An Interview With the Developer of Tube Archivist, [2023-03-30] [[link](https://noted.lol/dev-debrief-tube-archivist/)]
 * **console.substack.com**: Interview With Simon of Tube Archivist, [2023-01-29] [[link](https://console.substack.com/p/console-142#%C2%A7interview-with-simon-of-tube-archivist)]
 * **reddit.com**: Tube Archivist v0.3.0 - Now Archiving Comments, [2022-12-02] [[link](https://www.reddit.com/r/selfhosted/comments/zaonzp/tube_archivist_v030_now_archiving_comments/)]
 * **reddit.com**: Tube Archivist v0.2 - Now with Full Text Search, [2022-07-24] [[link](https://www.reddit.com/r/selfhosted/comments/w6jfa1/tube_archivist_v02_now_with_full_text_search/)]
@@ -299,3 +205,4 @@ Big thank you to [Digitalocean](https://www.digitalocean.com/) for generously do
     <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/PoweredByDO/DO_Powered_by_Badge_blue.svg" width="201px">
   </a>
 </p>
+

SHOWCASE.MD

@@ -0,0 +1,19 @@
+## Tube Archivist on YouTube
+[![ibracorp-youtube-video-thumb](assets/tube-archivist-ibracorp-O8H8Z01c0Ys.jpg)](https://www.youtube.com/watch?v=O8H8Z01c0Ys)
+Video featuring Tube Archivist generously created by [IBRACORP](https://www.youtube.com/@IBRACORP).
+
+## Screenshots
+![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")  
+*Home Page*: Your recent videos, continue watching incomplete videos.
+
+![channels screenshot](assets/tube-archivist-screenshot-channels.png?raw=true "Tube Archivist Channels")  
+*All Channels*: A list of all your indexed channels, filtered by subscribed only.
+
+![single channel screenshot](assets/tube-archivist-screenshot-single-channel.png?raw=true "Tube Archivist Single Channel")  
+*Single Channel*: Single channel page with additional metadata and sub pages.
+
+![video page screenshot](assets/tube-archivist-screenshot-video.png?raw=true "Tube Archivist Video Page")  
+*Video Page*: Stream your video directly from the interface.
+
+![video page screenshot](assets/tube-archivist-screenshot-download.png?raw=true "Tube Archivist Video Page")  
+*Downloads Page*: Add, control and monitor your download queue.

deploy.sh

@@ -26,6 +26,7 @@ function sync_blackhole {
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
         --exclude "db.sqlite3" \
+        --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
 
     ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist'
@@ -49,6 +50,7 @@ function sync_test {
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
         --exclude "db.sqlite3" \
+        --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
 
     # copy default docker-compose file if not exist
@@ -87,7 +89,7 @@ function validate {
     echo "running black"
     black --exclude "migrations/*" --diff --color --check -l 79 "$check_path"
     echo "running codespell"
-    codespell --skip="./.git,./package.json,./package-lock.json,./node_modules" "$check_path"
+    codespell --skip="./.git,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
     echo "running flake8"
     flake8 "$check_path" --exclude "migrations" --count --max-complexity=10 \
         --max-line-length=79 --show-source --statistics

docker-compose.yml

@@ -34,7 +34,7 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.6.2
+    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.7.0
     container_name: archivist-es
     restart: unless-stopped
     environment:

docs/Channels.md

@@ -1,49 +1,3 @@
-# Channels Overview and Channel Detail Page
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-The channels are organized on two different levels, similar as the [playlists](Playlists):
-
-## Channels Overview
-Accessible at `/channel/` of your Tube Archivist, the **Overview Page** shows a list of all channels you have indexed. 
-- You can filter that list to show or hide subscribed channels with the toggle. Clicking on the channel banner or the channel name will direct you to the *Channel Detail Page*.
-- If you are subscribed to a channel a *Unsubscribe* button will show, if you aren't subscribed, a *Subscribe* button will show instead. 
-
-The **Subscribe to Channels** button <img src="assets/icon-add.png?raw=true" alt="add icon" width="20px" style="margin:0 5px;"> opens a text field to subscribe to a channel. You have a few options:
-- Enter the YouTube channel ID, a 25 character alphanumeric string. For example *UCBa659QWEk1AI4Tg--mrJ2A*
-- Enter the URL to the channel page on YouTube. For example *https://www.youtube.com/channel/UCBa659QWEk1AI4Tg--mrJ2A* or alias url *https://www.youtube.com/@TomScottGo*
-- Enter a channel alias starting with *@*, for example: *@TomScottGo*
-- Enter the video URL for any video and let Tube Archivist extract the channel ID for you, for example *https://www.youtube.com/watch?v=2tdiKTSdE9Y*
-- Add one per line.
-
-To search your channels, click on the search icon <img src="assets/icon-search.png?raw=true" alt="search icon" width="20px" style="margin:0 5px;"> to reach the search page. Start your query with `channel:`, learn more on the [search](Search) page.
-
-## Channel Detail
-Each channel gets a set of channel detail pages.
-- If you are subscribed to the channel, an *Unsubscribe* button will show, else the *Subscribe* button will show.
-- The **Mark as Watched** button will mark all videos of this channel as watched.
-
-### Videos
-Accessible at `/channel/<channel-id>/`, this page shows all the videos you have downloaded from this channel.
-
-### Streams
-If you have any streams indexed, this page will become accessible at `/channel/<channel-id>/streams/`, this page shows all available live streams of that channel. 
-
-### Shorts
-If you have any shorts videos indexed, this page will become accessible at `/channel/<channel-id>/shorts/`, this page shows all the shorts videos of that channel.
-
-### Playlists
-If you have playlists from this channel indexed, this page will become accessible at `/channel/<channel-id>/playlist/`. Activate channel playlist indexing on the about page.
-
-### About
-On the *Channel About* page, accessible at `/channel/<channel-id>/about/`, you can see additional metadata.
-- The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata additionally this will also delete playlists metadata belonging to that channel.
-- The button **Reindex** will reindex all channel metadata. This will also categorize existing videos as shorts or streams.
-- The button **Reindex Videos** will reindex metadata for all videos in this channel.
-
-The channel customize form gives options to change settings on a per channel basis. Any configurations here will overwrite your configurations from the [settings](Settings) page.
-- **Download Format**: Overwrite the download quality for videos from this channel.
-- **Auto Delete**: Automatically delete watched videos from this channel after selected days.
-- **Index Playlists**: Automatically add all Playlists with at least a video downloaded to your index. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.
-- **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. Customize per channel: You can *disable* or *enable* SponsorBlock for certain channels only to overwrite the behavior set on the [Settings](settings) page. Selecting *unset* will remove the overwrite and your setting will fall back to the default on the settings page.
-
-### Downloads
-If you have any videos from this channel pending in the download queue, a *Downloads* link will show, bringing you directly to the [downloads](Downloads) page, filtering the list by the selected channel.
+Equivalent Channels page is: [here](https://docs.tubearchivist.com/channels/).

docs/Downloads.md

@@ -1,59 +1,3 @@
-# Downloads Page
-Accessible at `/downloads/` of your Tube Archivist, this page handles all the download functionality.
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-
-## Rescan Subscriptions
-The **Rescan Subscriptions** icon <img src="assets/icon-rescan.png?raw=true" alt="rescan icon" width="20px" style="margin:0 5px;"> will start a background task to look for new videos from the channels and playlists you are subscribed to.  
-Tube Archivist will get available *videos*, *shorts* and *streams* from each channel, you can define the channel and playlist page size on the [settings page](Settings#subscriptions). With the default page size, expect this process to take around 2-3 seconds for each channel or playlist you are subscribed to. A status message will show the progress.
-
-Then for every video found, **Tube Archivist** will skip the video if it has already been downloaded or if you added it to the *ignored* list before. All the other videos will get added to the download queue. Expect this to take around 2 seconds for each video as **Tube Archivist** needs to grab some additional metadata and artwork. New videos will get added at the bottom of the download queue.
-
-## Download Queue
-The **Start Download** icon <img src="assets/icon-download.png?raw=true" alt="download icon" width="20px" style="margin:0 5px;"> will start the download process starting from the top of the queue. Take a look at the relevant settings on the [Settings Page](Settings#downloads). Once the process started, a progress message will show with additional details and controls: 
-- The stop icon <img src="assets/icon-stop.png?raw=true" alt="stop icon" width="20px" style="margin:0 5px;"> will gracefully stop the download process, once the current video has been finished successfully.
-- The cancel icon <img src="assets/icon-close-red.png?raw=true" alt="close icon" width="20px" style="margin:0 5px;"> is equivalent to killing the process and will stop the download immediately. Any leftover files will get deleted, the canceled video will still be available in the download queue.
-
-After downloading, Tube Archivist tries to add new videos to already indexed playlists and if activated on the settings page, get comments for the new videos.
-
-## Add to Download Queue
-The **Add to Download Queue** icon <img src="assets/icon-add.png?raw=true" alt="add icon" width="20px" style="margin:0 5px;"> opens a text field to manually add videos to the download queue. Add one item per line. You have a few options:
-
-### Videos
-- Add a YouTube video ID, for example *2tdiKTSdE9Y*
-- Add a link to a YouTube video, for example *https://www.youtube.com/watch?v=2tdiKTSdE9Y*
-- Add a link to a YouTube video by providing the shortened URL, for example *https://youtu.be/2tdiKTSdE9Y*
-- Add a link to a shorts video, for example *https://www.youtube.com/shorts/UOfe6e0k7cQ*
-
-### Channels
-- When adding a channel, Tube Archivist will ignore the channel page size as described above, this is meant for an initial download of the whole channel. You can still ignore selected videos from the queue before starting the download.
-- Download a complete channel including shorts and streams by entering:
-  - Channel ID: *UCBa659QWEk1AI4Tg--mrJ2A*
-  - Channel URL: *https://www.youtube.com/channel/UCBa659QWEk1AI4Tg--mrJ2A*
-  - Channel *@* alias handler: For example *@TomScottGo*
-  - Channel alias URL: *https://www.youtube.com/@TomScottGo*
-- Download videos, live streams or shorts only, by providing a partial channel URL:
-  - Videos only: *https://www.youtube.com/@IBRACORP/videos*
-  - Shorts only: *https://www.youtube.com/@IBRACORP/shorts*
-  - Streams only: *https://www.youtube.com/@IBRACORP/streams*
-  - Every other channel sub page will default to download all, for example *https://www.youtube.com/@IBRACORP/featured* will download videos and shorts and streams.
-
-### Playlist
-- Add a playlist ID or URL to add every available video in the list to the download queue, for example *https://www.youtube.com/playlist?list=PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha* or *PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha*. 
-- When adding a playlist to the queue, this playlist will automatically get [indexed](Playlists#playlist-detail).
-- When you add a link to a video in a playlist, Tube Archivist assumes you want to download only the specific video and not the whole playlist, for example *https://www.youtube.com/watch?v=CINVwWHlzTY&list=PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha* will only add one video *CINVwWHlzTY* to the queue.
-
-## The Download Queue
-Below the three buttons you find the download queue. New items will get added at the bottom of the queue, the next video to download once you click on **Start Download** will be the first in the list.
-
-You can filter the download queue with the **filter** dropdown box, the filter will show once you have more than one channel in the download queue. Select the channel to filter by name, the number in parentheses indicates how many videos you have pending from this channel. Reset the filter by selecting *all* from the dropdown. This will generate links for the top 30 channels with pending videos.
-
-Every video in the download queue has two buttons:
-- **Ignore**: This will remove that video from the download queue and this video will not get added again, even when you **Rescan Subscriptions**.
-- **Download now**: This will give priority to this video. If the download process is already running, the prioritized video will get downloaded as soon as the current video is finished. If there is no download process running, this will start downloading this single video and stop after that.  
-
-You can flip the view by activating **Show Only Ignored Videos**. This will show all videos you have previously *ignored*.  
-Every video in the ignored list has two buttons:
-- **Forget**: This will delete the item from the ignored list.
-- **Add to Queue**: This will add the ignored video back to the download queue.  
-
-You can delete your download queue from the [Settings](Settings#actions) page.
+Equivalent Downloads page is: [here](https://docs.tubearchivist.com/downloads/).

docs/FAQ.md

@@ -1,36 +1,3 @@
-# Frequently Asked Questions
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-## 1. Scope of this project
-Tube Archivist is *Your self hosted YouTube media server*, which also defines the primary scope of what this project tries to do:
-- **Self hosted**: This assumes you have full control over the underlying operating system and hardware and can configure things to work properly with Docker, it's volumes and networks as well as whatever disk storage and filesystem you choose to use.
-- **YouTube**: Downloading, indexing and playing videos from YouTube, there are currently no plans to expand this to any additional platforms.
-- **Media server**: This project tries to be a stand alone media server in it's own web interface.
-
-Additionally to that, progress is also happening on:
-- **API**: Endpoints for additional integrations.
-- **Browser Extension**: To integrate between youtube.com and Tube Archivist.
-
-Defining the scope is important for the success of any project:
-- A scope too broad will result in development effort spreading too thin and will run into danger that his project tries to do too many things and none of them well.
-- A too narrow scope will make this project uninteresting and will exclude audiences that could also benefit from this project.
-- Not defining a scope will easily lead to misunderstandings and false hopes of where this project tries to go.
-
-Of course this is subject to change: The scope can be expanded as this project continues to grow and more people contribute.
-
-## 2. Emby/Plex/Jellyfin/Kodi integrations
-Although there are similarities between these excellent projects and Tube Archivist, they have a very different use case. Trying to fit the metadata relations and database structure of a YouTube archival project into these media servers that specialize in Movies and TV shows is always going to be limiting.
-
-Part of the scope is to be its own media server, so that's where the focus and effort of this project is. That being said, the nature of self hosted and open source software gives you all the possible freedom to use your media as you wish.
-
-## 3. To Docker or not to Docker
-This project is a classical docker application: There are multiple moving parts that need to be able to interact with each other and need to be compatible with multiple architectures and operating systems. Additionally Docker also drastically reduces development complexity which is highly appreciated.  
-
-So Docker is the only supported installation method. If you don't have any experience with Docker, consider investing the time to learn this very useful technology.
-
-## 4. Finetuning Elasticsearch
-A minimal configuration of Elasticsearch (ES) is provided in the example docker-compose.yml file. ES is highly configurable and very interesting to learn more about. Refer to the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) if you want to get into it.
-
-## 5. When I subscribe to a channel it only downloads the most recent 50 videos
-Subscribing to a channel is a different operation from downloading it. You can [add the channel to the download queue](https://github.com/tubearchivist/tubearchivist/wiki/Downloads#add-to-download-queue) to download all past videos.
-
-If you want to download the existing videos and also automatically download new videos, then you should both download the channel and subscribe to it.
+Equivalent FAQ page is: [here](https://docs.tubearchivist.com/faq/).

docs/Home.md

@@ -1,41 +1 @@
-# Tube Archivist Wiki
-Welcome to the official Tube Archivist Wiki. This is an up-to-date documentation of user functionality.
-
-Table of contents:
-* [FAQ](FAQ): Frequently asked questions what this project is and tries to do
-* [Channels](Channels): Browse your channels, handle channel subscriptions
-* [Playlists](Playlists): Browse your indexed playlists, handle playlist subscriptions
-* [Downloads](Downloads): Scanning subscriptions, handle download queue
-* [Settings](Settings): All the configuration options
-* [Video](Video): All details of a single video and playlist navigation.
-* [Users](Users): User management admin interface
-* [Search](Search): Search your archive
-* [Installation](Installation): Detailed installation instructions for various platforms.
-
-## Getting Started
-1. [Subscribe](Channels#channels-overview) to some of your favourite YouTube channels.
-2. [Scan](Downloads#rescan-subscriptions) subscriptions to add the latest videos to the download queue.
-3. [Add](Downloads#add-to-download-queue) additional videos, channels or playlist - ignore the ones you don't want to download.
-4. [Download](Downloads#download-queue) and let **Tube Archivist** do it's thing.
-5. Sit back and enjoy your archived and indexed collection!
-
-## General Navigation
-* Clicking on the channel name or the channel icon brings you to the dedicated channel page to show videos from that channel.
-* Clicking on a video title brings you to the dedicated video page and shows additional details.
-* Clicking on a video thumbnail opens the video player and starts streaming the selected video.
-* Clicking on the search icon <img src="assets/icon-search.png?raw=true" alt="gridview icon" width="20px" style="margin:0 5px;"> will open a dedicated search page to search over your complete index.
-* The pagination - if available - builds links for up to 10'000 results, use the search, sort or filter functionality to find what you are looking for.
-
-
-An empty checkbox icon <img src="assets/icon-unseen.png?raw=true" alt="unseen icon" width="20px" style="margin:0 5px;"> will show for videos you haven't marked as watched. Click on it and the icon will change to a filled checkbox <img src="assets/icon-seen.png?raw=true" alt="seen icon" width="20px" style="margin:0 5px;"> indicating it as watched - click again to revert.
-
-When available the <img src="assets/icon-gridview.png?raw=true" alt="gridview icon" width="20px" style="margin:0 5px;"> gridview icon will display the list in a grid. A grid row holds 3 items by default, use the <img src="assets/icon-add.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> icon to add more or the <img src="assets/icon-substract.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> icon to remove items per row, depending on your screen size. The <img src="assets/icon-listview.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> listview icon will arrange the items in a list. The sort icon <img src="assets/icon-sort.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> will open additional sort options.
-
-You can control the video player with the following keyboard shortcuts:
-- `?`: Show help
-- `m`: toggle mute
-- `c`: toggle subtitles if available
-- `>`: increase playback speed
-- `<`: decrease playback speed
-- `←` (left arrow): jump back 5 seconds
-- `→` (right arrow): jump forward 5 seconds
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.

docs/Installation.md

@@ -1,288 +1,3 @@
-# Detailed Installation Instructions for Various Platforms
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-## Table of Contents
-- [Podman](#podman)
-- [Unraid](#unraid)
-- [Truenas Scale](#truenas-scale)
-- [Synology](#synology)
-
-These are beginners guides installation instructions for additional platforms generously provided by users of these platforms. When in doubt, verify the details with the [Readme](https://github.com/tubearchivist/tubearchivist#installing-and-updating). If you see any issues here while using these instructions, please contribute. 
-
-## Podman
-Podman handles container hostname resolving slightly differently than docker, so you need to make a few changes to the `docker-compose.yml` to get up and running.
-
-### Step 1: Follow the installation instructions from the [README](https://github.com/tubearchivist/tubearchivist#installing-and-updating), with a few additional changes to the `docker-compose.yml`.
-
-Edit these additional changes to the `docker-compose.yml`:
- - under `tubearchivist`->`image`:
- 	- prefix the container name with `docker.io/` (or the url of your repo of choice).
- - under `tubearchivist`->`environment`:
-  - `ES_URL`: change `archivist-es` to the internal IP of the computer that will be running the containers.
-  - `REDIS_HOST`: change `archivist-redis` to the internal IP of the computer that will be running the containers (should be the same as above).
- - under `archivist-redis`->`image`: 
- 	- prefix the container name with `docker.io/` again.
- - under `archivist-redis`->`expose`: 
-  - change the whole entry from `expose: ["<PORT>"]` into `ports: ["<PORT>:<PORT>"].
- - under `archivist-es`->`image`: 
- 	- prefix the container name with `docker.io/` again.
- - under `archivist-es`->`expose`: 
-  - change the whole entry from `expose: ["<PORT>"]` into `ports: ["<PORT>:<PORT>"].
-
-### Step 2: Create service files (optional)
-
-Since podman doesn't run as a service, it can't start containers after reboots, at least not without some help.
-
-If you want to enable this behavior, you can follow [this example](https://techblog.jeppson.org/2020/04/create-podman-services-with-podman-compose/) to have `systemd` start up the containers with `podman-compose` when the computer boots up.
-
-
-## Unraid
-
-Tube Archivist, and all if it's dependencies are located in the [community applications](https://unraid.net/community/apps?q=tubearchivist) store. The three containers you will need are as follows:
-
--   **TubeArchivist-RedisJSON**: This container acts as a cache and temporary link between the application and the file system. Used to store and display messages and configuration variables.
--   **TubeArchivist-ES**: ElasticSearch stores video metadata and makes everything searchable. Also keeps track of the download queue.
--   **TubeArchivist**: Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface.
-
-### Step 1: Install  `TubeArchivist-RedisJSON`
-
-![TubeArchivist-RedisJSON](https://i.imgur.com/iGJZKYd.png)
-This is the easiest container to setup of the thee, just make sure that you do not have any port conflicts, and that your `/data` is mounted to the correct path. The other containers will map to the same root directory (/mnt/user/appdata/TubeArchivist).
-
-If you need to install `TubeArchivist-RedisJSON`on a different port, you'll have to follow [these steps](https://github.com/tubearchivist/tubearchivist#redis-on-a-custom-port) later on when installing the `TubeArchivist` container
-
-
-### Step 2: Install  `TubeArchivist-ES`
-![TubeArchivist-ES](https://i.imgur.com/RC38b3u.png)
-ElasticSeach is also pretty easy to setup. Again, make sure you have no port conflicts, make sure that you mapped the ElasticSearch Data to the same root directory as `RedisJSON` (/mnt/user/appdata/TubeArchivist), and make sure to change the default password to something more secure. 
-
-There is four additional settings in the "show more settings" area, but don't require any changes.
-
-
-### Step 3: Install  `TubeArchivist`
-
-![TubeArchivist](https://i.imgur.com/oLsmwBZ.png)
-It's finally time to set up TubeArchivist!
-
- - `HOST:`This is a list of IP addresses that you will host TA from. Example, 192.168.0.14 is the IP address of my Unraid server. If I was going to access TA from a VPN, or domain name, I'd put those next to my host IP with just a space separating the different addresses. More information [here.](https://github.com/tubearchivist/tubearchivist#installing-and-updating)
-
-
- - `Port:`Again, make sure that you have no port conflicts on 8000.
-   
- - `Youtube Media Path:` is where you'll download all of your videos to.
-   Make sure that this is an empty directory to not cause confusion when
-   starting the application. If you have existing videos that you'd like
-   to import into Tube Archivist, please checkout the [settings
-   wiki.](https://github.com/tubearchivist/tubearchivist/wiki/Settings#manual-media-files-import)
-   
-   
-- `Appdata:` This should be the same base path as the other two containers (/mnt/user/appdata/TubeArchivist).
-   
- - `TA Username:`This will be your username for TubeArchivist.
-   
- - `TA Password:`This will be your password for TubeArchivist.
-   
- - `Redis` This will be JUST the ip address of your redis container.
-
- - `ElasticSearch Password:`This is the password you defined in the `TubeArchivist-ES` container.
- - `ElasticSearch:` This seems to cause some confusion, but it's a pretty simple step, just replace the IP and Port to match you `TubeArchivist-ES` container.
-
- (example: if your IP is 192.168.0.14, the value should be http://192.168.0.14:9200)
-
- - `Time Zone:` This is an important step for your scheduler, to find your timezone, use a site like [TimeZoneConverter](http://www.timezoneconverter.com/cgi-bin/findzone.tzc) 
-
-### From there, you should be able to start up your containers and you're good to go!
-If you're still having trouble, join us on [discord](https://www.tubearchivist.com/discord) and come to the #support channel.
-
-<br />
-<br />
-
-## Truenas Scale 
-
-Truenas Scale can be a bit confusing, with its k3s kubernetes implementation. 
-
-However, there is a step by step guide available for it's users here:
-
-https://heavysetup.info/applications/tube-archivist/dataset/
-
-- Ensure you are navigating the columns under `Tube Archivist` on the left hand side of the screen
-
-<br />
-<br />
-
-## Synology
-
-There are several different methods to install TubeArchivist on Synology platforms. This will focus on the available `docker` package and `docker-compose` implementations.
-
-### Prepare Directories/Folders
-Before we setup TubeArchivist, we need to setup the directories/folders. You are assumed to be logged into the Synology NAS.
-#### 1. Docker Base Folder
-   1. Open the `File Station` utility.
-   2. Click on the **Create🔽** button and choose *Create New Shared Folder*.
-   3. **Name** the folder "Docker".
-   4. Add a **Description**.
-   5. Select the **Volume Location**. 
-      > Note: By default, this will be where all data is stored. Change the folders as best meets your requirements.
-   6. Select the appropriate options from the remaining checkbox configurations.
-![Synology - Create Docker Folder](assets/Synology_0.2.0_Docker-Folder-Create.png)
-   7. Click the **Next** button.
-   8. If you are going to **Encrypt** your folder, check the appropriate box and provide the Encryption Key and its confirmation.
-   9. Click the **Next** button.
-   10. On the **Advanced Settings** page, you can select the *Enable data checksum for advanced data integrity* setting. This may cause a performance impact, but will allow for potential file self-healing. **This cannot be changed later.**
-       > Note: This is not recommended, as we will be hosting databases within this folder.
-   11. If you are enabling a quota for how large the folder can get, you can select the *Enabled shared folder quota* setting and choose the maximum size this folder can grow. This can be changed later.
-   12. Click the **Next** button.
-   13. Confirm the settings, then click the **Apply** button. This will create the folder.
-#### 2. TubeArchivist Base Folder
-   1. Open the `File Station` utility.
-   2. Select the "Docker" folder on the left-hand side.
-   3. Click on the `Create🔽` button and choose *create Folder*.
-   4. **Name** the folder "TubeArchivist".
-#### 3. Redis Data
-   1. Open the `File Station` utility.
-   2. Select the "Docker" folder on the left-hand side.
-   3. Select the "TubeArchivist" folder beneath "Docker".
-   4. Click on the `Create🔽` button and choose *create Folder*.
-   5. **Name** the folder "redis".
-#### 4. Elastic Search Data
-   1. Open the `File Station` utility.
-   2. Select the "Docker" folder on the left-hand side.
-   3. Select the "TubeArchivist" folder beneath "Docker".
-   4. Click on the `Create🔽` button and choose *create Folder*.
-   5. **Name** the folder "es".
-#### 5. TubeArchivist Cache
-   1. Open the `File Station` utility.
-   2. Select the "Docker" folder on the left-hand side.
-   3. Select the "TubeArchivist" folder beneath "Docker".
-   4. Click on the `Create🔽` button and choose *create Folder*.
-   5. **Name** the folder "cache".
-#### 6. TubeArchivist Output
-   1. Open the `File Station` utility.
-   2. Select the "Docker" folder on the left-hand side.
-   3. Select the "TubeArchivist" folder beneath "Docker".
-   4. Click on the `Create🔽` button and choose *create Folder*.
-   5. **Name** the folder "media".
-#### 7. Confirm Folder Structure
-Once all of the folders have been created, it should have a folder structure within Docker\TubeArchivist that includes "cache", "es", "media", and "redis" folders.
-![Synology - Docker Folder Structure](assets/Synology_0.2.0_Docker-Folder-Structure.png)
-
-#### 8. Change Permissions - CLI Required
-> If you do not have SSH access enabled for CLI, [enable it](https://kb.synology.com/en-sg/DSM/tutorial/How_to_login_to_DSM_with_root_permission_via_SSH_Telnet) before continuing.
-   1. Open the SSH connection to the Synology. Login as your primary `Admin` user, or the user that was enabled for SSH access.
-   2. Elevate your access to `root`. Steps are provided [here](https://kb.synology.com/en-sg/DSM/tutorial/How_to_login_to_DSM_with_root_permission_via_SSH_Telnet).
-   3. Change directories to the **Volume** where the "Docker" folder resides.
-      </br>Example: `cd /volume1`
-   4. Change directories to the "Docker" folder.
-      </br>Example: `cd Docker`
-   5. Change directories to the "TubeArchivist" folder.
-      </br>Example: `cd TubeArchivist`
-   6. Change the owner of the "redis" folder. *If correct, this does not have an output.*
-      </br>Example: `chown 999:100 redis`
-   7. Change the owner of the "es" folder. *If correct, this does not have an output.*
-      </br>Example: `chown 1000:1000 es`
-   8. Confirm that the folders have the correct permissions.
-      </br>Example: `ls -hl`
-![Synology - Docker Folder Permissions Command](assets/Synology_0.2.0_Docker-Folder-Permissions-Commands.png)
-   9. Logout from root.
-      </br>Example: `logout`
-   10. Disconnect from the SSH connection.
-      </br>Example: `exit`
-### Docker Setup
-1. Install the `Docker` Synology Package.
-  1. Log in to your Synology NAS.
-  2. Open the `Package Center` utility.
-  3. Search for `Docker`.
-  4. Click `Install`.
-
-![Synology - Install Docker Utility](assets/Synology_0.2.0_Docker-Install.png)
-
-2. After `Docker` is installed, open the `Docker` utility.
-3. Go to the `Registry` tab.
-4. Search for the following `images` and download them. Follow the recommended versions for each of the images.
-  - `redislabs/rejson`
-![Synology - Redis Image Search](assets/Synology_0.2.0_Docker-Redis-Search.png)
-  - `bbilly1/tubearchivist-es`
-![Synology - ElasticSearch Image Search](assets/Synology_0.2.0_Docker-ES-Search.png)
-  - `bbilly1/tubearchivist`
-![Synology - TubeArchivist Image Search](assets/Synology_0.2.0_Docker-TA-Search.png)
-
-
-
-5. Go to the `Image` tab. From here, create an container based on each image with the associated configurations below.
-  - ElasticSearch
-    1. Select the associated image.
-    2. Click the **Launch** button in the top.
-    3. Edit the **Container Name** to be "tubearchivist-es".
-    4. Click on the **Advanced Settings** button.
-    5. In the **Advanced Settings** tab, check the box for `Enable auto-restart`.
-    6. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/es" folder, then type in `/usr/share/elasticsearch/data` for the mount path.
-    7. In the **Network** tab, leave the default `bridge` Network (unless you have a specific Network design that you know how to implement).
-    8. In the **Port Settings** tab, replace the "Auto" entry under **Local Port** with the port that will be used to connect to ElasticSearch (default is 9200).
-    9. In the **Port Settings** tab, select the entryline for port 9300 and **➖ delete** the line. It is not needed for this container.
-    10. The **Links** tab does not require configuration for this container.
-    11. In the **Environment** tab, add in the following ElasticSearch specific environment variables that may apply.
-        - "discovery.type=single-node"
-        - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-        - "UID=1000"
-        - "GID=0"
-        - "xpack.security.enabled=true"
-        - "ELASTIC_PASSWORD=verysecret" 
-        - "path.repo=/usr/share/elasticsearch/data/snapshot"
-          > Do not use the default password as it is very insecure.
-          > Activating snapshots for backups should only be done *after* setting the `path.repo` setting.
-        ![Synology - ElasticSearch Environment Configurations](assets/Synology_0.2.0_Docker-ES-Env-Conf.png)
-    12. Click on the **Apply** button.
-    13. Back on the **Create Container** screen, click the **Next** button.
-    14. Review the settings to confirm, then click the **Apply** button.
-  - Redis
-    1. Select the associated image.
-    2. Click the **Launch** button in the top.
-    3. Edit the **Container Name** to be "tubearchivist-redis".
-    4. Click on the **Advanced Settings** button.
-    5. In the **Advanced Settings** tab, check the box for `Enable auto-restart`.
-    6. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/redis" folder, then type in `/data` for the mount path.
-    7. In the **Network** tab, leave the default `bridge` Network (unless you have a specific Network design that you know how to implement).
-    8. In the **Port Settings** tab, replace the "Auto" entry under **Local Port** with the port that will be used to connect to Redis (default is 6379).
-    9. In the **Links** tab, select the "tubearchivist-es" container from the **Container Name** dropdown and provide it the same alias, "tubearchivist-es".
-    10. In the **Environment** tab, add in any Redis specific environment variables that may apply (none by default).
-    11. Click on the **Apply** button.
-    12. Back on the **Create Container** screen, click the **Next** button.
-    13. Review the settings to confirm, then click the **Apply** button.
-  
-  - TubeArchivist
-    1. Select the associated image.
-    2. Click the **Launch** button in the top.
-    3. Edit the **Container Name** to be "tubearchivist".
-    4. Click on the **Advanced Settings** button.
-    5. In the **Advanced Settings** tab, check the box for `Enable auto-restart`.
-    6. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/cache" folder, then type in `/cache` for the mount path.
-    7. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/media" folder, then type in `/youtube` for the mount path.
-    8. In the **Network** tab, leave the default `bridge` Network (unless you have a specific Network design that you know how to implement).
-    9. In the **Port Settings** tab, replace the "Auto" entry under **Local Port** with the port that will be used to connect to TubeArchivist (default is 8000).
-    10. In the **Links** tab, select the "tubearchivist-es" container from the **Container Name** dropdown and provide it the same alias, "tubearchivist-es".
-    11. In the **Links** tab, select the "tubearchivist-redis" container from the **Container Name** dropdown and provide it the same alias, "tubearchivist-redis".
-    12. In the **Environment** tab, add in the following TubeArchivist specific environment variables that may apply. **Change the variables as-is appropriate to your use case. Follow the [README section](https://github.com/tubearchivist/tubearchivist#tube-archivist) for details on what to set each variable.**
-        - "TA_HOST=synology.local"
-        - "ES_URL=http://tubearchivist-es:9200"
-        - "REDIS_HOST=tubearchivist-redis"
-        - "HOST_UID=1000"
-        - "HOST_GID=1000"
-        - "TA_USERNAME=tubearchivist"
-        - "TA_PASSWORD=verysecret"
-        - "ELASTIC_PASSWORD=verysecret"
-        - "TZ=America/New_York"
-          > Do not use the default password as it is very insecure.
-          > Ensure that ELASTIC_PASSWORD matches the password used on the tubearchivist-es container.
-        ![Synology - TubeArchivist Environment Configurations](assets/Synology_0.2.0_Docker-TA-Env-Conf.png)
-    13. Click on the **Apply** button.
-    14. Back on the **Create Container** screen, click the **Next** button.
-    15. Review the settings to confirm, then click the **Apply** button.
-6. After the containers have been configured and started, you can go to the **Container** tab and monitor the containers.
-7. To review the logs to ensure that the system has started successfully, select the "tubearchivist" container and click on the **Details** button. In the new window, go to the **Log** tab. Monitor the logs until either an error occurs or the message `celery@tubearchivist ready.` is in the logs. This may take a few minutes, especially for a first time setup.
-   > Note: Synology Docker presents the logs in a pagination format. If you are not seeing the logs update, check if there are additional pages.
-8. After it has started, go to the location in the `TA_HOST`. This should give you the standard TubeArchivist login screen.
-<!-- 
-### Docker-Compose Setup -->
-<!-- This section is a Work In Progress -->
-
-### From there, you should be able to start up your containers and you're good to go!
-If you're still having trouble, join us on [discord](https://www.tubearchivist.com/discord) and come to the #support channel.
+Equivalent pages are located under *installation* on the left.

docs/Playlists.md

@@ -1,25 +1,3 @@
-# Playlist Overview and Playlist Detail Page
-The playlists are organized in two different levels, similar as the [channels](Channels):
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-## Playlist Overview
-Accessible at `/playlist/` of your Tube Archivist, this **Overview Page** shows a list of all playlists you have indexed over all your channels.
-- You can filter that list to show only subscribed to playlists with the toggle.
-
-You can index playlists of a channel from the channel detail page as described [here](Channels#channel-detail).
-
-The **Subscribe to Playlist** button <img src="assets/icon-add.png?raw=true" alt="add icon" width="20px" style="margin:0 5px;"> opens a text field to subscribe to playlists. You have a few options:
-- Enter the YouTube playlist id, for example: *PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha*
-- Enter the Youtube dedicated playlist url, for example: *https://www.youtube.com/playlist?list=PL96C35uN7xGLLeET0dOWaKHkAlPsrkcha*
-- Add one per line.
-- NOTE: It doesn't make sense to subscribe to a playlist if you are already subscribed the corresponding channel as this will slow down the **Rescan Subscriptions** [task](Downloads#rescan-subscriptions).
-
-You can search your indexed playlists by clicking on the search icon <img src="assets/icon-search.png?raw=true" alt="search icon" width="20px" style="margin:0 5px;">. This will open a dedicated page.
-
-## Playlist Detail
-Each playlist will get a dedicated playlist detail page accessible at `/playlist/<playlist-id>/` of your Tube Archivist. This page shows all the videos you have downloaded from this playlist.
-
-- If you are subscribed to the playlist, an Unsubscribe button will show, else the Subscribe button will show.
-- The **Mark as Watched** button will mark all videos of this playlist as watched.
-- The button **Reindex** will reindex the playlist metadata.
-- The button **Reindex Videos** will reindex all videos from this playlist.
-- The **Delete Playlist** button will give you the option to delete just the *metadata* which won't delete any media files or *delete all* which will delete metadata plus all videos belonging to this playlist.
+Equivalent Playlist page is: [here](https://docs.tubearchivist.com/playlists/).

docs/Search.md

@@ -1,58 +1,3 @@
-# Search Page
-Accessible at `/search/` of your **Tube Archivist**, search your archive for Videos, Channels and Playlists - or even full text search throughout your indexed subtitles.
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-Just start typing to start a **simple** search *or* **start your query with a primary keyword** to search for a specific type and narrow down the result with secondary keywords. Secondary keywords can be in any order. Use *yes* or *no* for boolean values.
-
-- This will return 30 results per query, pagination is not implemented yet.
-- All your queries are case insensitive and are normalized to lowercase.
-- All your queries are analyzed for the english language, this means *singular*, *plural* and word variations like *-ing*, *-ed*, *-able* etc are treated as synonyms.
-- Keyword value parsing begins with the `keyword:` name all the way until the end of query or the next keyword, e.g. in `video:learn python channel:corey`, the keyword `video` has value `learn python`.
-- Fuzzy search is activated for all your searches by default. This can catch typos in your queries or in the matching documents with one to two letters difference, depending on the query length. You can configure fuzziness with the secondary keyword `fuzzy:`, e.g:
-  - `fuzzy:0` or `fuzzy:no`: Deactivate fuzzy matching.
-  - `fuzzy:1`: Set fuzziness to one letter difference.
-  - `fuzzy:2`: Set fuzziness to two letters difference.
-- All text searches are ranked, meaning the better a match the higher ranked the result. Unless otherwise stated, queries with multiple words are processed with the `and` operator, meaning all words need to match so each word will narrow down the result.
-
-## Simple
-Start your query without a keyword to make a simple query (primary keyword `simple:` is implied). This will search in *video titles*, *channel names* and *playlist titles* and will return matching videos, channels and playlists. Keyword searches will return more results in a particular category due to the fact that more fields are searched for matches. Simple queries do not have any secondary keywords.
-
-## Video
-Start your query with the **primary keyword** `video:` to search for videos only. This will search through the *video titles*, *tags* and *category* fields. Narrow your search down with secondary keywords:
-- `channel:` search for videos matching the channel name.
-- `active:` is a boolean value, to search for videos that are still active on youtube or that are not active any more.
-
-**Example**:
-- `video:learn python channel:corey schafer active:yes`: This will return all videos with the term *Learn Python* from the channel *Corey Schafer* that are still *Active* on YouTube.
-- `video: channel:tom scott active:no`: Note the omitted term after the primary key, this will show all videos from the channel *Tom Scott* that are no longer active on YouTube.
-
-## Channel
-Start with the `channel:` **primary keyword** to search for channels matching your query. This will search through the *channel name* and *channel description* fields. Narrow your search down with secondary keywords:
-- `subscribed:` is a boolean value, search for channels that you are subscribed to or not.
-- `active:` is a boolean value, to search for channels that are still active on YouTube or that are no longer active.
-
-**Example**:
-- `channel:linux subscribed:yes`: Search for channels with the term *Linux* that you are subscribed to.
-- `channel: active:no`: Note the omitted term after the primary key, this will return all channels that are no longer active on YouTube.
-
-## Playlist
-Start your query with the **primary keyword** `playlist:` to search for playlists only. This will search through the *playlist title* and *playlist description* fields. Narrow down your search with these secondary keywords:
-- `subscribed`: is a boolean value, search for playlists that you are subscribed to or not.
-- `active:` is a boolean value, to search for playlists that are still active on YouTube or that are no longer active.
-
-**Example**:
-- `playlist:backend engineering subscribed:yes`: Search for playlists about *Backend Engineering* that you are subscribed to.
-- `playlist: active:yes subscribed:yes`: Note the omitted primary search term, this will return all playlists active on YouTube that you are subscribed to.
-- `playlist:html css active:yes`: Search for playlists containing *HTML CSS* that are still active on YouTube.
-
-## Full
-Start a full text search by beginning your query with the **primary keyword** `full:`. This will search through your indexed Subtitles showing segments with possible matches. This will only show any results if you have activated *subtitle download and index* on the settings page. The operator for full text searches is `or` meaning when searching for multiple words not all words need to match, but additional words will change the ranking of the result, the more words match and the better they match, the higher ranked the result. The matching words will get highlighted in the text preview.
-
-Clicking the play button on the thumbnail will open the inplace player at the timestamp from where the segment starts. Same when clicking the video title, this will open the video page and put the player at the segment timestamp. This will overwrite any previous playback position.
-
-Narrow down your search with these secondary keywords:
-- `lang`: Search for matches only within a language. Use the same two letter ISO country code as you have set on the settings page.
-- `source`: Can either be *auto* to search through auto generated subtitles only or *user* to search through user uploaded subtitles only.
-
-**Example**:
-- `full:contribute to open source lang:en` search for subtitle segments matching with the words *Contribute to Open Source* in the language *en*.
-- `full:flight simulator cockpit source:user` to search for the words *Flight Simulator Cockpit* from *user* uploaded subtitle segments.
+Equivalent Search page is: [here](https://docs.tubearchivist.com/search/).

docs/Settings.md

@@ -1,191 +1,3 @@
-# Settings Page
-Accessible at `/settings/` of your **Tube Archivist**, this page holds all the configurations and additional functionality related to the database.
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-Click on **Update Settings** at the bottom of the form to apply your configurations.
-
-## Color scheme
-Switch between the easy on the eyes dark theme and the burning bright theme.
-
-## Archive View
-- **Page Size**: Defines how many results get displayed on a given page. Same value goes for all archive views.
-
-## Subscriptions
-Settings related to the channel management. Disable shorts or streams by setting their page size to 0 (zero).
-- **Channel Page Size**: Defines how many pages will get analyzed by **Tube Archivist** each time you click on *Rescan Subscriptions*. The default page size used by yt-dlp is **50**, that's also the recommended value to set here. Any value higher will slow down the rescan process, for example if you set the value to 51, that means yt-dlp will have to go through 2 pages of results instead of 1 and by that doubling the time that process takes.
-- **Live Page Size**: Same as above, but for channel live streams.
-- **Shorts page Size**: Same as above, but for shorts videos.
-
-## Downloads
-Settings related to the download process.
-- **Download Limit**: Stop the download process after downloading the set quantity of videos.
-- **Download Speed Limit**: Set your download speed limit in KB/s. This will pass the option `--limit-rate` to yt-dlp.
-- **Throttled Rate Limit**: Restart download if the download speed drops below this value in KB/s. This will pass the option `--throttled-rate` to yt-dlp. Using this option might have a negative effect if you have an unstable or slow internet connection.
-- **Sleep Interval**: Time in seconds to sleep between requests to YouTube. It's a good idea to set this to **3** seconds. Might be necessary to avoid throttling.
-- **Auto Delete Watched Videos**: Automatically delete videos marked as watched after selected days. If activated, checks your videos after download task is finished.
-
-## Download Format
-Additional settings passed to yt-dlp.
-- **Format**: This controls which streams get downloaded and is equivalent to passing `--format` to yt-dlp. Use one of the recommended one or look at the documentation of [yt-dlp](https://github.com/yt-dlp/yt-dlp#format-selection). Please note: The option `--merge-output-format mp4` is automatically passed to yt-dlp to guarantee browser compatibility. Similar to that, `--check-formats` is passed as well to check that the selected formats are actually downloadable.
-- **Embed Metadata**: This saves the available tags directly into the media file by passing `--embed-metadata` to yt-dlp.
-- **Embed Thumbnail**: This will save the thumbnail into the media file by passing `--embed-thumbnail` to yt-dlp.
-
-## Subtitles
-- **Download Setting**: Select the subtitle language you like to download. Add a comma separated list for multiple languages.
-- **Source Settings**: User created subtitles are provided from the uploader and are usually the video script. Auto generated is from YouTube, quality varies, particularly for auto translated tracks.
-- **Index Settings**: Enabling subtitle indexing will add the lines to Elasticsearch and will make subtitles searchable. This will increase the index size and is not recommended on low-end hardware.
-
-## Comments
-- **Download and index comments**: Set your configuration for downloading and indexing comments. This takes the same values as documented in the `max_comments` section for the youtube extractor of [yt-dlp](https://github.com/yt-dlp/yt-dlp#youtube). Add without space between the four different fields: *max-comments,max-parents,max-replies,max-replies-per-thread*. Example:
-  - `all,100,all,30`: Get 100 max-parents and 30 max-replies-per-thread.
-  - `1000,all,all,50`: Get a total of 1000 comments over all, 50 replies per thread.
-- **Comment sort method**: Change sort method between *top* or *new*. The default is *top*, as decided by YouTube.
-- The [Refresh Metadata](#refresh-metadata) background task will get comments from your already archived videos, spreading the requests out over time.  
-
-Archiving comments is slow as only very few comments get returned per request with yt-dlp. Choose your configuration above wisely. Tube Archivist will download comments after the download queue finishes, your videos will be already available while the comments are getting downloaded.
-
-## Cookie
-Importing your YouTube Cookie into Tube Archivist allows yt-dlp to bypass age restrictions, gives access to private videos and your *watch later* or *liked videos*.
-
-### Security concerns
-Cookies are used to store your session and contain your access token to your google account, this information can be used to take over your account. Treat that data with utmost care as you would any other password or credential. *Tube Archivist* stores your cookie in Redis and will automatically append it to yt-dlp for every request.
-
-### Auto import
-Easiest way to import your cookie is to use the **Tube Archivist Companion** [browser extension](https://github.com/tubearchivist/browser-extension) for Firefox and Chrome.
-
-### Alternative Manual Export your cookie
-- Install **Cookies.txt** addon for [chrome](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid) or [firefox](https://addons.mozilla.org/firefox/addon/cookies-txt).
-- Visit YouTube and login with whichever YouTube account you wish to use to generate the cookies.
-- Click on the extension icon in the toolbar - it will drop down showing the active cookies for YT.
-- Click Export to export the cookies, filename is by default *cookies.google.txt*.
-
-### Alternative Manual Import your cookie
-Place the file *cookies.google.txt* into the *cache/import* folder of Tube Archivist and enable the cookie import. Once you click on *Update Application Configurations* to save your changes, your cookie will get imported and stored internally.
-
-Once imported, a **Validate Cookie File** button will show, where you can confirm if your cookie is working or not.
-
-### Use your cookie
-Once imported, additionally to the advantages above, your [Watch Later](https://www.youtube.com/playlist?list=WL) and [Liked Videos](https://www.youtube.com/playlist?list=LL) become a regular playlist you can download and subscribe to as any other [playlist](Playlists).
-
-### Limitation
-There is only one cookie per Tube Archivist instance, this will be shared between all users.
-
-## Integrations
-All third party integrations of TubeArchivist will **always** be *opt in*.
-- **API**: Your access token for the Tube Archivist API.
-- **returnyoutubedislike.com**: This will get return dislikes and average ratings for each video by integrating with the API from [returnyoutubedislike.com](https://www.returnyoutubedislike.com/).
-- **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. If a video doesn't have timestamps, or has unlocked timestamps, use the browser addon to contribute to this excellent project. Can also be activated and deactivated as a per [channel overwrite](Settings#channel-customize).
-
-## Snapshots
-System snapshots will automatically make daily snapshots of the Elasticsearch index. The task will start at 12pm your local time. Snapshots are deduplicated, meaning that each snapshot will only have to backup changes since the last snapshot. The initial snapshot may be slow, but subsequent runs will be much faster. There is also a cleanup function implemented, that will remove snapshots older than 30 days.
-
-This will make a snapshot of your metadata index only, no media files or additional configuration variables you have set on the settings page will be backed up.
-
-Due to these improvements compared to the previous backup solution, system snapshots will replace the current backup system in a future version.
-
-Before activating system snapshots, you'll have to add one additional environment variables to the *archivist-es* container:
-```
-path.repo=/usr/share/elasticsearch/data/snapshot
-```
-The variable `path.repo` will set the folder where the snapshots will go inside the Elasticsearch container, you can't change it, but the variable needs to be set. Rebuild the container for changes to take effect, e.g `docker compose up -d`.
-
-- **Create snapshot now**: Will start the snapshot process now, outside of the regular daily schedule.
-- **Restore**: Restore your index to that point in time.
-
-# Scheduler Setup
-Schedule settings expect a cron like format, where the first value is minute, second is hour and third is day of the week. Day 0 is Sunday, day 1 is Monday etc.
-
-Examples:
-- **0 15 \***: Run task every day at 15:00 in the afternoon.
-- **30 8 \*/2**: Run task every second day of the week (Sun, Tue, Thu, Sat) at 08:30 in the morning.
-- **0 \*/3,8-17 \***: Execute every hour divisible by 3, and every hour during office hours (8 in the morning - 5 in the afternoon).
-- **0 8,16 \***: Execute every day at 8 in the morning and at 4 in the afternoon.
-- **auto**: Sensible default.
-- **0**: (zero), deactivate that task.
-
-NOTE:
-- Changes in the scheduler settings require a container restart to take effect.
-- Cron format as *number*/*number* are none standard cron and are not supported by the scheduler, for example **0 0/12 \*** is invalid, use **0 \*/12 \*** instead.
-- Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason, the scheduler doesn't support schedules that trigger more than once per hour.
-
-## Rescan Subscriptions
-That's the equivalent task as run from the downloads page looking through your channel and playlist and add missing videos to the download queue.
-
-## Start download
-Start downloading all videos currently in the download queue.
-
-## Refresh Metadata
-Rescan videos, channels and playlists on youtube and update metadata periodically. This will also refresh your subtitles based on your current settings. If an item is no longer available on YouTube, this will deactivate it and exclude it from future refreshes. This task is meant to be run once per day, set your schedule accordingly.
-
-The field **Refresh older than x days** takes a number where TubeArchivist will consider an item as *outdated*. This value is used to calculate how many items need to be refreshed today based on the total indexed. This will spread out the requests to YouTube. Sensible value here is **90** days.
-
-## Thumbnail check
-This will check if all expected thumbnails are there and will delete any artwork without matching video.
-
-## ZIP file index backup
-Create a zip file of the metadata and select **Max auto backups to keep** to automatically delete old backups created from this task. For data consistency, make sure there aren't any other tasks running that will change the index during the backup process. This is very slow, particularly for large archives. Use snapshots instead. 
-
-
-# Actions
-Additional database functionality.
-
-## Delete download queue
-The button **Delete all queued** will delete all pending videos from the download queue. The button **Delete all ignored** will delete all videos you have previously ignored.
-
-## Manual Media Files Import
-NOTE: This is inherently error prone, as there are many variables, some outside of the control of this project. Read this carefully and use at your own risk. 
-
-Add the files you'd like to import to the */cache/import* folder. Only add files, don't add subdirectories. All files you are adding, need to have the same *base name* as the media file. Then start the process from the settings page *Manual Media Files Import*.
-
-Valid media extensions are *.mp4*, *.mkv* or *.webm*. If you have other file extensions or incompatible codecs, convert them first to mp4. **Tube Archivist** can identify the videos with one of the following methods.
-
-### Method 1:
-Add a matching *.info.json* file with the media file. Both files need to have the same base name, for example:
-- For the media file: `<base-name>.mp4`
-- For the JSON file: `<base-name>.info.json`
-
-The import process then looks for the 'id' key within the JSON file to identify the video.
-
-### Method 2:
-Detect the YouTube ID from filename, this accepts the default yt-dlp naming convention for file names like:
-- `<base-name>[<youtube-id>].mp4`
-- The YouTube ID in square brackets at the end of the filename is the crucial part.
-
-### Offline import:
-If the video you are trying to import is not available on YouTube any more, **Tube Archivist** can import the required metadata:
-- The file `<base-name>.info.json` is required to extract the required information.
-- Add the thumbnail as `<base-name>.<ext>`, where valid file extensions are *.jpg*, *.png* or *.webp*. If there is no thumbnail file, **Tube Archivist** will try to extract the embedded cover from the media file or will fallback to a default thumbnail.
-- Add subtitles as `<base-name>.<lang>.vtt` where *lang* is the two letter ISO country code. This will archive all subtitle files you add to the import folder, independent from your configurations. Subtitles can be archived and used in the player, but they can't be indexed or made searchable due to the fact, that they have a very different structure than the subtitles as **Tube Archivist** needs them.
-- For videos, where the whole channel is not available any more, you can add the `<channel-id>.info.json` file as generated by *youtube-dl/yt-dlp* to get the full metadata. Alternatively **Tube Archivist** will extract as much info as possible from the video info.json file. 
-
-### Some notes:
-- This will **consume** the files you put into the import folder: Files will get converted to mp4 if needed (this might take a long time...) and moved to the archive, *.json* files will get deleted upon completion to avoid having duplicates on the next run.
-- For best file transcoding quality, convert your media files with desired settings first before importing.
-- Maybe start with a subset of your files to import to make sure everything goes well...
-- Follow the logs to monitor progress and errors: `docker-compose logs -f tubearchivist`.
-
-## Embed thumbnails into media file
-This will write or overwrite all thumbnails in the media file using the downloaded thumbnail. This is only necessary if you didn't download the files with the option *Embed Thumbnail* enabled or want to make sure all media files get the newest thumbnail. Follow the docker-compose logs to monitor progress.
-
-## ZIP file index backup
-This will backup your metadata into a zip file. The file will get stored at *cache/backup* and will contain the necessary files to restore the Elasticsearch index formatted **nd-json** files. For data consistency, make sure there aren't any other tasks running that will change the index during the backup process. This is very slow, particularly for large archives.  
-
-BE AWARE: This will **not** backup any media files, just the metadata from the Elasticsearch.
-
-## Restore From Backup
-The restore functionality will expect the same zip file in *cache/backup* as created from the **Backup database** function. This will recreate the index from the snapshot. There will be a list of all available backup to choose from. The *source* tag can have these different values:
-- **manual**: For backups manually created from here on the settings page.
-- **auto**: For backups automatically created via a sceduled task.
-- **update**: For backups created after a Tube Archivist update due to changes in the index.
-- **False**: Undefined.
-
-BE AWARE: This will **replace** your current index with the one from the backup file. This won't restore any media files.
-
-## Rescan Filesystem
-This function will go through all your media files and looks at the whole index to try to find any issues:
-- Should the filename not match with the indexed media url, this will rename the video files correctly and update the index with the new link.
-- When you delete media files from the filesystem outside of the Tube Archivist interface, this will delete leftover metadata from the index.
-- When you have media files that are not indexed yet, this will grab the metadata from YouTube like it was a newly downloaded video. This can be useful when restoring from an older backup file with missing metadata but already downloaded mediafiles. NOTE: This only works if the media files are named in the same convention as Tube Archivist does, particularly the YouTube ID needs to be at the same index in the filename, alternatively see above for *Manual Media Files Import*.
-- The task will stop, when adding a video fails, for example if the video is no longer available on YouTube.
-- This will also check all of your thumbnails and download any that are missing.
-
-BE AWARE: There is no undo.
+Equivalent Settings page is: [here](https://docs.tubearchivist.com/settings/).

docs/Users.md

@@ -1,20 +1,3 @@
-# User Management
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-For now, **Tube Archivist** is a single user application. You can create multiple users with different names and passwords, they will share the same videos and permissions but some interface configurations are on a per user basis. *More is on the roadmap*.
-
-## Superuser
-The first user gets created with the environment variables **TA_USERNAME** and **TA_PASSWORD** from your docker-compose file. That first user will automatically have *superuser* privileges. 
-
-## Admin Interface
-When logged in from your *superuser* account, you are able to access the admin interface from the settings page or at `/admin/`. This interface holds all functionality for user management.
-
-## Create additional users
-From the admin interface when you click on *Accounts* you will get a list of all users. From there you can create additional users by clicking on *Add Account*, provide a name and confirm password and click on *Save* to create the user.
-
-## Changing users
-You can delete or change permissions and password of a user by clicking on the username from the *Accounts* list page and follow the interface from there. Changing the password of the *superuser* here will overwrite the password originally set with the environment variables.
-
-## Reset
-Delete all user configurations by deleting the file `cache/db.sqlite3` and restart the container. This will create the superuser again from the environment variables.
-
-NOTE: Future improvements here will most likely require such a reset.
+Equivalent Users page is: [here](https://docs.tubearchivist.com/users/).

docs/Video.md

@@ -1,25 +1,3 @@
-# Video Page
-Every video downloaded gets a dedicated page accessible at `/video/<video-id>/` of your Tube Archivist. Throughout the interface, click on a video title to access the video page.
+All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
 
-Clicking on the channel name or the channel icon will bring you to the dedicated channel detail [page](Channels#channel-detail).
-
-- The button **Reindex** will reindex the metadata of this video.
-- The button **Download File** will download the media file in the browser.
-- The button **Delete Video** will delete that video including the media file.
-
-If available, a tag cloud will show, representing the tags set by the uploader.
-
-The video description is truncated to the first few lines, click on *show more* to expand the whole description.
-
-## Playlist
-When available, a playlist navigation will show at the bottom. Clicking on the playlist name will bring you to the dedicated [Playlist Detail](Playlists#playlist-detail) page showing all videos downloaded from that playlist. The number in square brackets indicates the position of the current video in that playlist.
-
-Clicking on the next or previous video name or thumbnail will bring you to that dedicated video page.
-
-## Similar Videos
-Tube Archivist will show up to six similar videos in a grid. Similarity is detected from the **video title** and the **video tags**. This naturally will show some videos from the same channel, but can also return videos about the same topic from other channels.
-
-When playing a video from the similar section with the inline player, the current video will get replaced, refresh the page to reset that or click on the video title to avoid that behavior. 
-
-## Comments
-If activated on the settings page, this will show the indexed comments. Reveal the threads by clicking the *+ Replies* button. Comments with a heart symbol are favorited by the uploader, comments by the uploader are highlighted in a different color.
+Equivalent Video page is: [here](https://docs.tubearchivist.com/video/).

tubearchivist/api/README.md

@@ -1,441 +1,3 @@
 # TubeArchivist API
-Documentation of available API endpoints.  
 
-Note:  
-- This is very early stages and will change!
-- Check the commit history to see if a documented feature is already in your release
-
-## Table of contents
-- [Authentication](#authentication)
-- [Pagination](#pagination)
-
-**Video**
-- [Video List](#video-list-view)
-- [Video Single](#video-item-view)
-- [Video Comments](#video-comment-view)
-- [Video Similar](#video-similar-view)
-- [Video Single Progress](#video-progress-view)
-- [Video Single Sponsorblock](#sponsor-block-view) WIP
-
-**Channel**
-- [Channel List](#channel-list-view)
-- [Channel Single](#channel-item-view)
-- [Channel Video List](#channel-videos-view)
-
-**Playlist**
-- [Playlist List](#playlist-list-view)
-- [Playlist Single](#playlist-item-view)
-- [Playlist Videos List](#playlist-videos-view)
-
-**Download queue**
-- [Download Queue List](#download-queue-list-view)
-- [Download Queue Single](#download-queue-item-view)
-
-**Snapshot management**
-- [Snapshot List](#snapshot-list-view)
-- [Snapshot Single](#snapshot-item-view)
-
-**Additional**
-- [Login](#login-view)
-- [Task](#task-view) WIP
-- [Refresh](#refresh-view)
-- [Cookie](#cookie-view)
-- [Search](#search-view)
-- [Watched](#watched-view)
-- [Ping](#ping-view)
-
-## Authentication
-API token will get automatically created, accessible on the settings page. Token needs to be passed as an authorization header with every request. Additionally session based authentication is enabled too: When you are logged into your TubeArchivist instance, you'll have access to the api in the browser for testing.
-
-Curl example:
-```shell
-curl -v /api/video/<video-id>/ \
-    -H "Authorization: Token xxxxxxxxxx"
-```
-
-Python requests example:
-```python
-import requests
-
-url = "/api/video/<video-id>/"
-headers = {"Authorization": "Token xxxxxxxxxx"}
-response = requests.get(url, headers=headers)
-```
-
-## Pagination
-The list views return a paginate object with the following keys:
-- page_size: *int* current page size set in config
-- page_from: *int* first result idx
-- prev_pages: *array of ints* of previous pages, if available
-- current_page: *int* current page from query
-- max_hits: *bool* if max of 10k results is reached
-- params: *str* additional url encoded query parameters
-- last_page: *int* of last page link
-- next_pages: *array of ints* of next pages
-- total_hits: *int* total results
-
-Pass page number as a query parameter: `page=2`. Defaults to *0*, `page=1` is redundant and falls back to *0*. If a page query doesn't return any results, you'll get `HTTP 404 Not Found`.
-
-## Video List View
-/api/video/
-
-## Video Item View
-GET: /api/video/\<video_id>/  
-DELETE: /api/video/\<video_id>/
-
-## Video Comment View
-/api/video/\<video_id>/comment/  
-
-## Video Similar View
-/api/video/\<video_id>/similar/  
-
-## Video Progress View
-/api/video/\<video_id>/progress/  
-
-Progress is stored for each user.
-
-### Get last player position of a video
-GET /api/video/\<video_id>/progress/
-```json
-{
-    "youtube_id": "<video_id>",
-    "user_id": 1,
-    "position": 100
-}
-```
-
-### Post player position of video
-POST /api/video/\<video_id>/progress/
-```json
-{
-    "position": 100
-}
-```
-
-### Delete player position of video
-DELETE /api/video/\<video_id>/progress/  
-
-
-## Sponsor Block View
-/api/video/\<video_id>/sponsor/
-
-Integrate with sponsorblock
-
-### Get list of segments
-GET /api/video/\<video_id>/sponsor/
-
-
-### Vote on existing segment
-**This only simulates the request**  
-POST /api/video/\<video_id>/sponsor/
-```json
-{
-    "vote": {
-        "uuid": "<uuid>",
-        "yourVote": 1
-    }
-}
-```
-yourVote needs to be *int*: 0 for downvote, 1 for upvote, 20 to undo vote
-
-### Create new segment
-**This only simulates the request**  
-POST /api/video/\<video_id>/sponsor/
-```json
-{
-    "segment": {
-        "startTime": 5,
-        "endTime": 10
-    }
-}
-```
-Timestamps either *int* or *float*, end time can't be before start time.
-
-
-## Channel List View
-/api/channel/
-
-Parameter:
-- filter: subscribed
-
-### Subscribe to a list of channels
-POST /api/channel/
-```json
-{
-    "data": [
-        {"channel_id": "UC9-y-6csu5WGm29I7JiwpnA", "channel_subscribed": true}
-    ]
-}
-```
-
-## Channel Item View
-GET: /api/channel/\<channel_id>/  
-DELETE: /api/channel/\<channel_id>/
-- Will delete channel with all it's videos
-
-## Channel Videos View
-/api/channel/\<channel_id>/video/
-
-## Playlist List View
-/api/playlist/
-
-## Playlist Item View
-/api/playlist/\<playlist_id>/
-
-## Playlist Videos View
-/api/playlist/\<playlist_id>/video/
-
-## Download Queue List View
-GET /api/download/
-
-Parameter:
-- filter: pending, ignore
-- channel: channel-id
-
-### Add list of videos to download queue
-POST /api/download/
-```json
-{
-    "data": [
-        {"youtube_id": "NYj3DnI81AQ", "status": "pending"}
-    ]
-}
-```
-
-### Delete download queue items by filter
-DELETE /api/download/?filter=ignore  
-DELETE /api/download/?filter=pending
-
-## Download Queue Item View
-GET /api/download/\<video_id>/  
-POST /api/download/\<video_id>/
-
-Ignore video in download queue:
-```json
-{
-    "status": "ignore"
-}
-```
-
-Add to queue previously ignored video:
-```json
-{
-    "status": "pending"
-}
-```
-
-Download existing video now:
-```json
-{
-    "status": "priority"
-}
-```
-
-DELETE /api/download/\<video_id>/  
-Forget or delete from download queue
-
-## Snapshot List View
-GET /api/snapshot/  
-Return snapshot config and a list of available snapshots.
-
-```json
-{
-    "next_exec": epoch,
-    "next_exec_str": "date_str",
-    "expire_after": "30d",
-    "snapshots": []
-}
-```
-
-POST /api/snapshot/  
-Create new snapshot now, will return immediately, task will run async in the background, will return snapshot name: 
-```json
-{
-    "snapshot_name": "ta_daily_<random-id>
-}
-```
-
-## Snapshot Item View
-GET /api/snapshot/\<snapshot-id>/  
-Return metadata of a single snapshot
-```json
-{
-    "id": "ta_daily_<random-id>,
-    "state": "SUCCESS",
-    "es_version": "0.0.0",
-    "start_date": "date_str",
-    "end_date": "date_str",
-    "end_stamp": epoch,
-    "duration_s": 0
-}
-```
-
-GET /api/snapshot/\<snapshot-id>/  
-Restore this snapshot
-
-DELETE /api/snapshot/\<snapshot-id>/  
-Remove this snapshot from index
-
-## Login View
-Return token and user ID for username and password:  
-POST /api/login/
-```json
-{
-    "username": "tubearchivist",
-    "password": "verysecret"
-}
-```
-
-after successful login returns 
-```json
-{
-    "token": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
-    "user_id": 1
-}
-```
-
-## Task View
-GET /api/task/
-POST /api/task/
-
-Check if there is an ongoing task:
-GET /api/task/
-
-Returns:
-```json
-{
-    "rescan": false,
-    "downloading": false
-}
-```
-
-Start a background task  
-POST /api/task/
-```json
-{
-    "run": "task_name"
-}
-```
-
-List of valid task names:
-- **download_pending**: Start the download queue
-- **rescan_pending**: Rescan your subscriptions
-
-## Refresh View
-GET /api/refresh/  
-parameters:
-- **type**: one of *video*, *channel*, *playlist*, optional
-- **id**: item id, optional
-
-without specifying type: return total for all queued items:
-```json
-{
-    "total_queued": 2,
-    "type": "all",
-    "state": "running"
-}
-```
-
-specify type: return total items queue of this type:
-```json
-{
-    "total_queued": 2,
-    "type": "video",
-    "state": "running"
-}
-```
-
-specify type *and* id to get state of item in queue:
-```json
-{
-    "total_queued": 2,
-    "type": "video",
-    "state": "in_queue",
-    "id": "video-id"
-}
-```
-
-POST /api/refresh/  
-Parameter:
-- extract_videos: to refresh all videos for channels/playlists, default False
-
-Manually start a refresh task: post list of *video*, *channel*, *playlist* IDs.
-```json
-{
-    "video": ["video1", "video2", "video3"],
-    "channel": ["channel1", "channel2", "channel3"],
-    "playlist": ["playlist1", "playlist2"]
-}
-```
-
-## Cookie View
-Check your youtube cookie settings, *status* turns to `true` if cookie has been validated.  
-GET /api/cookie/
-```json
-{
-    "cookie_enabled": true,
-    "status": true,
-    "validated": <timestamp>,
-    "validated_str": "timestamp"
-}
-```
-
-POST /api/cookie/  
-Send empty post request to validate cookie.
-```json
-{
-    "cookie_validated": true
-}
-```
-
-PUT /api/cookie/  
-Send put request containing the cookie as a string:
-```json
-{
-    "cookie": "your-cookie-as-string"
-}
-```
-Imports and validates cookie, returns on success:
-```json
-{
-    "cookie_import": "done",
-    "cookie_validated": true
-}
-```
-Or returns status code 400 on failure:
-```json
-{
-    "cookie_import": "fail",
-    "cookie_validated": false
-}
-```
-
-## Search View
-GET /api/search/?query=\<query>
-
-Returns search results from your query.
-
-## Watched View
-POST /api/watched/  
-
-Change watched state, where the `id` can be a single video, or channel/playlist to change all videos belonging to that channel/playlist.
-
-```json
-{
-    "id": "xxxxxxx",
-    "is_watched": True
-}
-```
-
-
-## Ping View
-Validate your connection with the API  
-GET /api/ping/
-
-When valid returns message with user id: 
-```json
-{
-    "response": "pong",
-    "user": 1
-}
-```
+All API documentation has moved to [docs.tubearchivist.com](https://docs.tubearchivist.com/).

tubearchivist/api/src/search_processor.py

@@ -64,7 +64,7 @@ class SearchProcess:
                 "channel_last_refresh": date_str,
                 "channel_banner_url": f"{art_base}_banner.jpg",
                 "channel_thumb_url": f"{art_base}_thumb.jpg",
-                "channel_tvart_url": False,
+                "channel_tvart_url": f"{art_base}_tvart.jpg",
             }
         )
 

tubearchivist/api/src/task_processor.py

@@ -1,54 +0,0 @@
-"""
-Functionality:
-- process tasks from API
-- validate
-- handover to celery
-"""
-
-from home.src.ta.ta_redis import RedisArchivist
-from home.tasks import download_pending, update_subscribed
-
-
-class TaskHandler:
-    """handle tasks from api"""
-
-    def __init__(self, data):
-        self.data = data
-
-    def run_task(self):
-        """map data and run"""
-        task_name = self.data["run"]
-        try:
-            to_run = self.exec_map(task_name)
-        except KeyError as err:
-            print(f"invalid task name {task_name}")
-            raise ValueError from err
-
-        response = to_run()
-        response.update({"task": task_name})
-        return response
-
-    def exec_map(self, task_name):
-        """map dict key and return function to execute"""
-        exec_map = {
-            "download_pending": self._download_pending,
-            "rescan_pending": self._rescan_pending,
-        }
-
-        return exec_map[task_name]
-
-    @staticmethod
-    def _rescan_pending():
-        """look for new items in subscribed channels"""
-        print("rescan subscribed channels")
-        update_subscribed.delay()
-        return {"success": True}
-
-    @staticmethod
-    def _download_pending():
-        """start the download queue"""
-        print("download pending")
-        running = download_pending.delay()
-        print("set task id: " + running.id)
-        RedisArchivist().set_message("dl_queue_id", running.id)
-        return {"success": True}

tubearchivist/api/urls.py

@@ -1,138 +1,134 @@
 """all api urls"""
 
-from api.views import (
-    ChannelApiListView,
-    ChannelApiVideoView,
-    ChannelApiView,
-    CookieView,
-    DownloadApiListView,
-    DownloadApiView,
-    LoginApiView,
-    PingView,
-    PlaylistApiListView,
-    PlaylistApiVideoView,
-    PlaylistApiView,
-    RefreshView,
-    SearchView,
-    SnapshotApiListView,
-    SnapshotApiView,
-    TaskApiView,
-    VideoApiListView,
-    VideoApiView,
-    VideoCommentView,
-    VideoProgressView,
-    VideoSimilarView,
-    VideoSponsorView,
-    WatchedView,
-)
+from api import views
 from django.urls import path
 
 urlpatterns = [
-    path("ping/", PingView.as_view(), name="ping"),
-    path("login/", LoginApiView.as_view(), name="api-login"),
+    path("ping/", views.PingView.as_view(), name="ping"),
+    path("login/", views.LoginApiView.as_view(), name="api-login"),
     path(
         "video/",
-        VideoApiListView.as_view(),
+        views.VideoApiListView.as_view(),
         name="api-video-list",
     ),
     path(
         "video/<slug:video_id>/",
-        VideoApiView.as_view(),
+        views.VideoApiView.as_view(),
         name="api-video",
     ),
     path(
         "video/<slug:video_id>/progress/",
-        VideoProgressView.as_view(),
+        views.VideoProgressView.as_view(),
         name="api-video-progress",
     ),
     path(
         "video/<slug:video_id>/comment/",
-        VideoCommentView.as_view(),
+        views.VideoCommentView.as_view(),
         name="api-video-comment",
     ),
     path(
         "video/<slug:video_id>/similar/",
-        VideoSimilarView.as_view(),
+        views.VideoSimilarView.as_view(),
         name="api-video-similar",
     ),
     path(
         "video/<slug:video_id>/sponsor/",
-        VideoSponsorView.as_view(),
+        views.VideoSponsorView.as_view(),
         name="api-video-sponsor",
     ),
     path(
         "channel/",
-        ChannelApiListView.as_view(),
+        views.ChannelApiListView.as_view(),
         name="api-channel-list",
     ),
     path(
         "channel/<slug:channel_id>/",
-        ChannelApiView.as_view(),
+        views.ChannelApiView.as_view(),
         name="api-channel",
     ),
     path(
         "channel/<slug:channel_id>/video/",
-        ChannelApiVideoView.as_view(),
+        views.ChannelApiVideoView.as_view(),
         name="api-channel-video",
     ),
     path(
         "playlist/",
-        PlaylistApiListView.as_view(),
+        views.PlaylistApiListView.as_view(),
         name="api-playlist-list",
     ),
     path(
         "playlist/<slug:playlist_id>/",
-        PlaylistApiView.as_view(),
+        views.PlaylistApiView.as_view(),
         name="api-playlist",
     ),
     path(
         "playlist/<slug:playlist_id>/video/",
-        PlaylistApiVideoView.as_view(),
+        views.PlaylistApiVideoView.as_view(),
         name="api-playlist-video",
     ),
     path(
         "download/",
-        DownloadApiListView.as_view(),
+        views.DownloadApiListView.as_view(),
         name="api-download-list",
     ),
     path(
         "download/<slug:video_id>/",
-        DownloadApiView.as_view(),
+        views.DownloadApiView.as_view(),
         name="api-download",
     ),
     path(
         "refresh/",
-        RefreshView.as_view(),
+        views.RefreshView.as_view(),
         name="api-refresh",
     ),
-    path(
-        "task/",
-        TaskApiView.as_view(),
-        name="api-task",
-    ),
     path(
         "snapshot/",
-        SnapshotApiListView.as_view(),
+        views.SnapshotApiListView.as_view(),
         name="api-snapshot-list",
     ),
     path(
         "snapshot/<slug:snapshot_id>/",
-        SnapshotApiView.as_view(),
+        views.SnapshotApiView.as_view(),
         name="api-snapshot",
     ),
+    path(
+        "task-name/",
+        views.TaskListView.as_view(),
+        name="api-task-list",
+    ),
+    path(
+        "task-name/<slug:task_name>/",
+        views.TaskNameListView.as_view(),
+        name="api-task-name-list",
+    ),
+    path(
+        "task-id/<slug:task_id>/",
+        views.TaskIDView.as_view(),
+        name="api-task-id",
+    ),
     path(
         "cookie/",
-        CookieView.as_view(),
+        views.CookieView.as_view(),
         name="api-cookie",
     ),
     path(
         "watched/",
-        WatchedView.as_view(),
+        views.WatchedView.as_view(),
         name="api-watched",
     ),
     path(
         "search/",
-        SearchView.as_view(),
+        views.SearchView.as_view(),
         name="api-search",
     ),
+    path(
+        "token/",
+        views.TokenView.as_view(),
+        name="api-token",
+    ),
+    path(
+        "notification/",
+        views.NotificationView.as_view(),
+        name="api-notification",
+    ),
 ]

tubearchivist/api/views.py

@@ -1,7 +1,6 @@
 """all API views"""
 
 from api.src.search_processor import SearchProcess
-from api.src.task_processor import TaskHandler
 from home.src.download.queue import PendingInteract
 from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.connect import ElasticWrap
@@ -14,8 +13,15 @@ from home.src.index.reindex import ReindexProgress
 from home.src.index.video import SponsorBlock, YoutubeVideo
 from home.src.ta.config import AppConfig
 from home.src.ta.ta_redis import RedisArchivist, RedisQueue
+from home.src.ta.task_manager import TaskCommand, TaskManager
 from home.src.ta.urlparser import Parser
-from home.tasks import check_reindex, download_single, extrac_dl, subscribe_to
+from home.tasks import (
+    BaseTask,
+    check_reindex,
+    download_pending,
+    extrac_dl,
+    subscribe_to,
+)
 from rest_framework.authentication import (
     SessionAuthentication,
     TokenAuthentication,
@@ -424,14 +430,15 @@ class DownloadApiView(ApiBaseView):
             print(message)
             return Response({"message": message}, status=400)
 
-        pending_video, status_code = PendingInteract(video_id).get_item()
+        _, status_code = PendingInteract(video_id).get_item()
         if status_code == 404:
             message = f"{video_id}: item not found {status_code}"
             return Response({"message": message}, status=404)
 
         print(f"{video_id}: change status to {item_status}")
         if item_status == "priority":
-            download_single.delay(pending_video)
+            PendingInteract(youtube_id=video_id).prioritize()
+            download_pending.delay(from_queue=False)
         else:
             PendingInteract(video_id, item_status).update_status()
             RedisQueue(queue_name="dl_queue").clear_item(video_id)
@@ -554,29 +561,6 @@ class LoginApiView(ObtainAuthToken):
         return Response({"token": token.key, "user_id": user.pk})
 
 
-class TaskApiView(ApiBaseView):
-    """resolves to /api/task/
-    GET: check if ongoing background task
-    POST: start a new background task
-    """
-
-    @staticmethod
-    def get(request):
-        """handle get request"""
-        # pylint: disable=unused-argument
-        response = {"rescan": False, "downloading": False}
-        for key in response.keys():
-            response[key] = RedisArchivist().is_locked(key)
-
-        return Response(response)
-
-    def post(self, request):
-        """handle post request"""
-        response = TaskHandler(request.data).run_task()
-
-        return Response(response)
-
-
 class SnapshotApiListView(ApiBaseView):
     """resolves to /api/snapshot/
     GET: returns snashot config plus list of existing snapshots
@@ -641,6 +625,108 @@ class SnapshotApiView(ApiBaseView):
         return Response(response)
 
 
+class TaskListView(ApiBaseView):
+    """resolves to /api/task-name/
+    GET: return a list of all stored task results
+    """
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        all_results = TaskManager().get_all_results()
+
+        return Response(all_results)
+
+
+class TaskNameListView(ApiBaseView):
+    """resolves to /api/task-name/<task-name>/
+    GET: return a list of stored results of task
+    POST: start new background process
+    """
+
+    def get(self, request, task_name):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        if task_name not in BaseTask.TASK_CONFIG:
+            message = {"message": "invalid task name"}
+            return Response(message, status=404)
+
+        all_results = TaskManager().get_tasks_by_name(task_name)
+
+        return Response(all_results)
+
+    def post(self, request, task_name):
+        """
+        handle post request
+        404 for invalid task_name
+        400 if task can't be started here without argument
+        """
+        # pylint: disable=unused-argument
+        task_config = BaseTask.TASK_CONFIG.get(task_name)
+        if not task_config:
+            message = {"message": "invalid task name"}
+            return Response(message, status=404)
+
+        if not task_config.get("api-start"):
+            message = {"message": "can not start task through this endpoint"}
+            return Response(message, status=400)
+
+        message = TaskCommand().start(task_name)
+
+        return Response({"message": message})
+
+
+class TaskIDView(ApiBaseView):
+    """resolves to /api/task-id/<task-id>/
+    GET: return details of task id
+    """
+
+    valid_commands = ["stop", "kill"]
+
+    def get(self, request, task_id):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        task_result = TaskManager().get_task(task_id)
+        if not task_result:
+            message = {"message": "task id not found"}
+            return Response(message, status=404)
+
+        return Response(task_result)
+
+    def post(self, request, task_id):
+        """post command to task"""
+        command = request.data.get("command")
+        if not command or command not in self.valid_commands:
+            message = {"message": "no valid command found"}
+            return Response(message, status=400)
+
+        task_result = TaskManager().get_task(task_id)
+        if not task_result:
+            message = {"message": "task id not found"}
+            return Response(message, status=404)
+
+        task_conf = BaseTask.TASK_CONFIG.get(task_result.get("name"))
+        if command == "stop":
+            if not task_conf.get("api-stop"):
+                message = {"message": "task can not be stopped"}
+                return Response(message, status=400)
+
+            message_key = self._build_message_key(task_conf, task_id)
+            TaskCommand().stop(task_id, message_key)
+        if command == "kill":
+            if not task_conf.get("api-stop"):
+                message = {"message": "task can not be killed"}
+                return Response(message, status=400)
+
+            TaskCommand().kill(task_id)
+
+        return Response({"message": "command sent"})
+
+    def _build_message_key(self, task_conf, task_id):
+        """build message key to forward command to notification"""
+        return f"message:{task_conf.get('group')}:{task_id.split('-')[0]}"
+
+
 class RefreshView(ApiBaseView):
     """resolves to /api/refresh/
     GET: get refresh progress
@@ -760,3 +846,33 @@ class SearchView(ApiBaseView):
 
         search_results = SearchForm().multi_search(search_query)
         return Response(search_results)
+
+
+class TokenView(ApiBaseView):
+    """resolves to /api/token/
+    DELETE: revoke the token
+    """
+
+    @staticmethod
+    def delete(request):
+        print("revoke API token")
+        request.user.auth_token.delete()
+        return Response({"success": True})
+
+
+class NotificationView(ApiBaseView):
+    """resolves to /api/notification/
+    GET: returns a list of notifications
+    filter query to filter messages by group
+    """
+
+    valid_filters = ["download", "settings", "channel"]
+
+    def get(self, request):
+        """get all notifications"""
+        query = "message"
+        filter_by = request.GET.get("filter", None)
+        if filter_by in self.valid_filters:
+            query = f"{query}:{filter_by}"
+
+        return Response(RedisArchivist().list_items(query))

tubearchivist/config/management/commands/ta_connection.py

@@ -43,7 +43,6 @@ class Command(BaseCommand):
         self.stdout.write("[1] connect to Redis")
         redis_conn = RedisArchivist().conn
         for _ in range(5):
-            sleep(2)
             try:
                 pong = redis_conn.execute_command("PING")
                 if pong:
@@ -54,10 +53,12 @@ class Command(BaseCommand):
 
             except Exception:  # pylint: disable=broad-except
                 self.stdout.write("    ... retry Redis connection")
+                sleep(2)
 
         message = "    🗙 Redis connection failed"
         self.stdout.write(self.style.ERROR(f"{message}"))
         RedisArchivist().exec("PING")
+        sleep(60)
         raise CommandError(message)
 
     def _redis_config_set(self):
@@ -75,13 +76,13 @@ class Command(BaseCommand):
         self.stdout.write("[3] connect to Elastic Search")
         total = self.TIMEOUT // 5
         for i in range(total):
-            sleep(5)
             self.stdout.write(f"    ... waiting for ES [{i}/{total}]")
             try:
                 _, status_code = ElasticWrap("/").get(
                     timeout=1, print_error=False
                 )
             except requests.exceptions.ConnectionError:
+                sleep(5)
                 continue
 
             if status_code and status_code == 200:
@@ -98,6 +99,7 @@ class Command(BaseCommand):
         self.stdout.write(self.style.ERROR(f"{message}"))
         self.stdout.write(f"    error message: {response}")
         self.stdout.write(f"    status code: {status_code}")
+        sleep(60)
         raise CommandError(message)
 
     def _es_version_check(self):
@@ -118,6 +120,7 @@ class Command(BaseCommand):
             + f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}"
         )
         self.stdout.write(self.style.ERROR(f"{message}"))
+        sleep(60)
         raise CommandError(message)
 
     def _es_path_check(self):
@@ -137,4 +140,5 @@ class Command(BaseCommand):
                 + "    path.repo=/usr/share/elasticsearch/data/snapshot"
             )
             self.stdout.write(self.style.ERROR(f"{message}"))
+            sleep(60)
             raise CommandError(message)

tubearchivist/config/management/commands/ta_startup.py

@@ -14,6 +14,7 @@ from home.src.es.snapshot import ElasticSnapshot
 from home.src.ta.config import AppConfig, ReleaseVersion
 from home.src.ta.helper import clear_dl_cache
 from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.task_manager import TaskManager
 
 TOPIC = """
 
@@ -35,6 +36,7 @@ class Command(BaseCommand):
         self._sync_redis_state()
         self._make_folders()
         self._release_locks()
+        self._clear_tasks()
         self._clear_dl_cache()
         self._version_check()
         self._mig_index_setup()
@@ -96,9 +98,23 @@ class Command(BaseCommand):
         if not has_changed:
             self.stdout.write(self.style.SUCCESS("    no locks found"))
 
+    def _clear_tasks(self):
+        """clear tasks and messages"""
+        self.stdout.write("[4] clear task leftovers")
+        TaskManager().fail_pending()
+        redis_con = RedisArchivist()
+        to_delete = redis_con.list_keys("message:")
+        if to_delete:
+            for key in to_delete:
+                redis_con.del_message(key)
+
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {len(to_delete)} messages")
+            )
+
     def _clear_dl_cache(self):
         """clear leftover files from dl cache"""
-        self.stdout.write("[4] clear leftover files from dl cache")
+        self.stdout.write("[5] clear leftover files from dl cache")
         config = AppConfig().config
         leftover_files = clear_dl_cache(config)
         if leftover_files:
@@ -110,7 +126,7 @@ class Command(BaseCommand):
 
     def _version_check(self):
         """remove new release key if updated now"""
-        self.stdout.write("[5] check for first run after update")
+        self.stdout.write("[6] check for first run after update")
         new_version = ReleaseVersion().is_updated()
         if new_version:
             self.stdout.write(
@@ -175,4 +191,5 @@ class Command(BaseCommand):
                 message = f"    🗙 {index_name} vid_type update failed"
                 self.stdout.write(self.style.ERROR(message))
                 self.stdout.write(response)
+                sleep(60)
                 raise CommandError(message)

tubearchivist/config/settings.py

@@ -18,6 +18,7 @@ import ldap
 from corsheaders.defaults import default_headers
 from django_auth_ldap.config import LDAPSearch
 from home.src.ta.config import AppConfig
+from home.src.ta.helper import ta_host_parser
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -26,23 +27,13 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
 
-PW_HASH = hashlib.sha256(environ.get("TA_PASSWORD").encode())
+PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode())
 SECRET_KEY = PW_HASH.hexdigest()
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = bool(environ.get("DJANGO_DEBUG"))
 
-ALLOWED_HOSTS = []
-if environ.get("TA_HOST"):
-    ALLOWED_HOSTS = [i.strip() for i in environ.get("TA_HOST").split()]
-
-CSRF_TRUSTED_ORIGINS = []
-for host in ALLOWED_HOSTS:
-    if host.startswith("http://") or host.startswith("https://"):
-        CSRF_TRUSTED_ORIGINS.append(host)
-    else:
-        CSRF_TRUSTED_ORIGINS.append(f"http://{host}")
-
+ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(environ["TA_HOST"])
 
 # Application definition
 
@@ -265,4 +256,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.3.4"
+TA_VERSION = "v0.3.5"

tubearchivist/home/src/download/queue.py

@@ -18,7 +18,7 @@ from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import DurationConverter, is_shorts
-from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.ta_redis import RedisQueue
 
 
 class PendingIndex:
@@ -117,6 +117,16 @@ class PendingInteract:
         path = f"ta_download/_update/{self.youtube_id}"
         _, _ = ElasticWrap(path).post(data=data)
 
+    def prioritize(self):
+        """prioritize pending item in redis queue"""
+        pending_video, _ = self.get_item()
+        vid_type = pending_video.get("vid_type", VideoTypeEnum.VIDEOS.value)
+        to_add = {
+            "youtube_id": pending_video["youtube_id"],
+            "vid_type": vid_type,
+        }
+        RedisQueue(queue_name="dl_queue").add_priority(to_add)
+
     def get_item(self):
         """return pending item dict"""
         path = f"ta_download/_doc/{self.youtube_id}"
@@ -151,12 +161,14 @@ class PendingList(PendingIndex):
         "noplaylist": True,
         "writethumbnail": True,
         "simulate": True,
+        "check_formats": None,
     }
 
-    def __init__(self, youtube_ids=False):
+    def __init__(self, youtube_ids=False, task=False):
         super().__init__()
         self.config = AppConfig().config
         self.youtube_ids = youtube_ids
+        self.task = task
         self.to_skip = False
         self.missing_videos = False
 
@@ -165,16 +177,16 @@ class PendingList(PendingIndex):
         self.missing_videos = []
         self.get_download()
         self.get_indexed()
-        for entry in self.youtube_ids:
-            # notify
-            mess_dict = {
-                "status": "message:add",
-                "level": "info",
-                "title": "Adding to download queue.",
-                "message": "Extracting lists",
-            }
-            RedisArchivist().set_message("message:add", mess_dict, expire=True)
+        total = len(self.youtube_ids)
+        for idx, entry in enumerate(self.youtube_ids):
             self._process_entry(entry)
+            if not self.task:
+                continue
+
+            self.task.send_progress(
+                message_lines=[f"Extracting items {idx + 1}/{total}"],
+                progress=(idx + 1) / total,
+            )
 
     def _process_entry(self, entry):
         """process single entry from url list"""
@@ -228,9 +240,13 @@ class PendingList(PendingIndex):
         self.get_channels()
         bulk_list = []
 
+        total = len(self.missing_videos)
         for idx, (youtube_id, vid_type) in enumerate(self.missing_videos):
-            print(f"{youtube_id} ({vid_type}): add to download queue")
-            self._notify_add(idx)
+            if self.task and self.task.is_stopped():
+                break
+
+            print(f"{youtube_id}: [{idx + 1}/{total}]: add to queue")
+            self._notify_add(idx, total)
             video_details = self.get_youtube_details(youtube_id, vid_type)
             if not video_details:
                 continue
@@ -243,29 +259,34 @@ class PendingList(PendingIndex):
             url = video_details["vid_thumb_url"]
             ThumbManager(youtube_id).download_video_thumb(url)
 
-        if bulk_list:
-            # add last newline
-            bulk_list.append("\n")
-            query_str = "\n".join(bulk_list)
-            _, _ = ElasticWrap("_bulk").post(query_str, ndjson=True)
+            if len(bulk_list) >= 20:
+                self._ingest_bulk(bulk_list)
+                bulk_list = []
 
-    def _notify_add(self, idx):
+        self._ingest_bulk(bulk_list)
+
+    def _ingest_bulk(self, bulk_list):
+        """add items to queue in bulk"""
+        if not bulk_list:
+            return
+
+        # add last newline
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+        _, _ = ElasticWrap("_bulk").post(query_str, ndjson=True)
+
+    def _notify_add(self, idx, total):
         """send notification for adding videos to download queue"""
-        progress = f"{idx + 1}/{len(self.missing_videos)}"
-        mess_dict = {
-            "status": "message:add",
-            "level": "info",
-            "title": "Adding new videos to download queue.",
-            "message": "Progress: " + progress,
-        }
-        if idx + 1 == len(self.missing_videos):
-            expire = 4
-        else:
-            expire = True
+        if not self.task:
+            return
 
-        RedisArchivist().set_message("message:add", mess_dict, expire=expire)
-        if idx + 1 % 25 == 0:
-            print("adding to queue progress: " + progress)
+        self.task.send_progress(
+            message_lines=[
+                "Adding new videos to download queue.",
+                f"Extracting items {idx + 1}/{total}",
+            ],
+            progress=(idx + 1) / total,
+        )
 
     def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEOS):
         """get details from youtubedl for single pending video"""

tubearchivist/home/src/download/subscriptions.py

@@ -13,13 +13,15 @@ from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
 from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.urlparser import Parser
 
 
 class ChannelSubscription:
     """manage the list of channels subscribed"""
 
-    def __init__(self):
+    def __init__(self, task=False):
         self.config = AppConfig().config
+        self.task = task
 
     @staticmethod
     def get_channels(subscribed_only=True):
@@ -44,7 +46,7 @@ class ChannelSubscription:
 
         last_videos = []
 
-        for vid_type, limit_amount in queries:
+        for vid_type_enum, limit_amount in queries:
             obs = {
                 "skip_download": True,
                 "extract_flat": True,
@@ -52,9 +54,9 @@ class ChannelSubscription:
             if limit:
                 obs["playlistend"] = limit_amount
 
-            path = vid_type.value
+            vid_type = vid_type_enum.value
             channel = YtWrap(obs, self.config).extract(
-                f"https://www.youtube.com/channel/{channel_id}/{path}"
+                f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
             )
             if not channel:
                 continue
@@ -101,12 +103,16 @@ class ChannelSubscription:
     def find_missing(self):
         """add missing videos from subscribed channels to pending"""
         all_channels = self.get_channels()
+        if not all_channels:
+            return False
+
         pending = queue.PendingList()
         pending.get_download()
         pending.get_indexed()
 
         missing_videos = []
 
+        total = len(all_channels)
         for idx, channel in enumerate(all_channels):
             channel_id = channel["channel_id"]
             print(f"{channel_id}: find missing videos.")
@@ -116,21 +122,19 @@ class ChannelSubscription:
                 for video_id, _, vid_type in last_videos:
                     if video_id not in pending.to_skip:
                         missing_videos.append((video_id, vid_type))
-            # notify
-            message = {
-                "status": "message:rescan",
-                "level": "info",
-                "title": "Scanning channels: Looking for new videos.",
-                "message": f"Progress: {idx + 1}/{len(all_channels)}",
-            }
-            if idx + 1 == len(all_channels):
-                expire = 4
-            else:
-                expire = True
 
-            RedisArchivist().set_message(
-                "message:rescan", message=message, expire=expire
-            )
+            if not self.task:
+                continue
+
+            if self.task:
+                if self.task.is_stopped():
+                    self.task.send_progress(["Received Stop signal."])
+                    break
+
+                self.task.send_progress(
+                    message_lines=[f"Scanning Channel {idx + 1}/{total}"],
+                    progress=(idx + 1) / total,
+                )
 
         return missing_videos
 
@@ -147,8 +151,9 @@ class ChannelSubscription:
 class PlaylistSubscription:
     """manage the playlist download functionality"""
 
-    def __init__(self):
+    def __init__(self, task=False):
         self.config = AppConfig().config
+        self.task = task
 
     @staticmethod
     def get_playlists(subscribed_only=True):
@@ -232,14 +237,18 @@ class PlaylistSubscription:
     def find_missing(self):
         """find videos in subscribed playlists not downloaded yet"""
         all_playlists = [i["playlist_id"] for i in self.get_playlists()]
+        if not all_playlists:
+            return False
+
         to_ignore = self.get_to_ignore()
 
         missing_videos = []
+        total = len(all_playlists)
         for idx, playlist_id in enumerate(all_playlists):
             size_limit = self.config["subscriptions"]["channel_size"]
             playlist = YoutubePlaylist(playlist_id)
-            playlist.update_playlist()
-            if not playlist:
+            is_active = playlist.update_playlist()
+            if not is_active:
                 playlist.deactivate()
                 continue
 
@@ -249,19 +258,123 @@ class PlaylistSubscription:
 
             all_missing = [i for i in playlist_entries if not i["downloaded"]]
 
-            message = {
-                "status": "message:rescan",
-                "level": "info",
-                "title": "Scanning playlists: Looking for new videos.",
-                "message": f"Progress: {idx + 1}/{len(all_playlists)}",
-            }
-            RedisArchivist().set_message(
-                "message:rescan", message=message, expire=True
-            )
-
             for video in all_missing:
                 youtube_id = video["youtube_id"]
                 if youtube_id not in to_ignore:
                     missing_videos.append(youtube_id)
 
+            if not self.task:
+                continue
+
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
+                    progress=(idx + 1) / total,
+                )
+                if self.task.is_stopped():
+                    self.task.send_progress(["Received Stop signal."])
+                    break
+
         return missing_videos
+
+
+class SubscriptionScanner:
+    """add missing videos to queue"""
+
+    def __init__(self, task=False):
+        self.task = task
+        self.missing_videos = False
+
+    def scan(self):
+        """scan channels and playlists"""
+        if self.task:
+            self.task.send_progress(["Rescanning channels and playlists."])
+
+        self.missing_videos = []
+        self.scan_channels()
+        if self.task and not self.task.is_stopped():
+            self.scan_playlists()
+
+        return self.missing_videos
+
+    def scan_channels(self):
+        """get missing from channels"""
+        channel_handler = ChannelSubscription(task=self.task)
+        missing = channel_handler.find_missing()
+        if not missing:
+            return
+
+        for vid_id, vid_type in missing:
+            self.missing_videos.append(
+                {"type": "video", "vid_type": vid_type, "url": vid_id}
+            )
+
+    def scan_playlists(self):
+        """get missing from playlists"""
+        playlist_handler = PlaylistSubscription(task=self.task)
+        missing = playlist_handler.find_missing()
+        if not missing:
+            return
+
+        for i in missing:
+            self.missing_videos.append(
+                {
+                    "type": "video",
+                    "vid_type": VideoTypeEnum.VIDEOS.value,
+                    "url": i,
+                }
+            )
+
+
+class SubscriptionHandler:
+    """subscribe to channels and playlists from url_str"""
+
+    def __init__(self, url_str, task=False):
+        self.url_str = url_str
+        self.task = task
+        self.to_subscribe = False
+
+    def subscribe(self):
+        """subscribe to url_str items"""
+        if self.task:
+            self.task.send_progress(["Processing form content."])
+        self.to_subscribe = Parser(self.url_str).parse()
+
+        total = len(self.to_subscribe)
+        for idx, item in enumerate(self.to_subscribe):
+            if self.task:
+                self._notify(idx, item, total)
+
+            self.subscribe_type(item)
+
+    def subscribe_type(self, item):
+        """process single item"""
+        if item["type"] == "playlist":
+            PlaylistSubscription().process_url_str([item])
+            return
+
+        if item["type"] == "video":
+            # extract channel id from video
+            vid = queue.PendingList().get_youtube_details(item["url"])
+            channel_id = vid["channel_id"]
+        elif item["type"] == "channel":
+            channel_id = item["url"]
+        else:
+            raise ValueError("failed to subscribe to: " + item["url"])
+
+        self._subscribe(channel_id)
+
+    def _subscribe(self, channel_id):
+        """subscribe to channel"""
+        ChannelSubscription().change_subscribe(
+            channel_id, channel_subscribed=True
+        )
+
+    def _notify(self, idx, item, total):
+        """send notification message to redis"""
+        subscribe_type = item["type"].title()
+        message_lines = [
+            f"Subscribe to {subscribe_type}",
+            f"Progress: {idx + 1}/{total}",
+        ]
+        self.task.send_progress(message_lines, progress=(idx + 1) / total)

tubearchivist/home/src/download/thumbnails.py

@@ -10,8 +10,7 @@ from io import BytesIO
 from time import sleep
 
 import requests
-from home.src.download import queue  # partial import
-from home.src.es.connect import IndexPaginate
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.ta.config import AppConfig
 from mutagen.mp4 import MP4, MP4Cover
 from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
@@ -82,6 +81,9 @@ class ThumbManagerBase:
             "banner": os.path.join(
                 app_root, "static/img/default-channel-banner.jpg"
             ),
+            "tvart": os.path.join(
+                app_root, "static/img/default-channel-art.jpg"
+            ),
         }
 
         img_raw = Image.open(default_map[self.item_type])
@@ -150,9 +152,10 @@ class ThumbManager(ThumbManagerBase):
 
     def download_channel_art(self, urls, skip_existing=False):
         """pass tuple of channel thumbnails"""
-        channel_thumb, channel_banner = urls
+        channel_thumb, channel_banner, channel_tv = urls
         self._download_channel_thumb(channel_thumb, skip_existing)
         self._download_channel_banner(channel_banner, skip_existing)
+        self._download_channel_tv(channel_tv, skip_existing)
 
     def _download_channel_thumb(self, channel_thumb, skip_existing):
         """download channel thumbnail"""
@@ -181,6 +184,16 @@ class ThumbManager(ThumbManagerBase):
         img_raw = self.download_raw(channel_banner)
         img_raw.convert("RGB").save(banner_path)
 
+    def _download_channel_tv(self, channel_tv, skip_existing):
+        """download channel tv art"""
+        art_path = os.path.join(self.CHANNEL_DIR, self.item_id + "_tvart.jpg")
+        self.item_type = "tvart"
+        if skip_existing and os.path.exists(art_path):
+            return
+
+        img_raw = self.download_raw(channel_tv)
+        img_raw.convert("RGB").save(art_path)
+
     def download_playlist_thumb(self, url, skip_existing=False):
         """pass thumbnail url"""
         thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
@@ -257,6 +270,7 @@ class ValidatorCallback:
             urls = (
                 channel["_source"]["channel_thumb_url"],
                 channel["_source"]["channel_banner_url"],
+                channel["_source"]["channel_tvart_url"],
             )
             handler = ThumbManager(channel["_source"]["channel_id"])
             handler.download_channel_art(urls, skip_existing=True)
@@ -272,102 +286,122 @@ class ValidatorCallback:
 class ThumbValidator:
     """validate thumbnails"""
 
-    def download_missing(self):
-        """download all missing artwork"""
-        self.download_missing_videos()
-        self.download_missing_channels()
-        self.download_missing_playlists()
-
-    def download_missing_videos(self):
-        """get all missing video thumbnails"""
-        data = {
-            "query": {"term": {"active": {"value": True}}},
-            "sort": [{"youtube_id": {"order": "asc"}}],
-            "_source": ["vid_thumb_url", "youtube_id"],
-        }
-        paginate = IndexPaginate(
-            "ta_video", data, size=5000, callback=ValidatorCallback
-        )
-        _ = paginate.get_results()
-
-    def download_missing_channels(self):
-        """get all missing channel thumbnails"""
-        data = {
-            "query": {"term": {"channel_active": {"value": True}}},
-            "sort": [{"channel_id": {"order": "asc"}}],
-            "_source": {
-                "excludes": ["channel_description", "channel_overwrites"]
+    INDEX = [
+        {
+            "data": {
+                "query": {"term": {"active": {"value": True}}},
+                "_source": ["vid_thumb_url", "youtube_id"],
             },
-        }
-        paginate = IndexPaginate(
-            "ta_channel", data, callback=ValidatorCallback
-        )
-        _ = paginate.get_results()
+            "name": "ta_video",
+        },
+        {
+            "data": {
+                "query": {"term": {"channel_active": {"value": True}}},
+                "_source": {
+                    "excludes": ["channel_description", "channel_overwrites"]
+                },
+            },
+            "name": "ta_channel",
+        },
+        {
+            "data": {
+                "query": {"term": {"playlist_active": {"value": True}}},
+                "_source": ["playlist_id", "playlist_thumbnail"],
+            },
+            "name": "ta_playlist",
+        },
+    ]
 
-    def download_missing_playlists(self):
-        """get all missing playlist artwork"""
-        data = {
-            "query": {"term": {"playlist_active": {"value": True}}},
-            "sort": [{"playlist_id": {"order": "asc"}}],
-            "_source": ["playlist_id", "playlist_thumbnail"],
-        }
-        paginate = IndexPaginate(
-            "ta_playlist", data, callback=ValidatorCallback
-        )
-        _ = paginate.get_results()
+    def __init__(self, task):
+        self.task = task
+
+    def validate(self):
+        """validate all indexes"""
+        for index in self.INDEX:
+            total = self._get_total(index["name"])
+            if not total:
+                continue
+
+            paginate = IndexPaginate(
+                index_name=index["name"],
+                data=index["data"],
+                size=1000,
+                callback=ValidatorCallback,
+                task=self.task,
+                total=total,
+            )
+            _ = paginate.get_results()
+
+    @staticmethod
+    def _get_total(index_name):
+        """get total documents in index"""
+        path = f"{index_name}/_count"
+        response, _ = ElasticWrap(path).get()
+
+        return response.get("count")
 
 
 class ThumbFilesystem:
-    """filesystem tasks for thumbnails"""
+    """sync thumbnail files to media files"""
+
+    INDEX_NAME = "ta_video"
+
+    def __init__(self, task=False):
+        self.task = task
+
+    def embed(self):
+        """entry point"""
+        data = {
+            "query": {"match_all": {}},
+            "_source": ["media_url", "youtube_id"],
+        }
+        paginate = IndexPaginate(
+            index_name=self.INDEX_NAME,
+            data=data,
+            size=200,
+            callback=EmbedCallback,
+            task=self.task,
+            total=self._get_total(),
+        )
+        _ = paginate.get_results()
+
+    def _get_total(self):
+        """get total documents in index"""
+        path = f"{self.INDEX_NAME}/_count"
+        response, _ = ElasticWrap(path).get()
+
+        return response.get("count")
+
+
+class EmbedCallback:
+    """callback class to embed thumbnails"""
 
     CONFIG = AppConfig().config
     CACHE_DIR = CONFIG["application"]["cache_dir"]
     MEDIA_DIR = CONFIG["application"]["videos"]
-    VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
+    FORMAT = MP4Cover.FORMAT_JPEG
 
-    def sync(self):
-        """embed thumbnails to mediafiles"""
-        video_list = self.get_thumb_list()
-        self._embed_thumbs(video_list)
+    def __init__(self, source, index_name):
+        self.source = source
+        self.index_name = index_name
 
-    def get_thumb_list(self):
-        """get list of mediafiles and matching thumbnails"""
-        pending = queue.PendingList()
-        pending.get_download()
-        pending.get_indexed()
-
-        video_list = []
-        for video in pending.all_videos:
-            video_id = video["youtube_id"]
-            media_url = os.path.join(self.MEDIA_DIR, video["media_url"])
+    def run(self):
+        """run embed"""
+        for video in self.source:
+            video_id = video["_source"]["youtube_id"]
+            media_url = os.path.join(
+                self.MEDIA_DIR, video["_source"]["media_url"]
+            )
             thumb_path = os.path.join(
                 self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path()
             )
-            video_list.append(
-                {
-                    "media_url": media_url,
-                    "thumb_path": thumb_path,
-                }
-            )
+            if os.path.exists(thumb_path):
+                self.embed(media_url, thumb_path)
 
-        return video_list
+    def embed(self, media_url, thumb_path):
+        """embed thumb in single media file"""
+        video = MP4(media_url)
+        with open(thumb_path, "rb") as f:
+            video["covr"] = [MP4Cover(f.read(), imageformat=self.FORMAT)]
 
-    @staticmethod
-    def _embed_thumbs(video_list):
-        """rewrite the thumbnail into media file"""
-
-        counter = 1
-        for video in video_list:
-            # loop through all videos
-            media_url = video["media_url"]
-            thumb_path = video["thumb_path"]
-
-            mutagen_vid = MP4(media_url)
-            with open(thumb_path, "rb") as f:
-                mutagen_vid["covr"] = [
-                    MP4Cover(f.read(), imageformat=MP4Cover.FORMAT_JPEG)
-                ]
-            mutagen_vid.save()
-            if counter % 50 == 0:
-                print(f"thumbnail write progress {counter}/{len(video_list)}")
-            counter = counter + 1
+        video.save()

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -22,7 +22,7 @@ from home.src.index.video import YoutubeVideo, index_new_video
 from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import clean_string, ignore_filelist
-from home.src.ta.ta_redis import RedisArchivist, RedisQueue
+from home.src.ta.ta_redis import RedisQueue
 
 
 class DownloadPostProcess:
@@ -96,7 +96,7 @@ class DownloadPostProcess:
     def validate_playlists(self):
         """look for playlist needing to update"""
         for id_c, channel_id in enumerate(self.download.channels):
-            channel = YoutubeChannel(channel_id)
+            channel = YoutubeChannel(channel_id, task=self.download.task)
             overwrites = self.pending.channel_overwrites.get(channel_id, False)
             if overwrites and overwrites.get("index_playlists"):
                 # validate from remote
@@ -125,28 +125,22 @@ class DownloadPostProcess:
 
     def _notify_playlist_progress(self, all_channel_playlist, id_c, id_p):
         """notify to UI"""
-        title = (
-            "Processing playlists for channels: "
-            + f"{id_c + 1}/{len(self.download.channels)}"
-        )
-        message = f"Progress: {id_p + 1}/{len(all_channel_playlist)}"
-        key = "message:download"
-        mess_dict = {
-            "status": key,
-            "level": "info",
-            "title": title,
-            "message": message,
-        }
-        if id_p + 1 == len(all_channel_playlist):
-            expire = 4
-        else:
-            expire = True
+        if not self.download.task:
+            return
 
-        RedisArchivist().set_message(key, mess_dict, expire=expire)
+        total_channel = len(self.download.channels)
+        total_playlist = len(all_channel_playlist)
+
+        message = [
+            f"Post Processing Channels: {id_c}/{total_channel}",
+            f"Validate Playlists {id_p + 1}/{total_playlist}",
+        ]
+        progress = (id_c + 1) / total_channel
+        self.download.task.send_progress(message, progress=progress)
 
     def get_comments(self):
         """get comments from youtube"""
-        CommentList(self.download.videos).index(notify=True)
+        CommentList(self.download.videos, task=self.download.task).index()
 
 
 class VideoDownloader:
@@ -155,12 +149,11 @@ class VideoDownloader:
     if not initiated with list, take from queue
     """
 
-    MSG = "message:download"
-
-    def __init__(self, youtube_id_list=False):
+    def __init__(self, youtube_id_list=False, task=False):
         self.obs = False
         self.video_overwrites = False
         self.youtube_id_list = youtube_id_list
+        self.task = task
         self.config = AppConfig().config
         self._build_obs()
         self.channels = set()
@@ -169,7 +162,6 @@ class VideoDownloader:
     def run_queue(self):
         """setup download queue in redis loop until no more items"""
         self._setup_queue()
-
         queue = RedisQueue(queue_name="dl_queue")
 
         limit_queue = self.config["downloads"]["limit_count"]
@@ -178,14 +170,11 @@ class VideoDownloader:
 
         while True:
             youtube_data = queue.get_next()
-            if not youtube_data:
+            if self.task.is_stopped() or not youtube_data:
+                queue.clear()
                 break
 
-            try:
-                youtube_data = json.loads(youtube_data)
-            except json.JSONDecodeError:  # This many not be necessary
-                continue
-
+            youtube_data = json.loads(youtube_data)
             youtube_id = youtube_data.get("youtube_id")
 
             tmp_vid_type = youtube_data.get(
@@ -198,13 +187,13 @@ class VideoDownloader:
             if not success:
                 continue
 
-            mess_dict = {
-                "status": self.MSG,
-                "level": "info",
-                "title": "Indexing....",
-                "message": "Add video metadata to index.",
-            }
-            RedisArchivist().set_message(self.MSG, mess_dict, expire=120)
+            if self.task:
+                self.task.send_progress(
+                    [
+                        f"Processing video {youtube_id}",
+                        "Add video metadata to index.",
+                    ]
+                )
 
             vid_dict = index_new_video(
                 youtube_id,
@@ -213,29 +202,25 @@ class VideoDownloader:
             )
             self.channels.add(vid_dict["channel"]["channel_id"])
             self.videos.add(vid_dict["youtube_id"])
-            mess_dict = {
-                "status": self.MSG,
-                "level": "info",
-                "title": "Moving....",
-                "message": "Moving downloaded file to storage folder",
-            }
-            RedisArchivist().set_message(self.MSG, mess_dict)
+
+            if self.task:
+                self.task.send_progress(
+                    [
+                        f"Processing video {youtube_id}",
+                        "Move downloaded file to archive.",
+                    ]
+                )
+
+            self.move_to_archive(vid_dict)
 
             if queue.has_item():
                 message = "Continue with next video."
-                expire = False
             else:
                 message = "Download queue is finished."
-                expire = 10
 
-            self.move_to_archive(vid_dict)
-            mess_dict = {
-                "status": self.MSG,
-                "level": "info",
-                "title": "Completed",
-                "message": message,
-            }
-            RedisArchivist().set_message(self.MSG, mess_dict, expire=expire)
+            if self.task:
+                self.task.send_progress([message])
+
             self._delete_from_pending(youtube_id)
 
         # post processing
@@ -256,13 +241,9 @@ class VideoDownloader:
 
     def add_pending(self):
         """add pending videos to download queue"""
-        mess_dict = {
-            "status": self.MSG,
-            "level": "info",
-            "title": "Looking for videos to download",
-            "message": "Scanning your download queue.",
-        }
-        RedisArchivist().set_message(self.MSG, mess_dict)
+        if self.task:
+            self.task.send_progress(["Scanning your download queue."])
+
         pending = PendingList()
         pending.get_download()
         to_add = [
@@ -279,40 +260,32 @@ class VideoDownloader:
         if not to_add:
             # there is nothing pending
             print("download queue is empty")
-            mess_dict = {
-                "status": self.MSG,
-                "level": "error",
-                "title": "Download queue is empty",
-                "message": "Add some videos to the queue first.",
-            }
-            RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
+            if self.task:
+                self.task.send_progress(["Download queue is empty."])
+
             return
 
         RedisQueue(queue_name="dl_queue").add_list(to_add)
 
     def _progress_hook(self, response):
         """process the progress_hooks from yt_dlp"""
-        title = "Downloading: " + response["info_dict"]["title"]
-
+        progress = False
         try:
             size = response.get("_total_bytes_str")
             if size.strip() == "N/A":
                 size = response.get("_total_bytes_estimate_str", "N/A")
 
             percent = response["_percent_str"]
+            progress = float(percent.strip("%")) / 100
             speed = response["_speed_str"]
             eta = response["_eta_str"]
             message = f"{percent} of {size} at {speed} - time left: {eta}"
         except KeyError:
             message = "processing"
 
-        mess_dict = {
-            "status": self.MSG,
-            "level": "info",
-            "title": title,
-            "message": message,
-        }
-        RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
+        if self.task:
+            title = response["info_dict"]["title"]
+            self.task.send_progress([title, message], progress=progress)
 
     def _build_obs(self):
         """collection to build all obs passed to yt-dlp"""
@@ -431,8 +404,8 @@ class VideoDownloader:
         new_folder = os.path.join(videos, channel_name)
         if not os.path.exists(new_folder):
             os.makedirs(new_folder)
-            if host_uid and host_gid:
-                os.chown(new_folder, host_uid, host_gid)
+        if host_uid and host_gid:
+            os.chown(new_folder, host_uid, host_gid)
         # find real filename
         cache_dir = self.config["application"]["cache_dir"]
         all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))

tubearchivist/home/src/es/backup.py

@@ -18,12 +18,13 @@ from home.src.ta.helper import get_mapping, ignore_filelist
 class ElasticBackup:
     """dump index to nd-json files for later bulk import"""
 
-    def __init__(self, reason=False):
+    def __init__(self, reason=False, task=False):
         self.config = AppConfig().config
         self.cache_dir = self.config["application"]["cache_dir"]
         self.timestamp = datetime.now().strftime("%Y%m%d")
         self.index_config = get_mapping()
         self.reason = reason
+        self.task = task
 
     def backup_all_indexes(self):
         """backup all indexes, add reason to init"""
@@ -31,6 +32,7 @@ class ElasticBackup:
         if not self.reason:
             raise ValueError("missing backup reason in ElasticBackup")
 
+        self.task.send_progress(["Scanning your index."])
         for index in self.index_config:
             index_name = index["index_name"]
             print(f"backup: export in progress for {index_name}")
@@ -40,22 +42,31 @@ class ElasticBackup:
 
             self.backup_index(index_name)
 
+        self.task.send_progress(["Compress files to zip archive."])
         self.zip_it()
         if self.reason == "auto":
             self.rotate_backup()
 
-    @staticmethod
-    def backup_index(index_name):
+    def backup_index(self, index_name):
         """export all documents of a single index"""
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"_doc": {"order": "desc"}}],
-        }
         paginate = IndexPaginate(
-            f"ta_{index_name}", data, keep_source=True, callback=BackupCallback
+            f"ta_{index_name}",
+            data={"query": {"match_all": {}}},
+            keep_source=True,
+            callback=BackupCallback,
+            task=self.task,
+            total=self._get_total(index_name),
         )
         _ = paginate.get_results()
 
+    @staticmethod
+    def _get_total(index_name):
+        """get total documents in index"""
+        path = f"ta_{index_name}/_count"
+        response, _ = ElasticWrap(path).get()
+
+        return response.get("count")
+
     def zip_it(self):
         """pack it up into single zip file"""
         file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
@@ -142,7 +153,8 @@ class ElasticBackup:
         """go through the unpacked files and restore"""
         backup_dir = os.path.join(self.cache_dir, "backup")
 
-        for json_f in zip_content:
+        for idx, json_f in enumerate(zip_content):
+            self._notify_restore(idx, json_f, len(zip_content))
             file_name = os.path.join(backup_dir, json_f)
 
             if not json_f.startswith("es_") or not json_f.endswith(".json"):
@@ -153,6 +165,12 @@ class ElasticBackup:
             self.post_bulk_restore(file_name)
             os.remove(file_name)
 
+    def _notify_restore(self, idx, json_f, total_files):
+        """notify restore progress"""
+        message = [f"Restore index from json backup file {json_f}."]
+        progress = (idx + 1) / total_files
+        self.task.send_progress(message_lines=message, progress=progress)
+
     @staticmethod
     def index_exists(index_name):
         """check if index already exists to skip"""

tubearchivist/home/src/es/connect.py

@@ -97,8 +97,10 @@ class IndexPaginate:
     """use search_after to go through whole index
     kwargs:
     - size: int, overwrite DEFAULT_SIZE
-    - keep_source: bool, keep _source key from es resutls
-    - callback: obj, Class with run method collback for every loop
+    - keep_source: bool, keep _source key from es results
+    - callback: obj, Class implementing run method callback for every loop
+    - task: task object to send notification
+    - total: int, total items in index for progress message
     """
 
     DEFAULT_SIZE = 500
@@ -107,12 +109,10 @@ class IndexPaginate:
         self.index_name = index_name
         self.data = data
         self.pit_id = False
-        self.size = kwargs.get("size")
-        self.keep_source = kwargs.get("keep_source")
-        self.callback = kwargs.get("callback")
+        self.kwargs = kwargs
 
     def get_results(self):
-        """get all results"""
+        """get all results, add task and total for notifications"""
         self.get_pit()
         self.validate_data()
         all_results = self.run_loop()
@@ -130,7 +130,7 @@ class IndexPaginate:
         if "sort" not in self.data.keys():
             self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
 
-        self.data["size"] = self.size or self.DEFAULT_SIZE
+        self.data["size"] = self.kwargs.get("size") or self.DEFAULT_SIZE
         self.data["pit"] = {"id": self.pit_id, "keep_alive": "10m"}
 
     def run_loop(self):
@@ -140,30 +140,37 @@ class IndexPaginate:
         while True:
             response, _ = ElasticWrap("_search").get(data=self.data)
             all_hits = response["hits"]["hits"]
-            if all_hits:
-                for hit in all_hits:
-                    if self.keep_source:
-                        source = hit
-                    else:
-                        source = hit["_source"]
-
-                    if not self.callback:
-                        all_results.append(source)
-
-                if self.callback:
-                    self.callback(all_hits, self.index_name).run()
-                    if counter % 10 == 0:
-                        print(f"{self.index_name}: processing page {counter}")
-                    counter = counter + 1
-
-                # update search_after with last hit data
-                self.data["search_after"] = all_hits[-1]["sort"]
-            else:
+            if not all_hits:
                 break
 
+            for hit in all_hits:
+                if self.kwargs.get("keep_source"):
+                    all_results.append(hit)
+                else:
+                    all_results.append(hit["_source"])
+
+            if self.kwargs.get("callback"):
+                self.kwargs.get("callback")(all_hits, self.index_name).run()
+
+            if self.kwargs.get("task"):
+                print(f"{self.index_name}: processing page {counter}")
+                self._notify(len(all_results))
+
+            counter += 1
+
+            # update search_after with last hit data
+            self.data["search_after"] = all_hits[-1]["sort"]
+
         return all_results
 
+    def _notify(self, processed):
+        """send notification on task"""
+        total = self.kwargs.get("total")
+        progress = processed / total
+        index_clean = self.index_name.lstrip("ta_").title()
+        message = [f"Processing {index_clean}s {processed}/{total}"]
+        self.kwargs.get("task").send_progress(message, progress=progress)
+
     def clean_pit(self):
         """delete pit from elastic search"""
-        data = {"id": self.pit_id}
-        ElasticWrap("_pit").delete(data=data)
+        ElasticWrap("_pit").delete(data={"id": self.pit_id})

tubearchivist/home/src/frontend/api_calls.py

@@ -9,20 +9,9 @@ from home.src.download.subscriptions import (
     PlaylistSubscription,
 )
 from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.ta_redis import RedisArchivist, RedisQueue
+from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.urlparser import Parser
-from home.tasks import (
-    download_pending,
-    index_channel_playlists,
-    kill_dl,
-    re_sync_thumbs,
-    rescan_filesystem,
-    run_backup,
-    run_manual_import,
-    run_restore_backup,
-    subscribe_to,
-    update_subscribed,
-)
+from home.tasks import run_restore_backup, subscribe_to
 
 
 class PostData:
@@ -47,22 +36,14 @@ class PostData:
         exec_map = {
             "change_view": self._change_view,
             "change_grid": self._change_grid,
-            "rescan_pending": self._rescan_pending,
-            "dl_pending": self._dl_pending,
-            "queue": self._queue_handler,
             "unsubscribe": self._unsubscribe,
             "subscribe": self._subscribe,
             "sort_order": self._sort_order,
             "hide_watched": self._hide_watched,
             "show_subed_only": self._show_subed_only,
             "show_ignored_only": self._show_ignored_only,
-            "manual-import": self._manual_import,
-            "re-embed": self._re_embed,
-            "db-backup": self._db_backup,
             "db-restore": self._db_restore,
-            "fs-rescan": self._fs_rescan,
             "delete-playlist": self._delete_playlist,
-            "find-playlists": self._find_playlists,
         }
 
         return exec_map[self.to_exec]
@@ -86,39 +67,6 @@ class PostData:
         RedisArchivist().set_message(key, {"status": grid_items})
         return {"success": True}
 
-    @staticmethod
-    def _rescan_pending():
-        """look for new items in subscribed channels"""
-        print("rescan subscribed channels")
-        update_subscribed.delay()
-        return {"success": True}
-
-    @staticmethod
-    def _dl_pending():
-        """start the download queue"""
-        print("download pending")
-        running = download_pending.delay()
-        task_id = running.id
-        print(f"{task_id}: set task id")
-        RedisArchivist().set_message("dl_queue_id", task_id)
-        return {"success": True}
-
-    def _queue_handler(self):
-        """queue controls from frontend"""
-        to_execute = self.exec_val
-        if to_execute == "stop":
-            print("stopping download queue")
-            RedisQueue(queue_name="dl_queue").clear()
-        elif to_execute == "kill":
-            task_id = RedisArchivist().get_message("dl_queue_id")
-            if not isinstance(task_id, str):
-                task_id = False
-            else:
-                print("brutally killing " + task_id)
-            kill_dl(task_id)
-
-        return {"success": True}
-
     def _unsubscribe(self):
         """unsubscribe from channels or playlists"""
         id_unsub = self.exec_val
@@ -185,27 +133,6 @@ class PostData:
         RedisArchivist().set_message(key, value)
         return {"success": True}
 
-    @staticmethod
-    def _manual_import():
-        """run manual import from settings page"""
-        print("starting manual import")
-        run_manual_import.delay()
-        return {"success": True}
-
-    @staticmethod
-    def _re_embed():
-        """rewrite thumbnails into media files"""
-        print("start video thumbnail embed process")
-        re_sync_thumbs.delay()
-        return {"success": True}
-
-    @staticmethod
-    def _db_backup():
-        """backup es to zip from settings page"""
-        print("backing up database")
-        run_backup.delay("manual")
-        return {"success": True}
-
     def _db_restore(self):
         """restore es zip from settings page"""
         print("restoring index from backup zip")
@@ -213,13 +140,6 @@ class PostData:
         run_restore_backup.delay(filename)
         return {"success": True}
 
-    @staticmethod
-    def _fs_rescan():
-        """start file system rescan task"""
-        print("start filesystem scan")
-        rescan_filesystem.delay()
-        return {"success": True}
-
     def _delete_playlist(self):
         """delete playlist, only metadata or incl all videos"""
         playlist_dict = self.exec_val
@@ -232,9 +152,3 @@ class PostData:
             YoutubePlaylist(playlist_id).delete_videos_playlist()
 
         return {"success": True}
-
-    def _find_playlists(self):
-        """add all playlists of a channel"""
-        channel_id = self.exec_val
-        index_channel_playlists.delay(channel_id)
-        return {"success": True}

tubearchivist/home/src/index/channel.py

@@ -18,7 +18,6 @@ from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.generic import YouTubeItem
 from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.helper import clean_string, requests_headers
-from home.src.ta.ta_redis import RedisArchivist
 
 
 class ChannelScraper:
@@ -167,12 +166,12 @@ class YoutubeChannel(YouTubeItem):
     es_path = False
     index_name = "ta_channel"
     yt_base = "https://www.youtube.com/channel/"
-    msg = "message:playlistscan"
 
-    def __init__(self, youtube_id):
+    def __init__(self, youtube_id, task=False):
         super().__init__(youtube_id)
         self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.all_playlists = False
+        self.task = task
 
     def build_json(self, upload=False, fallback=False):
         """get from es or from youtube"""
@@ -241,6 +240,7 @@ class YoutubeChannel(YouTubeItem):
         urls = (
             self.json_data["channel_thumb_url"],
             self.json_data["channel_banner_url"],
+            self.json_data["channel_tvart_url"],
         )
         ThumbManager(self.youtube_id, item_type="channel").download(urls)
 
@@ -324,34 +324,29 @@ class YoutubeChannel(YouTubeItem):
         print(f"{self.youtube_id}: index all playlists")
         self.get_from_es()
         channel_name = self.json_data["channel_name"]
-        mess_dict = {
-            "status": self.msg,
-            "level": "info",
-            "title": "Looking for playlists",
-            "message": f"{channel_name}: Scanning channel in progress",
-        }
-        RedisArchivist().set_message(self.msg, mess_dict, expire=True)
+        self.task.send_progress([f"{channel_name}: Looking for Playlists"])
         self.get_all_playlists()
         if not self.all_playlists:
             print(f"{self.youtube_id}: no playlists found.")
             return
 
         all_youtube_ids = self.get_all_video_ids()
+        total = len(self.all_playlists)
         for idx, playlist in enumerate(self.all_playlists):
-            self._notify_single_playlist(idx, playlist)
-            self._index_single_playlist(playlist, all_youtube_ids)
+            if self.task:
+                self._notify_single_playlist(idx, total)
 
-    def _notify_single_playlist(self, idx, playlist):
+            self._index_single_playlist(playlist, all_youtube_ids)
+            print("add playlist: " + playlist[1])
+
+    def _notify_single_playlist(self, idx, total):
         """send notification"""
         channel_name = self.json_data["channel_name"]
-        mess_dict = {
-            "status": self.msg,
-            "level": "info",
-            "title": f"{channel_name}: Scanning channel for playlists",
-            "message": f"Progress: {idx + 1}/{len(self.all_playlists)}",
-        }
-        RedisArchivist().set_message(self.msg, mess_dict, expire=True)
-        print("add playlist: " + playlist[1])
+        message = [
+            f"{channel_name}: Scanning channel for playlists",
+            f"Progress: {idx + 1}/{total}",
+        ]
+        self.task.send_progress(message, progress=(idx + 1) / total)
 
     @staticmethod
     def _index_single_playlist(playlist, all_youtube_ids):

tubearchivist/home/src/index/comments.py

@@ -10,7 +10,6 @@ from datetime import datetime
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
-from home.src.ta.ta_redis import RedisArchivist
 
 
 class Comments:
@@ -24,14 +23,13 @@ class Comments:
         self.is_activated = False
         self.comments_format = False
 
-    def build_json(self, notify=False):
+    def build_json(self):
         """build json document for es"""
         print(f"{self.youtube_id}: get comments")
         self.check_config()
         if not self.is_activated:
             return
 
-        self._send_notification(notify)
         comments_raw, channel_id = self.get_yt_comments()
         if not comments_raw and not channel_id:
             return
@@ -52,23 +50,6 @@ class Comments:
 
         self.is_activated = bool(self.config["downloads"]["comment_max"])
 
-    @staticmethod
-    def _send_notification(notify):
-        """send notification for download post process message"""
-        if not notify:
-            return
-
-        key = "message:download"
-        idx, total_videos = notify
-        message = {
-            "status": key,
-            "level": "info",
-            "title": "Download and index comments",
-            "message": f"Progress: {idx + 1}/{total_videos}",
-        }
-
-        RedisArchivist().set_message(key, message)
-
     def build_yt_obs(self):
         """
         get extractor config
@@ -79,6 +60,7 @@ class Comments:
         comment_sort = self.config["downloads"]["comment_sort"]
 
         yt_obs = {
+            "check_formats": None,
             "skip_download": True,
             "getcomments": True,
             "extractor_args": {
@@ -200,38 +182,28 @@ class Comments:
 class CommentList:
     """interact with comments in group"""
 
-    def __init__(self, video_ids):
+    def __init__(self, video_ids, task=False):
         self.video_ids = video_ids
+        self.task = task
         self.config = AppConfig().config
 
-    def index(self, notify=False):
-        """index group of videos"""
+    def index(self):
+        """index comments for list, init with task object to notify"""
         if not self.config["downloads"].get("comment_max"):
             return
 
         total_videos = len(self.video_ids)
-        if notify:
-            self._notify(f"add comments for {total_videos} videos", False)
+        for idx, youtube_id in enumerate(self.video_ids):
+            if self.task:
+                self.notify(idx, total_videos)
 
-        for idx, video_id in enumerate(self.video_ids):
-            comment = Comments(video_id, config=self.config)
-            if notify:
-                notify = (idx, total_videos)
-            comment.build_json(notify=notify)
+            comment = Comments(youtube_id, config=self.config)
+            comment.build_json()
             if comment.json_data:
                 comment.upload_comments()
 
-        if notify:
-            self._notify(f"added comments for {total_videos} videos", 5)
-
-    @staticmethod
-    def _notify(message, expire):
-        """send notification"""
-        key = "message:download"
-        message = {
-            "status": key,
-            "level": "info",
-            "title": "Download and index comments finished",
-            "message": message,
-        }
-        RedisArchivist().set_message(key, message, expire=expire)
+    def notify(self, idx, total_videos):
+        """send notification on task"""
+        message = [f"Add comments for new videos {idx + 1}/{total_videos}"]
+        progress = (idx + 1) / total_videos
+        self.task.send_progress(message, progress=progress)

tubearchivist/home/src/index/filesystem.py

@@ -7,38 +7,37 @@ Functionality:
 
 import json
 import os
-import re
-import shutil
-import subprocess
 
 from home.src.download.queue import PendingList
-from home.src.download.thumbnails import ThumbManager
 from home.src.es.connect import ElasticWrap
 from home.src.index.comments import CommentList
-from home.src.index.video import YoutubeVideo, index_new_video
+from home.src.index.video import index_new_video
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import clean_string, ignore_filelist
-from PIL import Image, ImageFile
-from yt_dlp.utils import ISO639Utils
+from PIL import ImageFile
 
 ImageFile.LOAD_TRUNCATED_IMAGES = True
 
 
-class FilesystemScanner:
-    """handle scanning and fixing from filesystem"""
+class ScannerBase:
+    """scan the filesystem base class"""
 
     CONFIG = AppConfig().config
     VIDEOS = CONFIG["application"]["videos"]
 
     def __init__(self):
-        self.all_downloaded = self.get_all_downloaded()
-        self.all_indexed = self.get_all_indexed()
-        self.mismatch = None
-        self.to_rename = None
-        self.to_index = None
-        self.to_delete = None
+        self.to_index = False
+        self.to_delete = False
+        self.mismatch = False
+        self.to_rename = False
 
-    def get_all_downloaded(self):
+    def scan(self):
+        """entry point, scan and compare"""
+        all_downloaded = self._get_all_downloaded()
+        all_indexed = self._get_all_indexed()
+        self.list_comarison(all_downloaded, all_indexed)
+
+    def _get_all_downloaded(self):
         """get a list of all video files downloaded"""
         channels = os.listdir(self.VIDEOS)
         all_channels = ignore_filelist(channels)
@@ -56,7 +55,7 @@ class FilesystemScanner:
         return all_downloaded
 
     @staticmethod
-    def get_all_indexed():
+    def _get_all_indexed():
         """get a list of all indexed videos"""
         index_handler = PendingList()
         index_handler.get_download()
@@ -71,64 +70,77 @@ class FilesystemScanner:
             all_indexed.append((youtube_id, media_url, published, title))
         return all_indexed
 
-    def list_comarison(self):
+    def list_comarison(self, all_downloaded, all_indexed):
         """compare the lists to figure out what to do"""
-        self.find_unindexed()
-        self.find_missing()
-        self.find_bad_media_url()
+        self._find_unindexed(all_downloaded, all_indexed)
+        self._find_missing(all_downloaded, all_indexed)
+        self._find_bad_media_url(all_downloaded, all_indexed)
 
-    def find_unindexed(self):
+    def _find_unindexed(self, all_downloaded, all_indexed):
         """find video files without a matching document indexed"""
-        all_indexed_ids = [i[0] for i in self.all_indexed]
-        to_index = []
-        for downloaded in self.all_downloaded:
+        all_indexed_ids = [i[0] for i in all_indexed]
+        self.to_index = []
+        for downloaded in all_downloaded:
             if downloaded[2] not in all_indexed_ids:
-                to_index.append(downloaded)
+                self.to_index.append(downloaded)
 
-        self.to_index = to_index
-
-    def find_missing(self):
+    def _find_missing(self, all_downloaded, all_indexed):
         """find indexed videos without matching media file"""
-        all_downloaded_ids = [i[2] for i in self.all_downloaded]
-        to_delete = []
-        for video in self.all_indexed:
+        all_downloaded_ids = [i[2] for i in all_downloaded]
+        self.to_delete = []
+        for video in all_indexed:
             youtube_id = video[0]
             if youtube_id not in all_downloaded_ids:
-                to_delete.append(video)
+                self.to_delete.append(video)
 
-        self.to_delete = to_delete
-
-    def find_bad_media_url(self):
+    def _find_bad_media_url(self, all_downloaded, all_indexed):
         """rename media files not matching the indexed title"""
-        to_fix = []
-        to_rename = []
-        for downloaded in self.all_downloaded:
+        self.mismatch = []
+        self.to_rename = []
+
+        for downloaded in all_downloaded:
             channel, filename, downloaded_id = downloaded
             # find in indexed
-            for indexed in self.all_indexed:
+            for indexed in all_indexed:
                 indexed_id, media_url, published, title = indexed
                 if indexed_id == downloaded_id:
                     # found it
-                    title_c = clean_string(title)
                     pub = published.replace("-", "")
-                    expected_filename = f"{pub}_{indexed_id}_{title_c}.mp4"
-                    new_url = os.path.join(channel, expected_filename)
-                    if expected_filename != filename:
+                    expected = f"{pub}_{indexed_id}_{clean_string(title)}.mp4"
+                    new_url = os.path.join(channel, expected)
+                    if expected != filename:
                         # file to rename
-                        to_rename.append(
-                            (channel, filename, expected_filename)
-                        )
+                        self.to_rename.append((channel, filename, expected))
                     if media_url != new_url:
                         # media_url to update in es
-                        to_fix.append((indexed_id, new_url))
+                        self.mismatch.append((indexed_id, new_url))
 
                     break
 
-        self.mismatch = to_fix
-        self.to_rename = to_rename
+
+class Filesystem(ScannerBase):
+    """handle scanning and fixing from filesystem"""
+
+    def __init__(self, task=False):
+        super().__init__()
+        self.task = task
+
+    def process(self):
+        """entry point"""
+        self.task.send_progress(["Scanning your archive and index."])
+        self.scan()
+        self.rename_files()
+        self.send_mismatch_bulk()
+        self.delete_from_index()
+        self.add_missing()
 
     def rename_files(self):
         """rename media files as identified by find_bad_media_url"""
+        if not self.to_rename:
+            return
+
+        total = len(self.to_rename)
+        self.task.send_progress([f"Rename {total} media files."])
         for bad_filename in self.to_rename:
             channel, filename, expected_filename = bad_filename
             print(f"renaming [{filename}] to [{expected_filename}]")
@@ -138,6 +150,11 @@ class FilesystemScanner:
 
     def send_mismatch_bulk(self):
         """build bulk update"""
+        if not self.mismatch:
+            return
+
+        total = len(self.mismatch)
+        self.task.send_progress([f"Fix media urls for {total} files"])
         bulk_list = []
         for video_mismatch in self.mismatch:
             youtube_id, media_url = video_mismatch
@@ -153,457 +170,32 @@ class FilesystemScanner:
 
     def delete_from_index(self):
         """find indexed but deleted mediafile"""
+        if not self.to_delete:
+            return
+
+        total = len(self.to_delete)
+        self.task.send_progress([f"Clean up {total} items from index."])
         for indexed in self.to_delete:
             youtube_id = indexed[0]
             print(f"deleting {youtube_id} from index")
             path = f"ta_video/_doc/{youtube_id}"
             _, _ = ElasticWrap(path).delete()
 
-
-class ImportFolderScanner:
-    """import and indexing existing video files
-    - identify all media files belonging to a video
-    - identify youtube id
-    - convert if needed
-    """
-
-    CONFIG = AppConfig().config
-    CACHE_DIR = CONFIG["application"]["cache_dir"]
-    IMPORT_DIR = os.path.join(CACHE_DIR, "import")
-
-    EXT_MAP = {
-        "media": [".mp4", ".mkv", ".webm"],
-        "metadata": [".json"],
-        "thumb": [".jpg", ".png", ".webp"],
-        "subtitle": [".vtt"],
-    }
-
-    def __init__(self):
-        self.to_import = False
-
-    def scan(self):
-        """scan and match media files"""
-        all_files = self.get_all_files()
-        self.match_files(all_files)
-        self.process_videos()
-
-        return self.to_import
-
-    def get_all_files(self):
-        """get all files in /import"""
-        rel_paths = ignore_filelist(os.listdir(self.IMPORT_DIR))
-        all_files = [os.path.join(self.IMPORT_DIR, i) for i in rel_paths]
-        all_files.sort()
-
-        return all_files
-
-    @staticmethod
-    def _get_template():
-        """base dict for video"""
-        return {
-            "media": False,
-            "video_id": False,
-            "metadata": False,
-            "thumb": False,
-            "subtitle": [],
-        }
-
-    def match_files(self, all_files):
-        """loop through all files, join what matches"""
-        self.to_import = []
-
-        current_video = self._get_template()
-        last_base = False
-
-        for file_path in all_files:
-            base_name, ext = self._detect_base_name(file_path)
-            key, file_path = self._detect_type(file_path, ext)
-            if not key or not file_path:
-                continue
-
-            if base_name != last_base:
-                if last_base:
-                    print(f"manual import: {current_video}")
-                    self.to_import.append(current_video)
-
-                current_video = self._get_template()
-                last_base = base_name
-
-            if key == "subtitle":
-                current_video["subtitle"].append(file_path)
-            else:
-                current_video[key] = file_path
-
-        if current_video.get("media"):
-            print(f"manual import: {current_video}")
-            self.to_import.append(current_video)
-
-    def _detect_base_name(self, file_path):
-        """extract base_name and ext for matching"""
-        base_name_raw, ext = os.path.splitext(file_path)
-        base_name, ext2 = os.path.splitext(base_name_raw)
-
-        if ext2:
-            if ISO639Utils.short2long(ext2.strip(".")) or ext2 == ".info":
-                # valid secondary extension
-                return base_name, ext
-
-        return base_name_raw, ext
-
-    def _detect_type(self, file_path, ext):
-        """detect metadata type for file"""
-
-        for key, value in self.EXT_MAP.items():
-            if ext in value:
-                return key, file_path
-
-        return False, False
-
-    def process_videos(self):
-        """loop through all videos"""
-        for current_video in self.to_import:
-            if not current_video["media"]:
-                print(f"{current_video}: no matching media file found.")
-                raise ValueError
-
-            self._detect_youtube_id(current_video)
-            self._dump_thumb(current_video)
-            self._convert_thumb(current_video)
-            self._get_subtitles(current_video)
-            self._convert_video(current_video)
-            print(f"manual import: {current_video}")
-
-            ManualImport(current_video, self.CONFIG).run()
-
-    def _detect_youtube_id(self, current_video):
-        """find video id from filename or json"""
-        youtube_id = self._extract_id_from_filename(current_video["media"])
-        if youtube_id:
-            current_video["video_id"] = youtube_id
+    def add_missing(self):
+        """add missing videos to index"""
+        video_ids = [i[2] for i in self.to_index]
+        if not video_ids:
             return
 
-        youtube_id = self._extract_id_from_json(current_video["metadata"])
-        if youtube_id:
-            current_video["video_id"] = youtube_id
-            return
-
-        raise ValueError("failed to find video id")
-
-    @staticmethod
-    def _extract_id_from_filename(file_name):
-        """
-        look at the file name for the youtube id
-        expects filename ending in [<youtube_id>].<ext>
-        """
-        base_name, _ = os.path.splitext(file_name)
-        id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
-        if id_search:
-            youtube_id = id_search.group(1)
-            return youtube_id
-
-        print(f"id extraction failed from filename: {file_name}")
-
-        return False
-
-    def _extract_id_from_json(self, json_file):
-        """open json file and extract id"""
-        json_path = os.path.join(self.CACHE_DIR, "import", json_file)
-        with open(json_path, "r", encoding="utf-8") as f:
-            json_content = f.read()
-
-        youtube_id = json.loads(json_content)["id"]
-
-        return youtube_id
-
-    def _dump_thumb(self, current_video):
-        """extract embedded thumb before converting"""
-        if current_video["thumb"]:
-            return
-
-        media_path = current_video["media"]
-        _, ext = os.path.splitext(media_path)
-
-        new_path = False
-        if ext == ".mkv":
-            idx, thumb_type = self._get_mkv_thumb_stream(media_path)
-            if idx is not None:
-                new_path = self.dump_mpv_thumb(media_path, idx, thumb_type)
-
-        elif ext == ".mp4":
-            thumb_type = self.get_mp4_thumb_type(media_path)
-            if thumb_type:
-                new_path = self.dump_mp4_thumb(media_path, thumb_type)
-
-        if new_path:
-            current_video["thumb"] = new_path
-
-    def _get_mkv_thumb_stream(self, media_path):
-        """get stream idx of thumbnail for mkv files"""
-        streams = self._get_streams(media_path)
-        attachments = [
-            i for i in streams["streams"] if i["codec_type"] == "attachment"
-        ]
-
-        for idx, stream in enumerate(attachments):
-            tags = stream["tags"]
-            if "mimetype" in tags and tags["filename"].startswith("cover"):
-                _, ext = os.path.splitext(tags["filename"])
-                return idx, ext
-
-        return None, None
-
-    @staticmethod
-    def dump_mpv_thumb(media_path, idx, thumb_type):
-        """write cover to disk for mkv"""
-        _, media_ext = os.path.splitext(media_path)
-        new_path = f"{media_path.rstrip(media_ext)}{thumb_type}"
-        subprocess.run(
-            [
-                "ffmpeg",
-                "-v",
-                "quiet",
-                f"-dump_attachment:t:{idx}",
-                new_path,
-                "-i",
-                media_path,
-            ],
-            check=False,
-        )
-
-        return new_path
-
-    def get_mp4_thumb_type(self, media_path):
-        """detect filetype of embedded thumbnail"""
-        streams = self._get_streams(media_path)
-
-        for stream in streams["streams"]:
-            if stream["codec_name"] in ["png", "jpg"]:
-                return stream["codec_name"]
-
-        return False
-
-    def _convert_thumb(self, current_video):
-        """convert all thumbnails to jpg"""
-        if not current_video["thumb"]:
-            return
-
-        thumb_path = current_video["thumb"]
-
-        base_path, ext = os.path.splitext(thumb_path)
-        if ext == ".jpg":
-            return
-
-        new_path = f"{base_path}.jpg"
-        img_raw = Image.open(thumb_path)
-        img_raw.convert("RGB").save(new_path)
-
-        os.remove(thumb_path)
-        current_video["thumb"] = new_path
-
-    def _get_subtitles(self, current_video):
-        """find all subtitles in media file"""
-        if current_video["subtitle"]:
-            return
-
-        media_path = current_video["media"]
-        streams = self._get_streams(media_path)
-        base_path, ext = os.path.splitext(media_path)
-
-        if ext == ".webm":
-            print(f"{media_path}: subtitle extract from webm not supported")
-            return
-
-        for idx, stream in enumerate(streams["streams"]):
-            if stream["codec_type"] == "subtitle":
-                lang = ISO639Utils.long2short(stream["tags"]["language"])
-                sub_path = f"{base_path}.{lang}.vtt"
-                self._dump_subtitle(idx, media_path, sub_path)
-                current_video["subtitle"].append(sub_path)
-
-    @staticmethod
-    def _dump_subtitle(idx, media_path, sub_path):
-        """extract subtitle from media file"""
-        subprocess.run(
-            ["ffmpeg", "-i", media_path, "-map", f"0:{idx}", sub_path],
-            check=True,
-        )
-
-    @staticmethod
-    def _get_streams(media_path):
-        """return all streams from media_path"""
-        streams_raw = subprocess.run(
-            [
-                "ffprobe",
-                "-v",
-                "error",
-                "-show_streams",
-                "-print_format",
-                "json",
-                media_path,
-            ],
-            capture_output=True,
-            check=True,
-        )
-        streams = json.loads(streams_raw.stdout.decode())
-
-        return streams
-
-    @staticmethod
-    def dump_mp4_thumb(media_path, thumb_type):
-        """save cover to disk"""
-        _, ext = os.path.splitext(media_path)
-        new_path = f"{media_path.rstrip(ext)}.{thumb_type}"
-
-        subprocess.run(
-            [
-                "ffmpeg",
-                "-i",
-                media_path,
-                "-map",
-                "0:v",
-                "-map",
-                "-0:V",
-                "-c",
-                "copy",
-                new_path,
-            ],
-            check=True,
-        )
-
-        return new_path
-
-    def _convert_video(self, current_video):
-        """convert if needed"""
-        current_path = current_video["media"]
-        base_path, ext = os.path.splitext(current_path)
-        if ext == ".mp4":
-            return
-
-        new_path = base_path + ".mp4"
-        subprocess.run(
-            [
-                "ffmpeg",
-                "-i",
-                current_path,
-                new_path,
-                "-loglevel",
-                "warning",
-                "-stats",
-            ],
-            check=True,
-        )
-        current_video["media"] = new_path
-        os.remove(current_path)
-
-
-class ManualImport:
-    """import single identified video"""
-
-    def __init__(self, current_video, config):
-        self.current_video = current_video
-        self.config = config
-
-    def run(self):
-        """run all"""
-        json_data = self.index_metadata()
-        self._move_to_archive(json_data)
-        self._cleanup(json_data)
-
-    def index_metadata(self):
-        """get metadata from yt or json"""
-        video_id = self.current_video["video_id"]
-        video = YoutubeVideo(video_id)
-        video.build_json(
-            youtube_meta_overwrite=self._get_info_json(),
-            media_path=self.current_video["media"],
-        )
-        if not video.json_data:
-            print(f"{video_id}: manual import failed, and no metadata found.")
-            raise ValueError
-
-        video.check_subtitles(subtitle_files=self.current_video["subtitle"])
-        video.upload_to_es()
-
-        if video.offline_import and self.current_video["thumb"]:
-            old_path = self.current_video["thumb"]
-            thumbs = ThumbManager(video_id)
-            new_path = thumbs.vid_thumb_path(absolute=True, create_folder=True)
-            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
-        else:
-            url = video.json_data["vid_thumb_url"]
-            ThumbManager(video_id).download_video_thumb(url)
-
-        return video.json_data
-
-    def _get_info_json(self):
-        """read info_json from file"""
-        if not self.current_video["metadata"]:
-            return False
-
-        with open(self.current_video["metadata"], "r", encoding="utf-8") as f:
-            info_json = json.loads(f.read())
-
-        return info_json
-
-    def _move_to_archive(self, json_data):
-        """move identified media file to archive"""
-        videos = self.config["application"]["videos"]
-
-        channel, file = os.path.split(json_data["media_url"])
-        channel_folder = os.path.join(videos, channel)
-        if not os.path.exists(channel_folder):
-            os.makedirs(channel_folder)
-
-        old_path = self.current_video["media"]
-        new_path = os.path.join(channel_folder, file)
-        shutil.move(old_path, new_path, copy_function=shutil.copyfile)
-
-        base_name, _ = os.path.splitext(new_path)
-        for old_path in self.current_video["subtitle"]:
-            lang = old_path.split(".")[-2]
-            new_path = f"{base_name}.{lang}.vtt"
-            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
-
-    def _cleanup(self, json_data):
-        """cleanup leftover files"""
-        meta_data = self.current_video["metadata"]
-        if meta_data and os.path.exists(meta_data):
-            os.remove(meta_data)
-
-        thumb = self.current_video["thumb"]
-        if thumb and os.path.exists(thumb):
-            os.remove(thumb)
-
-        for subtitle_file in self.current_video["subtitle"]:
-            if os.path.exists(subtitle_file):
-                os.remove(subtitle_file)
-
-        channel_info = os.path.join(
-            self.config["application"]["cache_dir"],
-            "import",
-            f"{json_data['channel']['channel_id']}.info.json",
-        )
-        if os.path.exists(channel_info):
-            os.remove(channel_info)
-
-
-def scan_filesystem():
-    """grouped function to delete and update index"""
-    filesystem_handler = FilesystemScanner()
-    filesystem_handler.list_comarison()
-    if filesystem_handler.to_rename:
-        print("renaming files")
-        filesystem_handler.rename_files()
-    if filesystem_handler.mismatch:
-        print("fixing media urls in index")
-        filesystem_handler.send_mismatch_bulk()
-    if filesystem_handler.to_delete:
-        print("delete metadata from index")
-        filesystem_handler.delete_from_index()
-    if filesystem_handler.to_index:
-        print("index new videos")
-        video_ids = [i[2] for i in filesystem_handler.to_index]
-        for youtube_id in video_ids:
+        total = len(video_ids)
+        for idx, youtube_id in enumerate(video_ids):
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[
+                        f"Index missing video {youtube_id}, {idx}/{total}"
+                    ],
+                    progress=(idx + 1) / total,
+                )
             index_new_video(youtube_id)
 
-        CommentList(video_ids).index()
+        CommentList(video_ids, task=self.task).index()

tubearchivist/home/src/index/manual.py

@@ -0,0 +1,477 @@
+"""
+Functionality:
+- Handle manual import task
+- Scan and identify media files in import folder
+- Process import media files
+"""
+
+import json
+import os
+import re
+import shutil
+import subprocess
+
+from home.src.download.thumbnails import ThumbManager
+from home.src.index.comments import CommentList
+from home.src.index.video import YoutubeVideo
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import ignore_filelist
+from PIL import Image
+from yt_dlp.utils import ISO639Utils
+
+
+class ImportFolderScanner:
+    """import and indexing existing video files
+    - identify all media files belonging to a video
+    - identify youtube id
+    - convert if needed
+    """
+
+    CONFIG = AppConfig().config
+    CACHE_DIR = CONFIG["application"]["cache_dir"]
+    IMPORT_DIR = os.path.join(CACHE_DIR, "import")
+
+    EXT_MAP = {
+        "media": [".mp4", ".mkv", ".webm"],
+        "metadata": [".json"],
+        "thumb": [".jpg", ".png", ".webp"],
+        "subtitle": [".vtt"],
+    }
+
+    def __init__(self, task=False):
+        self.task = task
+        self.to_import = False
+
+    def scan(self):
+        """scan and match media files"""
+        if self.task:
+            self.task.send_progress(["Scanning your import folder."])
+
+        all_files = self.get_all_files()
+        self.match_files(all_files)
+        self.process_videos()
+
+        return self.to_import
+
+    def get_all_files(self):
+        """get all files in /import"""
+        rel_paths = ignore_filelist(os.listdir(self.IMPORT_DIR))
+        all_files = [os.path.join(self.IMPORT_DIR, i) for i in rel_paths]
+        all_files.sort()
+
+        return all_files
+
+    @staticmethod
+    def _get_template():
+        """base dict for video"""
+        return {
+            "media": False,
+            "video_id": False,
+            "metadata": False,
+            "thumb": False,
+            "subtitle": [],
+        }
+
+    def match_files(self, all_files):
+        """loop through all files, join what matches"""
+        self.to_import = []
+
+        current_video = self._get_template()
+        last_base = False
+
+        for file_path in all_files:
+            base_name, ext = self._detect_base_name(file_path)
+            key, file_path = self._detect_type(file_path, ext)
+            if not key or not file_path:
+                continue
+
+            if base_name != last_base:
+                if last_base:
+                    print(f"manual import: {current_video}")
+                    self.to_import.append(current_video)
+
+                current_video = self._get_template()
+                last_base = base_name
+
+            if key == "subtitle":
+                current_video["subtitle"].append(file_path)
+            else:
+                current_video[key] = file_path
+
+        if current_video.get("media"):
+            print(f"manual import: {current_video}")
+            self.to_import.append(current_video)
+
+    def _detect_base_name(self, file_path):
+        """extract base_name and ext for matching"""
+        base_name_raw, ext = os.path.splitext(file_path)
+        base_name, ext2 = os.path.splitext(base_name_raw)
+
+        if ext2:
+            if ISO639Utils.short2long(ext2.strip(".")) or ext2 == ".info":
+                # valid secondary extension
+                return base_name, ext
+
+        return base_name_raw, ext
+
+    def _detect_type(self, file_path, ext):
+        """detect metadata type for file"""
+
+        for key, value in self.EXT_MAP.items():
+            if ext in value:
+                return key, file_path
+
+        return False, False
+
+    def process_videos(self):
+        """loop through all videos"""
+        for idx, current_video in enumerate(self.to_import):
+            if not current_video["media"]:
+                print(f"{current_video}: no matching media file found.")
+                raise ValueError
+
+            if self.task:
+                self._notify(idx, current_video)
+
+            self._detect_youtube_id(current_video)
+            self._dump_thumb(current_video)
+            self._convert_thumb(current_video)
+            self._get_subtitles(current_video)
+            self._convert_video(current_video)
+            print(f"manual import: {current_video}")
+
+            ManualImport(current_video, self.CONFIG).run()
+
+        video_ids = [i["video_id"] for i in self.to_import]
+        CommentList(video_ids, task=self.task).index()
+
+    def _notify(self, idx, current_video):
+        """send notification back to task"""
+        filename = os.path.split(current_video["media"])[-1]
+        if len(filename) > 50:
+            filename = filename[:50] + "..."
+
+        message = [
+            f"Import queue processing video {idx + 1}/{len(self.to_import)}",
+            filename,
+        ]
+        progress = (idx + 1) / len(self.to_import)
+        self.task.send_progress(message, progress=progress)
+
+    def _detect_youtube_id(self, current_video):
+        """find video id from filename or json"""
+        youtube_id = self._extract_id_from_filename(current_video["media"])
+        if youtube_id:
+            current_video["video_id"] = youtube_id
+            return
+
+        youtube_id = self._extract_id_from_json(current_video["metadata"])
+        if youtube_id:
+            current_video["video_id"] = youtube_id
+            return
+
+        raise ValueError("failed to find video id")
+
+    @staticmethod
+    def _extract_id_from_filename(file_name):
+        """
+        look at the file name for the youtube id
+        expects filename ending in [<youtube_id>].<ext>
+        """
+        base_name, _ = os.path.splitext(file_name)
+        id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
+        if id_search:
+            youtube_id = id_search.group(1)
+            return youtube_id
+
+        print(f"id extraction failed from filename: {file_name}")
+
+        return False
+
+    def _extract_id_from_json(self, json_file):
+        """open json file and extract id"""
+        json_path = os.path.join(self.CACHE_DIR, "import", json_file)
+        with open(json_path, "r", encoding="utf-8") as f:
+            json_content = f.read()
+
+        youtube_id = json.loads(json_content)["id"]
+
+        return youtube_id
+
+    def _dump_thumb(self, current_video):
+        """extract embedded thumb before converting"""
+        if current_video["thumb"]:
+            return
+
+        media_path = current_video["media"]
+        _, ext = os.path.splitext(media_path)
+
+        new_path = False
+        if ext == ".mkv":
+            idx, thumb_type = self._get_mkv_thumb_stream(media_path)
+            if idx is not None:
+                new_path = self.dump_mpv_thumb(media_path, idx, thumb_type)
+
+        elif ext == ".mp4":
+            thumb_type = self.get_mp4_thumb_type(media_path)
+            if thumb_type:
+                new_path = self.dump_mp4_thumb(media_path, thumb_type)
+
+        if new_path:
+            current_video["thumb"] = new_path
+
+    def _get_mkv_thumb_stream(self, media_path):
+        """get stream idx of thumbnail for mkv files"""
+        streams = self._get_streams(media_path)
+        attachments = [
+            i for i in streams["streams"] if i["codec_type"] == "attachment"
+        ]
+
+        for idx, stream in enumerate(attachments):
+            tags = stream["tags"]
+            if "mimetype" in tags and tags["filename"].startswith("cover"):
+                _, ext = os.path.splitext(tags["filename"])
+                return idx, ext
+
+        return None, None
+
+    @staticmethod
+    def dump_mpv_thumb(media_path, idx, thumb_type):
+        """write cover to disk for mkv"""
+        _, media_ext = os.path.splitext(media_path)
+        new_path = f"{media_path.rstrip(media_ext)}{thumb_type}"
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-v",
+                "quiet",
+                f"-dump_attachment:t:{idx}",
+                new_path,
+                "-i",
+                media_path,
+            ],
+            check=False,
+        )
+
+        return new_path
+
+    def get_mp4_thumb_type(self, media_path):
+        """detect filetype of embedded thumbnail"""
+        streams = self._get_streams(media_path)
+
+        for stream in streams["streams"]:
+            if stream["codec_name"] in ["png", "jpg"]:
+                return stream["codec_name"]
+
+        return False
+
+    def _convert_thumb(self, current_video):
+        """convert all thumbnails to jpg"""
+        if not current_video["thumb"]:
+            return
+
+        thumb_path = current_video["thumb"]
+
+        base_path, ext = os.path.splitext(thumb_path)
+        if ext == ".jpg":
+            return
+
+        new_path = f"{base_path}.jpg"
+        img_raw = Image.open(thumb_path)
+        img_raw.convert("RGB").save(new_path)
+
+        os.remove(thumb_path)
+        current_video["thumb"] = new_path
+
+    def _get_subtitles(self, current_video):
+        """find all subtitles in media file"""
+        if current_video["subtitle"]:
+            return
+
+        media_path = current_video["media"]
+        streams = self._get_streams(media_path)
+        base_path, ext = os.path.splitext(media_path)
+
+        if ext == ".webm":
+            print(f"{media_path}: subtitle extract from webm not supported")
+            return
+
+        for idx, stream in enumerate(streams["streams"]):
+            if stream["codec_type"] == "subtitle":
+                lang = ISO639Utils.long2short(stream["tags"]["language"])
+                sub_path = f"{base_path}.{lang}.vtt"
+                self._dump_subtitle(idx, media_path, sub_path)
+                current_video["subtitle"].append(sub_path)
+
+    @staticmethod
+    def _dump_subtitle(idx, media_path, sub_path):
+        """extract subtitle from media file"""
+        subprocess.run(
+            ["ffmpeg", "-i", media_path, "-map", f"0:{idx}", sub_path],
+            check=True,
+        )
+
+    @staticmethod
+    def _get_streams(media_path):
+        """return all streams from media_path"""
+        streams_raw = subprocess.run(
+            [
+                "ffprobe",
+                "-v",
+                "error",
+                "-show_streams",
+                "-print_format",
+                "json",
+                media_path,
+            ],
+            capture_output=True,
+            check=True,
+        )
+        streams = json.loads(streams_raw.stdout.decode())
+
+        return streams
+
+    @staticmethod
+    def dump_mp4_thumb(media_path, thumb_type):
+        """save cover to disk"""
+        _, ext = os.path.splitext(media_path)
+        new_path = f"{media_path.rstrip(ext)}.{thumb_type}"
+
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-i",
+                media_path,
+                "-map",
+                "0:v",
+                "-map",
+                "-0:V",
+                "-c",
+                "copy",
+                new_path,
+            ],
+            check=True,
+        )
+
+        return new_path
+
+    def _convert_video(self, current_video):
+        """convert if needed"""
+        current_path = current_video["media"]
+        base_path, ext = os.path.splitext(current_path)
+        if ext == ".mp4":
+            return
+
+        new_path = base_path + ".mp4"
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-i",
+                current_path,
+                new_path,
+                "-loglevel",
+                "warning",
+                "-stats",
+            ],
+            check=True,
+        )
+        current_video["media"] = new_path
+        os.remove(current_path)
+
+
+class ManualImport:
+    """import single identified video"""
+
+    def __init__(self, current_video, config):
+        self.current_video = current_video
+        self.config = config
+
+    def run(self):
+        """run all"""
+        json_data = self.index_metadata()
+        self._move_to_archive(json_data)
+        self._cleanup(json_data)
+
+    def index_metadata(self):
+        """get metadata from yt or json"""
+        video_id = self.current_video["video_id"]
+        video = YoutubeVideo(video_id)
+        video.build_json(
+            youtube_meta_overwrite=self._get_info_json(),
+            media_path=self.current_video["media"],
+        )
+        if not video.json_data:
+            print(f"{video_id}: manual import failed, and no metadata found.")
+            raise ValueError
+
+        video.check_subtitles(subtitle_files=self.current_video["subtitle"])
+        video.upload_to_es()
+
+        if video.offline_import and self.current_video["thumb"]:
+            old_path = self.current_video["thumb"]
+            thumbs = ThumbManager(video_id)
+            new_path = thumbs.vid_thumb_path(absolute=True, create_folder=True)
+            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+        else:
+            url = video.json_data["vid_thumb_url"]
+            ThumbManager(video_id).download_video_thumb(url)
+
+        return video.json_data
+
+    def _get_info_json(self):
+        """read info_json from file"""
+        if not self.current_video["metadata"]:
+            return False
+
+        with open(self.current_video["metadata"], "r", encoding="utf-8") as f:
+            info_json = json.loads(f.read())
+
+        return info_json
+
+    def _move_to_archive(self, json_data):
+        """move identified media file to archive"""
+        videos = self.config["application"]["videos"]
+        host_uid = self.config["application"]["HOST_UID"]
+        host_gid = self.config["application"]["HOST_GID"]
+
+        channel, file = os.path.split(json_data["media_url"])
+        channel_folder = os.path.join(videos, channel)
+        if not os.path.exists(channel_folder):
+            os.makedirs(channel_folder)
+
+        if host_uid and host_gid:
+            os.chown(channel_folder, host_uid, host_gid)
+
+        old_path = self.current_video["media"]
+        new_path = os.path.join(channel_folder, file)
+        shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+        if host_uid and host_gid:
+            os.chown(new_path, host_uid, host_gid)
+
+        base_name, _ = os.path.splitext(new_path)
+        for old_path in self.current_video["subtitle"]:
+            lang = old_path.split(".")[-2]
+            new_path = f"{base_name}.{lang}.vtt"
+            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+
+    def _cleanup(self, json_data):
+        """cleanup leftover files"""
+        meta_data = self.current_video["metadata"]
+        if meta_data and os.path.exists(meta_data):
+            os.remove(meta_data)
+
+        thumb = self.current_video["thumb"]
+        if thumb and os.path.exists(thumb):
+            os.remove(thumb)
+
+        for subtitle_file in self.current_video["subtitle"]:
+            if os.path.exists(subtitle_file):
+                os.remove(subtitle_file)
+
+        channel_info = os.path.join(
+            self.config["application"]["cache_dir"],
+            "import",
+            f"{json_data['channel']['channel_id']}.info.json",
+        )
+        if os.path.exists(channel_info):
+            os.remove(channel_info)

tubearchivist/home/src/index/reindex.py

@@ -21,7 +21,7 @@ from home.src.index.comments import Comments
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo
 from home.src.ta.config import AppConfig
-from home.src.ta.ta_redis import RedisArchivist, RedisQueue
+from home.src.ta.ta_redis import RedisQueue
 
 
 class ReindexBase:
@@ -49,6 +49,7 @@ class ReindexBase:
     }
 
     MULTIPLY = 1.2
+    DAYS3 = 60 * 60 * 24 * 3
 
     def __init__(self):
         self.config = AppConfig().config
@@ -62,13 +63,34 @@ class ReindexBase:
         RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
 
 
-class ReindexOutdated(ReindexBase):
-    """add outdated documents to reindex queue"""
+class ReindexPopulate(ReindexBase):
+    """add outdated and recent documents to reindex queue"""
 
     def __init__(self):
         super().__init__()
         self.interval = self.config["scheduler"]["check_reindex_days"]
 
+    def add_recent(self):
+        """add recent videos to refresh"""
+        gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
+        must_list = [
+            {"term": {"active": {"value": True}}},
+            {"range": {"published": {"gte": gte}}},
+        ]
+        data = {
+            "size": 10000,
+            "query": {"bool": {"must": must_list}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+        hits = response["hits"]["hits"]
+        if not hits:
+            return
+
+        all_ids = [i["_source"]["youtube_id"] for i in hits]
+        reindex_config = self.REINDEX_CONFIG.get("video")
+        self.populate(all_ids, reindex_config)
+
     def add_outdated(self):
         """add outdated documents"""
         for reindex_config in self.REINDEX_CONFIG.values():
@@ -201,8 +223,9 @@ class ReindexManual(ReindexBase):
 class Reindex(ReindexBase):
     """reindex all documents from redis queue"""
 
-    def __init__(self):
+    def __init__(self, task=False):
         super().__init__()
+        self.task = task
         self.all_indexed_ids = False
 
     def reindex_all(self):
@@ -211,22 +234,22 @@ class Reindex(ReindexBase):
             print("[reindex] cookie invalid, exiting...")
             return
 
-        for index_config in self.REINDEX_CONFIG.values():
+        for name, index_config in self.REINDEX_CONFIG.items():
             if not RedisQueue(index_config["queue_name"]).has_item():
                 continue
 
+            total = RedisQueue(index_config["queue_name"]).length()
             while True:
-                has_next = self.reindex_index(index_config)
+                has_next = self.reindex_index(name, index_config, total)
                 if not has_next:
                     break
 
-        RedisArchivist().set_message("last_reindex", self.now)
-
-    def reindex_index(self, index_config):
+    def reindex_index(self, name, index_config, total):
         """reindex all of a single index"""
         reindex = self.get_reindex_map(index_config["index_name"])
         youtube_id = RedisQueue(index_config["queue_name"]).get_next()
         if youtube_id:
+            self._notify(name, index_config, total)
             reindex(youtube_id)
             sleep_interval = self.config["downloads"].get("sleep_interval", 0)
             sleep(sleep_interval)
@@ -243,6 +266,14 @@ class Reindex(ReindexBase):
 
         return def_map.get(index_name)
 
+    def _notify(self, name, index_config, total):
+        """send notification back to task"""
+        remaining = RedisQueue(index_config["queue_name"]).length()
+        idx = total - remaining
+        message = [f"Reindexing {name.title()}s {idx}/{total}"]
+        progress = idx / total
+        self.task.send_progress(message, progress=progress)
+
     def _reindex_single_video(self, youtube_id):
         """wrapper to handle channel name changes"""
         try:
@@ -511,7 +542,7 @@ class ChannelFullScan:
                 print(f"{video_id}: no remote match found")
                 continue
 
-            expected_type = remote_match[0][-1].value
+            expected_type = remote_match[0][-1]
             if video["vid_type"] != expected_type:
                 self.to_update.append(
                     {

tubearchivist/home/src/index/subtitle.py

@@ -115,7 +115,7 @@ class YoutubeSubtitle:
             source = subtitle["source"]
             lang = subtitle.get("lang")
             response = requests.get(
-                subtitle["url"], headers=requests_headers()
+                subtitle["url"], headers=requests_headers(), timeout=30
             )
             if not response.ok:
                 print(f"{self.video.youtube_id}: failed to download subtitle")
@@ -137,14 +137,18 @@ class YoutubeSubtitle:
 
         return indexed
 
-    @staticmethod
-    def _write_subtitle_file(dest_path, subtitle_str):
+    def _write_subtitle_file(self, dest_path, subtitle_str):
         """write subtitle file to disk"""
         # create folder here for first video of channel
         os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
         with open(dest_path, "w", encoding="utf-8") as subfile:
             subfile.write(subtitle_str)
 
+        host_uid = self.video.config["application"]["HOST_UID"]
+        host_gid = self.video.config["application"]["HOST_GID"]
+        if host_uid and host_gid:
+            os.chown(dest_path, host_uid, host_gid)
+
     @staticmethod
     def _index_subtitle(query_str):
         """send subtitle to es for indexing"""

tubearchivist/home/src/ta/helper.py

@@ -11,6 +11,7 @@ import string
 import subprocess
 import unicodedata
 from datetime import datetime
+from urllib.parse import urlparse
 
 import requests
 
@@ -148,6 +149,22 @@ def is_shorts(youtube_id):
     return response.status_code == 200
 
 
+def ta_host_parser(ta_host):
+    """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
+    allowed_hosts = []
+    csrf_trusted_origins = []
+    for host in ta_host.split():
+        host_clean = host.strip()
+        if not host_clean.startswith("http"):
+            host_clean = f"http://{host_clean}"
+
+        parsed = urlparse(host_clean)
+        allowed_hosts.append(f"{parsed.hostname}")
+        csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}")
+
+    return allowed_hosts, csrf_trusted_origins
+
+
 class DurationConverter:
     """
     using ffmpeg to get and parse duration from filepath

tubearchivist/home/src/ta/ta_redis.py

@@ -2,6 +2,7 @@
 functionality:
 - interact with redis
 - hold temporary download queue in redis
+- interact with celery tasks results
 """
 
 import json
@@ -13,9 +14,9 @@ import redis
 class RedisBase:
     """connection base for redis"""
 
-    REDIS_HOST = os.environ.get("REDIS_HOST")
-    REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
-    NAME_SPACE = "ta:"
+    REDIS_HOST: str = str(os.environ.get("REDIS_HOST"))
+    REDIS_PORT: int = int(os.environ.get("REDIS_PORT") or 6379)
+    NAME_SPACE: str = "ta:"
 
     def __init__(self):
         self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
@@ -24,7 +25,7 @@ class RedisBase:
 class RedisArchivist(RedisBase):
     """collection of methods to interact with redis"""
 
-    CHANNELS = [
+    CHANNELS: list[str] = [
         "download",
         "add",
         "rescan",
@@ -34,7 +35,13 @@ class RedisArchivist(RedisBase):
         "setting",
     ]
 
-    def set_message(self, key, message, path=".", expire=False):
+    def set_message(
+        self,
+        key: str,
+        message: dict,
+        path: str = ".",
+        expire: bool | int = False,
+    ) -> None:
         """write new message to redis"""
         self.conn.execute_command(
             "JSON.SET", self.NAME_SPACE + key, path, json.dumps(message)
@@ -42,69 +49,47 @@ class RedisArchivist(RedisBase):
 
         if expire:
             if isinstance(expire, bool):
-                secs = 20
+                secs: int = 20
             else:
                 secs = expire
             self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
 
-    def get_message(self, key):
+    def get_message(self, key: str) -> dict:
         """get message dict from redis"""
         reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
         if reply:
-            json_str = json.loads(reply)
-        else:
-            json_str = {"status": False}
+            return json.loads(reply)
 
-        return json_str
+        return {"status": False}
 
-    def list_items(self, query):
-        """list all matches"""
+    def list_keys(self, query: str) -> list:
+        """return all key matches"""
         reply = self.conn.execute_command(
             "KEYS", self.NAME_SPACE + query + "*"
         )
-        all_matches = [i.decode().lstrip(self.NAME_SPACE) for i in reply]
-        all_results = []
-        for match in all_matches:
-            json_str = self.get_message(match)
-            all_results.append(json_str)
+        if not reply:
+            return []
 
-        return all_results
+        return [i.decode().lstrip(self.NAME_SPACE) for i in reply]
 
-    def del_message(self, key):
+    def list_items(self, query: str) -> list:
+        """list all matches"""
+        all_matches = self.list_keys(query)
+        if not all_matches:
+            return []
+
+        return [self.get_message(i) for i in all_matches]
+
+    def del_message(self, key: str) -> bool:
         """delete key from redis"""
         response = self.conn.execute_command("DEL", self.NAME_SPACE + key)
         return response
 
-    def get_lock(self, lock_key):
-        """handle lock for task management"""
-        redis_lock = self.conn.lock(self.NAME_SPACE + lock_key)
-        return redis_lock
-
-    def is_locked(self, lock_key):
-        """check if lock is set"""
-        lock_name = self.NAME_SPACE + lock_key
-        lock_status = bool(self.conn.execute_command("GET", lock_name))
-        return lock_status
-
-    def get_progress(self):
-        """get a list of all progress messages"""
-        all_messages = []
-        for channel in self.CHANNELS:
-            key = "message:" + channel
-            reply = self.conn.execute_command(
-                "JSON.GET", self.NAME_SPACE + key
-            )
-            if reply:
-                json_str = json.loads(reply)
-                all_messages.append(json_str)
-
-        return all_messages
-
 
 class RedisQueue(RedisBase):
     """dynamically interact with queues in redis"""
 
-    def __init__(self, queue_name):
+    def __init__(self, queue_name: str):
         super().__init__()
         self.key = f"{self.NAME_SPACE}{queue_name}"
 
@@ -114,7 +99,11 @@ class RedisQueue(RedisBase):
         all_elements = [i.decode() for i in result]
         return all_elements
 
-    def in_queue(self, element):
+    def length(self) -> int:
+        """return total elements in list"""
+        return self.conn.execute_command("LLEN", self.key)
+
+    def in_queue(self, element) -> str | bool:
         """check if element is in list"""
         result = self.conn.execute_command("LPOS", self.key, element)
         if result is not None:
@@ -126,12 +115,13 @@ class RedisQueue(RedisBase):
         """add list to queue"""
         self.conn.execute_command("RPUSH", self.key, *to_add)
 
-    def add_priority(self, to_add):
+    def add_priority(self, to_add: str) -> None:
         """add single video to front of queue"""
-        self.clear_item(to_add)
-        self.conn.execute_command("LPUSH", self.key, to_add)
+        item: str = json.dumps(to_add)
+        self.clear_item(item)
+        self.conn.execute_command("LPUSH", self.key, item)
 
-    def get_next(self):
+    def get_next(self) -> str | bool:
         """return next element in the queue, False if none"""
         result = self.conn.execute_command("LPOP", self.key)
         if not result:
@@ -140,19 +130,74 @@ class RedisQueue(RedisBase):
         next_element = result.decode()
         return next_element
 
-    def clear(self):
+    def clear(self) -> None:
         """delete list from redis"""
         self.conn.execute_command("DEL", self.key)
 
-    def clear_item(self, to_clear):
+    def clear_item(self, to_clear: str) -> None:
         """remove single item from list if it's there"""
         self.conn.execute_command("LREM", self.key, 0, to_clear)
 
-    def trim(self, size):
+    def trim(self, size: int) -> None:
         """trim the queue based on settings amount"""
         self.conn.execute_command("LTRIM", self.key, 0, size)
 
-    def has_item(self):
+    def has_item(self) -> bool:
         """check if queue as at least one pending item"""
         result = self.conn.execute_command("LRANGE", self.key, 0, 0)
         return bool(result)
+
+
+class TaskRedis(RedisBase):
+    """interact with redis tasks"""
+
+    BASE: str = "celery-task-meta-"
+    EXPIRE: int = 60 * 60 * 24
+    COMMANDS: list[str] = ["STOP", "KILL"]
+
+    def get_all(self) -> list:
+        """return all tasks"""
+        all_keys = self.conn.execute_command("KEYS", f"{self.BASE}*")
+        return [i.decode().replace(self.BASE, "") for i in all_keys]
+
+    def get_single(self, task_id: str) -> dict:
+        """return content of single task"""
+        result = self.conn.execute_command("GET", self.BASE + task_id)
+        if not result:
+            return {}
+
+        return json.loads(result.decode())
+
+    def set_key(
+        self, task_id: str, message: dict, expire: bool | int = False
+    ) -> None:
+        """set value for lock, initial or update"""
+        key: str = f"{self.BASE}{task_id}"
+        self.conn.execute_command("SET", key, json.dumps(message))
+
+        if expire:
+            self.conn.execute_command("EXPIRE", key, self.EXPIRE)
+
+    def set_command(self, task_id: str, command: str) -> None:
+        """set task command"""
+        if command not in self.COMMANDS:
+            print(f"{command} not in valid commands {self.COMMANDS}")
+            raise ValueError
+
+        message = self.get_single(task_id)
+        if not message:
+            print(f"{task_id} not found")
+            raise KeyError
+
+        message.update({"command": command})
+        self.set_key(task_id, message)
+
+    def del_task(self, task_id: str) -> None:
+        """delete task result by id"""
+        self.conn.execute_command("DEL", f"{self.BASE}{task_id}")
+
+    def del_all(self) -> None:
+        """delete all task results"""
+        all_tasks = self.get_all()
+        for task_id in all_tasks:
+            self.del_task(task_id)

tubearchivist/home/src/ta/task_manager.py

@@ -0,0 +1,116 @@
+"""
+functionality:
+- interact with in redis stored task results
+- handle threads and locks
+"""
+
+from home import tasks as ta_tasks
+from home.src.ta.ta_redis import RedisArchivist, TaskRedis
+
+
+class TaskManager:
+    """manage tasks"""
+
+    def get_all_results(self):
+        """return all task results"""
+        handler = TaskRedis()
+        all_keys = handler.get_all()
+        if not all_keys:
+            return False
+
+        return [handler.get_single(i) for i in all_keys]
+
+    def get_tasks_by_name(self, task_name):
+        """get all tasks by name"""
+        all_results = self.get_all_results()
+        if not all_results:
+            return False
+
+        return [i for i in all_results if i.get("name") == task_name]
+
+    def get_task(self, task_id):
+        """get single task"""
+        return TaskRedis().get_single(task_id)
+
+    def is_pending(self, task):
+        """check if task_name is pending, pass task object"""
+        tasks = self.get_tasks_by_name(task.name)
+        if not tasks:
+            return False
+
+        return bool([i for i in tasks if i.get("status") == "PENDING"])
+
+    def is_stopped(self, task_id):
+        """check if task_id has received STOP command"""
+        task = self.get_task(task_id)
+
+        return task.get("command") == "STOP"
+
+    def get_pending(self, task_name):
+        """get all pending tasks of task_name"""
+        tasks = self.get_tasks_by_name(task_name)
+        if not tasks:
+            return False
+
+        return [i for i in tasks if i.get("status") == "PENDING"]
+
+    def init(self, task):
+        """pass task object from bind task to set initial pending message"""
+        message = {
+            "status": "PENDING",
+            "result": None,
+            "traceback": None,
+            "date_done": False,
+            "name": task.name,
+            "task_id": task.request.id,
+        }
+        TaskRedis().set_key(task.request.id, message)
+
+    def fail_pending(self):
+        """
+        mark all pending as failed,
+        run at startup to recover from hard reset
+        """
+        all_results = self.get_all_results()
+        if not all_results:
+            return
+
+        for result in all_results:
+            if result.get("status") == "PENDING":
+                result["status"] = "FAILED"
+                TaskRedis().set_key(result["task_id"], result, expire=True)
+
+
+class TaskCommand:
+    """run commands on task"""
+
+    def start(self, task_name):
+        """start task by task_name, only pass task that don't take args"""
+        task = ta_tasks.app.tasks.get(task_name).delay()
+        message = {
+            "task_id": task.id,
+            "status": task.status,
+            "task_name": task.name,
+        }
+
+        return message
+
+    def stop(self, task_id, message_key):
+        """
+        send stop signal to task_id,
+        needs to be implemented in task to take effect
+        """
+        print(f"[task][{task_id}]: received STOP signal.")
+        handler = TaskRedis()
+
+        task = handler.get_single(task_id)
+        if not task["name"] in ta_tasks.BaseTask.TASK_CONFIG:
+            raise ValueError
+
+        handler.set_command(task_id, "STOP")
+        RedisArchivist().set_message(message_key, "STOP", path=".command")
+
+    def kill(self, task_id):
+        """send kill signal to task_id"""
+        print(f"[task][{task_id}]: received KILL signal.")
+        ta_tasks.app.control.revoke(task_id, terminate=True)

tubearchivist/home/src/ta/urlparser.py

@@ -103,6 +103,7 @@ class Parser:
     def _extract_channel_name(url):
         """find channel id from channel name with yt-dlp help"""
         obs_request = {
+            "check_formats": None,
             "skip_download": True,
             "extract_flat": True,
             "playlistend": 0,

tubearchivist/home/tasks.py

@@ -6,304 +6,322 @@ Functionality:
   because tasks are initiated at application start
 """
 
-import json
 import os
 
-from celery import Celery, shared_task
+from celery import Celery, Task, shared_task
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import (
-    ChannelSubscription,
-    PlaylistSubscription,
+    SubscriptionHandler,
+    SubscriptionScanner,
 )
 from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator
 from home.src.download.yt_dlp_handler import VideoDownloader
 from home.src.es.backup import ElasticBackup
 from home.src.es.index_setup import ElasitIndexWrap
 from home.src.index.channel import YoutubeChannel
-from home.src.index.filesystem import ImportFolderScanner, scan_filesystem
-from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated
-from home.src.index.video_constants import VideoTypeEnum
+from home.src.index.filesystem import Filesystem
+from home.src.index.manual import ImportFolderScanner
+from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
 from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
-from home.src.ta.helper import clear_dl_cache
-from home.src.ta.ta_redis import RedisArchivist, RedisQueue
-from home.src.ta.urlparser import Parser
+from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.task_manager import TaskManager
 
 CONFIG = AppConfig().config
 REDIS_HOST = os.environ.get("REDIS_HOST")
 REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
-app = Celery("tasks", broker=f"redis://{REDIS_HOST}:{REDIS_PORT}")
+app = Celery(
+    "tasks",
+    broker=f"redis://{REDIS_HOST}:{REDIS_PORT}",
+    backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
+    result_extended=True,
+)
 app.config_from_object("django.conf:settings", namespace="ta:")
 app.autodiscover_tasks()
 app.conf.timezone = os.environ.get("TZ") or "UTC"
 
 
-@shared_task(name="update_subscribed")
-def update_subscribed():
-    """look for missing videos and add to pending"""
-    message = {
-        "status": "message:rescan",
-        "level": "info",
-        "title": "Rescanning channels and playlists.",
-        "message": "Looking for new videos.",
+class BaseTask(Task):
+    """base class to inherit each class from"""
+
+    # pylint: disable=abstract-method
+
+    TASK_CONFIG = {
+        "update_subscribed": {
+            "title": "Rescan your Subscriptions",
+            "group": "download:scan",
+            "api-start": True,
+            "api-stop": True,
+        },
+        "download_pending": {
+            "title": "Downloading",
+            "group": "download:run",
+            "api-start": True,
+            "api-stop": True,
+        },
+        "extract_download": {
+            "title": "Add to download queue",
+            "group": "download:add",
+            "api-stop": True,
+        },
+        "check_reindex": {
+            "title": "Reindex Documents",
+            "group": "reindex:run",
+        },
+        "manual_import": {
+            "title": "Manual video import",
+            "group": "setting:import",
+            "api-start": True,
+        },
+        "run_backup": {
+            "title": "Index Backup",
+            "group": "setting:backup",
+            "api-start": True,
+        },
+        "restore_backup": {
+            "title": "Restore Backup",
+            "group": "setting:restore",
+        },
+        "rescan_filesystem": {
+            "title": "Rescan your Filesystem",
+            "group": "setting:filesystemscan",
+            "api-start": True,
+        },
+        "thumbnail_check": {
+            "title": "Check your Thumbnails",
+            "group": "setting:thumbnailcheck",
+            "api-start": True,
+        },
+        "resync_thumbs": {
+            "title": "Sync Thumbnails to Media Files",
+            "group": "setting:thumbnailsync",
+            "api-start": True,
+        },
+        "index_playlists": {
+            "title": "Index Channel Playlist",
+            "group": "channel:indexplaylist",
+        },
+        "subscribe_to": {
+            "title": "Add Subscription",
+            "group": "subscription:add",
+        },
     }
-    RedisArchivist().set_message("message:rescan", message, expire=True)
 
-    have_lock = False
-    my_lock = RedisArchivist().get_lock("rescan")
+    def on_failure(self, exc, task_id, args, kwargs, einfo):
+        """callback for task failure"""
+        print(f"{task_id} Failed callback")
+        message, key = self._build_message(level="error")
+        message.update({"messages": ["Task failed"]})
+        RedisArchivist().set_message(key, message, expire=20)
 
-    try:
-        have_lock = my_lock.acquire(blocking=False)
-        if have_lock:
-            channel_handler = ChannelSubscription()
-            missing_from_channels = channel_handler.find_missing()
-            playlist_handler = PlaylistSubscription()
-            missing_from_playlists = playlist_handler.find_missing()
-            if missing_from_channels or missing_from_playlists:
-                channel_videos = [
-                    {"type": "video", "vid_type": vid_type, "url": vid_id}
-                    for vid_id, vid_type in missing_from_channels
-                ]
-                playlist_videos = [
-                    {
-                        "type": "video",
-                        "vid_type": VideoTypeEnum.VIDEOS,
-                        "url": i,
-                    }
-                    for i in missing_from_playlists
-                ]
-                pending_handler = PendingList(
-                    youtube_ids=channel_videos + playlist_videos
-                )
-                pending_handler.parse_url_list()
-                pending_handler.add_to_pending()
+    def on_success(self, retval, task_id, args, kwargs):
+        """callback task completed successfully"""
+        print(f"{task_id} success callback")
+        message, key = self._build_message()
+        message.update({"messages": ["Task completed successfully"]})
+        RedisArchivist().set_message(key, message, expire=5)
 
-        else:
-            print("Did not acquire rescan lock.")
+    def before_start(self, task_id, args, kwargs):
+        """callback before initiating task"""
+        print(f"{self.name} create callback")
+        message, key = self._build_message()
+        message.update({"messages": ["New task received."]})
+        RedisArchivist().set_message(key, message)
 
-    finally:
-        if have_lock:
-            my_lock.release()
-
-
-@shared_task(name="download_pending")
-def download_pending():
-    """download latest pending videos"""
-    have_lock = False
-    my_lock = RedisArchivist().get_lock("downloading")
-
-    try:
-        have_lock = my_lock.acquire(blocking=False)
-        if have_lock:
-            downloader = VideoDownloader()
-            downloader.add_pending()
-            downloader.run_queue()
-        else:
-            print("Did not acquire download lock.")
-
-    finally:
-        if have_lock:
-            my_lock.release()
-
-
-@shared_task
-def download_single(pending_video):
-    """start download single video now"""
-    queue = RedisQueue(queue_name="dl_queue")
-
-    to_add = {
-        "youtube_id": pending_video["youtube_id"],
-        "vid_type": pending_video.get("vid_type", VideoTypeEnum.VIDEOS.value),
-    }
-    queue.add_priority(json.dumps(to_add))
-    print(f"Added to queue with priority: {to_add}")
-    # start queue if needed
-    have_lock = False
-    my_lock = RedisArchivist().get_lock("downloading")
-
-    try:
-        have_lock = my_lock.acquire(blocking=False)
-        if have_lock:
-            key = "message:download"
-            mess_dict = {
-                "status": key,
-                "level": "info",
-                "title": "Download single video",
-                "message": "processing",
+    def send_progress(self, message_lines, progress=False, title=False):
+        """send progress message"""
+        message, key = self._build_message()
+        message.update(
+            {
+                "messages": message_lines,
+                "progress": progress,
             }
-            RedisArchivist().set_message(key, mess_dict, expire=True)
-            VideoDownloader().run_queue()
-        else:
-            print("Download queue already running.")
+        )
+        if title:
+            message["title"] = title
 
-    finally:
-        # release if only single run
-        if have_lock and not queue.get_next():
-            my_lock.release()
+        RedisArchivist().set_message(key, message)
+
+    def _build_message(self, level="info"):
+        """build message dict"""
+        task_id = self.request.id
+        message = self.TASK_CONFIG.get(self.name).copy()
+        message.update({"level": level, "id": task_id})
+        task_result = TaskManager().get_task(task_id)
+        if task_result:
+            command = task_result.get("command", False)
+            message.update({"command": command})
+
+        key = f"message:{message.get('group')}:{task_id.split('-')[0]}"
+        return message, key
+
+    def is_stopped(self):
+        """check if task is stopped"""
+        return TaskManager().is_stopped(self.request.id)
 
 
-@shared_task
-def extrac_dl(youtube_ids):
+@shared_task(name="update_subscribed", bind=True, base=BaseTask)
+def update_subscribed(self):
+    """look for missing videos and add to pending"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] rescan already running")
+        self.send_progress("Rescan already in progress.")
+        return
+
+    manager.init(self)
+    missing_videos = SubscriptionScanner(task=self).scan()
+    if missing_videos:
+        print(missing_videos)
+        extrac_dl.delay(missing_videos)
+
+
+@shared_task(name="download_pending", bind=True, base=BaseTask)
+def download_pending(self, from_queue=True):
+    """download latest pending videos"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] download queue already running")
+        self.send_progress("Download Queue is already running.")
+        return
+
+    manager.init(self)
+    downloader = VideoDownloader(task=self)
+    if from_queue:
+        downloader.add_pending()
+    downloader.run_queue()
+
+
+@shared_task(name="extract_download", bind=True, base=BaseTask)
+def extrac_dl(self, youtube_ids):
     """parse list passed and add to pending"""
-    pending_handler = PendingList(youtube_ids=youtube_ids)
+    TaskManager().init(self)
+    pending_handler = PendingList(youtube_ids=youtube_ids, task=self)
     pending_handler.parse_url_list()
     pending_handler.add_to_pending()
 
 
-@shared_task(name="check_reindex")
-def check_reindex(data=False, extract_videos=False):
+@shared_task(bind=True, name="check_reindex", base=BaseTask)
+def check_reindex(self, data=False, extract_videos=False):
     """run the reindex main command"""
     if data:
+        # started from frontend through API
+        print(f"[task][{self.name}] reindex {data}")
+        self.send_progress("Add items to the reindex Queue.")
         ReindexManual(extract_videos=extract_videos).extract_data(data)
 
-    have_lock = False
-    reindex_lock = RedisArchivist().get_lock("reindex")
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] reindex queue is already running")
+        self.send_progress("Reindex Queue is already running.")
+        return
 
-    try:
-        have_lock = reindex_lock.acquire(blocking=False)
-        if have_lock:
-            if not data:
-                ReindexOutdated().add_outdated()
+    manager.init(self)
+    if not data:
+        # started from scheduler
+        populate = ReindexPopulate()
+        print(f"[task][{self.name}] reindex outdated documents")
+        self.send_progress("Add recent documents to the reindex Queue.")
+        populate.add_recent()
+        self.send_progress("Add outdated documents to the reindex Queue.")
+        populate.add_outdated()
 
-            Reindex().reindex_all()
-        else:
-            print("Did not acquire reindex lock.")
-
-    finally:
-        if have_lock:
-            reindex_lock.release()
+    Reindex(task=self).reindex_all()
 
 
-@shared_task
-def run_manual_import():
+@shared_task(bind=True, name="manual_import", base=BaseTask)
+def run_manual_import(self):
     """called from settings page, to go through import folder"""
-    print("starting media file import")
-    have_lock = False
-    my_lock = RedisArchivist().get_lock("manual_import")
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] manual import is already running")
+        self.send_progress("Manual import is already running.")
+        return
 
-    try:
-        have_lock = my_lock.acquire(blocking=False)
-        if have_lock:
-            ImportFolderScanner().scan()
-        else:
-            print("Did not acquire lock form import.")
-
-    finally:
-        if have_lock:
-            my_lock.release()
+    manager.init(self)
+    ImportFolderScanner(task=self).scan()
 
 
-@shared_task(name="run_backup")
-def run_backup(reason="auto"):
+@shared_task(bind=True, name="run_backup", base=BaseTask)
+def run_backup(self, reason="auto"):
     """called from settings page, dump backup to zip file"""
-    have_lock = False
-    my_lock = RedisArchivist().get_lock("run_backup")
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] backup is already running")
+        self.send_progress("Backup is already running.")
+        return
 
-    try:
-        have_lock = my_lock.acquire(blocking=False)
-        if have_lock:
-            ElasticBackup(reason=reason).backup_all_indexes()
-        else:
-            print("Did not acquire lock for backup task.")
-    finally:
-        if have_lock:
-            my_lock.release()
-            print("backup finished")
+    manager.init(self)
+    ElasticBackup(reason=reason, task=self).backup_all_indexes()
 
 
-@shared_task
-def run_restore_backup(filename):
+@shared_task(bind=True, name="restore_backup", base=BaseTask)
+def run_restore_backup(self, filename):
     """called from settings page, dump backup to zip file"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] restore is already running")
+        self.send_progress("Restore is already running.")
+        return
+
+    manager.init(self)
+    self.send_progress(["Reset your Index"])
     ElasitIndexWrap().reset()
-    ElasticBackup().restore(filename)
+    ElasticBackup(task=self).restore(filename)
     print("index restore finished")
 
 
-def kill_dl(task_id):
-    """kill download worker task by ID"""
-    if task_id:
-        app.control.revoke(task_id, terminate=True)
-
-    _ = RedisArchivist().del_message("dl_queue_id")
-    RedisQueue(queue_name="dl_queue").clear()
-
-    _ = clear_dl_cache(CONFIG)
-
-    # notify
-    mess_dict = {
-        "status": "message:download",
-        "level": "error",
-        "title": "Canceling download process",
-        "message": "Canceling download queue now.",
-    }
-    RedisArchivist().set_message("message:download", mess_dict, expire=True)
-
-
-@shared_task
-def rescan_filesystem():
+@shared_task(bind=True, name="rescan_filesystem", base=BaseTask)
+def rescan_filesystem(self):
     """check the media folder for mismatches"""
-    scan_filesystem()
-    ThumbValidator().download_missing()
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] filesystem rescan already running")
+        self.send_progress("Filesystem Rescan is already running.")
+        return
+
+    manager.init(self)
+    Filesystem(task=self).process()
+    ThumbValidator(task=self).validate()
 
 
-@shared_task(name="thumbnail_check")
-def thumbnail_check():
+@shared_task(bind=True, name="thumbnail_check", base=BaseTask)
+def thumbnail_check(self):
     """validate thumbnails"""
-    ThumbValidator().download_missing()
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] thumbnail check is already running")
+        self.send_progress("Thumbnail check is already running.")
+        return
+
+    manager.init(self)
+    ThumbValidator(task=self).validate()
 
 
-@shared_task
-def re_sync_thumbs():
+@shared_task(bind=True, name="resync_thumbs", base=BaseTask)
+def re_sync_thumbs(self):
     """sync thumbnails to mediafiles"""
-    ThumbFilesystem().sync()
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] thumb re-embed is already running")
+        self.send_progress("Thumbnail re-embed is already running.")
+        return
+
+    manager.init(self)
+    ThumbFilesystem(task=self).embed()
 
 
-@shared_task
-def subscribe_to(url_str):
+@shared_task(bind=True, name="subscribe_to", base=BaseTask)
+def subscribe_to(self, url_str):
     """take a list of urls to subscribe to"""
-    to_subscribe_list = Parser(url_str).parse()
-    for idx, item in enumerate(to_subscribe_list):
-        to_sub_id = item["url"]
-        if item["type"] == "playlist":
-            PlaylistSubscription().process_url_str([item])
-            continue
-
-        if item["type"] == "video":
-            vid_details = PendingList().get_youtube_details(to_sub_id)
-            channel_id_sub = vid_details["channel_id"]
-        elif item["type"] == "channel":
-            channel_id_sub = to_sub_id
-        else:
-            raise ValueError("failed to subscribe to: " + to_sub_id)
-
-        ChannelSubscription().change_subscribe(
-            channel_id_sub, channel_subscribed=True
-        )
-        # notify for channels
-        key = "message:subchannel"
-        message = {
-            "status": key,
-            "level": "info",
-            "title": "Subscribing to Channels",
-            "message": f"Processing {idx + 1} of {len(to_subscribe_list)}",
-        }
-        RedisArchivist().set_message(key, message=message, expire=True)
+    SubscriptionHandler(url_str, task=self).subscribe()
 
 
-@shared_task
-def index_channel_playlists(channel_id):
+@shared_task(bind=True, name="index_playlists", base=BaseTask)
+def index_channel_playlists(self, channel_id):
     """add all playlists of channel to index"""
-    channel = YoutubeChannel(channel_id)
-    # notify
-    key = "message:playlistscan"
-    mess_dict = {
-        "status": key,
-        "level": "info",
-        "title": "Looking for playlists",
-        "message": f"{channel_id}: Channel scan in progress",
-    }
-    RedisArchivist().set_message(key, mess_dict, expire=True)
+    channel = YoutubeChannel(channel_id, task=self)
     channel.index_channel_playlists()
 
 

tubearchivist/home/templates/home/about.html

@@ -8,7 +8,7 @@
     <div class="about-section">
         <h2>Useful Links</h2>
         <p>This project is in active and constant development, take a look at the <a href="https://github.com/tubearchivist/tubearchivist#roadmap" target="_blank">roadmap</a> for a overview.</p>
-        <p>For any questions on what a button or a function does, You can find the up-to-date user documentation on <a href="https://github.com/tubearchivist/tubearchivist/wiki" target="_blank">Github</a>.</p>
+        <p>All functionality is documented in our up-to-date <a href="https://docs.tubearchivist.com" target="_blank">user guide</a>.</p>
         <p>All contributions are welcome: Open an <a href="https://github.com/tubearchivist/tubearchivist/issues" target="_blank">issue</a> for any bugs and errors, join us on <a href="https://www.tubearchivist.com/discord" target="_blank">Discord</a> to discuss details. The <a href="https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md" target="_blank">contributing</a> page is a good place to get started.</p>
     </div>
     <div class="about-section">

tubearchivist/home/templates/home/channel.html

@@ -18,7 +18,7 @@
             </div>
         </div>
     </div>
-    <div id="notifications" data="channel"></div>
+    <div id="notifications" data="subscription"></div>
     <div class="view-controls">
         <div class="toggle">
             <span>Show subscribed only:</span>

tubearchivist/home/templates/home/channel_id.html

@@ -22,7 +22,7 @@
             <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
         {% endif %}
     </div>
-    <div id="notifications" data="channel_id"></div>
+    <div id="notifications" data="channel reindex"></div>
     <div class="info-box info-box-2">
         <div class="info-box-item">
             <div class="round-img">

tubearchivist/home/templates/home/channel_id_about.html

@@ -22,6 +22,7 @@
             <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
         {% endif %}
     </div>
+    <div id="notifications" data="channel reindex"></div>
     <div class="info-box info-box-3">
         <div class="info-box-item">
             <div class="round-img">
@@ -130,4 +131,5 @@
         </div>
     </div>
 </div>
+<script type="text/javascript" src="{% static 'progress.js' %}"></script>
 {% endblock content %}

tubearchivist/home/templates/home/channel_id_playlist.html

@@ -22,6 +22,7 @@
             <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
         {% endif %}
     </div>
+    <div id="notifications" data="channel reindex"></div>
     <div class="view-controls">
         <div class="toggle">
             <span>Show subscribed only:</span>
@@ -65,4 +66,5 @@
         {% endif %}
     </div>
 </div>
+<script type="text/javascript" src="{% static 'progress.js' %}"></script>
 {% endblock content %}

tubearchivist/home/templates/home/playlist.html

@@ -17,7 +17,7 @@
             </div>
         </div>
     </div>
-    <div id="notifications" data="playlist"></div>
+    <div id="notifications" data="subscription"></div>
     <div class="view-controls">
         <div class="toggle">
             <span>Show subscribed only:</span>

tubearchivist/home/templates/home/search.html

@@ -47,7 +47,7 @@
         </div>
         <div>
             <h2>Keywords cheatsheet</h2>
-            <p>For detailed usage check <a href="https://github.com/tubearchivist/tubearchivist/wiki/Search" target="_blank">wiki</a>.</p>
+            <p>For detailed usage check <a href="https://docs.tubearchivist.com/search/" target="_blank">wiki</a>.</p>
             <div>
                 <ul>
                     <li><span>simple:</span> (implied) — search in video titles, channel names and playlist titles</li>

tubearchivist/home/templates/home/settings.html

@@ -2,7 +2,7 @@
 {% load static %}
 {% block content %}
 <div class="boxed-content">
-    <div id="notifications" data="setting"></div>
+    <div id="notifications" data="setting reindex"></div>
     <div class="title-bar">
         <h1>User Configurations</h1>
     </div>
@@ -148,7 +148,7 @@
             <div class="settings-item">
                 <p>Import YouTube cookie: <span class="settings-current">{{ config.downloads.cookie_import }}</span><br></p>
                 <p>For automatic cookie import use <b>Tube Archivist Companion</b> <a href="https://github.com/tubearchivist/browser-extension" target="_blank">browser extension</a>.</p> 
-                <i>For manual cookie import, place your cookie file named <span class="settings-current">cookies.google.txt</span> in <span class="settings-current">cache/import</span> before enabling. Instructions in the <a href="https://github.com/tubearchivist/tubearchivist/wiki/Settings" target="_blank">Wiki.</a></i><br>
+                <i>For manual cookie import, place your cookie file named <span class="settings-current">cookies.google.txt</span> in <span class="settings-current">cache/import</span> before enabling. Instructions in the <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki.</a></i><br>
                 {{ app_form.downloads_cookie_import }}<br>
                 {% if config.downloads.cookie_import %}
                     <div id="cookieMessage">
@@ -181,7 +181,7 @@
             <h2 id="snapshots">Snapshots</h2>
             <div class="settings-item">
                 <p>Current system snapshot: <span class="settings-current">{{ config.application.enable_snapshot }}</span></p>
-                <i>Automatically create daily deduplicated snapshots of the index, stored in Elasticsearch. Read first before activating: <a target="_blank" href="https://github.com/tubearchivist/tubearchivist/wiki/Settings#snapshots">Wiki</a>.</i><br>
+                <i>Automatically create daily deduplicated snapshots of the index, stored in Elasticsearch. Read first before activating: <a target="_blank" href="https://docs.tubearchivist.com/settings/#snapshots">Wiki</a>.</i><br>
                 {{ app_form.application_enable_snapshot }}
             </div>
             <div>
@@ -228,6 +228,7 @@
                         False
                     {% endif %}
                 </span></p>
+                <p>Become a sponsor and join <a href="https://members.tubearchivist.com/" target="_blank">members.tubearchivist.com</a> to get access to <span class="settings-current">real time</span> notifications for new videos uploaded by your favorite channels.</p>
                 <p>Periodically rescan your subscriptions:</p>
                 {{ scheduler_form.update_subscribed }}
             </div>
@@ -320,7 +321,7 @@
     </div>
     <div class="settings-group">
         <h2>Manual media files import.</h2>
-        <p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://github.com/tubearchivist/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
+        <p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki</a>.</p>
         <div id="manual-import">
             <button onclick="manualImport()">Start import</button>
         </div>
@@ -366,7 +367,7 @@
     <div class="settings-group">
         <h2>Rescan filesystem</h2>
         <p><span class="danger-zone">Danger Zone</span>: This will delete the metadata of deleted videos from the filesystem.</p>
-        <p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://github.com/tubearchivist/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
+        <p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki</a>.</p>
         <div id="fs-rescan">
             <button onclick="fsRescan()">Rescan filesystem</button>
         </div>

tubearchivist/home/urls.py

@@ -4,78 +4,73 @@ from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.views import LogoutView
 from django.urls import path
-from home.views import (
-    AboutView,
-    ChannelIdAboutView,
-    ChannelIdLiveView,
-    ChannelIdPlaylistView,
-    ChannelIdShortsView,
-    ChannelIdView,
-    ChannelView,
-    DownloadView,
-    HomeView,
-    LoginView,
-    PlaylistIdView,
-    PlaylistView,
-    SearchView,
-    SettingsView,
-    VideoView,
-    process,
-    progress,
-)
+from home import views
 
 urlpatterns = [
-    path("", login_required(HomeView.as_view()), name="home"),
-    path("login/", LoginView.as_view(), name="login"),
+    path("", login_required(views.HomeView.as_view()), name="home"),
+    path("login/", views.LoginView.as_view(), name="login"),
     path(
         "logout/",
         LogoutView.as_view(),
         {"next_page": settings.LOGOUT_REDIRECT_URL},
         name="logout",
     ),
-    path("about/", AboutView.as_view(), name="about"),
+    path("about/", views.AboutView.as_view(), name="about"),
     path(
-        "downloads/", login_required(DownloadView.as_view()), name="downloads"
+        "downloads/",
+        login_required(views.DownloadView.as_view()),
+        name="downloads",
+    ),
+    path(
+        "settings/",
+        login_required(views.SettingsView.as_view()),
+        name="settings",
+    ),
+    path("process/", login_required(views.process), name="process"),
+    path(
+        "channel/",
+        login_required(views.ChannelView.as_view()),
+        name="channel",
     ),
-    path("settings/", login_required(SettingsView.as_view()), name="settings"),
-    path("process/", login_required(process), name="process"),
-    path("progress/", login_required(progress), name="progress"),
-    path("channel/", login_required(ChannelView.as_view()), name="channel"),
     path(
         "channel/<slug:channel_id>/",
-        login_required(ChannelIdView.as_view()),
+        login_required(views.ChannelIdView.as_view()),
         name="channel_id",
     ),
     path(
         "channel/<slug:channel_id>/streams/",
-        login_required(ChannelIdLiveView.as_view()),
+        login_required(views.ChannelIdLiveView.as_view()),
         name="channel_id_live",
     ),
     path(
         "channel/<slug:channel_id>/shorts/",
-        login_required(ChannelIdShortsView.as_view()),
+        login_required(views.ChannelIdShortsView.as_view()),
         name="channel_id_shorts",
     ),
     path(
         "channel/<slug:channel_id>/about/",
-        login_required(ChannelIdAboutView.as_view()),
+        login_required(views.ChannelIdAboutView.as_view()),
         name="channel_id_about",
     ),
     path(
         "channel/<slug:channel_id>/playlist/",
-        login_required(ChannelIdPlaylistView.as_view()),
+        login_required(views.ChannelIdPlaylistView.as_view()),
         name="channel_id_playlist",
     ),
     path(
         "video/<slug:video_id>/",
-        login_required(VideoView.as_view()),
+        login_required(views.VideoView.as_view()),
         name="video",
     ),
-    path("playlist/", login_required(PlaylistView.as_view()), name="playlist"),
+    path(
+        "playlist/",
+        login_required(views.PlaylistView.as_view()),
+        name="playlist",
+    ),
     path(
         "playlist/<slug:playlist_id>/",
-        login_required(PlaylistIdView.as_view()),
+        login_required(views.PlaylistIdView.as_view()),
         name="playlist_id",
     ),
-    path("search/", login_required(SearchView.as_view()), name="search"),
+    path("search/", login_required(views.SearchView.as_view()), name="search"),
 ]

tubearchivist/home/views.py

@@ -619,21 +619,6 @@ class ChannelIdView(ChannelIdBaseView):
             to_append = {"term": {"player.watched": {"value": False}}}
             self.data["query"]["bool"]["must"].append(to_append)
 
-    @staticmethod
-    def post(request, channel_id):
-        """handle post request"""
-        print(f"handle post from {channel_id}")
-        channel_overwrite_form = ChannelOverwriteForm(request.POST)
-        if channel_overwrite_form.is_valid():
-            overwrites = channel_overwrite_form.cleaned_data
-            print(f"{channel_id}: set overwrites {overwrites}")
-            channel_overwrites(channel_id, overwrites=overwrites)
-            if overwrites.get("index_playlists") == "1":
-                index_channel_playlists.delay(channel_id)
-
-        sleep(1)
-        return redirect("channel_id", channel_id, permanent=True)
-
 
 class ChannelIdLiveView(ChannelIdView):
     """resolves to /channel/<channel-id>/streams/
@@ -769,14 +754,6 @@ class ChannelView(ArchivistResultsView):
         """handle http post requests"""
         subscribe_form = SubscribeToChannelForm(data=request.POST)
         if subscribe_form.is_valid():
-            key = "message:subchannel"
-            message = {
-                "status": key,
-                "level": "info",
-                "title": "Subscribing to Channels",
-                "message": "Parsing form data",
-            }
-            RedisArchivist().set_message(key, message=message, expire=True)
             url_str = request.POST.get("subscribe")
             print(url_str)
             subscribe_to.delay(url_str)
@@ -922,14 +899,6 @@ class PlaylistView(ArchivistResultsView):
         if subscribe_form.is_valid():
             url_str = request.POST.get("subscribe")
             print(url_str)
-            key = "message:subplaylist"
-            message = {
-                "status": key,
-                "level": "info",
-                "title": "Subscribing to Playlists",
-                "message": "Parsing form data",
-            }
-            RedisArchivist().set_message(key, message=message, expire=True)
             subscribe_to.delay(url_str)
 
         sleep(1)
@@ -1131,26 +1100,11 @@ class SettingsView(MinView):
         RedisArchivist().set_message(key, message=message, expire=True)
 
 
-def progress(request):
-    # pylint: disable=unused-argument
-    """resolves to /progress/
-    return list of messages for frontend
-    """
-    all_messages = RedisArchivist().get_progress()
-    json_data = {"messages": all_messages}
-    return JsonResponse(json_data)
-
-
 def process(request):
     """handle all the buttons calls via POST ajax"""
     if request.method == "POST":
         current_user = request.user.id
         post_dict = json.loads(request.body.decode())
-        if post_dict.get("reset-token"):
-            print("revoke API token")
-            request.user.auth_token.delete()
-            return JsonResponse({"success": True})
-
         post_handler = PostData(post_dict, current_user)
         if post_handler.to_exec:
             task_result = post_handler.run_task()

tubearchivist/requirements.txt

@@ -1,13 +1,13 @@
-beautifulsoup4==4.11.2
+beautifulsoup4==4.12.2
 celery==5.2.7
-Django==4.1.7
-django-auth-ldap==4.1.0
+Django==4.2
+django-auth-ldap==4.2.0
 django-cors-headers==3.14.0
 djangorestframework==3.14.0
-Pillow==9.4.0
-redis==4.5.1
+Pillow==9.5.0
+redis==4.5.4
 requests==2.28.2
 ryd-client==0.0.6
 uWSGI==2.0.21
 whitenoise==6.4.0
-yt_dlp==2023.3.3
+yt_dlp==2023.3.4

tubearchivist/static/css/style.css

@@ -395,6 +395,10 @@ button:hover {
     position: relative; /* needed for modal */
 }
 
+#notifications {
+    position: relative;
+}
+
 .notifications {
     text-align: center;
     width: 80%;
@@ -473,7 +477,8 @@ video:-webkit-full-screen {
     align-items: center;
 }
 
-.video-progress-bar {
+.video-progress-bar,
+.notification-progress-bar {
     position: absolute;
     background-color: var(--accent-font-dark);
     height: 7px;
@@ -1017,13 +1022,12 @@ video:-webkit-full-screen {
     cursor: pointer;
 }
 
-.dl-control-icons {
+.task-control-icons {
     display: flex;
     justify-content: center;
-    padding-bottom: 10px;
 }
 
-.dl-control-icons img {
+.task-control-icons img {
     width: 30px;
     cursor: pointer;
     margin: 5px;
@@ -1044,6 +1048,7 @@ video:-webkit-full-screen {
 
 /* status message */
 .notification {
+    position: relative;
     background-color: var(--highlight-bg);
     text-align: center;
     padding: 30px 0 15px 0;

tubearchivist/static/progress.js

@@ -5,16 +5,9 @@
 
 'use strict';
 
-checkMessages();
+/* globals apiRequest animate */
 
-// page map to notification status
-const messageTypes = {
-  download: ['message:download', 'message:add', 'message:rescan', 'message:playlistscan'],
-  channel: ['message:subchannel'],
-  channel_id: ['message:playlistscan'],
-  playlist: ['message:subplaylist'],
-  setting: ['message:setting'],
-};
+checkMessages();
 
 // start to look for messages
 function checkMessages() {
@@ -25,82 +18,121 @@ function checkMessages() {
   }
 }
 
-// get messages for page on timer
 function getMessages(dataOrigin) {
-  fetch('/progress/')
-    .then(response => response.json())
-    .then(responseData => {
-      const messages = buildMessage(responseData, dataOrigin);
-      if (messages.length > 0) {
-        // restart itself
-        setTimeout(() => getMessages(dataOrigin), 500);
-      }
-    });
+  let apiEndpoint = '/api/notification/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let messages = buildMessage(responseData, dataOrigin);
+  if (messages.length > 0) {
+    // restart itself
+    setTimeout(() => getMessages(dataOrigin), 500);
+  }
 }
 
-// make div for all messages, return relevant
 function buildMessage(responseData, dataOrigin) {
-  // filter relevan messages
-  let allMessages = responseData['messages'];
-  let messages = allMessages.filter(function (value) {
-    return messageTypes[dataOrigin].includes(value['status']);
+  // filter relevant messages
+  let messages = responseData.filter(function (value) {
+    return dataOrigin.split(' ').includes(value.group.split(':')[0]);
   }, dataOrigin);
-  // build divs
-  let notificationDiv = document.getElementById('notifications');
-  let nots = notificationDiv.childElementCount;
-  notificationDiv.innerHTML = '';
+  let notifications = document.getElementById('notifications');
+  let currentNotifications = notifications.childElementCount;
+
   for (let i = 0; i < messages.length; i++) {
-    let messageData = messages[i];
-    let messageStatus = messageData['status'];
-    let messageBox = document.createElement('div');
-    let title = document.createElement('h3');
-    title.innerHTML = messageData['title'];
-    let message = document.createElement('p');
-    message.innerHTML = messageData['message'];
-    messageBox.appendChild(title);
-    messageBox.appendChild(message);
-    messageBox.classList.add(messageData['level'], 'notification');
-    notificationDiv.appendChild(messageBox);
-    if (messageStatus === 'message:download') {
-      checkDownloadIcons();
+    const messageData = messages[i];
+    if (!document.getElementById(messageData.id)) {
+      let messageBox = buildPlainBox(messageData);
+      notifications.appendChild(messageBox);
+    }
+    updateMessageBox(messageData);
+    if (messageData.group.startsWith('download:')) {
+      animateIcons(messageData.group);
     }
   }
-  // reload page when no more notifications
-  if (nots > 0 && messages.length === 0) {
+  clearNotifications(responseData);
+  if (currentNotifications > 0 && messages.length === 0) {
     location.reload();
   }
   return messages;
 }
 
-// check if download icons are needed
-function checkDownloadIcons() {
-  let iconBox = document.getElementById('downloadControl');
-  if (iconBox.childElementCount === 0) {
-    let downloadIcons = buildDownloadIcons();
-    iconBox.appendChild(downloadIcons);
+function buildPlainBox(messageData) {
+  let messageBox = document.createElement('div');
+  messageBox.classList.add(messageData.level, 'notification');
+  messageBox.id = messageData.id;
+  messageBox.innerHTML = `
+  <h3></h3>
+  <p></p>
+  <div class="task-control-icons"></div>
+  <div class="notification-progress-bar"></div>`;
+  return messageBox;
+}
+
+function updateMessageBox(messageData) {
+  let messageBox = document.getElementById(messageData.id);
+  let children = messageBox.children;
+  children[0].textContent = messageData.title;
+  children[1].innerHTML = messageData.messages.join('<br>');
+  if (
+    !messageBox.querySelector('#stop-icon') &&
+    messageData['api-stop'] &&
+    messageData.command !== 'STOP'
+  ) {
+    children[2].appendChild(buildStopIcon(messageData.id));
+  }
+  if (messageData.progress) {
+    children[3].style.width = `${messageData.progress * 100 || 0}%`;
   }
 }
 
-// add dl control icons
-function buildDownloadIcons() {
-  let downloadIcons = document.createElement('div');
-  downloadIcons.classList = 'dl-control-icons';
-  // stop icon
+function animateIcons(group) {
+  let rescanIcon = document.getElementById('rescan-icon');
+  let dlIcon = document.getElementById('download-icon');
+  switch (group) {
+    case 'download:scan':
+      if (rescanIcon && !rescanIcon.classList.contains('rotate-img')) {
+        animate('rescan-icon', 'rotate-img');
+      }
+      break;
+
+    case 'download:run':
+      if (dlIcon && !dlIcon.classList.contains('bounce-img')) {
+        animate('download-icon', 'bounce-img');
+      }
+      break;
+
+    default:
+      break;
+  }
+}
+
+function buildStopIcon(taskId) {
   let stopIcon = document.createElement('img');
   stopIcon.setAttribute('id', 'stop-icon');
-  stopIcon.setAttribute('title', 'Stop Download Queue');
+  stopIcon.setAttribute('data', taskId);
+  stopIcon.setAttribute('title', 'Stop Task');
   stopIcon.setAttribute('src', '/static/img/icon-stop.svg');
   stopIcon.setAttribute('alt', 'stop icon');
-  stopIcon.setAttribute('onclick', 'stopQueue()');
-  // kill icon
+  stopIcon.setAttribute('onclick', 'stopTask(this)');
+  return stopIcon;
+}
+
+function buildKillIcon(taskId) {
   let killIcon = document.createElement('img');
   killIcon.setAttribute('id', 'kill-icon');
-  killIcon.setAttribute('title', 'Kill Download Queue');
+  killIcon.setAttribute('data', taskId);
+  killIcon.setAttribute('title', 'Kill Task');
   killIcon.setAttribute('src', '/static/img/icon-close.svg');
   killIcon.setAttribute('alt', 'kill icon');
-  killIcon.setAttribute('onclick', 'killQueue()');
-  // stich together
-  downloadIcons.appendChild(stopIcon);
-  downloadIcons.appendChild(killIcon);
-  return downloadIcons;
+  killIcon.setAttribute('onclick', 'killTask(this)');
+  return killIcon;
+}
+
+function clearNotifications(responseData) {
+  let allIds = Array.from(responseData, x => x.id);
+  let allBoxes = document.getElementsByClassName('notification');
+  for (let i = 0; i < allBoxes.length; i++) {
+    const notificationBox = allBoxes[i];
+    if (!allIds.includes(notificationBox.id)) {
+      notificationBox.remove();
+    }
+  }
 }

tubearchivist/static/script.js

@@ -55,19 +55,6 @@ function createWatchStatusIndicator(videoId, videoWatchStatus) {
   return watchStatusIndicator;
 }
 
-// function isWatched(youtube_id) {
-//     var payload = JSON.stringify({'watched': youtube_id});
-//     sendPost(payload);
-//     var seenIcon = document.createElement('img');
-//     seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
-//     seenIcon.setAttribute('alt', 'seen-icon');
-//     seenIcon.setAttribute('id', youtube_id);
-//     seenIcon.setAttribute('title', "Mark as unwatched");
-//     seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
-//     seenIcon.classList = 'seen-icon';
-//     document.getElementById(youtube_id).replaceWith(seenIcon);
-// }
-
 // Removes the progress bar when passed a video id
 function removeProgressBar(videoId) {
   setProgressBar(videoId, 0, 1);
@@ -84,20 +71,6 @@ function isWatchedButton(button) {
   }, 1000);
 }
 
-// function isUnwatched(youtube_id) {
-//     postVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
-//     var payload = JSON.stringify({'un_watched': youtube_id});
-//     sendPost(payload);
-//     var unseenIcon = document.createElement('img');
-//     unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
-//     unseenIcon.setAttribute('alt', 'unseen-icon');
-//     unseenIcon.setAttribute('id', youtube_id);
-//     unseenIcon.setAttribute('title', "Mark as watched");
-//     unseenIcon.setAttribute('onclick', "isWatched(this.id)");
-//     unseenIcon.classList = 'unseen-icon';
-//     document.getElementById(youtube_id).replaceWith(unseenIcon);
-// }
-
 function unsubscribe(id_unsub) {
   let payload = JSON.stringify({ unsubscribe: id_unsub });
   sendPost(payload);
@@ -163,22 +136,25 @@ function reindex(button) {
   let message = document.createElement('p');
   message.innerText = 'Reindex scheduled';
   document.getElementById('reindex-button').replaceWith(message);
+  setTimeout(function () {
+    checkMessages();
+  }, 500);
 }
 
 // download page buttons
 function rescanPending() {
-  let payload = JSON.stringify({ rescan_pending: true });
+  let apiEndpoint = '/api/task-name/update_subscribed/';
+  apiRequest(apiEndpoint, 'POST');
   animate('rescan-icon', 'rotate-img');
-  sendPost(payload);
   setTimeout(function () {
     checkMessages();
   }, 500);
 }
 
 function dlPending() {
-  let payload = JSON.stringify({ dl_pending: true });
+  let apiEndpoint = '/api/task-name/download_pending/';
+  apiRequest(apiEndpoint, 'POST');
   animate('download-icon', 'bounce-img');
-  sendPost(payload);
   setTimeout(function () {
     checkMessages();
   }, 500);
@@ -228,50 +204,64 @@ function deleteQueue(button) {
   document.getElementById(button.id).replaceWith(message);
 }
 
-function stopQueue() {
-  let payload = JSON.stringify({ queue: 'stop' });
-  sendPost(payload);
-  document.getElementById('stop-icon').remove();
+function stopTask(icon) {
+  let taskId = icon.getAttribute('data');
+  let apiEndpoint = `/api/task-id/${taskId}/`;
+  apiRequest(apiEndpoint, 'POST', { command: 'stop' });
+  icon.remove();
 }
 
-function killQueue() {
-  let payload = JSON.stringify({ queue: 'kill' });
-  sendPost(payload);
-  document.getElementById('kill-icon').remove();
+function killTask(icon) {
+  let taskId = icon.getAttribute('data');
+  let apiEndpoint = `/api/task-id/${taskId}/`;
+  apiRequest(apiEndpoint, 'POST', { command: 'kill' });
+  icon.remove();
 }
 
 // settings page buttons
 function manualImport() {
-  let payload = JSON.stringify({ 'manual-import': true });
-  sendPost(payload);
+  let apiEndpoint = '/api/task-name/manual_import/';
+  apiRequest(apiEndpoint, 'POST');
   // clear button
   let message = document.createElement('p');
   message.innerText = 'processing import';
   let toReplace = document.getElementById('manual-import');
   toReplace.innerHTML = '';
   toReplace.appendChild(message);
+  setTimeout(function () {
+    location.replace('#notifications');
+    checkMessages();
+  }, 200);
 }
 
 function reEmbed() {
-  let payload = JSON.stringify({ 're-embed': true });
-  sendPost(payload);
+  let apiEndpoint = '/api/task-name/resync_thumbs/';
+  apiRequest(apiEndpoint, 'POST');
   // clear button
   let message = document.createElement('p');
   message.innerText = 'processing thumbnails';
   let toReplace = document.getElementById('re-embed');
   toReplace.innerHTML = '';
   toReplace.appendChild(message);
+  setTimeout(function () {
+    location.replace('#notifications');
+    checkMessages();
+  }, 200);
 }
 
 function dbBackup() {
-  let payload = JSON.stringify({ 'db-backup': true });
-  sendPost(payload);
+  let apiEndpoint = '/api/task-name/run_backup/';
+  apiRequest(apiEndpoint, 'POST');
   // clear button
   let message = document.createElement('p');
   message.innerText = 'backing up archive';
   let toReplace = document.getElementById('db-backup');
   toReplace.innerHTML = '';
   toReplace.appendChild(message);
+  setTimeout(function () {
+    location.replace('#notifications');
+    checkMessages();
+  }, 200);
 }
 
 function dbRestore(button) {
@@ -284,25 +274,37 @@ function dbRestore(button) {
   let toReplace = document.getElementById(fileName);
   toReplace.innerHTML = '';
   toReplace.appendChild(message);
+  setTimeout(function () {
+    location.replace('#notifications');
+    checkMessages();
+  }, 200);
 }
 
 function fsRescan() {
-  let payload = JSON.stringify({ 'fs-rescan': true });
-  sendPost(payload);
+  let apiEndpoint = '/api/task-name/rescan_filesystem/';
+  apiRequest(apiEndpoint, 'POST');
   // clear button
   let message = document.createElement('p');
   message.innerText = 'File system scan in progress';
   let toReplace = document.getElementById('fs-rescan');
   toReplace.innerHTML = '';
   toReplace.appendChild(message);
+  setTimeout(function () {
+    location.replace('#notifications');
+    checkMessages();
+  }, 200);
 }
 
 function resetToken() {
-  let payload = JSON.stringify({ 'reset-token': true });
-  sendPost(payload);
-  let message = document.createElement('p');
-  message.innerText = 'Token revoked';
-  document.getElementById('text-reveal').replaceWith(message);
+  let apiEndpoint = '/api/token/';
+  let result = apiRequest(apiEndpoint, 'DELETE');
+  if (result && result.success) {
+    let message = document.createElement('p');
+    message.innerText = 'Token revoked';
+    document.getElementById('text-reveal').replaceWith(message);
+  } else {
+    console.error('unable to revoke token');
+  }
 }
 
 // restore from snapshot
@@ -1402,7 +1404,8 @@ function recordTextTrackChanges() {
 }
 
 // keyboard shortcuts for the video player
-document.addEventListener('keydown', doShortcut);
+// need useCapture so we can prevent events from reaching the player
+document.addEventListener('keydown', doShortcut, true);
 
 let modalHideTimeout = -1;
 function showModal(html, duration) {
@@ -1458,19 +1461,13 @@ function doShortcut(e) {
       break;
     }
     case 'ArrowLeft': {
-      if (targetName === 'video') {
-        // hitting arrows while the video is focused will use the built-in skip
-        break;
-      }
+      e.preventDefault();
       showModal('- 5 seconds', 500);
       player.currentTime -= 5;
       break;
     }
     case 'ArrowRight': {
-      if (targetName === 'video') {
-        // hitting space while the video is focused will use the built-in skip
-        break;
-      }
+      e.preventDefault();
       showModal('+ 5 seconds', 500);
       player.currentTime += 5;
       break;
@@ -1493,10 +1490,6 @@ function doShortcut(e) {
       break;
     }
     case ' ': {
-      if (targetName === 'video') {
-        // hitting space while the video is focused will toggle it anyway
-        break;
-      }
       e.preventDefault();
       if (player.paused) {
         player.play();