bump TA_VERSION for release

Changed:
- Bumped yt-dlp
- Added migration to update TZ in schedule when changed
- Fixed issue where version check would never trigger

.eslintrc.js

@@ -0,0 +1,17 @@
+'use strict';
+module.exports = {
+  extends: ['eslint:recommended', 'eslint-config-prettier'],
+  parserOptions: {
+    ecmaVersion: 2020,
+  },
+  env: {
+    browser: true,
+  },
+  rules: {
+    strict: ['error', 'global'],
+    'no-unused-vars': ['error', { vars: 'local' }],
+    eqeqeq: ['error', 'always', { null: 'ignore' }],
+    curly: ['error', 'multi-line'],
+    'no-var': 'error',
+  },
+};

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+github: bbilly1
+ko_fi: bbilly1
+custom: https://paypal.me/bbilly1

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -0,0 +1,55 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to help improve this project! Please read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide carefully before continuing.
+
+  - type: checkboxes
+    id: latest
+    attributes:
+      label: "I've read the documentation"
+      options:
+        - label: I'm running the latest version of Tube Archivist and have read the [release notes](https://github.com/tubearchivist/tubearchivist/releases/latest).
+          required: true
+        - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [bug report](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#bug-report) section.
+          required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: On what operating system are you running Tube Archivist on?
+      placeholder: ex. Linux, Unraid, Synology...
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: Your Bug Report
+      value: "## Describe the bug\n\n## Steps To Reproduce\n\n## Expected behavior"
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Anything else?
+      description: |
+        Links? References? Anything that will give us more context about the issue you are encountering!
+
+        Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -0,0 +1,34 @@
+name: Feature Request
+description: I have an idea for a great addition to this project
+title: "[Feature Request]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to help improve this project! This project is *very* selective with accepting new feature requests. Please read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide carefully before continuing.
+
+  - type: checkboxes
+    id: already
+    attributes:
+      label: "I've read the documentation"
+      options:
+        - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [feature request](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#feature-request) section.
+          required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Your Feature Request
+      value: "## Is your feature request related to a problem? Please describe.\n\n## Describe the solution you'd like\n\n## Additional context"
+      placeholder: Tell us what you see!
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: help
+    attributes:
+      label: Your help is needed!
+      description: This project is ambitious as it is, please contribute.
+      options:
+        - label: Yes I will work on this in the next few days or weeks.

.github/ISSUE_TEMPLATE/FRONTEND-MIGRATION.yml

@@ -0,0 +1,23 @@
+name: Frontend Migration
+description: Tracking our new React based frontend
+title: "[Frontend Migration]: "
+labels: ["react migration"]
+
+body:
+  - type: dropdown
+    id: domain
+    attributes:
+      label: Domain
+      options:
+        - Frontend
+        - Backend
+        - Combined
+    validations:
+      required: true
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      placeholder: Organizing our React frontend migration
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml

@@ -0,0 +1,43 @@
+name: Installation Help
+description: There is an issue with installing or configuring Tube Archivist
+title: "[Installation]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This is most likely not the best place to ask for installation help. There is a helpful community on [Discord](https://github.com/tubearchivist/tubearchivist#connect).
+
+  - type: checkboxes
+    id: instructions
+    attributes:
+      label: Installation instructions
+      options:
+        - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [installation help](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#installation-help) section.
+          required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: On what operating system do you want to install Tube Archivist on?
+      placeholder: ex. Linux, Unraid, Synology...
+    validations:
+      required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Your installation or setup question
+      placeholder: Describe your issue
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
+      render: Shell
+    validations:
+      required: true

.github/pull_request_template.md

@@ -0,0 +1,3 @@
+Thank you for taking the time to improve this project. Please take a look at the [How to make a Pull Request](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-make-a-pull-request) section to help get your contribution merged.
+
+You can delete this text before submitting.

.github/workflows/lint_js.yml

@@ -0,0 +1,22 @@
+name: lint_js
+
+on:
+  push:
+    paths:
+      - '**/*.js'
+  pull_request:
+    paths:
+      - '**/*.js'
+
+jobs:
+  check:
+    name: lint_js
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+      - run: npm ci
+      - run: npm run lint
+      - run: npm run format -- --check

.github/workflows/lint_python.yml

@@ -0,0 +1,42 @@
+name: lint_python
+
+on:
+  push:
+    paths:
+      - '**/*.py'
+  pull_request:
+    paths:
+      - '**/*.py'
+
+jobs:
+  lint_python:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc libldap2-dev libsasl2-dev libssl-dev
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tubearchivist/requirements-dev.txt
+
+      - name: Run Linter
+        run: ./deploy.sh validate

.github/workflows/unit_tests.yml

@@ -0,0 +1,43 @@
+name: python_unit_tests
+
+on:
+  push:
+    paths:
+      - '**/*.py'
+  pull_request:
+    paths:
+      - '**/*.py'
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc libldap2-dev libsasl2-dev libssl-dev
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tubearchivist/requirements-dev.txt
+
+      - name: Run unit tests
+        run: pytest tubearchivist

.gitignore

@@ -1,8 +1,12 @@
 # python testing cache
 __pycache__
+.venv
 
 # django testing db
 db.sqlite3
 
-# frontend fonts
-*ttf.woff
+# vscode custom conf
+.vscode
+
+# JavaScript stuff
+node_modules

.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-    "python.linting.pylintEnabled": true,
-    "python.linting.pycodestyleEnabled": false,
-    "python.linting.enabled": true
-}

ARCHITECTURE.md

@@ -0,0 +1,48 @@
+# The Inner Workings of Tube Archivist
+This is a high level overview of the architecture of Tube Archivist, intended for interested contributors to find your way around quickly.
+
+```
+                        Tube Archivist
+        ______________________|_____________________
+        |                     |                    |
+-------------------    ---------------    -------------------
+|                 |    |             |    |                 |
+|  DjangoProject  |    |  RedisJson  |    |  ElasticSearch  |
+|                 |    |             |    |                 |
+-------------------    ---------------    -------------------
+```
+
+## DjangoProject
+This is the main Python application. Django serves its data container internally with **Uwsgi** on port 8080, the interface is served with **Nginx** on the public port 8000.
+
+Users created static files like media files and artwork as well as application artwork like logos and fonts are served directly from Nginx, while the rest of the application uses uwsgi_pass to proxy the requests to uwsgi.
+
+Config files are located in the `docker_assets` folder. The script `run.sh` is the container `CMD` command and entry point, validating env vars, connection to ElasticSearch (ES) and will start the application.
+
+Compared to other Django projects, this application doesn't make use of the database models, due to a lack of integration with ES. This project has its own abstractions and integrations, treating ES as a REST API.
+
+Long running application tasks are handed off to **Celery** - using **Redis** as a broker - to run asynchronously from the main threads. 
+- All tasks are defined in the `home.tasks.py` module.
+
+There are three Django apps:
+- **config**: The root app, routing the main endpoints and the main `settings.py` file
+- **api**: The API app with its views and functionality
+- **home**: Most of the application logic, templates and views, will probably get split up further in the future.
+
+The *home* app is split up into packages in the `src` directory:
+- **download**: All download related classes, interact with yt-dlp, download artwork, handle the download queue and post processing tasks.
+- **es**: All index setup and validation classes, handles mapping validations and makes mapping changes, wrapper functions to simplify interactions with Elasticsearch, backup and restore.
+- **frontend**: All direct interactions with the frontend, like Django forms, searching, watched state changes, and legacy api_calls in the process of moving to the api app.
+- **index**: Contains all functionality for scraping and indexing videos, channels, playlists, comments, subtitles, etc...
+- **ta**: Loose collection of functions and classes, handle application config and contains redis wrapper classes.
+
+## RedisJson
+Holds the main application config json object that gets dynamically edited from the frontend, serves as a message broker for **Celery**. Redis serves as a temporary and thread safe link between Django and the frontend, storing progress messages and temporary queues for processing. Used to store locking keys for threads and execution details for tasks.
+
+- Wrapper classes to interact with Redis are located in the `home.src.ta.ta_redis.py` module.
+
+## ElasticSearch (ES)
+Is used to store and index all metadata, functions as an application database and makes it all searchable. The mapping defines which fields are indexed as searchable text fields and which fields are used for match filtering.
+
+- The index setup and validation is handled in the `home.src.es.index_setup.py` module.
+- Wrapper classes for making requests to ES are located in the `home.src.es.connect.py` module.

CONTRIBUTING.md

@@ -0,0 +1,208 @@
+# Contributing to Tube Archivist
+
+Welcome, and thanks for showing interest in improving Tube Archivist!  
+
+## Table of Content
+- [Next Steps](#next-steps)
+- [Beta Testing](#beta-testing)
+- [How to open an issue](#how-to-open-an-issue)
+  - [Bug Report](#bug-report)
+  - [Feature Request](#feature-request)
+  - [Installation Help](#installation-help)
+- [How to make a Pull Request](#how-to-make-a-pull-request)
+- [Contributions beyond the scope](#contributions-beyond-the-scope)
+- [User Scripts](#user-scripts)
+- [Improve to the Documentation](#improve-to-the-documentation)
+- [Development Environment](#development-environment)
+---
+
+## Next Steps
+Going forward, this project will focus on developing a new modern frontend.
+
+- For the time being, don't open any new PRs that are not towards the new frontend.
+- New features requests likely won't get accepted during this process.
+- Depending on the severity, bug reports may or may not get fixed during this time.
+- When in doubt, reach out.
+
+Join us on [Discord](https://tubearchivist.com/discord) if you want to help with that process.
+
+## Beta Testing
+Be the first to help test new features and improvements and provide feedback! There are regular `:unstable` builds for easy access. That's for the tinkerers and the breave. Ideally use a testing environment first, before a release be the first to install it on your main system.
+
+There is always something that can get missed during development. Look at the commit messages tagged with `#build`, these are the unstable builds and give a quick overview what has changed.
+
+- Test the features mentioned, play around, try to break it.
+- Test the update path by installing the `:latest` release first, the upgrade to `:unstable` to check for any errors.
+- Test the unstable build on a fresh install.
+
+Then provide feedback, if there is a problem but also if there is no problem. Reach out on [Discord](https://tubearchivist.com/discord) in the `#beta-testing` channel with your findings.
+
+This will help with a smooth update for the regular release. Plus you get to test things out early! 
+
+## How to open an issue
+Please read this carefully before opening any [issue](https://github.com/tubearchivist/tubearchivist/issues) on GitHub. Make sure you read [Next Steps](#next-steps) above.
+
+**Do**:
+- Do provide details and context, this matters a lot and makes it easier for people to help.
+- Do familiarize yourself with the project first, some questions answer themselves when using the project for some time. Familiarize yourself with the [Readme](https://github.com/tubearchivist/tubearchivist) and the [documentation](https://docs.tubearchivist.com/), this covers a lot of the common questions, particularly the [FAQ](https://docs.tubearchivist.com/faq/).
+- Do respond to questions within a day or two so issues can progress. If the issue doesn't move forward due to a lack of response, we'll assume it's solved and we'll close it after some time to keep the list fresh.
+
+**Don't**:
+- Don't open *duplicates*, that includes open and closed issues.
+- Don't open an issue for something that's already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), this needs your help to implement it, not another issue.
+- Don't open an issue for something that's a [known limitation](https://github.com/tubearchivist/tubearchivist#known-limitations). These are *known* by definition and don't need another reminder. Some limitations may be solved in the future, maybe by you?
+- Don't overwrite the *issue template*, they are there for a reason. Overwriting that shows that you don't really care about this project. It shows that you have a misunderstanding how open source collaboration works and just want to push your ideas through. Overwriting the template may result in a ban.
+
+### Bug Report
+Bug reports are highly welcome! This project has improved a lot due to your help by providing feedback when something doesn't work as expected. The developers can't possibly cover all edge cases in an ever changing environment like YouTube and yt-dlp.
+
+Please keep in mind:
+- Docker logs are the easiest way to understand what's happening when something goes wrong, *always* provide the logs upfront.
+- Set the environment variable `DJANGO_DEBUG=True` to Tube Archivist and reproduce the bug for a better log output. Don't forget to remove that variable again after.
+- A bug that can't be reproduced, is difficult or sometimes even impossible to fix. Provide very clear steps *how to reproduce*.
+
+### Feature Request
+This project needs your help to grow further. There is no shortage of ideas, see the open [issues on GH](https://github.com/tubearchivist/tubearchivist/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement) and the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), what this project lacks is contributors interested in helping with overall improvements of the application. Focus is *not* on adding new features, but improving existing ones.
+
+Existing ideas are easily *multiple years* worth of development effort, at least at current speed. This project is *very* selective with accepting new feature requests at this point.  
+
+Good feature requests usually fall into one or more of these categories:
+- You want to work on your own small scoped idea within the next few days or weeks.
+- Your idea is beneficial for a wide range of users, not just for you.
+- Your idea extends the current project by building on and improving existing functionality.
+- Your idea is quick and easy to implement, for an experienced as well as for a first time contributor.
+
+Your request is likely going to be rejected if:
+- Your idea requires multiple days worth of development time and is unrealistic to be implemented any time soon.
+- There are already other ways to do what you are trying to do.
+- You are trying to do something that only applies to your platform, your specific workflow or your specific setup.
+- Your idea would fundamentally change how the project works or it wouldn't be able to be implemented with backwards compatibility.
+- Your idea is not a good fit for this project.
+
+### Installation Help
+GitHub is most likely not the best place to ask for installation help. That's inherently individual and one on one.
+1. First step is always, help yourself. Start at the [Readme](https://github.com/tubearchivist/tubearchivist) or the additional platform specific installation pages in the [docs](https://docs.tubearchivist.com/).
+2. If that doesn't answer your question, open a `#support` thread on [Discord](https://www.tubearchivist.com/discord).
+3. Only if that is not an option, open an issue here.
+
+IMPORTANT: When receiving help, contribute back to the community by improving the installation instructions with your newly gained knowledge.
+
+---
+
+## How to make a Pull Request
+
+Make sure you read [Next Steps](#next-steps) above.
+
+Thank you for contributing and helping improve this project. Focus for the foreseeable future is on improving and building on existing functionality, *not* on adding and expanding the application.
+
+This is a quick checklist to help streamline the process:
+
+- For **code changes**, make your PR against the [testing branch](https://github.com/tubearchivist/tubearchivist/tree/testing). That's where all active development happens. This simplifies the later merging into *master*, minimizes any conflicts and usually allows for easy and convenient *fast-forward* merging.
+- For **documentation changes**, make your PR directly against the *master* branch.
+- Show off your progress, even if not yet complete, by creating a [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) PR first and switch it as *ready* when you are ready.
+- Make sure all your code is linted and formatted correctly, see below. The automatic GH action unfortunately needs to be triggered manually by a maintainer for first time contributors, but will trigger automatically for existing contributors.
+
+### Making changes to the JavaScript
+
+The JavaScript does not require any build step; you just edit the files directly. However, there is config for eslint and prettier (a linter and formatter respectively); their use is recommended but not required. To use them, install `node`, run `npm i` from the root directory of this repository to install dependencies, then run `npm run lint` and `npm run format` to run eslint and prettier respectively.
+
+### Code formatting and linting
+
+To keep things clean and consistent for everybody, there is a github action setup to lint and check the changes. You can test your code locally first if you want. For example if you made changes in the **video** module, run
+
+```shell
+./deploy.sh validate tubearchivist/home/src/index/video.py
+```
+
+to validate your changes. If you omit the path, all the project files will get checked. This is subject to change as the codebase improves.
+
+---
+
+## Contributions beyond the scope
+
+As you have read the [FAQ](https://docs.tubearchivist.com/faq/) and the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and have gotten an idea what this project tries to do, there will be some obvious shortcomings that stand out, that have been explicitly excluded from the scope of this project, at least for the time being.
+
+Extending the scope of this project will only be feasible with more [regular contributors](https://github.com/tubearchivist/tubearchivist/graphs/contributors) that are willing to help improve this project in the long run. Contributors that have an overall improvement of the project in mind and not just about implementing this *one* thing.  
+
+Small minor additions, or making a PR for a documented feature request or bug, even if that was and will be your only contribution to this project, are always welcome and is *not* what this is about.
+
+Beyond that, general rules to consider:
+
+- Maintainability is key: It's not just about implementing something and being done with it, it's about maintaining it, fixing bugs as they occur, improving on it and supporting it in the long run.
+- Others can do it better: Some problems have been solved by very talented developers. These things don't need to be reinvented again here in this project.
+- Develop for the 80%: New features and additions *should* be beneficial for 80% of the users. If you are trying to solve your own problem that only applies to you, maybe that would be better to do in your own fork or if possible by a standalone implementation using the API.
+- If all of that sounds too strict for you, as stated above, start becoming a regular contributor to this project.
+
+---
+
+## User Scripts
+Some of you might have created useful scripts or API integrations around this project. Sharing is caring! Please add a link to your script to the Readme [here](https://github.com/tubearchivist/tubearchivist#user-scripts).
+- Your repo should have a `LICENSE` file with one of the common open source licenses. People are expected to fork, adapt and build upon your great work.
+- Your script should not modify the *official* files of Tube Archivist. E.g. your symlink script should build links *outside* of your `/youtube` folder. Or your fancy script that creates a beautiful artwork gallery should do that *outside* of the `/cache` folder. Modifying the *official* files and folders of TA are probably not supported.
+- On the top of the repo you should have a mention and a link back to the Tube Archivist repo. Clearly state to **not** to open any issues on the main TA repo regarding your script.
+- Example template:
+  - `[<user>/<repo>](https://linktoyourrepo.com)`: A short one line description.
+
+---
+
+## Improve to the Documentation
+
+The documentation available at [docs.tubearchivist.com](https://docs.tubearchivist.com/) and is build from a separate repo [tubearchivist/docs](https://github.com/tubearchivist/docs). The Readme has additional instructions on how to make changes.
+
+---
+
+## Development Environment
+
+I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. You might be able to run the application directly, but this document assumes you're using docker.
+
+### Instructions
+
+Set up docker on your development machine.
+
+Clone this repository.
+
+Functional changes should be made against the unstable `testing` branch, so check that branch out, then make a new branch for your work.
+
+Edit the `docker-compose.yml` file and replace the [`image: bbilly1/tubearchivist` line](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L7) with `build: .`. Also make any other changes to the environment variables and so on necessary to run the application, just like you're launching the application as normal.
+
+Run `docker compose up --build`. This will bring up the application. Kill it with `ctrl-c` or by running `docker compose down` from a new terminal window in the same directory.
+
+Make your changes locally and re-run `docker compose up --build`. The `Dockerfile` is structured in a way that the actual application code is in the last layer so rebuilding the image with only code changes utilizes the build cache for everything else and will just take a few seconds.
+
+### Develop environment inside a VM
+
+You may find it nice to run everything inside of a VM, though this is not necessary. There's a `deploy.sh` script which has some helpers for this use case. YMMV, this is what one of the developers does:
+
+- Clone the repo, work on it with your favorite code editor in your local filesystem. *testing* branch is where all the changes are happening, might be unstable and is WIP.
+- Then I have a VM running standard Ubuntu Server LTS with docker installed. The VM keeps my projects separate and offers convenient snapshot functionality. The VM also offers ways to simulate low end environments by limiting CPU cores and memory. You can use this [Ansible Docker Ubuntu](https://github.com/bbilly1/ansible-playbooks) playbook to get started quickly. But you could also just run docker on your host system.
+- I have my local DNS resolve `tubearchivist.local` to the IP of the VM for convenience. To deploy the latest changes and rebuild the application to the testing VM run:
+```bash
+./deploy.sh test
+```
+- The command above will call the docker build command with `--build-arg INSTALL_DEBUG=1` to install additional useful debug tools.
+- The `test` argument takes another optional argument to build for a specific architecture valid options are: `amd64`, `arm64` and `multi`, default is `amd64`.
+- This `deploy.sh` script is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking.
+
+### Working with Elasticsearch
+Additionally to the required services as listed in the example docker-compose file, the **Dev Tools** of [Kibana](https://www.elastic.co/guide/en/kibana/current/docker.html) are invaluable for running and testing Elasticsearch queries.
+
+**Quick start**  
+Generate your access token in Elasitcsearch:
+```bash
+bin/elasticsearch-service-tokens create elastic/kibana kibana
+```
+
+Example docker compose, use same version as for Elasticsearch:
+```yml
+services:
+  kibana:
+    image: docker.elastic.co/kibana/kibana:0.0.0
+    container_name: kibana
+    environment:
+    - "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
+    - "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
+    ports:
+    - "5601:5601"
+```
+
+If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.

Dockerfile

@@ -1,32 +1,64 @@
-# build the tube archivist image from default python slim image
+# multi stage to build tube archivist
+# build python wheel, download and extract ffmpeg, copy into final image
 
-FROM python:3.9.7-slim-bullseye
+
+# First stage to build python wheel
+FROM python:3.11.8-slim-bookworm AS builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev git
+
+# install requirements
+COPY ./tubearchivist/requirements.txt /requirements.txt
+RUN pip install --user -r requirements.txt
+
+# build ffmpeg
+FROM python:3.11.8-slim-bookworm as ffmpeg-builder
+
+ARG TARGETPLATFORM
+
+COPY docker_assets/ffmpeg_download.py ffmpeg_download.py
+RUN python ffmpeg_download.py $TARGETPLATFORM
+
+# build final image
+FROM python:3.11.8-slim-bookworm as tubearchivist
+
+ARG INSTALL_DEBUG
 
 ENV PYTHONUNBUFFERED 1
 
+# copy build requirements
+COPY --from=builder /root/.local /root/.local
+ENV PATH=/root/.local/bin:$PATH
+
+# copy ffmpeg
+COPY --from=ffmpeg-builder ./ffmpeg/ffmpeg /usr/bin/ffmpeg
+COPY --from=ffmpeg-builder ./ffprobe/ffprobe /usr/bin/ffprobe
+
 # install distro packages needed
 RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
-    build-essential \
-    ffmpeg \
     nginx \
+    atomicparsley \
     curl && rm -rf /var/lib/apt/lists/*
 
-# copy config files
-COPY nginx.conf /etc/nginx/conf.d/
+# install debug tools for testing environment
+RUN if [ "$INSTALL_DEBUG" ] ; then \
+        apt-get -y update && apt-get -y install --no-install-recommends \
+        vim htop bmon net-tools iputils-ping procps \
+        && pip install --user ipython pytest pytest-django \
+    ; fi
 
 # make folders
-RUN mkdir /cache
-RUN mkdir /youtube
-RUN mkdir /app
+RUN mkdir /cache /youtube /app
 
-# install python dependencies
-COPY ./tubearchivist/requirements.txt /requirements.txt
-RUN pip install --no-cache-dir -r requirements.txt --src /usr/local/src
+# copy config files
+COPY docker_assets/nginx.conf /etc/nginx/sites-available/default
+RUN sed -i 's/^user www\-data\;$/user root\;/' /etc/nginx/nginx.conf
 
 # copy application into container
 COPY ./tubearchivist /app
-COPY ./run.sh /app
-COPY ./uwsgi.ini /app
+COPY ./docker_assets/run.sh /app
+COPY ./docker_assets/uwsgi.ini /app
 
 # volumes
 VOLUME /cache

README.md

@@ -1,129 +1,234 @@
-![Tube Archivist](assets/tube-archivist-banner.jpg?raw=true "Tube Archivist Banner")  
+![Tube Archivist](assets/tube-archivist-front.jpg?raw=true "Tube Archivist Banner")  
+[*more screenshots and video*](SHOWCASE.MD)
 
-<center><h1>Your self hosted Youtube media server</h1></center>
+<div align="center">
+<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-docker.png" alt="tubearchivist-docker" title="Tube Archivist Docker Pulls" height="50" width="190"/></a>
+<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-star.png" alt="tubearchivist-github-star" title="Tube Archivist GitHub Stars" height="50" width="190"/></a>
+<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-forks.png" alt="tubearchivist-github-forks" title="Tube Archivist GitHub Forks" height="50" width="190"/></a>
+<a href="https://www.tubearchivist.com/discord" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-discord.png" alt="tubearchivist-discord" title="TA Discord Server Members" height="50" width="190"/></a>
+</div>
 
+## Table of contents:
+* [Docs](https://docs.tubearchivist.com/) with [FAQ](https://docs.tubearchivist.com/faq/), and API documentation
+* [Core functionality](#core-functionality)
+* [Resources](#resources)
+* [Installing](#installing)
+* [Getting Started](#getting-started)
+* [Known limitations](#known-limitations)
+* [Port Collisions](#port-collisions)
+* [Common Errors](#common-errors)
+* [Roadmap](#roadmap)
+* [Donate](#donate)
+
+------------------------
 
 ## Core functionality
-* Subscribe to your favourite Youtube channels
+Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. This includes:  
+* Subscribe to your favorite YouTube channels
 * Download Videos using **yt-dlp**
 * Index and make videos searchable
 * Play videos
 * Keep track of viewed and unviewed videos
-
-## Screenshots
-![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")  
-*Home Page*
-
-![channels screenshot](assets/tube-archivist-screenshot-channels.png?raw=true "Tube Archivist Channels")  
-*All Channels*
-
-![single channel screenshot](assets/tube-archivist-screenshot-single-channel.png?raw=true "Tube Archivist Single Channel")  
-*Single Channel*
-
-![video page screenshot](assets/tube-archivist-screenshot-video.png?raw=true "Tube Archivist Video Page")  
-*Video Page*
-
-![video page screenshot](assets/tube-archivist-screenshot-download.png?raw=true "Tube Archivist Video Page")  
-*Downloads Page*
   
-## Problem Tube Archivist tries to solve
-Once your Youtube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from Youtube, you can organize, search and enjoy your archived Youtube videos without hassle offline through a convenient web interface.
+## Resources
+- [Discord](https://www.tubearchivist.com/discord): Connect with us on our Discord server.
+- [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit.
+- [Browser Extension](https://github.com/tubearchivist/browser-extension) Tube Archivist Companion, for [Firefox](https://addons.mozilla.org/addon/tubearchivist-companion/) and [Chrome](https://chrome.google.com/webstore/detail/tubearchivist-companion/jjnkmicfnfojkkgobdfeieblocadmcie)
+- [Jellyfin Plugin](https://github.com/tubearchivist/tubearchivist-jf-plugin): Add your videos to Jellyfin
+- [Plex Plugin](https://github.com/tubearchivist/tubearchivist-plex): Add your videos to Plex
 
-## Installation
-Take a look at the example `docker-compose.yml` file provided. Tube Archivist depends on three main components split up into separate docker containers:  
+## Installing
+For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. Minimal with dual core with 4 threads, better quad core plus. 
+This project requires docker. Ensure it is installed and running on your system.  
 
-### Tube Archivist
-The main Python application that displays and serves your video collection, built with Django.
-  - Serves the interface on port `8000`
-  - Needs a mandatory volume for the video archive at **/youtube**
-  - And another recommended volume to save the cache for thumbnails and artwork at **/cache**.
-  - The environment variables `ES_URL` and `REDIS_HOST` are needed to tell Tube Archivist where Elasticsearch and Redis respectively are located.
-  - The environment variables `HOST_UID` and `HOST_GID` allows Tube Archivist to `chown` the video files to the main host system user instead of the container user.
+The documentation has additional user provided instructions for [Unraid](https://docs.tubearchivist.com/installation/unraid/), [Synology](https://docs.tubearchivist.com/installation/synology/) and [Podman](https://docs.tubearchivist.com/installation/podman/).
 
-### Elasticsearch
-Stores video metadata and makes everything searchable. Also keeps track of the download queue.
-  - Needs to be accessible over the default port `9200`
-  - Needs a volume at **/usr/share/elasticsearch/data** to store data
+The instructions here should get you up and running quickly, for Docker beginners and full explanation about each environment variable, see the [docs](https://docs.tubearchivist.com/installation/docker-compose/).
 
-Follow the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html) for additional installation details.
+Take a look at the example [docker-compose.yml](https://github.com/tubearchivist/tubearchivist/blob/master/docker-compose.yml) and configure the required environment variables.
 
-### Redis JSON
-Functions as a cache and temporary link between the application and the filesystem. Used to store and display messages and configuration variables.
-  - Needs to be accessible over the default port `6379`
-  - Takes an optional volume at **/data** to make your configuration changes permanent.
+**TubeArchivist**:  
+| Environment Var | Value |  |
+| ----------- | ----------- | ----------- |
+| TA_HOST | Server IP or hostname | Required |
+| TA_USERNAME | Initial username when logging into TA | Required |
+| TA_PASSWORD | Initial password when logging into TA | Required |
+| ELASTIC_PASSWORD | Password for ElasticSearch | Required |
+| REDIS_HOST | Hostname for Redis | Required |
+| TZ | Set your timezone for the scheduler | Required |
+| TA_PORT | Overwrite Nginx port | Optional |
+| TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
+| TA_ENABLE_AUTH_PROXY | Enables support for forwarding auth in reverse proxies | [Read more](https://docs.tubearchivist.com/configuration/forward-auth/) |
+| TA_AUTH_PROXY_USERNAME_HEADER | Header containing username to log in | Optional |
+| TA_AUTH_PROXY_LOGOUT_URL | Logout URL for forwarded auth | Optional |
+| ES_URL | URL That ElasticSearch runs on | Optional |
+| ES_DISABLE_VERIFY_SSL | Disable ElasticSearch SSL certificate verification | Optional |
+| ES_SNAPSHOT_DIR | Custom path where elastic search stores snapshots for master/data nodes | Optional |
+| HOST_GID | Allow TA to own the video files instead of container user | Optional |
+| HOST_UID | Allow TA to own the video files instead of container user | Optional |
+| ELASTIC_USER | Change the default ElasticSearch user | Optional |
+| REDIS_PORT | Port that Redis runs on | Optional |
+| TA_LDAP | Configure TA to use LDAP Authentication | [Read more](https://docs.tubearchivist.com/configuration/ldap/) |
+| ENABLE_CAST | Enable casting support | [Read more](https://docs.tubearchivist.com/configuration/cast/) |
+| DJANGO_DEBUG | Return additional error messages, for debug only |  |
+
+**ElasticSearch**  
+| Environment Var | Value | State |
+| ----------- | ----------- | ----------- |
+| ELASTIC_PASSWORD | Matching password `ELASTIC_PASSWORD` from TubeArchivist | Required |
+| http.port | Change the port ElasticSearch runs on | Optional |
+
+
+## Update
+Always use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tags are only for your testing environment, there might not be an update path for these testing builds. 
+
+You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. To update, you need to update the docker images, the method for which will depend on your platform. For example, if you're using `docker-compose`, run `docker-compose pull` and then restart with `docker-compose up -d`. After updating, check the footer to verify you are running the expected version.  
+
+  - This project is tested for updates between one or two releases maximum. Further updates back may or may not be supported and you might have to reset your index and configurations to update. Ideally apply new updates at least once per month.  
+  - There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.  
+  - All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version.   
 
 ## Getting Started
-1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default.
-2. Subscribe to some of your favourite Youtube channels on the **channels** page. 
+1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS and some other browsers a compatible format must be specified. For example:
+```
+bestvideo[vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4
+```
+2. Subscribe to some of your favorite YouTube channels on the **channels** page. 
 3. On the **downloads** page, click on *Rescan subscriptions* to add videos from the subscribed channels to your Download queue or click on *Add to download queue* to manually add Video IDs, links, channels or playlists.
-4. Click on *Download queue* and let Tube Archivist to it's thing. 
+4. Click on *Start download* and let **Tube Archivist** to it's thing. 
 5. Enjoy your archived collection!
-  
-## Import your existing library
-So far this depends on the video you are trying to import to be still available on youtube to get the metadata. Add the files you like to import to the */cache/import* folder. Then start the process from the settings page *Manual media files import*. Make sure to follow one of the two methods below.
-
-### Method 1:
-Add a matching *.json* file with the media file. Both files need to have the same base name, for example:
-- For the media file: \<base-name>.mp4
-- For the JSON file: \<base-name>.info.json
-- Alternate JSON file: \<base-name>.json
-
-**Tube Archivist** then looks for the 'id' key within the JSON file to identify the video.
-
-### Method 2:
-Detect the Youtube ID from filename, this accepts the default yt-dlp naming convention for file names like:
-- \<base-name>[\<youtube-id>].mp4
-- The Youtube ID in square brackets at the end of the filename is the crucial part.
-
-### Some notes:
-- This will **consume** the files you put into the import folder: Files will get converted to mp4 if needed (this might take a long time...) and moved to the archive, *.json* files will get deleted upon completion to avoid having doublicates on the next run.
-- Maybe start with a subset of your files to import to make sure everything goes well...
-- Follow the logs to monitor progress and errors: `docker-compose logs -f tubearchivist`.
 
 
-## Potential pitfalls
-### vm.max_map_count
+### Port Collisions  
+If you have a collision on port `8000`, best solution is to use dockers *HOST_PORT* and *CONTAINER_PORT* distinction: To for example change the interface to port 9000 use `9000:8000` in your docker-compose file.  
+
+For more information on port collisions, check the docs.
+
+## Common Errors  
+Here is a list of common errors and their solutions.  
+
+### `vm.max_map_count`
 **Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.
 
 To temporary set the value run:  
 ```
 sudo sysctl -w vm.max_map_count=262144
 ```  
-
 To apply the change permanently depends on your host operating system:  
-- For example on Ubuntu Server add `vm.max_map_count = 262144` to the file */etc/sysctl.conf*.
-- On Arch based systems create a file */etc/sysctl.d/max_map_count.conf* with the content `vm.max_map_count = 262144`. 
-- On any other platform look up in the documentation on how to pass kernel parameters.
+
+ - For example on Ubuntu Server add `vm.max_map_count = 262144` to the file `/etc/sysctl.conf`.
+ - On Arch based systems create a file `/etc/sysctl.d/max_map_count.conf` with the content `vm.max_map_count = 262144`. 
+ - On any other platform look up in the documentation on how to pass kernel parameters.  
+
 
 ### Permissions for elasticsearch
-If you see a message similar to `AccessDeniedException[/usr/share/elasticsearch/data/nodes]` when initially starting elasticsearch, that means the container is not allowed to write files to the volume.  
-That's most likely the case when you run `docker-compose` as an unprivileged user. To fix that issue, shutdown the container and on your host machine run:
+If you see a message similar to `Unable to access 'path.repo' (/usr/share/elasticsearch/data/snapshot)` or `failed to obtain node locks, tried [/usr/share/elasticsearch/data]` and `maybe these locations are not writable` when initially starting elasticsearch, that probably means the container is not allowed to write files to the volume.  
+To fix that issue, shutdown the container and on your host machine run:
 ```
-chown 1000:0 /path/to/mount/point
+chown 1000:0 -R /path/to/mount/point
 ```
-This will match the permissions with the **UID** and **GID** of elasticsearch within the container and should fix the issue.
+This will match the permissions with the **UID** and **GID** of elasticsearch process within the container and should fix the issue.  
 
-## Roadmap
-This should be considered as a **minimal viable product**, there is an extensive list of future functions and improvements planned.
 
-### Functionality
-- [ ] Access control
-- [ ] User roles
-- [ ] Delete videos and channel
-- [ ] Create playlists
-- [ ] Backup and restore
-- [ ] Podcast mode to serve channel as mp3
-- [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage
-- [X] Scan your filesystem to index already downloaded videos [2021-09-14]
+### Disk usage
+The Elasticsearch index will turn to ***read only*** if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`.  
 
-### UI
-- [ ] Show similar videos on video page
-- [ ] Multi language support
-- [ ] Grid and list view for both channel and video list pages
-- [ ] Show total video downloaded vs total videos available in channel
+Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
 
+## `error setting rlimit`
+If you are seeing errors like `failed to create shim: OCI runtime create failed` and `error during container init: error setting rlimits`, this means docker can't set these limits, usually because they are set at another place or are incompatible because of other reasons. Solution is to remove the `ulimits` key from the ES container in your docker compose and start again.
+
+This can happen if you have nested virtualizations, e.g. LXC running Docker in Proxmox.
 
 ## Known limitations
-- Video files created by Tube Archivist need to be **mp4** video files for best browser compatibility.
+- Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
 - Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
-- For now this is meant to be run in a trusted network environment.
+- There is no flexibility in naming of the media files.
+
+## Roadmap
+We have come far, nonetheless we are not short of ideas on how to improve and extend this project. Issues waiting for you to be tackled in no particular order:
+
+- [ ] User roles
+- [ ] Audio download
+- [ ] Podcast mode to serve channel as mp3
+- [ ] Random and repeat controls ([#108](https://github.com/tubearchivist/tubearchivist/issues/108), [#220](https://github.com/tubearchivist/tubearchivist/issues/220))
+- [ ] Auto play or play next link ([#226](https://github.com/tubearchivist/tubearchivist/issues/226))
+- [ ] Multi language support
+- [ ] Show total video downloaded vs total videos available in channel
+- [ ] Download or Ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163))
+- [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144))
+- [ ] Search comments
+- [ ] Search download queue
+- [ ] Configure shorts, streams and video sizes per channel
+
+Implemented:
+- [X] User created playlists [2024-04-10]
+- [X] Add statistics of index [2023-09-03]
+- [X] Implement [Apprise](https://github.com/caronc/apprise) for notifications [2023-08-05]
+- [X] Download video comments [2022-11-30]
+- [X] Show similar videos on video page [2022-11-30]
+- [X] Implement complete offline media file import from json file [2022-08-20]
+- [X] Filter and query in search form, search by url query [2022-07-23]
+- [X] Make items in grid row configurable to use more of the screen [2022-06-04]
+- [X] Add passing browser cookies to yt-dlp [2022-05-08]
+- [X] Add [SponsorBlock](https://sponsor.ajay.app/) integration [2022-04-16]
+- [X] Implement per channel settings [2022-03-26]
+- [X] Subtitle download & indexing [2022-02-13]
+- [X] Fancy advanced unified search interface [2022-01-08]
+- [X] Auto rescan and auto download on a schedule [2021-12-17]
+- [X] Optional automatic deletion of watched items after a specified time [2021-12-17]
+- [X] Create playlists [2021-11-27]
+- [X] Access control [2021-11-01]
+- [X] Delete videos and channel [2021-10-16]
+- [X] Add thumbnail embed option [2021-10-16]
+- [X] Create a github wiki for user documentation [2021-10-03]
+- [X] Grid and list view for both channel and video list pages [2021-10-03]
+- [X] Un-ignore videos [2021-10-03]
+- [X] Dynamic download queue [2021-09-26]
+- [X] Backup and restore [2021-09-22]
+- [X] Scan your file system to index already downloaded videos [2021-09-14]
+
+## User Scripts
+This is a list of useful user scripts, generously created from folks like you to extend this project and its functionality. Make sure to check the respective repository links for detailed license information.  
+
+This is your time to shine, [read this](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#user-scripts) then open a PR to add your script here.
+
+- [danieljue/ta_dl_page_script](https://github.com/danieljue/ta_dl_page_script): Helper browser script to prioritize a channels' videos in download queue.
+- [dot-mike/ta-scripts](https://github.com/dot-mike/ta-scripts): A collection of personal scripts for managing TubeArchivist.
+- [DarkFighterLuke/ta_base_url_nginx](https://gist.github.com/DarkFighterLuke/4561b6bfbf83720493dc59171c58ac36): Set base URL with Nginx when you can't use subdomains.
+- [lamusmaser/ta_migration_helper](https://github.com/lamusmaser/ta_migration_helper): Advanced helper script for migration issues to TubeArchivist v0.4.4 or later.
+- [lamusmaser/create_info_json](https://gist.github.com/lamusmaser/837fb58f73ea0cad784a33497932e0dd): Script to generate `.info.json` files using `ffmpeg` collecting information from downloaded videos.
+- [lamusmaser/ta_fix_for_video_redirection](https://github.com/lamusmaser/ta_fix_for_video_redirection): Script to fix videos that were incorrectly indexed by YouTube's "Video is Unavailable" response. 
+- [RoninTech/ta-helper](https://github.com/RoninTech/ta-helper): Helper script to provide a symlink association to reference TubeArchivist videos with their original titles.
+
+## Donate
+The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.  
+Second best way to support the development is to provide for caffeinated beverages:
+* [GitHub Sponsor](https://github.com/sponsors/bbilly1) become a sponsor here on GitHub
+* [Paypal.me](https://paypal.me/bbilly1) for a one time coffee
+* [Paypal Subscription](https://www.paypal.com/webapps/billing/plans/subscribe?plan_id=P-03770005GR991451KMFGVPMQ) for a monthly coffee
+* [ko-fi.com](https://ko-fi.com/bbilly1) for an alternative platform
+
+## Notable mentions
+This is a selection of places where this project has been featured on reddit, in the news, blogs or any other online media, newest on top.
+* **ycombinator**: Tube Archivist on Hackernews front page, [2023-07-16][[link](https://news.ycombinator.com/item?id=36744395)]
+* **linux-community.de**: Tube Archivist bringt Ordnung in die Youtube-Sammlung, [German][2023-05-01][[link](https://www.linux-community.de/ausgaben/linuxuser/2023/05/tube-archivist-bringt-ordnung-in-die-youtube-sammlung/)]
+* **noted.lol**: Dev Debrief, An Interview With the Developer of Tube Archivist, [2023-03-30] [[link](https://noted.lol/dev-debrief-tube-archivist/)]
+* **console.substack.com**: Interview With Simon of Tube Archivist, [2023-01-29] [[link](https://console.substack.com/p/console-142#%C2%A7interview-with-simon-of-tube-archivist)]
+* **reddit.com**: Tube Archivist v0.3.0 - Now Archiving Comments, [2022-12-02] [[link](https://www.reddit.com/r/selfhosted/comments/zaonzp/tube_archivist_v030_now_archiving_comments/)]
+* **reddit.com**: Tube Archivist v0.2 - Now with Full Text Search, [2022-07-24] [[link](https://www.reddit.com/r/selfhosted/comments/w6jfa1/tube_archivist_v02_now_with_full_text_search/)]
+* **noted.lol**: How I Control What Media My Kids Watch Using Tube Archivist, [2022-03-27] [[link](https://noted.lol/how-i-control-what-media-my-kids-watch-using-tube-archivist/)]
+* **thehomelab.wiki**: Tube Archivist - A Youtube-DL Alternative on Steroids, [2022-01-27] [[link](https://thehomelab.wiki/books/news/page/tube-archivist-a-youtube-dl-alternative-on-steroids)]
+* **reddit.com**: Celebrating TubeArchivist v0.1, [2022-01-09] [[link](https://www.reddit.com/r/selfhosted/comments/rzh084/celebrating_tubearchivist_v01/)]
+* **linuxunplugged.com**: Pick: tubearchivist — Your self-hosted YouTube media server, [2021-09-11] [[link](https://linuxunplugged.com/425)] and [2021-10-05] [[link](https://linuxunplugged.com/426)]
+* **reddit.com**: Introducing Tube Archivist, your self hosted Youtube media server, [2021-09-12] [[link](https://www.reddit.com/r/selfhosted/comments/pmj07b/introducing_tube_archivist_your_self_hosted/)]
+
+
+## Sponsor
+Big thank you to [Digitalocean](https://www.digitalocean.com/) for generously donating credit for the tubearchivist.com VPS and buildserver. 
+<p>
+  <a href="https://www.digitalocean.com/">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/PoweredByDO/DO_Powered_by_Badge_blue.svg" width="201px">
+  </a>
+</p>
+

SHOWCASE.MD

@@ -0,0 +1,25 @@
+## Tube Archivist on YouTube
+[![ibracorp-youtube-video-thumb](assets/tube-archivist-ibracorp-O8H8Z01c0Ys.jpg)](https://www.youtube.com/watch?v=O8H8Z01c0Ys)
+Video featuring Tube Archivist generously created by [IBRACORP](https://www.youtube.com/@IBRACORP).
+
+## Screenshots
+![login screenshot](assets/tube-archivist-login.png?raw=true "Tube Archivist Login")  
+*Login Page*: Secure way to access your media collection.
+
+![home screenshot](assets/tube-archivist-home.png?raw=true "Tube Archivist Home")  
+*Home Page*: Your recent videos, continue watching incomplete videos.
+
+![channels screenshot](assets/tube-archivist-channels.png?raw=true "Tube Archivist Channels")  
+*All Channels*: A list of all your indexed channels, filtered by subscribed only.
+
+![single channel screenshot](assets/tube-archivist-single-channel.png?raw=true "Tube Archivist Single Channel")  
+*Single Channel*: Single channel page with additional metadata and sub pages.
+
+![video page screenshot](assets/tube-archivist-video.png?raw=true "Tube Archivist Video Page")  
+*Video Page*: Stream your video directly from the interface.
+
+![video page screenshot](assets/tube-archivist-download.png?raw=true "Tube Archivist Video Page")  
+*Downloads Page*: Add, control, and monitor your download queue.
+
+![search page screenshot](assets/tube-archivist-search.png?raw=true "Tube Archivist Search Page")  
+*Search Page*. Use expressions to quickly search through your collection.

deploy.sh

@@ -3,79 +3,230 @@
 # deploy all needed project files to different servers:
 # test for local vm for testing
 # blackhole for local production
-# docker to publish
+# unstable to publish intermediate releases
+# docker to publish regular release
+
+# create builder:
+# docker buildx create --name tubearchivist
+# docker buildx use tubearchivist
+# docker buildx inspect --bootstrap
+
+# more details:
+# https://github.com/tubearchivist/tubearchivist/issues/6
 
 set -e
 
 function sync_blackhole {
 
-    # docker commands need sudo
     host="blackhole.local"
-
-    read -sp 'Password: ' remote_pw
-    export PASS=$remote_pw
     
     rsync -a --progress --delete-after \
         --exclude ".git" \
         --exclude ".gitignore" \
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
+        --exclude ".venv" \
         --exclude "db.sqlite3" \
+        --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
 
-    echo "$PASS" | ssh "$host" 'sudo -S docker build -t bbilly1/tubearchivist:latest tubearchivist 2>/dev/null'
-    echo "$PASS" | ssh "$host" 'sudo -S docker-compose up -d 2>/dev/null'
+    ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist'
+    ssh "$host" 'docker compose up -d'
 
 }
 
 function sync_test {
 
     # docker commands don't need sudo in testing vm
-    host="tubearchivist.local"
+    # pass argument to build for specific platform
 
+    host="tubearchivist.local"
+    # make base folder
+    ssh "$host" "mkdir -p docker"
+
+    # copy project files to build image
     rsync -a --progress --delete-after \
         --exclude ".git" \
         --exclude ".gitignore" \
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
+        --exclude "**/.pytest_cache/" \
+        --exclude ".venv" \
         --exclude "db.sqlite3" \
+        --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
 
-    rsync -r --progress --delete docker-compose.yml -e ssh "$host":docker
+    # copy default docker-compose file if not exist
+    rsync --progress --ignore-existing docker-compose.yml -e ssh "$host":docker
 
-    ssh "$host" 'docker build -t bbilly1/tubearchivist:latest tubearchivist'
-    ssh "$host" 'docker-compose -f docker/docker-compose.yml up -d'
+    if [[ $1 = "amd64" ]]; then
+        platform="linux/amd64"
+    elif [[ $1 = "arm64" ]]; then
+        platform="linux/arm64"
+    elif [[ $1 = "multi" ]]; then
+        platform="linux/amd64,linux/arm64"
+    else
+        platform="linux/amd64"
+    fi
 
-    ssh "$host" 'docker cp tubearchivist/tubearchivist/testing.sh tubearchivist:/app/testing.sh'
-    ssh "$host" 'docker exec tubearchivist chmod +x /app/testing.sh'
+    ssh "$host" "docker buildx build --build-arg INSTALL_DEBUG=1 --platform $platform -t bbilly1/tubearchivist:latest tubearchivist --load"
+    ssh "$host" 'docker compose -f docker/docker-compose.yml up -d'
 
 }
 
 
-function sync_docker {
+# run same tests and checks as with github action but locally
+# takes filename to validate as optional argument
+function validate {
+
+    if [[ $1 ]]; then
+        check_path="$1"
+    else
+        check_path="."
+    fi
+
+    echo "run validate on $check_path"
+
+    # note: this logic is duplicated in the `./github/workflows/lint_python.yml` config
+    # if you update this file, you should update that as well
+    echo "running black"
+    black --force-exclude "migrations/*" --diff --color --check -l 79 "$check_path"
+    echo "running codespell"
+    codespell --skip="./.git,./.venv,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
+    echo "running flake8"
+    flake8 "$check_path" --exclude "migrations,.venv" --count --max-complexity=10 \
+        --max-line-length=79 --show-source --statistics
+    echo "running isort"
+    isort --skip "migrations" --skip ".venv" --check-only --diff --profile black -l 79 "$check_path"
+    printf "    \n> all validations passed\n"
+
+}
+
+
+# update latest tag compatible es for set and forget
+function sync_latest_es {
+
+    VERSION=$(grep "bbilly1/tubearchivist-es" docker-compose.yml | awk '{print $NF}')
+    printf "\nsync new ES version %s\nContinue?\n" "$VERSION"
+    read -rn 1
 
     if [[ $(systemctl is-active docker) != 'active' ]]; then
         echo "starting docker"
         sudo systemctl start docker
     fi
 
-    sudo docker build -t bbilly1/tubearchivist:latest .
-    sudo docker push bbilly1/tubearchivist:latest
+    sudo docker image pull docker.elastic.co/elasticsearch/elasticsearch:"$VERSION"
+
+    sudo docker tag \
+        docker.elastic.co/elasticsearch/elasticsearch:"$VERSION" \
+        bbilly1/tubearchivist-es
+
+    sudo docker tag \
+        docker.elastic.co/elasticsearch/elasticsearch:"$VERSION" \
+        bbilly1/tubearchivist-es:"$VERSION"
+
+    sudo docker push bbilly1/tubearchivist-es
+    sudo docker push bbilly1/tubearchivist-es:"$VERSION"
 
 }
 
-# check package versions in requirements.txt for updates
-python version_check.py
+
+# publish unstable tag to docker
+function sync_unstable {
+
+    if [[ $(systemctl is-active docker) != 'active' ]]; then
+        echo "starting docker"
+        sudo systemctl start docker
+    fi
+
+    # start amd64 build
+    sudo docker buildx build \
+        --platform linux/amd64 \
+        -t bbilly1/tubearchivist:unstable --push .
+
+}
+
+
+# new function, sync only tag, build with build server
+function sync_docker {
+
+    # check things
+    if [[ $(git branch --show-current) != 'master' ]]; then
+        echo 'you are not on master, dummy!'
+        return
+    fi
+
+    echo "latest tags:"
+    git tag | tail -n 5 | sort -r
+
+    printf "\ncreate new version:\n"
+    read -r VERSION
+
+    echo "push new tag: $VERSION?"
+    read -rn 1
+
+    # create release tag
+    echo "commits since last version:"
+    git log "$(git describe --tags --abbrev=0)"..HEAD --oneline
+    git tag -a "$VERSION" -m "new release version $VERSION"
+    git push origin "$VERSION"
+
+}
+
+
+# old builder, sync tag, build and push locally
+function sync_docker_old {
+
+    # check things
+    if [[ $(git branch --show-current) != 'master' ]]; then
+        echo 'you are not on master, dummy!'
+        return
+    fi
+
+    if [[ $(systemctl is-active docker) != 'active' ]]; then
+        echo "starting docker"
+        sudo systemctl start docker
+    fi
+
+    echo "latest tags:"
+    git tag | tail -n 5 | sort -r
+
+    printf "\ncreate new version:\n"
+    read -r VERSION
+
+    echo "build and push $VERSION?"
+    read -rn 1
+
+    # start build
+    sudo docker buildx build \
+        --platform linux/amd64,linux/arm64 \
+        -t bbilly1/tubearchivist \
+        -t bbilly1/tubearchivist:unstable \
+        -t bbilly1/tubearchivist:"$VERSION" --push .
+
+    # create release tag
+    echo "commits since last version:"
+    git log "$(git describe --tags --abbrev=0)"..HEAD --oneline
+    git tag -a "$VERSION" -m "new release version $VERSION"
+    git push origin "$VERSION"
+
+}
 
 
 if [[ $1 == "blackhole" ]]; then
     sync_blackhole
 elif [[ $1 == "test" ]]; then
-    sync_test
+    sync_test "$2"
+elif [[ $1 == "validate" ]]; then
+    validate "$2"
 elif [[ $1 == "docker" ]]; then
     sync_docker
+elif [[ $1 == "unstable" ]]; then
+    sync_unstable
+elif [[ $1 == "es" ]]; then
+    sync_latest_es
 else
-    echo "valid options are: blackhole | test | docker"
+    echo "valid options are: blackhole | test | validate | docker | unstable | es"
 fi
 
 

docker-compose.yml

@@ -1,45 +1,65 @@
-version: '3.3'
+version: '3.5'
 
 services:
   tubearchivist:
     container_name: tubearchivist
-    restart: always
-    image: bbilly1/tubearchivist:latest
+    restart: unless-stopped
+    image: bbilly1/tubearchivist
     ports:
       - 8000:8000
     volumes:
-      - ./volumes/tubearchivist/media:/youtube
-      - ./volumes/tubearchivist/cache:/cache
+      - media:/youtube
+      - cache:/cache
     environment:
-      - ES_URL=http://archivist-es:9200
-      - REDIS_HOST=archivist-redis
+      - ES_URL=http://archivist-es:9200     # needs protocol e.g. http and port
+      - REDIS_HOST=archivist-redis          # don't add protocol
       - HOST_UID=1000
       - HOST_GID=1000
+      - TA_HOST=tubearchivist.local         # set your host name
+      - TA_USERNAME=tubearchivist           # your initial TA credentials
+      - TA_PASSWORD=verysecret              # your initial TA credentials
+      - ELASTIC_PASSWORD=verysecret         # set password for Elasticsearch
+      - TZ=America/New_York                 # set your time zone
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 2m
+      timeout: 10s
+      retries: 3
+      start_period: 30s
     depends_on:
       - archivist-es
       - archivist-redis
   archivist-redis:
-    image: redislabs/rejson:latest
+    image: redis/redis-stack-server
     container_name: archivist-redis
-    restart: always
-    ports:
-      - 6379:6379
+    restart: unless-stopped
+    expose:
+      - "6379"
     volumes:
-      - ./volumes/tubearchivist/redis:/data
+      - redis:/data
     depends_on:
       - archivist-es
   archivist-es:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.14.1
+    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.13.2
     container_name: archivist-es
-    restart: always
+    restart: unless-stopped
     environment:
+      - "ELASTIC_PASSWORD=verysecret"       # matching Elasticsearch password
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      - "xpack.security.enabled=true"
       - "discovery.type=single-node"
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "path.repo=/usr/share/elasticsearch/data/snapshot"
     ulimits:
       memlock:
         soft: -1
         hard: -1
     volumes:
-      - ./volumes/tubearchivist/es:/usr/share/elasticsearch/data
-    ports:
-      - 9200:9200
+      - es:/usr/share/elasticsearch/data    # check for permission error when using bind mount, see readme
+    expose:
+      - "9200"
+
+volumes:
+  media:
+  cache:
+  redis:
+  es:

docker_assets/ffmpeg_download.py

@@ -0,0 +1,71 @@
+"""
+ffmpeg link builder
+copied as into build step in Dockerfile
+"""
+
+import json
+import os
+import sys
+import tarfile
+import urllib.request
+from enum import Enum
+
+API_URL = "https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest"
+BINARIES = ["ffmpeg", "ffprobe"]
+
+
+class PlatformFilter(Enum):
+    """options"""
+
+    ARM64 = "linuxarm64"
+    AMD64 = "linux64"
+
+
+def get_assets():
+    """get all available assets from latest build"""
+    with urllib.request.urlopen(API_URL) as f:
+        all_links = json.loads(f.read().decode("utf-8"))
+
+    return all_links
+
+
+def pick_url(all_links, platform):
+    """pick url for platform"""
+    filter_by = PlatformFilter[platform.split("/")[1].upper()].value
+    options = [i for i in all_links["assets"] if filter_by in i["name"]]
+    if not options:
+        raise ValueError(f"no valid asset found for filter {filter_by}")
+
+    url_pick = options[0]["browser_download_url"]
+
+    return url_pick
+
+
+def download_extract(url):
+    """download and extract binaries"""
+    print("download file")
+    filename, _ = urllib.request.urlretrieve(url)
+    print("extract file")
+    with tarfile.open(filename, "r:xz") as tar:
+        for member in tar.getmembers():
+            member.name = os.path.basename(member.name)
+            if member.name in BINARIES:
+                print(f"extract {member.name}")
+                tar.extract(member, member.name)
+
+
+def main():
+    """entry point"""
+    args = sys.argv
+    if len(args) == 1:
+        platform = "linux/amd64"
+    else:
+        platform = args[1]
+
+    all_links = get_assets()
+    url = pick_url(all_links, platform)
+    download_extract(url)
+
+
+if __name__ == "__main__":
+    main()

docker_assets/nginx.conf

@@ -0,0 +1,33 @@
+server {
+
+    listen 8000;
+    
+    location /cache/videos/ {
+        auth_request /api/ping/;
+        alias /cache/videos/;
+    }
+    
+    location /cache/channels/ {
+        auth_request /api/ping/;
+        alias /cache/channels/;
+    }
+    
+    location /cache/playlists/ {
+        auth_request /api/ping/;
+        alias /cache/playlists/;
+    }
+    
+    location /media/ {
+        auth_request /api/ping/;
+        alias /youtube/;
+        types {
+            text/vtt vtt;
+        }
+    }
+    
+    location / {
+        include uwsgi_params;
+        uwsgi_pass localhost:8080;
+    }
+    
+}

docker_assets/run.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# startup script inside the container for tubearchivist
+
+set -e
+
+# django setup
+python manage.py migrate
+
+if [[ -z "$DJANGO_DEBUG" ]]; then
+    python manage.py collectstatic --noinput -c
+fi
+
+# ta setup
+python manage.py ta_envcheck
+python manage.py ta_connection
+python manage.py ta_startup
+
+# start all tasks
+nginx &
+celery -A home.celery worker --loglevel=INFO --max-tasks-per-child 10 &
+celery -A home beat --loglevel=INFO \
+    --scheduler django_celery_beat.schedulers:DatabaseScheduler &
+uwsgi --ini uwsgi.ini

docker_assets/uwsgi.ini

@@ -0,0 +1,11 @@
+[uwsgi]
+module = config.wsgi:application
+master = True
+pidfile = /tmp/project-master.pid
+vacuum = True
+max-requests = 5000
+socket = :8080
+buffer-size = 8192
+log-5xx = true
+log-4xx = true
+disable-logging = true

nginx.conf

@@ -1,18 +0,0 @@
-server {
-
-    listen 8000;
-
-    location /cache/ {
-        alias /cache/;
-    }
-
-    location /media/ {
-        alias /youtube/;
-    }
-
-    location / {
-        include uwsgi_params;
-        uwsgi_pass localhost:8080;
-    }
-
-}

package.json

@@ -0,0 +1,17 @@
+{
+  "private": true,
+  "scripts": {
+    "lint": "eslint 'tubearchivist/static/**/*.js'",
+    "format": "prettier --write 'tubearchivist/static/**/*.js'"
+  },
+  "devDependencies": {
+    "eslint": "^8.26.0",
+    "prettier": "^2.7.1",
+    "eslint-config-prettier": "^8.5.0"
+  },
+  "prettier": {
+    "singleQuote": true,
+    "arrowParens": "avoid",
+    "printWidth": 100
+  }
+}

run.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-# startup script inside the container for tubearchivist
-
-counter=0
-until curl "$ES_URL" -fs; do
-    echo "waiting for elastic search to start"
-    counter=$((counter+1))
-    if [[ $counter -eq 12 ]]; then
-        # fail after 1 min
-        echo "failed to connect to elastic search, exiting..."
-        exit 1
-    fi
-    sleep 5
-done
-
-python manage.py migrate
-python manage.py collectstatic --noinput -c
-nginx &
-celery -A home.tasks worker --loglevel=INFO &
-uwsgi --ini uwsgi.ini

tubearchivist/api/README.md

@@ -0,0 +1,3 @@
+# TubeArchivist API
+
+All API documentation has moved to [docs.tubearchivist.com](https://docs.tubearchivist.com/).

tubearchivist/api/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin  # noqa: F401
+
+# Register your models here.

tubearchivist/api/apps.py

@@ -0,0 +1,10 @@
+"""apps file for api package"""
+
+from django.apps import AppConfig
+
+
+class ApiConfig(AppConfig):
+    """app config"""
+
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "api"

tubearchivist/api/models.py

@@ -0,0 +1,3 @@
+"""api models"""
+
+# from django.db import models

tubearchivist/api/src/aggs.py

@@ -0,0 +1,351 @@
+"""aggregations"""
+
+from home.src.es.connect import ElasticWrap
+from home.src.ta.helper import get_duration_str
+from home.src.ta.settings import EnvironmentSettings
+
+
+class AggBase:
+    """base class for aggregation calls"""
+
+    path: str = ""
+    data: dict = {}
+    name: str = ""
+
+    def get(self):
+        """make get call"""
+        response, _ = ElasticWrap(self.path).get(self.data)
+        print(f"[agg][{self.name}] took {response.get('took')} ms to process")
+
+        return response.get("aggregations")
+
+    def process(self):
+        """implement in subclassess"""
+        raise NotImplementedError
+
+
+class Video(AggBase):
+    """get video stats"""
+
+    name = "video_stats"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "video_type": {
+                "terms": {"field": "vid_type"},
+                "aggs": {
+                    "media_size": {"sum": {"field": "media_size"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                },
+            },
+            "video_active": {
+                "terms": {"field": "active"},
+                "aggs": {
+                    "media_size": {"sum": {"field": "media_size"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                },
+            },
+            "video_media_size": {"sum": {"field": "media_size"}},
+            "video_count": {"value_count": {"field": "youtube_id"}},
+            "duration": {"sum": {"field": "player.duration"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+
+        duration = int(aggregations["duration"]["value"])
+        response = {
+            "doc_count": aggregations["video_count"]["value"],
+            "media_size": int(aggregations["video_media_size"]["value"]),
+            "duration": duration,
+            "duration_str": get_duration_str(duration),
+        }
+        for bucket in aggregations["video_type"]["buckets"]:
+            duration = int(bucket["duration"].get("value"))
+            response.update(
+                {
+                    f"type_{bucket['key']}": {
+                        "doc_count": bucket.get("doc_count"),
+                        "media_size": int(bucket["media_size"].get("value")),
+                        "duration": duration,
+                        "duration_str": get_duration_str(duration),
+                    }
+                }
+            )
+
+        for bucket in aggregations["video_active"]["buckets"]:
+            duration = int(bucket["duration"].get("value"))
+            response.update(
+                {
+                    f"active_{bucket['key_as_string']}": {
+                        "doc_count": bucket.get("doc_count"),
+                        "media_size": int(bucket["media_size"].get("value")),
+                        "duration": duration,
+                        "duration_str": get_duration_str(duration),
+                    }
+                }
+            )
+
+        return response
+
+
+class Channel(AggBase):
+    """get channel stats"""
+
+    name = "channel_stats"
+    path = "ta_channel/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "channel_count": {"value_count": {"field": "channel_id"}},
+            "channel_active": {"terms": {"field": "channel_active"}},
+            "channel_subscribed": {"terms": {"field": "channel_subscribed"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+
+        response = {
+            "doc_count": aggregations["channel_count"].get("value"),
+        }
+        for bucket in aggregations["channel_active"]["buckets"]:
+            key = f"active_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+        for bucket in aggregations["channel_subscribed"]["buckets"]:
+            key = f"subscribed_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
+class Playlist(AggBase):
+    """get playlist stats"""
+
+    name = "playlist_stats"
+    path = "ta_playlist/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "playlist_count": {"value_count": {"field": "playlist_id"}},
+            "playlist_active": {"terms": {"field": "playlist_active"}},
+            "playlist_subscribed": {"terms": {"field": "playlist_subscribed"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+        response = {"doc_count": aggregations["playlist_count"].get("value")}
+        for bucket in aggregations["playlist_active"]["buckets"]:
+            key = f"active_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+        for bucket in aggregations["playlist_subscribed"]["buckets"]:
+            key = f"subscribed_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
+class Download(AggBase):
+    """get downloads queue stats"""
+
+    name = "download_queue_stats"
+    path = "ta_download/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "status": {"terms": {"field": "status"}},
+            "video_type": {
+                "filter": {"term": {"status": "pending"}},
+                "aggs": {"type_pending": {"terms": {"field": "vid_type"}}},
+            },
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+        response = {}
+        for bucket in aggregations["status"]["buckets"]:
+            response.update({bucket["key"]: bucket.get("doc_count")})
+
+        for bucket in aggregations["video_type"]["type_pending"]["buckets"]:
+            key = f"pending_{bucket['key']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
+class WatchProgress(AggBase):
+    """get watch progress"""
+
+    name = "watch_progress"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            name: {
+                "terms": {"field": "player.watched"},
+                "aggs": {
+                    "watch_docs": {
+                        "filter": {"terms": {"player.watched": [True, False]}},
+                        "aggs": {
+                            "true_count": {"value_count": {"field": "_index"}},
+                            "duration": {"sum": {"field": "player.duration"}},
+                        },
+                    },
+                },
+            },
+            "total_duration": {"sum": {"field": "player.duration"}},
+            "total_vids": {"value_count": {"field": "_index"}},
+        },
+    }
+
+    def process(self):
+        """make the call"""
+        aggregations = self.get()
+        buckets = aggregations[self.name]["buckets"]
+
+        response = {}
+        all_duration = int(aggregations["total_duration"].get("value"))
+        response.update(
+            {
+                "total": {
+                    "duration": all_duration,
+                    "duration_str": get_duration_str(all_duration),
+                    "items": aggregations["total_vids"].get("value"),
+                }
+            }
+        )
+
+        for bucket in buckets:
+            response.update(self._build_bucket(bucket, all_duration))
+
+        return response
+
+    @staticmethod
+    def _build_bucket(bucket, all_duration):
+        """parse bucket"""
+
+        duration = int(bucket["watch_docs"]["duration"]["value"])
+        duration_str = get_duration_str(duration)
+        items = bucket["watch_docs"]["true_count"]["value"]
+        if bucket["key_as_string"] == "false":
+            key = "unwatched"
+        else:
+            key = "watched"
+
+        bucket_parsed = {
+            key: {
+                "duration": duration,
+                "duration_str": duration_str,
+                "progress": duration / all_duration if all_duration else 0,
+                "items": items,
+            }
+        }
+
+        return bucket_parsed
+
+
+class DownloadHist(AggBase):
+    """get downloads histogram last week"""
+
+    name = "videos_last_week"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            name: {
+                "date_histogram": {
+                    "field": "date_downloaded",
+                    "calendar_interval": "day",
+                    "format": "yyyy-MM-dd",
+                    "order": {"_key": "desc"},
+                    "time_zone": EnvironmentSettings.TZ,
+                },
+                "aggs": {
+                    "total_videos": {"value_count": {"field": "youtube_id"}},
+                    "media_size": {"sum": {"field": "media_size"}},
+                },
+            }
+        },
+        "query": {
+            "range": {
+                "date_downloaded": {
+                    "gte": "now-7d/d",
+                    "time_zone": EnvironmentSettings.TZ,
+                }
+            }
+        },
+    }
+
+    def process(self):
+        """process query"""
+        aggregations = self.get()
+        buckets = aggregations[self.name]["buckets"]
+
+        response = [
+            {
+                "date": i.get("key_as_string"),
+                "count": i.get("doc_count"),
+                "media_size": i["media_size"].get("value"),
+            }
+            for i in buckets
+        ]
+
+        return response
+
+
+class BiggestChannel(AggBase):
+    """get channel aggregations"""
+
+    def __init__(self, order):
+        self.data["aggs"][self.name]["multi_terms"]["order"] = {order: "desc"}
+
+    name = "channel_stats"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            name: {
+                "multi_terms": {
+                    "terms": [
+                        {"field": "channel.channel_name.keyword"},
+                        {"field": "channel.channel_id"},
+                    ],
+                    "order": {"doc_count": "desc"},
+                },
+                "aggs": {
+                    "doc_count": {"value_count": {"field": "_index"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                    "media_size": {"sum": {"field": "media_size"}},
+                },
+            },
+        },
+    }
+    order_choices = ["doc_count", "duration", "media_size"]
+
+    def process(self):
+        """process aggregation, order_by validated in the view"""
+
+        aggregations = self.get()
+        buckets = aggregations[self.name]["buckets"]
+
+        response = [
+            {
+                "id": i["key"][1],
+                "name": i["key"][0].title(),
+                "doc_count": i["doc_count"]["value"],
+                "duration": i["duration"]["value"],
+                "duration_str": get_duration_str(int(i["duration"]["value"])),
+                "media_size": i["media_size"]["value"],
+            }
+            for i in buckets
+        ]
+
+        return response

tubearchivist/api/src/search_processor.py

@@ -0,0 +1,176 @@
+"""
+Functionality:
+- processing search results for frontend
+- this is duplicated code from home.src.frontend.searching.SearchHandler
+"""
+
+import urllib.parse
+
+from home.src.download.thumbnails import ThumbManager
+from home.src.ta.helper import date_parser, get_duration_str
+from home.src.ta.settings import EnvironmentSettings
+
+
+class SearchProcess:
+    """process search results"""
+
+    CACHE_DIR = EnvironmentSettings.CACHE_DIR
+
+    def __init__(self, response):
+        self.response = response
+        self.processed = False
+
+    def process(self):
+        """detect type and process"""
+        if "_source" in self.response.keys():
+            # single
+            self.processed = self._process_result(self.response)
+
+        elif "hits" in self.response.keys():
+            # multiple
+            self.processed = []
+            all_sources = self.response["hits"]["hits"]
+            for result in all_sources:
+                self.processed.append(self._process_result(result))
+
+        return self.processed
+
+    def _process_result(self, result):
+        """detect which type of data to process"""
+        index = result["_index"]
+        processed = False
+        if index == "ta_video":
+            processed = self._process_video(result["_source"])
+        if index == "ta_channel":
+            processed = self._process_channel(result["_source"])
+        if index == "ta_playlist":
+            processed = self._process_playlist(result["_source"])
+        if index == "ta_download":
+            processed = self._process_download(result["_source"])
+        if index == "ta_comment":
+            processed = self._process_comment(result["_source"])
+        if index == "ta_subtitle":
+            processed = self._process_subtitle(result)
+
+        if isinstance(processed, dict):
+            processed.update(
+                {
+                    "_index": index,
+                    "_score": round(result.get("_score") or 0, 2),
+                }
+            )
+
+        return processed
+
+    @staticmethod
+    def _process_channel(channel_dict):
+        """run on single channel"""
+        channel_id = channel_dict["channel_id"]
+        art_base = f"/cache/channels/{channel_id}"
+        date_str = date_parser(channel_dict["channel_last_refresh"])
+        channel_dict.update(
+            {
+                "channel_last_refresh": date_str,
+                "channel_banner_url": f"{art_base}_banner.jpg",
+                "channel_thumb_url": f"{art_base}_thumb.jpg",
+                "channel_tvart_url": f"{art_base}_tvart.jpg",
+            }
+        )
+
+        return dict(sorted(channel_dict.items()))
+
+    def _process_video(self, video_dict):
+        """run on single video dict"""
+        video_id = video_dict["youtube_id"]
+        media_url = urllib.parse.quote(video_dict["media_url"])
+        vid_last_refresh = date_parser(video_dict["vid_last_refresh"])
+        published = date_parser(video_dict["published"])
+        vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
+        channel = self._process_channel(video_dict["channel"])
+
+        if "subtitles" in video_dict:
+            for idx, _ in enumerate(video_dict["subtitles"]):
+                url = video_dict["subtitles"][idx]["media_url"]
+                video_dict["subtitles"][idx]["media_url"] = f"/media/{url}"
+
+        video_dict.update(
+            {
+                "channel": channel,
+                "media_url": f"/media/{media_url}",
+                "vid_last_refresh": vid_last_refresh,
+                "published": published,
+                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+            }
+        )
+
+        return dict(sorted(video_dict.items()))
+
+    @staticmethod
+    def _process_playlist(playlist_dict):
+        """run on single playlist dict"""
+        playlist_id = playlist_dict["playlist_id"]
+        playlist_last_refresh = date_parser(
+            playlist_dict["playlist_last_refresh"]
+        )
+        playlist_dict.update(
+            {
+                "playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg",
+                "playlist_last_refresh": playlist_last_refresh,
+            }
+        )
+
+        return dict(sorted(playlist_dict.items()))
+
+    def _process_download(self, download_dict):
+        """run on single download item"""
+        video_id = download_dict["youtube_id"]
+        vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
+        published = date_parser(download_dict["published"])
+
+        download_dict.update(
+            {
+                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+                "published": published,
+            }
+        )
+        return dict(sorted(download_dict.items()))
+
+    def _process_comment(self, comment_dict):
+        """run on all comments, create reply thread"""
+        all_comments = comment_dict["comment_comments"]
+        processed_comments = []
+
+        for comment in all_comments:
+            if comment["comment_parent"] == "root":
+                comment.update({"comment_replies": []})
+                processed_comments.append(comment)
+            else:
+                processed_comments[-1]["comment_replies"].append(comment)
+
+        return processed_comments
+
+    def _process_subtitle(self, result):
+        """take complete result dict to extract highlight"""
+        subtitle_dict = result["_source"]
+        highlight = result.get("highlight")
+        if highlight:
+            # replace lines with the highlighted markdown
+            subtitle_line = highlight.get("subtitle_line")[0]
+            subtitle_dict.update({"subtitle_line": subtitle_line})
+
+        thumb_path = ThumbManager(subtitle_dict["youtube_id"]).vid_thumb_path()
+        subtitle_dict.update({"vid_thumb_url": f"/cache/{thumb_path}"})
+
+        return subtitle_dict
+
+
+def process_aggs(response):
+    """convert aggs duration to str"""
+
+    if response.get("aggregations"):
+        aggs = response["aggregations"]
+        if "total_duration" in aggs:
+            duration_sec = int(aggs["total_duration"]["value"])
+            aggs["total_duration"].update(
+                {"value_str": get_duration_str(duration_sec)}
+            )

tubearchivist/api/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase  # noqa: F401
+
+# Create your tests here.

tubearchivist/api/urls.py

@@ -0,0 +1,199 @@
+"""all api urls"""
+
+from api import views
+from django.urls import path
+
+urlpatterns = [
+    path("ping/", views.PingView.as_view(), name="ping"),
+    path("login/", views.LoginApiView.as_view(), name="api-login"),
+    path(
+        "video/",
+        views.VideoApiListView.as_view(),
+        name="api-video-list",
+    ),
+    path(
+        "video/<slug:video_id>/",
+        views.VideoApiView.as_view(),
+        name="api-video",
+    ),
+    path(
+        "video/<slug:video_id>/progress/",
+        views.VideoProgressView.as_view(),
+        name="api-video-progress",
+    ),
+    path(
+        "video/<slug:video_id>/comment/",
+        views.VideoCommentView.as_view(),
+        name="api-video-comment",
+    ),
+    path(
+        "video/<slug:video_id>/similar/",
+        views.VideoSimilarView.as_view(),
+        name="api-video-similar",
+    ),
+    path(
+        "video/<slug:video_id>/sponsor/",
+        views.VideoSponsorView.as_view(),
+        name="api-video-sponsor",
+    ),
+    path(
+        "channel/",
+        views.ChannelApiListView.as_view(),
+        name="api-channel-list",
+    ),
+    path(
+        "channel/search/",
+        views.ChannelApiSearchView.as_view(),
+        name="api-channel-search",
+    ),
+    path(
+        "channel/<slug:channel_id>/",
+        views.ChannelApiView.as_view(),
+        name="api-channel",
+    ),
+    path(
+        "channel/<slug:channel_id>/video/",
+        views.ChannelApiVideoView.as_view(),
+        name="api-channel-video",
+    ),
+    path(
+        "playlist/",
+        views.PlaylistApiListView.as_view(),
+        name="api-playlist-list",
+    ),
+    path(
+        "playlist/<slug:playlist_id>/",
+        views.PlaylistApiView.as_view(),
+        name="api-playlist",
+    ),
+    path(
+        "playlist/<slug:playlist_id>/video/",
+        views.PlaylistApiVideoView.as_view(),
+        name="api-playlist-video",
+    ),
+    path(
+        "download/",
+        views.DownloadApiListView.as_view(),
+        name="api-download-list",
+    ),
+    path(
+        "download/<slug:video_id>/",
+        views.DownloadApiView.as_view(),
+        name="api-download",
+    ),
+    path(
+        "refresh/",
+        views.RefreshView.as_view(),
+        name="api-refresh",
+    ),
+    path(
+        "snapshot/",
+        views.SnapshotApiListView.as_view(),
+        name="api-snapshot-list",
+    ),
+    path(
+        "snapshot/<slug:snapshot_id>/",
+        views.SnapshotApiView.as_view(),
+        name="api-snapshot",
+    ),
+    path(
+        "backup/",
+        views.BackupApiListView.as_view(),
+        name="api-backup-list",
+    ),
+    path(
+        "backup/<str:filename>/",
+        views.BackupApiView.as_view(),
+        name="api-backup",
+    ),
+    path(
+        "task-name/",
+        views.TaskListView.as_view(),
+        name="api-task-list",
+    ),
+    path(
+        "task-name/<slug:task_name>/",
+        views.TaskNameListView.as_view(),
+        name="api-task-name-list",
+    ),
+    path(
+        "task-id/<slug:task_id>/",
+        views.TaskIDView.as_view(),
+        name="api-task-id",
+    ),
+    path(
+        "schedule/",
+        views.ScheduleView.as_view(),
+        name="api-schedule",
+    ),
+    path(
+        "schedule/notification/",
+        views.ScheduleNotification.as_view(),
+        name="api-schedule-notification",
+    ),
+    path(
+        "config/user/",
+        views.UserConfigView.as_view(),
+        name="api-config-user",
+    ),
+    path(
+        "cookie/",
+        views.CookieView.as_view(),
+        name="api-cookie",
+    ),
+    path(
+        "watched/",
+        views.WatchedView.as_view(),
+        name="api-watched",
+    ),
+    path(
+        "search/",
+        views.SearchView.as_view(),
+        name="api-search",
+    ),
+    path(
+        "token/",
+        views.TokenView.as_view(),
+        name="api-token",
+    ),
+    path(
+        "notification/",
+        views.NotificationView.as_view(),
+        name="api-notification",
+    ),
+    path(
+        "stats/video/",
+        views.StatVideoView.as_view(),
+        name="api-stats-video",
+    ),
+    path(
+        "stats/channel/",
+        views.StatChannelView.as_view(),
+        name="api-stats-channel",
+    ),
+    path(
+        "stats/playlist/",
+        views.StatPlaylistView.as_view(),
+        name="api-stats-playlist",
+    ),
+    path(
+        "stats/download/",
+        views.StatDownloadView.as_view(),
+        name="api-stats-download",
+    ),
+    path(
+        "stats/watch/",
+        views.StatWatchProgress.as_view(),
+        name="api-stats-watch",
+    ),
+    path(
+        "stats/downloadhist/",
+        views.StatDownloadHist.as_view(),
+        name="api-stats-downloadhist",
+    ),
+    path(
+        "stats/biggestchannels/",
+        views.StatBiggestChannel.as_view(),
+        name="api-stats-biggestchannels",
+    ),
+]

tubearchivist/config/asgi.py

@@ -11,6 +11,6 @@ import os
 
 from django.core.asgi import get_asgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
 
 application = get_asgi_application()

tubearchivist/config/management/commands/ta_connection.py

@@ -0,0 +1,163 @@
+"""
+Functionality:
+- check that all connections are working
+"""
+
+from time import sleep
+
+import requests
+from django.core.management.base import BaseCommand, CommandError
+from home.src.es.connect import ElasticWrap
+from home.src.ta.settings import EnvironmentSettings
+from home.src.ta.ta_redis import RedisArchivist
+
+TOPIC = """
+
+#######################
+#  Connection check   #
+#######################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    TIMEOUT = 120
+    MIN_MAJOR, MAX_MAJOR = 8, 8
+    MIN_MINOR = 0
+
+    # pylint: disable=no-member
+    help = "Check connections"
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(TOPIC)
+        self._redis_connection_check()
+        self._redis_config_set()
+        self._es_connection_check()
+        self._es_version_check()
+        self._es_path_check()
+
+    def _redis_connection_check(self):
+        """check ir redis connection is established"""
+        self.stdout.write("[1] connect to Redis")
+        redis_conn = RedisArchivist().conn
+        for _ in range(5):
+            try:
+                pong = redis_conn.execute_command("PING")
+                if pong:
+                    self.stdout.write(
+                        self.style.SUCCESS("    ✓ Redis connection verified")
+                    )
+                    return
+
+            except Exception:  # pylint: disable=broad-except
+                self.stdout.write("    ... retry Redis connection")
+                sleep(2)
+
+        message = "    🗙 Redis connection failed"
+        self.stdout.write(self.style.ERROR(f"{message}"))
+        RedisArchivist().exec("PING")
+        sleep(60)
+        raise CommandError(message)
+
+    def _redis_config_set(self):
+        """set config for redis if not set already"""
+        self.stdout.write("[2] set Redis config")
+        redis_conn = RedisArchivist().conn
+        timeout_is = int(redis_conn.config_get("timeout").get("timeout"))
+        if not timeout_is:
+            redis_conn.config_set("timeout", 3600)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ Redis config set"))
+
+    def _es_connection_check(self):
+        """wait for elasticsearch connection"""
+        self.stdout.write("[3] connect to Elastic Search")
+        total = self.TIMEOUT // 5
+        for i in range(total):
+            self.stdout.write(f"    ... waiting for ES [{i}/{total}]")
+            try:
+                _, status_code = ElasticWrap("/").get(
+                    timeout=1, print_error=False
+                )
+            except (
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout,
+            ):
+                sleep(5)
+                continue
+
+            if status_code and status_code == 200:
+                path = "_cluster/health?wait_for_status=yellow&timeout=60s"
+                _, _ = ElasticWrap(path).get(timeout=60)
+                self.stdout.write(
+                    self.style.SUCCESS("    ✓ ES connection established")
+                )
+                return
+
+        response, status_code = ElasticWrap("/").get(
+            timeout=1, print_error=False
+        )
+
+        message = "    🗙 ES connection failed"
+        self.stdout.write(self.style.ERROR(f"{message}"))
+        self.stdout.write(f"    error message: {response}")
+        self.stdout.write(f"    status code: {status_code}")
+        sleep(60)
+        raise CommandError(message)
+
+    def _es_version_check(self):
+        """check for minimal elasticsearch version"""
+        self.stdout.write("[4] Elastic Search version check")
+        response, _ = ElasticWrap("/").get()
+        version = response["version"]["number"]
+        major = int(version.split(".")[0])
+
+        if self.MIN_MAJOR <= major <= self.MAX_MAJOR:
+            self.stdout.write(
+                self.style.SUCCESS("    ✓ ES version check passed")
+            )
+            return
+
+        message = (
+            "    🗙 ES version check failed. "
+            + f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}"
+        )
+        self.stdout.write(self.style.ERROR(f"{message}"))
+        sleep(60)
+        raise CommandError(message)
+
+    def _es_path_check(self):
+        """check that path.repo var is set"""
+        self.stdout.write("[5] check ES path.repo env var")
+        response, _ = ElasticWrap("_nodes/_all/settings").get()
+        snaphost_roles = [
+            "data",
+            "data_cold",
+            "data_content",
+            "data_frozen",
+            "data_hot",
+            "data_warm",
+            "master",
+        ]
+        for node in response["nodes"].values():
+            if not (set(node["roles"]) & set(snaphost_roles)):
+                continue
+
+            if node["settings"]["path"].get("repo"):
+                self.stdout.write(
+                    self.style.SUCCESS("    ✓ path.repo env var is set")
+                )
+                return
+
+            message = (
+                "    🗙 path.repo env var not found. "
+                + "set the following env var to the ES container:\n"
+                + "    path.repo="
+                + EnvironmentSettings.ES_SNAPSHOT_DIR
+            )
+            self.stdout.write(self.style.ERROR(message))
+            sleep(60)
+            raise CommandError(message)

tubearchivist/config/management/commands/ta_envcheck.py

@@ -0,0 +1,193 @@
+"""
+Functionality:
+- Check environment at startup
+- Process config file overwrites from env var
+- Stop startup on error
+- python management.py ta_envcheck
+"""
+
+import os
+import re
+
+from django.core.management.base import BaseCommand, CommandError
+from home.models import Account
+from home.src.ta.settings import EnvironmentSettings
+
+LOGO = """
+
+                         ....  .....
+                  ...'',;:cc,. .;::;;,'...
+               ..,;:cccllclc,  .:ccllllcc;,..
+            ..,:cllcc:;,'.',.  ....'',;ccllc:,..
+          ..;cllc:,'..                ...,:cccc:'.
+         .;cccc;..                        ..,:ccc:'.
+       .ckkkOkxollllllllllllc.      .,:::;.  .,cclc;
+      .:0MMMMMMMMMMMMMMMMMMMX:     .cNMMMWx.   .;clc:
+     .;lOXK0000KNMMMMX00000KO;     ;KMMMMMNl.   .;ccl:,.
+     .;:c:'.....kMMMNo........    'OMMMWMMMK:    '::;;'.
+   .......     .xMMMNl           .dWMMXdOMMMO'   ........
+   .:cc:;.     .xMMMNc          .lNMMNo.:XMMWx.    .:cl:.
+   .:llc,.     .:xxxd,          ;KMMMk. .oWMMNl.   .:llc'
+   .cll:.     .;:;;:::,.       'OMMMK:';''kWMMK:   .;llc,
+   .cll:.     .,;;;;;;,.     .,xWMMNl.:l:.;KMMMO'  .;llc'
+   .:llc.      .cOOOk;      .lKNMMWx..:l:..lNMMWx. .:llc'
+   .;lcc,.     .xMMMNc      :KMMMM0, .:lc. .xWMMNl.'ccl:.
+    .cllc.     .xMMMNc     'OMMMMXc...:lc...,0MMMKl:lcc,.
+    .,ccl:.    .xMMMNc    .xWMMMWo.,;;:lc;;;.cXMMMXdcc;.
+     .,clc:.   .xMMMNc   .lNMMMWk. .':clc:,. .dWMMW0o;.
+      .,clcc,. .ckkkx;   .okkkOx,    .';,.    'kKKK0l.
+       .':lcc:'.....      .  ..            ..,;cllc,.
+         .,cclc,....                     ....;clc;..
+          ..,:,..,c:'..              ...';:,..,:,.
+            ....:lcccc:;,'''.....'',;;:clllc,....
+               .'',;:cllllllccccclllllcc:,'..
+                   ...'',,;;;;;;;;;,''...
+                            .....
+
+"""
+
+TOPIC = """
+#######################
+#  Environment Setup  #
+#######################
+
+"""
+
+EXPECTED_ENV_VARS = [
+    "TA_USERNAME",
+    "TA_PASSWORD",
+    "ELASTIC_PASSWORD",
+    "ES_URL",
+    "TA_HOST",
+]
+INST = "https://github.com/tubearchivist/tubearchivist#installing-and-updating"
+NGINX = "/etc/nginx/sites-available/default"
+UWSGI = "/app/uwsgi.ini"
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+    help = "Check environment before startup"
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(LOGO)
+        self.stdout.write(TOPIC)
+        self._expected_vars()
+        self._elastic_user_overwrite()
+        self._ta_port_overwrite()
+        self._ta_uwsgi_overwrite()
+        self._enable_cast_overwrite()
+        self._create_superuser()
+
+    def _expected_vars(self):
+        """check if expected env vars are set"""
+        self.stdout.write("[1] checking expected env vars")
+        env = os.environ
+        for var in EXPECTED_ENV_VARS:
+            if not env.get(var):
+                message = f"    🗙 expected env var {var} not set\n    {INST}"
+                self.stdout.write(self.style.ERROR(message))
+                raise CommandError(message)
+
+        message = "    ✓ all expected env vars are set"
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _elastic_user_overwrite(self):
+        """check for ELASTIC_USER overwrite"""
+        self.stdout.write("[2] check ES user overwrite")
+        env = EnvironmentSettings.ES_USER
+        self.stdout.write(self.style.SUCCESS(f"    ✓ ES user is set to {env}"))
+
+    def _ta_port_overwrite(self):
+        """set TA_PORT overwrite for nginx"""
+        self.stdout.write("[3] check TA_PORT overwrite")
+        overwrite = EnvironmentSettings.TA_PORT
+        if not overwrite:
+            self.stdout.write(self.style.SUCCESS("    TA_PORT is not set"))
+            return
+
+        regex = re.compile(r"listen [0-9]{1,5}")
+        to_overwrite = f"listen {overwrite}"
+        changed = file_overwrite(NGINX, regex, to_overwrite)
+        if changed:
+            message = f"    ✓ TA_PORT changed to {overwrite}"
+        else:
+            message = f"    ✓ TA_PORT already set to {overwrite}"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _ta_uwsgi_overwrite(self):
+        """set TA_UWSGI_PORT overwrite"""
+        self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
+        overwrite = EnvironmentSettings.TA_UWSGI_PORT
+        if not overwrite:
+            message = "    TA_UWSGI_PORT is not set"
+            self.stdout.write(self.style.SUCCESS(message))
+            return
+
+        # nginx
+        regex = re.compile(r"uwsgi_pass localhost:[0-9]{1,5}")
+        to_overwrite = f"uwsgi_pass localhost:{overwrite}"
+        changed = file_overwrite(NGINX, regex, to_overwrite)
+
+        # uwsgi
+        regex = re.compile(r"socket = :[0-9]{1,5}")
+        to_overwrite = f"socket = :{overwrite}"
+        changed = file_overwrite(UWSGI, regex, to_overwrite)
+
+        if changed:
+            message = f"    ✓ TA_UWSGI_PORT changed to {overwrite}"
+        else:
+            message = f"    ✓ TA_UWSGI_PORT already set to {overwrite}"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _enable_cast_overwrite(self):
+        """cast workaround, remove auth for static files in nginx"""
+        self.stdout.write("[5] check ENABLE_CAST overwrite")
+        overwrite = EnvironmentSettings.ENABLE_CAST
+        if not overwrite:
+            self.stdout.write(self.style.SUCCESS("    ENABLE_CAST is not set"))
+            return
+
+        regex = re.compile(r"[^\S\r\n]*auth_request /api/ping/;\n")
+        changed = file_overwrite(NGINX, regex, "")
+        if changed:
+            message = "    ✓ process nginx to enable Cast"
+        else:
+            message = "    ✓ Cast is already enabled in nginx"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _create_superuser(self):
+        """create superuser if not exist"""
+        self.stdout.write("[6] create superuser")
+        is_created = Account.objects.filter(is_superuser=True)
+        if is_created:
+            message = "    superuser already created"
+            self.stdout.write(self.style.SUCCESS(message))
+            return
+
+        name = EnvironmentSettings.TA_USERNAME
+        password = EnvironmentSettings.TA_PASSWORD
+        Account.objects.create_superuser(name, password)
+        message = f"    ✓ new superuser with name {name} created"
+        self.stdout.write(self.style.SUCCESS(message))
+
+
+def file_overwrite(file_path, regex, overwrite):
+    """change file content from old to overwrite, return true when changed"""
+    with open(file_path, "r", encoding="utf-8") as f:
+        file_content = f.read()
+
+    changed = re.sub(regex, overwrite, file_content)
+    if changed == file_content:
+        return False
+
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(changed)
+
+    return True

tubearchivist/config/management/commands/ta_fix_channels.py

@@ -0,0 +1,44 @@
+"""
+channel fix for update from v0.4.7 to v0.4.8
+reindex channels with 0 subscriber count
+python manage.py ta_fix_channels
+"""
+
+from django.core.management.base import BaseCommand
+from home.src.es.connect import IndexPaginate
+from home.tasks import check_reindex
+
+
+class Command(BaseCommand):
+    """fix comment link"""
+
+    def handle(self, *args, **options):
+        """run command"""
+        self.stdout.write("reindex failed channels")
+        channels = self._get_channels()
+        if not channels:
+            self.stdout.write("did not find any failed channels")
+            return
+
+        self.stdout.write(f"add {len(channels)} channels(s) to queue")
+        to_reindex = {"channel": [i["channel_id"] for i in channels]}
+        check_reindex.delay(data=to_reindex)
+        self.stdout.write(self.style.SUCCESS("    ✓ task queued\n"))
+
+    def _get_channels(self):
+        """get failed channels"""
+        self.stdout.write("search for failed channels")
+        es_query = {
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {"channel_subs": {"value": 0}}},
+                        {"term": {"channel_active": {"value": True}}},
+                    ]
+                },
+            },
+            "_source": ["channel_id"],
+        }
+        channels = IndexPaginate("ta_channel", es_query).get_results()
+
+        return channels

tubearchivist/config/management/commands/ta_fix_comment_link.py

@@ -0,0 +1,76 @@
+"""
+comment link fix for update from v0.4.7 to v0.4.8
+scan your videos and comments to fix comment_count field
+python manage.py ta_fix_comment_link
+"""
+
+from django.core.management.base import BaseCommand, CommandError
+from home.src.es.connect import ElasticWrap, IndexPaginate
+
+
+class Command(BaseCommand):
+    """fix comment link"""
+
+    def handle(self, *args, **options):
+        """run command"""
+        self.stdout.write("run comment link fix")
+        expected_count = self._get_comment_indexed()
+        all_videos = self._get_videos()
+
+        self.stdout.write(f"checking {len(all_videos)} video(s)")
+        videos_updated = []
+        for video in all_videos:
+            video_id = video["youtube_id"]
+            comment_count = expected_count.get(video_id)
+            if not comment_count:
+                continue
+
+            data = {"doc": {"comment_count": comment_count}}
+            path = f"ta_video/_update/{video_id}"
+            response, status_code = ElasticWrap(path).post(data=data)
+
+            if status_code != 200:
+                message = (
+                    "failed to add comment count to video"
+                    + f"response code: {status_code}"
+                    + response
+                )
+                raise CommandError(message)
+
+            videos_updated.append(video_id)
+
+        self.stdout.write(f"fixed {len(videos_updated)} video(s)")
+        self.stdout.write(self.style.SUCCESS("    ✓ task completed\n"))
+
+    def _get_comment_indexed(self):
+        """get comment count by index"""
+        self.stdout.write("get comments")
+        src = "params['_source']['comment_comments'].length"
+        data = {
+            "script_fields": {
+                "comments_length": {
+                    "script": {"source": src, "lang": "painless"}
+                }
+            }
+        }
+        all_comments = IndexPaginate(
+            "ta_comment", data=data, keep_source=True
+        ).get_results()
+
+        expected_count = {
+            i["_id"]: i["fields"]["comments_length"][0] for i in all_comments
+        }
+
+        return expected_count
+
+    def _get_videos(self):
+        """get videos without comment_count"""
+        self.stdout.write("get videos")
+        data = {
+            "query": {
+                "bool": {"must_not": [{"exists": {"field": "comment_count"}}]}
+            }
+        }
+        all_videos = IndexPaginate("ta_video", data).get_results()
+
+        return all_videos

tubearchivist/config/management/commands/ta_migpath.py

@@ -0,0 +1,185 @@
+"""
+filepath migration from v0.3.6 to v0.3.7
+not getting called at startup any more, to run manually if needed:
+python manage.py ta_migpath
+"""
+
+import json
+import os
+import shutil
+
+from django.core.management.base import BaseCommand
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.ta.helper import ignore_filelist
+from home.src.ta.settings import EnvironmentSettings
+
+TOPIC = """
+
+########################
+# Filesystem Migration #
+########################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """run commands"""
+        self.stdout.write(TOPIC)
+
+        handler = FolderMigration()
+        to_migrate = handler.get_to_migrate()
+        if not to_migrate:
+            self.stdout.write(
+                self.style.SUCCESS("    no channel migration needed\n")
+            )
+            return
+
+        self.stdout.write(self.style.SUCCESS("    migrating channels"))
+        total_channels = handler.create_folders(to_migrate)
+        self.stdout.write(
+            self.style.SUCCESS(f"    created {total_channels} channels")
+        )
+
+        self.stdout.write(
+            self.style.SUCCESS(f"    migrating {len(to_migrate)} videos")
+        )
+        handler.migrate_videos(to_migrate)
+        self.stdout.write(self.style.SUCCESS("    update videos in index"))
+        handler.send_bulk()
+
+        self.stdout.write(self.style.SUCCESS("    cleanup old folders"))
+        handler.delete_old()
+
+        self.stdout.write(self.style.SUCCESS("    ✓ migration completed\n"))
+
+
+class FolderMigration:
+    """migrate video archive folder"""
+
+    def __init__(self):
+        self.videos = EnvironmentSettings.MEDIA_DIR
+        self.bulk_list = []
+
+    def get_to_migrate(self):
+        """get videos to migrate"""
+        script = (
+            "doc['media_url'].value == "
+            + "doc['channel.channel_id'].value + '/'"
+            + " + doc['youtube_id'].value + '.mp4'"
+        )
+        data = {
+            "query": {"bool": {"must_not": [{"script": {"script": script}}]}},
+            "_source": [
+                "youtube_id",
+                "media_url",
+                "channel.channel_id",
+                "subtitles",
+            ],
+        }
+        response = IndexPaginate("ta_video", data).get_results()
+
+        return response
+
+    def create_folders(self, to_migrate):
+        """create required channel folders"""
+        host_uid = EnvironmentSettings.HOST_UID
+        host_gid = EnvironmentSettings.HOST_GID
+        all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate}
+
+        for channel_id in all_channel_ids:
+            new_folder = os.path.join(self.videos, channel_id)
+            os.makedirs(new_folder, exist_ok=True)
+            if host_uid and host_gid:
+                os.chown(new_folder, host_uid, host_gid)
+
+        return len(all_channel_ids)
+
+    def migrate_videos(self, to_migrate):
+        """migrate all videos of channel"""
+        total = len(to_migrate)
+        for idx, video in enumerate(to_migrate):
+            new_media_url = self._move_video_file(video)
+            if not new_media_url:
+                continue
+
+            all_subtitles = self._move_subtitles(video)
+            action = {
+                "update": {"_id": video["youtube_id"], "_index": "ta_video"}
+            }
+            source = {"doc": {"media_url": new_media_url}}
+            if all_subtitles:
+                source["doc"].update({"subtitles": all_subtitles})
+
+            self.bulk_list.append(json.dumps(action))
+            self.bulk_list.append(json.dumps(source))
+            if idx % 1000 == 0:
+                print(f"processing migration [{idx}/{total}]")
+                self.send_bulk()
+
+    def _move_video_file(self, video):
+        """move video file to new location"""
+        old_path = os.path.join(self.videos, video["media_url"])
+        if not os.path.exists(old_path):
+            print(f"did not find expected video at {old_path}")
+            return False
+
+        new_media_url = os.path.join(
+            video["channel"]["channel_id"], video["youtube_id"] + ".mp4"
+        )
+        new_path = os.path.join(self.videos, new_media_url)
+        os.rename(old_path, new_path)
+
+        return new_media_url
+
+    def _move_subtitles(self, video):
+        """move subtitle files to new location"""
+        all_subtitles = video.get("subtitles")
+        if not all_subtitles:
+            return False
+
+        for subtitle in all_subtitles:
+            old_path = os.path.join(self.videos, subtitle["media_url"])
+            if not os.path.exists(old_path):
+                print(f"did not find expected subtitle at {old_path}")
+                continue
+
+            new_media_url = os.path.join(
+                video["channel"]["channel_id"],
+                f"{video.get('youtube_id')}.{subtitle.get('lang')}.vtt",
+            )
+            new_path = os.path.join(self.videos, new_media_url)
+            os.rename(old_path, new_path)
+            subtitle["media_url"] = new_media_url
+
+        return all_subtitles
+
+    def send_bulk(self):
+        """send bulk request to update index with new urls"""
+        if not self.bulk_list:
+            print("nothing to update")
+            return
+
+        self.bulk_list.append("\n")
+        path = "_bulk?refresh=true"
+        data = "\n".join(self.bulk_list)
+        response, status = ElasticWrap(path).post(data=data, ndjson=True)
+        if not status == 200:
+            print(response)
+
+        self.bulk_list = []
+
+    def delete_old(self):
+        """delete old empty folders"""
+        all_folders = ignore_filelist(os.listdir(self.videos))
+        for folder in all_folders:
+            folder_path = os.path.join(self.videos, folder)
+            if not os.path.isdir(folder_path):
+                continue
+
+            if not ignore_filelist(os.listdir(folder_path)):
+                shutil.rmtree(folder_path)

tubearchivist/config/management/commands/ta_startup.py

@@ -0,0 +1,417 @@
+"""
+Functionality:
+- Application startup
+- Apply migrations
+"""
+
+import os
+from datetime import datetime
+from random import randint
+from time import sleep
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import dateformat
+from django_celery_beat.models import CrontabSchedule, PeriodicTasks
+from home.models import CustomPeriodicTask
+from home.src.es.connect import ElasticWrap
+from home.src.es.index_setup import ElasitIndexWrap
+from home.src.es.snapshot import ElasticSnapshot
+from home.src.ta.config import AppConfig, ReleaseVersion
+from home.src.ta.config_schedule import ScheduleBuilder
+from home.src.ta.helper import clear_dl_cache
+from home.src.ta.notify import Notifications
+from home.src.ta.settings import EnvironmentSettings
+from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.task_config import TASK_CONFIG
+from home.src.ta.task_manager import TaskManager
+from home.tasks import version_check
+
+TOPIC = """
+
+#######################
+#  Application Start  #
+#######################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(TOPIC)
+        self._sync_redis_state()
+        self._make_folders()
+        self._clear_redis_keys()
+        self._clear_tasks()
+        self._clear_dl_cache()
+        self._version_check()
+        self._mig_index_setup()
+        self._mig_snapshot_check()
+        self._mig_schedule_store()
+        self._mig_custom_playlist()
+        self._mig_add_missing_timestamp()
+        self._create_default_schedules()
+        self._update_schedule_tz()
+
+    def _sync_redis_state(self):
+        """make sure redis gets new config.json values"""
+        self.stdout.write("[1] set new config.json values")
+        needs_update = AppConfig().load_new_defaults()
+        if needs_update:
+            self.stdout.write(
+                self.style.SUCCESS("    ✓ new config values set")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no new config values"))
+
+    def _make_folders(self):
+        """make expected cache folders"""
+        self.stdout.write("[2] create expected cache folders")
+        folders = [
+            "backup",
+            "channels",
+            "download",
+            "import",
+            "playlists",
+            "videos",
+        ]
+        cache_dir = EnvironmentSettings.CACHE_DIR
+        for folder in folders:
+            folder_path = os.path.join(cache_dir, folder)
+            os.makedirs(folder_path, exist_ok=True)
+
+        self.stdout.write(self.style.SUCCESS("    ✓ expected folders created"))
+
+    def _clear_redis_keys(self):
+        """make sure there are no leftover locks or keys set in redis"""
+        self.stdout.write("[3] clear leftover keys in redis")
+        all_keys = [
+            "dl_queue_id",
+            "dl_queue",
+            "downloading",
+            "manual_import",
+            "reindex",
+            "rescan",
+            "run_backup",
+            "startup_check",
+            "reindex:ta_video",
+            "reindex:ta_channel",
+            "reindex:ta_playlist",
+        ]
+
+        redis_con = RedisArchivist()
+        has_changed = False
+        for key in all_keys:
+            if redis_con.del_message(key):
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ cleared key {key}")
+                )
+                has_changed = True
+
+        if not has_changed:
+            self.stdout.write(self.style.SUCCESS("    no keys found"))
+
+    def _clear_tasks(self):
+        """clear tasks and messages"""
+        self.stdout.write("[4] clear task leftovers")
+        TaskManager().fail_pending()
+        redis_con = RedisArchivist()
+        to_delete = redis_con.list_keys("message:")
+        if to_delete:
+            for key in to_delete:
+                redis_con.del_message(key)
+
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {len(to_delete)} messages")
+            )
+
+    def _clear_dl_cache(self):
+        """clear leftover files from dl cache"""
+        self.stdout.write("[5] clear leftover files from dl cache")
+        leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
+        if leftover_files:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ cleared {leftover_files} files")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no files found"))
+
+    def _version_check(self):
+        """remove new release key if updated now"""
+        self.stdout.write("[6] check for first run after update")
+        new_version = ReleaseVersion().is_updated()
+        if new_version:
+            self.stdout.write(
+                self.style.SUCCESS(f"    ✓ update to {new_version} completed")
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS("    no new update found"))
+
+        version_task = CustomPeriodicTask.objects.filter(name="version_check")
+        if not version_task.exists():
+            return
+
+        if not version_task.first().last_run_at:
+            self.style.SUCCESS("    ✓ send initial version check task")
+            version_check.delay()
+
+    def _mig_index_setup(self):
+        """migration: validate index mappings"""
+        self.stdout.write("[MIGRATION] validate index mappings")
+        ElasitIndexWrap().setup()
+
+    def _mig_snapshot_check(self):
+        """migration setup snapshots"""
+        self.stdout.write("[MIGRATION] setup snapshots")
+        ElasticSnapshot().setup()
+
+    def _mig_schedule_store(self):
+        """
+        update from 0.4.7 to 0.4.8
+        migrate schedule task store to CustomCronSchedule
+        """
+        self.stdout.write("[MIGRATION] migrate schedule store")
+        config = AppConfig().config
+        current_schedules = config.get("scheduler")
+        if not current_schedules:
+            self.stdout.write(
+                self.style.SUCCESS("    no schedules to migrate")
+            )
+            return
+
+        self._mig_update_subscribed(current_schedules)
+        self._mig_download_pending(current_schedules)
+        self._mig_check_reindex(current_schedules)
+        self._mig_thumbnail_check(current_schedules)
+        self._mig_run_backup(current_schedules)
+        self._mig_version_check()
+
+        del config["scheduler"]
+        RedisArchivist().set_message("config", config, save=True)
+
+    def _mig_update_subscribed(self, current_schedules):
+        """create update_subscribed schedule"""
+        task_name = "update_subscribed"
+        update_subscribed_schedule = current_schedules.get(task_name)
+        if update_subscribed_schedule:
+            self._create_task(task_name, update_subscribed_schedule)
+
+        self._create_notifications(task_name, current_schedules)
+
+    def _mig_download_pending(self, current_schedules):
+        """create download_pending schedule"""
+        task_name = "download_pending"
+        download_pending_schedule = current_schedules.get(task_name)
+        if download_pending_schedule:
+            self._create_task(task_name, download_pending_schedule)
+
+        self._create_notifications(task_name, current_schedules)
+
+    def _mig_check_reindex(self, current_schedules):
+        """create check_reindex schedule"""
+        task_name = "check_reindex"
+        check_reindex_schedule = current_schedules.get(task_name)
+        if check_reindex_schedule:
+            task_config = {}
+            days = current_schedules.get("check_reindex_days")
+            if days:
+                task_config.update({"days": days})
+
+            self._create_task(
+                task_name,
+                check_reindex_schedule,
+                task_config=task_config,
+            )
+
+        self._create_notifications(task_name, current_schedules)
+
+    def _mig_thumbnail_check(self, current_schedules):
+        """create thumbnail_check schedule"""
+        thumbnail_check_schedule = current_schedules.get("thumbnail_check")
+        if thumbnail_check_schedule:
+            self._create_task("thumbnail_check", thumbnail_check_schedule)
+
+    def _mig_run_backup(self, current_schedules):
+        """create run_backup schedule"""
+        run_backup_schedule = current_schedules.get("run_backup")
+        if run_backup_schedule:
+            task_config = False
+            rotate = current_schedules.get("run_backup_rotate")
+            if rotate:
+                task_config = {"rotate": rotate}
+
+            self._create_task(
+                "run_backup", run_backup_schedule, task_config=task_config
+            )
+
+    def _mig_version_check(self):
+        """create version_check schedule"""
+        version_check_schedule = {
+            "minute": randint(0, 59),
+            "hour": randint(0, 23),
+            "day_of_week": "*",
+        }
+        self._create_task("version_check", version_check_schedule)
+
+    def _create_task(self, task_name, schedule, task_config=False):
+        """create task"""
+        description = TASK_CONFIG[task_name].get("title")
+        schedule, _ = CrontabSchedule.objects.get_or_create(**schedule)
+        schedule.timezone = settings.TIME_ZONE
+        schedule.save()
+
+        task, _ = CustomPeriodicTask.objects.get_or_create(
+            crontab=schedule,
+            name=task_name,
+            description=description,
+            task=task_name,
+        )
+        if task_config:
+            task.task_config = task_config
+            task.save()
+
+        self.stdout.write(
+            self.style.SUCCESS(f"    ✓ new task created: '{task}'")
+        )
+
+    def _create_notifications(self, task_name, current_schedules):
+        """migrate notifications of task"""
+        notifications = current_schedules.get(f"{task_name}_notify")
+        if not notifications:
+            return
+
+        urls = [i.strip() for i in notifications.split()]
+        if not urls:
+            return
+
+        self.stdout.write(
+            self.style.SUCCESS(f"    ✓ migrate notifications: '{urls}'")
+        )
+        handler = Notifications(task_name)
+        for url in urls:
+            handler.add_url(url)
+
+    def _mig_custom_playlist(self):
+        """add playlist_type for migration from v0.4.6 to v0.4.7"""
+        self.stdout.write("[MIGRATION] custom playlist")
+        data = {
+            "query": {
+                "bool": {"must_not": [{"exists": {"field": "playlist_type"}}]}
+            },
+            "script": {"source": "ctx._source['playlist_type'] = 'regular'"},
+        }
+        path = "ta_playlist/_update_by_query"
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code == 200:
+            updated = response.get("updated", 0)
+            if updated:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"    ✓ {updated} playlist_type updated in ta_playlist"
+                    )
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        "    no playlist_type needed updating in ta_playlist"
+                    )
+                )
+            return
+
+        message = "    🗙 ta_playlist playlist_type update failed"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)
+
+    def _mig_add_missing_timestamp(self) -> None:
+        """
+        add missing timestamp for versioncheck
+        migrate from v0.4.8 to v0.4.9
+        """
+        version_tasks = CustomPeriodicTask.objects.filter(name="version_check")
+        if not version_tasks.exists():
+            return
+
+        version_task = version_tasks.first()
+        if not version_task.last_run_at:
+            self.style.SUCCESS("    ✓ send initial version check task")
+            version_check.delay()
+            version_task.last_run_at = dateformat.make_aware(datetime.now())
+            version_task.save()
+
+    def _create_default_schedules(self) -> None:
+        """
+        create default schedules for new installations
+        needs to be called after _mig_schedule_store
+        """
+        self.stdout.write("[7] create initial schedules")
+        init_has_run = CustomPeriodicTask.objects.filter(
+            name="version_check"
+        ).exists()
+
+        if init_has_run:
+            self.stdout.write(
+                self.style.SUCCESS(
+                    "    schedule init already done, skipping..."
+                )
+            )
+            return
+
+        builder = ScheduleBuilder()
+        check_reindex = builder.get_set_task(
+            "check_reindex", schedule=builder.SCHEDULES["check_reindex"]
+        )
+        check_reindex.task_config.update({"days": 90})
+        check_reindex.last_run_at = dateformat.make_aware(datetime.now())
+        check_reindex.save()
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {check_reindex}"
+            )
+        )
+
+        thumbnail_check = builder.get_set_task(
+            "thumbnail_check", schedule=builder.SCHEDULES["thumbnail_check"]
+        )
+        thumbnail_check.last_run_at = dateformat.make_aware(datetime.now())
+        thumbnail_check.save()
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {thumbnail_check}"
+            )
+        )
+        daily_random = f"{randint(0, 59)} {randint(0, 23)} *"
+        version_check_task = builder.get_set_task(
+            "version_check", schedule=daily_random
+        )
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"    ✓ created new default schedule: {version_check_task}"
+            )
+        )
+        self.stdout.write(
+            self.style.SUCCESS("    ✓ all default schedules created")
+        )
+
+    def _update_schedule_tz(self) -> None:
+        """update timezone for Schedule instances"""
+        tz = EnvironmentSettings.TZ
+        to_update = CrontabSchedule.objects.exclude(timezone=tz)
+
+        if not to_update.exists():
+            self.stdout.write(
+                self.style.SUCCESS("    all schedules have correct TZ")
+            )
+            return
+
+        updated = to_update.update(timezone=tz)
+        self.stdout.write(
+            self.style.SUCCESS(f"    ✓ updated {updated} schedules to {tz}.")
+        )
+        PeriodicTasks.update_changed()

tubearchivist/config/settings.py

@@ -10,8 +10,15 @@ For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.2/ref/settings/
 """
 
+import hashlib
+from os import environ, path
 from pathlib import Path
-from os import environ
+
+import ldap
+from corsheaders.defaults import default_headers
+from django_auth_ldap.config import LDAPSearch
+from home.src.ta.helper import ta_host_parser
+from home.src.ta.settings import EnvironmentSettings
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -20,68 +27,168 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
 
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'Fvid^aUL6LohRZz*kZFvq85B&JW&kB9o*#jdzWsdWE8*XkCLR8'
+PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode())
+SECRET_KEY = PW_HASH.hexdigest()
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = bool(environ.get('DJANGO_DEBUG'))
-
-ALLOWED_HOSTS = ['*']
+DEBUG = bool(environ.get("DJANGO_DEBUG"))
 
+ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(
+    environ.get("TA_HOST", "localhost")
+)
 
 # Application definition
 
 INSTALLED_APPS = [
-    'home.apps.HomeConfig',
-    'django.contrib.admin',
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.messages',
-    'whitenoise.runserver_nostatic',
-    'django.contrib.staticfiles',
-    'django.contrib.humanize'
+    "django_celery_beat",
+    "home.apps.HomeConfig",
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "corsheaders",
+    "whitenoise.runserver_nostatic",
+    "django.contrib.staticfiles",
+    "django.contrib.humanize",
+    "rest_framework",
+    "rest_framework.authtoken",
+    "api",
+    "config",
 ]
 
 MIDDLEWARE = [
-    'django.middleware.security.SecurityMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'whitenoise.middleware.WhiteNoiseMiddleware',
-    'django.middleware.common.CommonMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "corsheaders.middleware.CorsMiddleware",
+    "whitenoise.middleware.WhiteNoiseMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "home.src.ta.health.HealthCheckMiddleware",
 ]
 
-ROOT_URLCONF = 'config.urls'
+ROOT_URLCONF = "config.urls"
 
 TEMPLATES = [
     {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [],
-        'APP_DIRS': True,
-        'OPTIONS': {
-            'context_processors': [
-                'django.template.context_processors.debug',
-                'django.template.context_processors.request',
-                'django.contrib.auth.context_processors.auth',
-                'django.contrib.messages.context_processors.messages',
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
             ],
         },
     },
 ]
 
-WSGI_APPLICATION = 'config.wsgi.application'
+WSGI_APPLICATION = "config.wsgi.application"
 
+if bool(environ.get("TA_LDAP")):
+    # pylint: disable=global-at-module-level
+    global AUTH_LDAP_SERVER_URI
+    AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
+
+    global AUTH_LDAP_BIND_DN
+    AUTH_LDAP_BIND_DN = environ.get("TA_LDAP_BIND_DN")
+
+    global AUTH_LDAP_BIND_PASSWORD
+    AUTH_LDAP_BIND_PASSWORD = environ.get("TA_LDAP_BIND_PASSWORD")
+
+    """
+    Since these are new environment variables, taking the opporunity to use
+    more accurate env names.
+    Given Names are *_technically_* different from Personal names, as people
+    who change their names have different given names and personal names,
+    and they go by personal names. Additionally, "LastName" is actually
+    incorrect for many cultures, such as Korea, where the
+    family name comes first, and the personal name comes last.
+
+    But we all know people are going to try to guess at these, so still want
+    to include names that people will guess, hence using first/last as well.
+    """
+
+    # Attribute mapping options
+
+    global AUTH_LDAP_USER_ATTR_MAP_USERNAME
+    AUTH_LDAP_USER_ATTR_MAP_USERNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_USERNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_UID")
+        or "uid"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME
+    AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_PERSONALNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_FIRSTNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_GIVENNAME")
+        or "givenName"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_SURNAME
+    AUTH_LDAP_USER_ATTR_MAP_SURNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_SURNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_LASTNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_FAMILYNAME")
+        or "sn"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_EMAIL
+    AUTH_LDAP_USER_ATTR_MAP_EMAIL = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_EMAIL")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_MAIL")
+        or "mail"
+    )
+
+    global AUTH_LDAP_USER_BASE
+    AUTH_LDAP_USER_BASE = environ.get("TA_LDAP_USER_BASE")
+
+    global AUTH_LDAP_USER_FILTER
+    AUTH_LDAP_USER_FILTER = environ.get("TA_LDAP_USER_FILTER")
+
+    global AUTH_LDAP_USER_SEARCH
+    # pylint: disable=no-member
+    AUTH_LDAP_USER_SEARCH = LDAPSearch(
+        AUTH_LDAP_USER_BASE,
+        ldap.SCOPE_SUBTREE,
+        "(&("
+        + AUTH_LDAP_USER_ATTR_MAP_USERNAME
+        + "=%(user)s)"
+        + AUTH_LDAP_USER_FILTER
+        + ")",
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP
+    AUTH_LDAP_USER_ATTR_MAP = {
+        "username": AUTH_LDAP_USER_ATTR_MAP_USERNAME,
+        "first_name": AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME,
+        "last_name": AUTH_LDAP_USER_ATTR_MAP_SURNAME,
+        "email": AUTH_LDAP_USER_ATTR_MAP_EMAIL,
+    }
+
+    if bool(environ.get("TA_LDAP_DISABLE_CERT_CHECK")):
+        global AUTH_LDAP_GLOBAL_OPTIONS
+        AUTH_LDAP_GLOBAL_OPTIONS = {
+            ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
+        }
+
+    AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
 
 # Database
 # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
 
+CACHE_DIR = EnvironmentSettings.CACHE_DIR
+DB_PATH = path.join(CACHE_DIR, "db.sqlite3")
 DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': BASE_DIR / 'db.sqlite3',
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": DB_PATH,
     }
 }
 
@@ -91,54 +198,82 @@ DATABASES = {
 
 AUTH_PASSWORD_VALIDATORS = [
     {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",  # noqa: E501
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",  # noqa: E501
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",  # noqa: E501
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",  # noqa: E501
     },
 ]
 
+AUTH_USER_MODEL = "home.Account"
+
+# Forward-auth authentication
+if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
+    TA_AUTH_PROXY_USERNAME_HEADER = (
+        environ.get("TA_AUTH_PROXY_USERNAME_HEADER") or "HTTP_REMOTE_USER"
+    )
+    TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")
+
+    MIDDLEWARE.append("home.src.ta.auth.HttpRemoteUserMiddleware")
+
+    AUTHENTICATION_BACKENDS = (
+        "django.contrib.auth.backends.RemoteUserBackend",
+    )
+
 
 # Internationalization
 # https://docs.djangoproject.com/en/3.2/topics/i18n/
 
-LANGUAGE_CODE = 'en-us'
-
-TIME_ZONE = 'UTC'
-
+LANGUAGE_CODE = "en-us"
+TIME_ZONE = EnvironmentSettings.TZ
 USE_I18N = True
-
 USE_L10N = True
-
 USE_TZ = True
 
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
 
-STATIC_URL = '/static/'
-
-# STATICFILES_DIRS = [
-#     str(BASE_DIR.joinpath('static')),
-#     '/cache/'
-# ]
-
-# STATIC_URL = '/static/'
-STATICFILES_DIRS = (str(BASE_DIR.joinpath('static')),)
-# MEDIA_ROOT = str(BASE_DIR.joinpath('media'))
-# MEDIA_URL = '/media/'
-
-STATIC_ROOT = str(BASE_DIR.joinpath('staticfiles'))
-
-STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
+STATIC_URL = "/static/"
+STATICFILES_DIRS = (str(BASE_DIR.joinpath("static")),)
+STATIC_ROOT = str(BASE_DIR.joinpath("staticfiles"))
+STORAGES = {
+    "staticfiles": {
+        "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage",
+    },
+}
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
 
-DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+LOGIN_URL = "/login/"
+LOGOUT_REDIRECT_URL = "/login/"
+
+# Cors needed for browser extension
+# background.js makes the request so HTTP_ORIGIN will be from extension
+if environ.get("DISABLE_CORS"):
+    # disable cors
+    CORS_ORIGIN_ALLOW_ALL = True
+else:
+    CORS_ALLOWED_ORIGIN_REGEXES = [
+        r"moz-extension://*",
+        r"chrome-extension://*",
+    ]
+    CORS_ALLOWED_ORIGINS = ["http://localhost:3000"]
+
+
+CORS_ALLOW_HEADERS = list(default_headers) + [
+    "mode",
+]
+
+# TA application settings
+TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
+TA_VERSION = "v0.4.9"

tubearchivist/config/urls.py

@@ -13,10 +13,12 @@ Including another URLconf
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
+
 from django.contrib import admin
-from django.urls import path, include
+from django.urls import include, path
 
 urlpatterns = [
-    path('', include('home.urls')),
-    path('admin/', admin.site.urls),
+    path("", include("home.urls")),
+    path("api/", include("api.urls")),
+    path("admin/", admin.site.urls),
 ]

tubearchivist/config/wsgi.py

@@ -11,6 +11,6 @@ import os
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
 
 application = get_wsgi_application()

tubearchivist/home/__init__.py

@@ -1,40 +1,7 @@
-""" handle startup """
+"""start celery app"""
 
-import os
+from __future__ import absolute_import, unicode_literals
 
-from home.src.config import AppConfig
-from home.src.helper import set_message
-from home.src.index_management import index_check
+from home.celery import app as celery_app
 
-from .tasks import app as celery_app
-
-
-def sync_redis_state():
-    """ make sure redis gets the config.json values """
-    print('sync redis')
-    config = AppConfig().config
-    sort_order = config['archive']['sort']
-    set_message('sort_order', sort_order, expire=False)
-    hide_watched = bool(int(config['archive']['hide_watched']))
-    set_message('hide_watched', hide_watched, expire=False)
-    show_subed_only = bool(int(config['archive']['show_subed_only']))
-    set_message('show_subed_only', show_subed_only, expire=False)
-
-
-def make_folders():
-    """ make needed cache folders here so docker doesn't mess it up """
-    folders = ['download', 'channels', 'videos', 'import', 'backup']
-    config = AppConfig().config
-    cache_dir = config['application']['cache_dir']
-    for folder in folders:
-        folder_path = os.path.join(cache_dir, folder)
-        try:
-            os.makedirs(folder_path)
-        except FileExistsError:
-            continue
-
-
-__all__ = ('celery_app',)
-make_folders()
-sync_redis_state()
-index_check()
+__all__ = ("celery_app",)

tubearchivist/home/admin.py

@@ -1,3 +1,46 @@
-from django.contrib import admin
+"""custom admin classes"""
 
-# Register your models here.
+from django.contrib import admin
+from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
+from django_celery_beat import models as BeatModels
+
+from .models import Account
+
+
+class HomeAdmin(BaseUserAdmin):
+    """register in admin page"""
+
+    list_display = ("name", "is_staff", "is_superuser")
+    list_filter = ("is_superuser",)
+
+    fieldsets = (
+        (None, {"fields": ("is_staff", "is_superuser", "password")}),
+        ("Personal info", {"fields": ("name",)}),
+        ("Groups", {"fields": ("groups",)}),
+        ("Permissions", {"fields": ("user_permissions",)}),
+    )
+    add_fieldsets = (
+        (
+            None,
+            {"fields": ("is_staff", "is_superuser", "password1", "password2")},
+        ),
+        ("Personal info", {"fields": ("name",)}),
+        ("Groups", {"fields": ("groups",)}),
+        ("Permissions", {"fields": ("user_permissions",)}),
+    )
+
+    search_fields = ("name",)
+    ordering = ("name",)
+    filter_horizontal = ()
+
+
+admin.site.register(Account, HomeAdmin)
+admin.site.unregister(
+    [
+        BeatModels.ClockedSchedule,
+        BeatModels.CrontabSchedule,
+        BeatModels.IntervalSchedule,
+        BeatModels.PeriodicTask,
+        BeatModels.SolarSchedule,
+    ]
+)

tubearchivist/home/apps.py

@@ -1,6 +1,10 @@
+"""handle custom startup functions"""
+
 from django.apps import AppConfig
 
 
 class HomeConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'home'
+    """call startup funcs"""
+
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "home"

tubearchivist/home/celery.py

@@ -0,0 +1,22 @@
+"""initiate celery"""
+
+import os
+
+from celery import Celery
+from home.src.ta.settings import EnvironmentSettings
+
+REDIS_HOST = EnvironmentSettings.REDIS_HOST
+REDIS_PORT = EnvironmentSettings.REDIS_PORT
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+app = Celery(
+    "tasks",
+    broker=f"redis://{REDIS_HOST}:{REDIS_PORT}",
+    backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
+    result_extended=True,
+)
+app.config_from_object(
+    "django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE
+)
+app.autodiscover_tasks()
+app.conf.timezone = EnvironmentSettings.TZ

tubearchivist/home/config.json

@@ -1,25 +1,30 @@
 {
-    "archive": {
-        "sort": "published",
-        "hide_watched": false,
-        "show_subed_only": false,
-        "page_size": 12
-    },
     "subscriptions": {
-        "auto_search": false,
-        "auto_download": false,
-        "channel_size": 50
+        "channel_size": 50,
+        "live_channel_size": 50,
+        "shorts_channel_size": 50,
+        "auto_start": false
     },
     "downloads": {
-        "limit_count": 5,
         "limit_speed": false,
         "sleep_interval": 3,
-        "format": false
+        "autodelete_days": false,
+        "format": false,
+        "format_sort": false,
+        "add_metadata": false,
+        "add_thumbnail": false,
+        "subtitle": false,
+        "subtitle_source": false,
+        "subtitle_index": false,
+        "comment_max": false,
+        "comment_sort": "top",
+        "cookie_import": false,
+        "throttledratelimit": false,
+        "extractor_lang": false,
+        "integrate_ryd": false,
+        "integrate_sponsorblock": false
     },
     "application": {
-        "cache_dir": "/cache",
-        "videos": "/youtube",
-        "file_template": "%(id)s_%(title)s.mp4",
-        "colors": "dark"
+        "enable_snapshot": true
     }
-}
+}

tubearchivist/home/migrations/0001_initial.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.1.5 on 2023-02-02 06:49
+
+from django.db import migrations, models
+import home.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('auth', '0012_alter_user_first_name_max_length'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Account',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('password', models.CharField(max_length=128, verbose_name='password')),
+                ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')),
+                ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')),
+                ('name', models.CharField(max_length=150, unique=True)),
+                ('is_staff', models.BooleanField(default=False)),
+                ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')),
+                ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')),
+            ],
+            options={
+                'abstract': False,
+            },
+            managers=[
+                ('objects', home.models.AccountManager()),
+            ],
+        ),
+    ]

tubearchivist/home/migrations/0002_customperiodictask.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.7 on 2023-12-05 13:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_celery_beat', '0018_improve_crontab_helptext'),
+        ('home', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CustomPeriodicTask',
+            fields=[
+                ('periodictask_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='django_celery_beat.periodictask')),
+                ('task_config', models.JSONField(default=dict)),
+            ],
+            bases=('django_celery_beat.periodictask',),
+        ),
+    ]

tubearchivist/home/models.py

@@ -1,3 +1,61 @@
-from django.db import models
+"""custom models"""
 
-# Create your models here.
+from django.contrib.auth.models import (
+    AbstractBaseUser,
+    BaseUserManager,
+    PermissionsMixin,
+)
+from django.db import models
+from django_celery_beat.models import PeriodicTask
+
+
+class AccountManager(BaseUserManager):
+    """manage user creation methods"""
+
+    use_in_migrations = True
+
+    def _create_user(self, name, password, **extra_fields):
+        """create regular user private"""
+        values = [name, password]
+        field_value_map = dict(zip(self.model.REQUIRED_FIELDS, values))
+        for field_name, value in field_value_map.items():
+            if not value:
+                raise ValueError(f"The {field_name} value must be set")
+
+        user = self.model(name=name, **extra_fields)
+        user.set_password(password)
+        user.save(using=self._db)
+        return user
+
+    def create_user(self, name, password):
+        """create regular user public"""
+        return self._create_user(name, password)
+
+    def create_superuser(self, name, password, **extra_fields):
+        """create super user"""
+        extra_fields.setdefault("is_staff", True)
+        extra_fields.setdefault("is_superuser", True)
+
+        if extra_fields.get("is_staff") is not True:
+            raise ValueError("Superuser must have is_staff=True.")
+        if extra_fields.get("is_superuser") is not True:
+            raise ValueError("Superuser must have is_superuser=True.")
+
+        return self._create_user(name, password, **extra_fields)
+
+
+class Account(AbstractBaseUser, PermissionsMixin):
+    """handle account creation"""
+
+    name = models.CharField(max_length=150, unique=True)
+    is_staff = models.BooleanField(default=False)
+    objects = AccountManager()
+
+    USERNAME_FIELD = "name"
+    REQUIRED_FIELDS = ["password"]
+
+
+class CustomPeriodicTask(PeriodicTask):
+    """add custom metadata to to task"""
+
+    task_config = models.JSONField(default=dict)

tubearchivist/home/src/config.py

@@ -1,60 +0,0 @@
-"""
-Functionality:
-- read and write config
-- load config variables into redis
-- needs to be a separate module to avoid circular import
-"""
-
-import json
-import os
-
-from home.src.helper import get_message, set_message
-
-
-class AppConfig:
-    """ handle user settings and application variables """
-
-    def __init__(self):
-        self.config = self.get_config()
-
-    def get_config(self):
-        """ get config from default file or redis if changed """
-        config = self.get_config_redis()
-        if not config:
-            with open('home/config.json', 'r', encoding="utf-8") as f:
-                config_str = f.read()
-                config = json.loads(config_str)
-
-        config['application']['REDIS_HOST'] = os.environ.get('REDIS_HOST')
-        config['application']['es_url'] = os.environ.get('ES_URL')
-        config['application']['HOST_UID'] = int(os.environ.get('HOST_UID'))
-        config['application']['HOST_GID'] = int(os.environ.get('HOST_GID'))
-        return config
-
-    @staticmethod
-    def get_config_redis():
-        """ read config json set from redis to overwrite defaults """
-        config = get_message('config')
-        if not list(config.values())[0]:
-            return False
-
-        return config
-
-    def update_config(self, form_post):
-        """ update config values from settings form """
-        config = self.config
-        for key, value in form_post.items():
-            to_write = value[0]
-            if len(to_write):
-                if to_write == '0':
-                    to_write = False
-                elif to_write.isdigit():
-                    to_write = int(to_write)
-
-                config_dict, config_value = key.split('.')
-                config[config_dict][config_value] = to_write
-
-        with open('home/config.json', 'w', encoding="utf-8") as f:
-            f.write(json.dumps(config))
-
-        set_message('config', config, expire=False)

tubearchivist/home/src/download.py

@@ -1,505 +0,0 @@
-"""
-Functionality:
-- handele the download queue
-- manage subscriptions to channels
-- downloading videos
-"""
-
-import json
-import shutil
-import os
-
-from datetime import datetime
-from time import sleep
-
-import requests
-import yt_dlp as youtube_dl
-
-from home.src.index import YoutubeChannel, index_new_video
-from home.src.config import AppConfig
-from home.src.helper import clean_string, DurationConverter, set_message
-
-
-class PendingList:
-    """ manage the pending videos list """
-
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG['application']['es_url']
-    VIDEOS = CONFIG['application']['videos']
-
-    @staticmethod
-    def parse_url_list(youtube_ids):
-        """ extract youtube ids from list """
-        missing_videos = []
-        for entry in youtube_ids:
-            # notify
-            mess_dict = {
-                "status": "pending",
-                "level": "info",
-                "title": "Adding to download queue.",
-                "message": 'Extracting lists'
-            }
-            set_message('progress:download', mess_dict)
-            # extract
-            url = entry['url']
-            url_type = entry['type']
-            if url_type == 'video':
-                missing_videos.append(url)
-            elif url_type == 'channel':
-                youtube_ids = ChannelSubscription().get_last_youtube_videos(
-                    url, limit=False
-                )
-                missing_videos = missing_videos + youtube_ids
-            elif url_type == 'playlist':
-                youtube_ids = playlist_extractor(url)
-                missing_videos = missing_videos + youtube_ids
-
-        return missing_videos
-
-    def add_to_pending(self, missing_videos):
-        """ build the bulk json data from pending """
-        # check if channel is indexed
-        channel_handler = ChannelSubscription()
-        all_indexed = channel_handler.get_channels(subscribed_only=False)
-        all_channel_ids = [i['channel_id'] for i in all_indexed]
-        # check if already there
-        all_downloaded = self.get_all_downloaded()
-        # loop
-        bulk_list = []
-        for video in missing_videos:
-            if isinstance(video, str):
-                youtube_id = video
-            elif isinstance(video, tuple):
-                youtube_id = video[0]
-            if youtube_id in all_downloaded:
-                # skip already downlaoded
-                continue
-            video = self.get_youtube_details(youtube_id)
-            # skip on download error
-            if not video:
-                continue
-
-            if video['channel_id'] in all_channel_ids:
-                video['channel_indexed'] = True
-            else:
-                video['channel_indexed'] = False
-            video['status'] = "pending"
-            action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(video))
-            # notify
-            mess_dict = {
-                "status": "pending",
-                "level": "info",
-                "title": "Adding to download queue.",
-                "message": 'Processing IDs...'
-            }
-            set_message('progress:download', mess_dict)
-        # add last newline
-        bulk_list.append('\n')
-        query_str = '\n'.join(bulk_list)
-        headers = {'Content-type': 'application/x-ndjson'}
-        url = self.ES_URL + '/_bulk'
-        request = requests.post(url, data=query_str, headers=headers)
-        if not request.ok:
-            print(request)
-
-    @staticmethod
-    def get_youtube_details(youtube_id):
-        """ get details from youtubedl for single pending video """
-        obs = {
-            'default_search': 'ytsearch',
-            'quiet': True,
-            'skip_download': True,
-        }
-        try:
-            vid = youtube_dl.YoutubeDL(obs).extract_info(youtube_id)
-        except youtube_dl.utils.DownloadError:
-            print('failed to extract info for: ' + youtube_id)
-            return False
-        # parse response
-        seconds = vid['duration']
-        duration_str = DurationConverter.get_str(seconds)
-        upload_date = vid['upload_date']
-        upload_dt = datetime.strptime(upload_date, "%Y%m%d")
-        published = upload_dt.strftime("%Y-%m-%d")
-        # build dict
-        youtube_details = {
-            "youtube_id": youtube_id,
-            "channel_name": vid['channel'],
-            "vid_thumb_url": vid['thumbnail'],
-            "title": vid['title'],
-            "channel_id": vid['channel_id'],
-            "duration": duration_str,
-            "published": published,
-            "timestamp": int(datetime.now().strftime("%s"))
-        }
-        return youtube_details
-
-    def get_all_pending(self):
-        """ get a list of all pending videos in ta_download """
-        headers = {'Content-type': 'application/json'}
-        # get PIT ID
-        url = self.ES_URL + '/ta_download/_pit?keep_alive=1m'
-        response = requests.post(url)
-        json_data = json.loads(response.text)
-        pit_id = json_data['id']
-        # query
-        data = {
-            "size": 50, "query": {"match_all": {}},
-            "pit": {"id": pit_id, "keep_alive": "1m"},
-            "sort": [{"timestamp": {"order": "desc"}}]
-        }
-        query_str = json.dumps(data)
-        url = self.ES_URL + '/_search'
-        all_pending = []
-        all_ignore = []
-        while True:
-            response = requests.get(url, data=query_str, headers=headers)
-            json_data = json.loads(response.text)
-            all_hits = json_data['hits']['hits']
-            if all_hits:
-                for hit in all_hits:
-                    youtube_id = hit['_source']['youtube_id']
-                    status = hit['_source']['status']
-                    if status == 'pending':
-                        all_pending.append(hit['_source'])
-                    elif status == 'ignore':
-                        all_ignore.append(youtube_id)
-                    search_after = hit['sort']
-                # update search_after with last hit data
-                data['search_after'] = search_after
-                query_str = json.dumps(data)
-            else:
-                break
-        # clean up PIT
-        query_str = json.dumps({"id": pit_id})
-        requests.delete(self.ES_URL + '/_pit', data=query_str, headers=headers)
-        return all_pending, all_ignore
-
-    def get_all_indexed(self):
-        """ get a list of all videos indexed """
-        headers = {'Content-type': 'application/json'}
-        # get PIT ID
-        url = self.ES_URL + '/ta_video/_pit?keep_alive=1m'
-        response = requests.post(url)
-        json_data = json.loads(response.text)
-        pit_id = json_data['id']
-        # query
-        data = {
-            "size": 500, "query": {"match_all": {}},
-            "pit": {"id": pit_id, "keep_alive": "1m"},
-            "sort": [{"published": {"order": "desc"}}]
-        }
-        query_str = json.dumps(data)
-        url = self.ES_URL + '/_search'
-        all_indexed = []
-        while True:
-            response = requests.get(url, data=query_str, headers=headers)
-            json_data = json.loads(response.text)
-            all_hits = json_data['hits']['hits']
-            if all_hits:
-                for hit in all_hits:
-                    all_indexed.append(hit)
-                    search_after = hit['sort']
-                # update search_after with last hit data
-                data['search_after'] = search_after
-                query_str = json.dumps(data)
-            else:
-                break
-        # clean up PIT
-        query_str = json.dumps({"id": pit_id})
-        requests.delete(self.ES_URL + '/_pit', data=query_str, headers=headers)
-        return all_indexed
-
-    def get_all_downloaded(self):
-        """ get a list of all videos in archive """
-        all_channel_folders = os.listdir(self.VIDEOS)
-        all_downloaded = []
-        for channel_folder in all_channel_folders:
-            channel_path = os.path.join(self.VIDEOS, channel_folder)
-            all_videos = os.listdir(channel_path)
-            youtube_vids = [i[9:20] for i in all_videos]
-            for youtube_id in youtube_vids:
-                all_downloaded.append(youtube_id)
-        return all_downloaded
-
-    def delete_from_pending(self, youtube_id):
-        """ delete the youtube_id from ta_download """
-        url = f'{self.ES_URL}/ta_download/_doc/{youtube_id}'
-        response = requests.delete(url)
-        if not response.ok:
-            print(response.text)
-
-    def ignore_from_pending(self, ignore_list):
-        """ build the bulk query string """
-
-        stamp = int(datetime.now().strftime("%s"))
-        bulk_list = []
-
-        for youtube_id in ignore_list:
-            action = {"update": {"_id": youtube_id, "_index": "ta_download"}}
-            source = {"doc": {"status": 'ignore', "timestamp": stamp}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(source))
-
-        # add last newline
-        bulk_list.append('\n')
-        query_str = '\n'.join(bulk_list)
-
-        headers = {'Content-type': 'application/x-ndjson'}
-        url = self.ES_URL + '/_bulk'
-        request = requests.post(url, data=query_str, headers=headers)
-        mess_dict = {
-            "status": "ignore",
-            "level": "info",
-            "title": "Added to ignore list",
-            "message": ''
-        }
-        set_message('progress:download', mess_dict)
-        if not request.ok:
-            print(request)
-
-
-class ChannelSubscription:
-    """ manage the list of channels subscribed """
-
-    def __init__(self):
-        config = AppConfig().config
-        self.es_url = config['application']['es_url']
-        self.channel_size = config['subscriptions']['channel_size']
-
-    def get_channels(self, subscribed_only=True):
-        """ get a list of all channels subscribed to """
-        headers = {'Content-type': 'application/json'}
-        # get PIT ID
-        url = self.es_url + '/ta_channel/_pit?keep_alive=1m'
-        response = requests.post(url)
-        json_data = json.loads(response.text)
-        pit_id = json_data['id']
-        # query
-        if subscribed_only:
-            data = {
-                "query": {"term": {"channel_subscribed": {"value": True}}},
-                "size": 50, "pit": {"id": pit_id, "keep_alive": "1m"},
-                "sort": [{"channel_name.keyword": {"order": "asc"}}]
-            }
-        else:
-            data = {
-                "query": {"match_all": {}},
-                "size": 50, "pit": {"id": pit_id, "keep_alive": "1m"},
-                "sort": [{"channel_name.keyword": {"order": "asc"}}]
-            }
-        query_str = json.dumps(data)
-        url = self.es_url + '/_search'
-        all_channels = []
-        while True:
-            response = requests.get(url, data=query_str, headers=headers)
-            json_data = json.loads(response.text)
-            all_hits = json_data['hits']['hits']
-            if all_hits:
-                for hit in all_hits:
-                    source = hit['_source']
-                    search_after = hit['sort']
-                    all_channels.append(source)
-                # update search_after with last hit data
-                data['search_after'] = search_after
-                query_str = json.dumps(data)
-            else:
-                break
-        # clean up PIT
-        query_str = json.dumps({"id": pit_id})
-        requests.delete(self.es_url + '/_pit', data=query_str, headers=headers)
-        return all_channels
-
-    def get_last_youtube_videos(self, channel_id, limit=True):
-        """ get a list of last videos from channel """
-        url = f'https://www.youtube.com/channel/{channel_id}/videos'
-        obs = {
-            'default_search': 'ytsearch', 'quiet': True,
-            'skip_download': True, 'extract_flat': True
-        }
-        if limit:
-            obs['playlistend'] = self.channel_size
-        chan = youtube_dl.YoutubeDL(obs).extract_info(url, download=False)
-        last_videos = [(i['id'], i['title']) for i in chan['entries']]
-        return last_videos
-
-    def find_missing(self):
-        """ add missing videos from subscribed channels to pending """
-        all_channels = self.get_channels()
-        pending_handler = PendingList()
-        all_pending, all_ignore = pending_handler.get_all_pending()
-        all_pending_ids = [i['youtube_id'] for i in all_pending]
-        all_downloaded = pending_handler.get_all_downloaded()
-        to_ignore = all_pending_ids + all_ignore + all_downloaded
-        missing_videos = []
-        counter = 1
-        for channel in all_channels:
-            channel_id = channel['channel_id']
-            last_videos = self.get_last_youtube_videos(channel_id)
-            set_message('progress:download', {
-                "status": "rescan",
-                "level": "info",
-                "title": "Rescanning: Looking for new videos.",
-                "message": f'Progress: {counter}/{len(all_channels)}'
-                }
-            )
-            for video in last_videos:
-                youtube_id = video[0]
-                if youtube_id not in to_ignore:
-                    missing_videos.append(youtube_id)
-            counter = counter + 1
-
-        return missing_videos
-
-    def change_subscribe(self, channel_id, channel_subscribed):
-        """ subscribe or unsubscribe from channel and update """
-        if not isinstance(channel_subscribed, bool):
-            print('invalid status, should be bool')
-            return
-        headers = {'Content-type': 'application/json'}
-        channel_handler = YoutubeChannel(channel_id)
-        channel_dict = channel_handler.channel_dict
-        channel_dict['channel_subscribed'] = channel_subscribed
-        if channel_subscribed:
-            # handle subscribe
-            url = self.es_url + '/ta_channel/_doc/' + channel_id
-            payload = json.dumps(channel_dict)
-            print(channel_dict)
-        else:
-            url = self.es_url + '/ta_channel/_update/' + channel_id
-            payload = json.dumps({'doc': channel_dict})
-        # update channel
-        request = requests.post(url, data=payload, headers=headers)
-        if not request.ok:
-            print(request.text)
-        # sync to videos
-        channel_handler.sync_to_videos()
-
-
-def playlist_extractor(playlist_id):
-    """ return youtube_ids from a playlist_id """
-    url = 'https://www.youtube.com/playlist?list=' + playlist_id
-    obs = {
-        'default_search': 'ytsearch', 'quiet': True, 'ignoreerrors': True,
-        'skip_download': True, 'extract_flat': True
-    }
-    playlist = youtube_dl.YoutubeDL(obs).extract_info(url, download=False)
-    playlist_vids = [(i['id'], i['title']) for i in playlist['entries']]
-    return playlist_vids
-
-
-class VideoDownloader:
-    """ handle the video download functionality """
-
-    def __init__(self, youtube_id_list):
-        self.youtube_id_list = youtube_id_list
-        self.config = AppConfig().config
-
-    def download_list(self):
-        """ download the list of youtube_ids """
-        limit_count = self.config['downloads']['limit_count']
-        if limit_count:
-            self.youtube_id_list = self.youtube_id_list[:limit_count]
-
-        for youtube_id in self.youtube_id_list:
-            try:
-                self.dl_single_vid(youtube_id)
-            except youtube_dl.utils.DownloadError:
-                print('failed to download ' + youtube_id)
-                continue
-            vid_dict = index_new_video(youtube_id)
-            self.move_to_archive(vid_dict)
-            self.delete_from_pending(youtube_id)
-            if self.config['downloads']['sleep_interval']:
-                sleep(self.config['downloads']['sleep_interval'])
-
-    @staticmethod
-    def progress_hook(response):
-        """ process the progress_hooks from youtube_dl """
-        # title
-        filename = response['filename'][12:].replace('_', ' ')
-        title = "Downloading: " + os.path.split(filename)[-1]
-        # message
-        try:
-            percent = response['_percent_str']
-            size = response['_total_bytes_str']
-            speed = response['_speed_str']
-            eta = response['_eta_str']
-            message = f'{percent} of {size} at {speed} - time left: {eta}'
-        except KeyError:
-            message = ''
-        mess_dict = {
-            "status": "downloading",
-            "level": "info",
-            "title": title,
-            "message": message
-        }
-        set_message('progress:download', mess_dict)
-
-    def dl_single_vid(self, youtube_id):
-        """ download single video """
-        obs = {
-            'default_search': 'ytsearch',
-            'merge_output_format': 'mp4', 'restrictfilenames': True,
-            'outtmpl': (self.config['application']['cache_dir'] +
-                        '/download/' +
-                        self.config['application']['file_template']),
-            'progress_hooks': [self.progress_hook],
-            'quiet': True, 'continuedl': True, 'retries': 3
-        }
-        if self.config['downloads']['format']:
-            obs['format'] = self.config['downloads']['format']
-        if self.config['downloads']['limit_speed']:
-            obs['ratelimit'] = self.config['downloads']['limit_speed'] * 1024
-        external = False
-        if external:
-            obs['external_downloader'] = 'aria2c'
-        # check if already in cache to continue from there
-        cache_dir = self.config['application']['cache_dir']
-        all_cached = os.listdir(cache_dir + '/download/')
-        for file_name in all_cached:
-            if youtube_id in file_name:
-                obs['outtmpl'] = cache_dir + '/download/' + file_name
-        with youtube_dl.YoutubeDL(obs) as ydl:
-            try:
-                ydl.download([youtube_id])
-            except youtube_dl.utils.DownloadError:
-                print('retry failed download: ' + youtube_id)
-                sleep(10)
-                ydl.download([youtube_id])
-
-    def move_to_archive(self, vid_dict):
-        """ move downloaded video from cache to archive """
-        videos = self.config['application']['videos']
-        channel_name = vid_dict['channel']['channel_name']
-        channel_name_clean = clean_string(channel_name)
-        media_url = vid_dict['media_url']
-        youtube_id = vid_dict['youtube_id']
-        # make archive folder
-        videos = self.config['application']['videos']
-        new_folder = os.path.join(videos, channel_name_clean)
-        os.makedirs(new_folder, exist_ok=True)
-        # find real filename
-        cache_dir = self.config['application']['cache_dir']
-        for file_str in os.listdir(cache_dir + '/download'):
-            if youtube_id in file_str:
-                old_file = file_str
-        old_file_path = os.path.join(cache_dir, 'download', old_file)
-        new_file_path = os.path.join(videos, media_url)
-        # move and fix permission
-        shutil.move(old_file_path, new_file_path)
-        os.chown(
-            new_file_path,
-            self.config['application']['HOST_UID'],
-            self.config['application']['HOST_GID']
-        )
-
-    def delete_from_pending(self, youtube_id):
-        """ delete downloaded video from pending index if its there """
-        es_url = self.config['application']['es_url']
-        url = f'{es_url}/ta_download/_doc/{youtube_id}'
-        response = requests.delete(url)
-        if not response.ok and not response.status_code == 404:
-            print(response.text)

tubearchivist/home/src/download/queue.py

@@ -0,0 +1,370 @@
+"""
+Functionality:
+- handle download queue
+- linked with ta_dowload index
+"""
+
+import json
+from datetime import datetime
+
+from home.src.download.subscriptions import ChannelSubscription
+from home.src.download.thumbnails import ThumbManager
+from home.src.download.yt_dlp_base import YtWrap
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.index.playlist import YoutubePlaylist
+from home.src.index.video_constants import VideoTypeEnum
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import get_duration_str, is_shorts
+
+
+class PendingIndex:
+    """base class holding all export methods"""
+
+    def __init__(self):
+        self.all_pending = False
+        self.all_ignored = False
+        self.all_videos = False
+        self.all_channels = False
+        self.channel_overwrites = False
+        self.video_overwrites = False
+        self.to_skip = False
+
+    def get_download(self):
+        """get a list of all pending videos in ta_download"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"timestamp": {"order": "asc"}}],
+        }
+        all_results = IndexPaginate("ta_download", data).get_results()
+
+        self.all_pending = []
+        self.all_ignored = []
+        self.to_skip = []
+
+        for result in all_results:
+            self.to_skip.append(result["youtube_id"])
+            if result["status"] == "pending":
+                self.all_pending.append(result)
+            elif result["status"] == "ignore":
+                self.all_ignored.append(result)
+
+    def get_indexed(self):
+        """get a list of all videos indexed"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        self.all_videos = IndexPaginate("ta_video", data).get_results()
+        for video in self.all_videos:
+            self.to_skip.append(video["youtube_id"])
+
+    def get_channels(self):
+        """get a list of all channels indexed"""
+        self.all_channels = []
+        self.channel_overwrites = {}
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"channel_id": {"order": "asc"}}],
+        }
+        channels = IndexPaginate("ta_channel", data).get_results()
+
+        for channel in channels:
+            channel_id = channel["channel_id"]
+            self.all_channels.append(channel_id)
+            if channel.get("channel_overwrites"):
+                self.channel_overwrites.update(
+                    {channel_id: channel.get("channel_overwrites")}
+                )
+
+        self._map_overwrites()
+
+    def _map_overwrites(self):
+        """map video ids to channel ids overwrites"""
+        self.video_overwrites = {}
+        for video in self.all_pending:
+            video_id = video["youtube_id"]
+            channel_id = video["channel_id"]
+            overwrites = self.channel_overwrites.get(channel_id, False)
+            if overwrites:
+                self.video_overwrites.update({video_id: overwrites})
+
+
+class PendingInteract:
+    """interact with items in download queue"""
+
+    def __init__(self, youtube_id=False, status=False):
+        self.youtube_id = youtube_id
+        self.status = status
+
+    def delete_item(self):
+        """delete single item from pending"""
+        path = f"ta_download/_doc/{self.youtube_id}"
+        _, _ = ElasticWrap(path).delete(refresh=True)
+
+    def delete_by_status(self):
+        """delete all matching item by status"""
+        data = {"query": {"term": {"status": {"value": self.status}}}}
+        path = "ta_download/_delete_by_query"
+        _, _ = ElasticWrap(path).post(data=data)
+
+    def update_status(self):
+        """update status of pending item"""
+        if self.status == "priority":
+            data = {
+                "doc": {
+                    "status": "pending",
+                    "auto_start": True,
+                    "message": None,
+                }
+            }
+        else:
+            data = {"doc": {"status": self.status}}
+
+        path = f"ta_download/_update/{self.youtube_id}/?refresh=true"
+        _, _ = ElasticWrap(path).post(data=data)
+
+    def get_item(self):
+        """return pending item dict"""
+        path = f"ta_download/_doc/{self.youtube_id}"
+        response, status_code = ElasticWrap(path).get()
+        return response["_source"], status_code
+
+    def get_channel(self):
+        """
+        get channel metadata from queue to not depend on channel to be indexed
+        """
+        data = {
+            "size": 1,
+            "query": {"term": {"channel_id": {"value": self.youtube_id}}},
+        }
+        response, _ = ElasticWrap("ta_download/_search").get(data=data)
+        hits = response["hits"]["hits"]
+        if not hits:
+            channel_name = "NA"
+        else:
+            channel_name = hits[0]["_source"].get("channel_name", "NA")
+
+        return {
+            "channel_id": self.youtube_id,
+            "channel_name": channel_name,
+        }
+
+
+class PendingList(PendingIndex):
+    """manage the pending videos list"""
+
+    yt_obs = {
+        "noplaylist": True,
+        "writethumbnail": True,
+        "simulate": True,
+        "check_formats": None,
+    }
+
+    def __init__(self, youtube_ids=False, task=False):
+        super().__init__()
+        self.config = AppConfig().config
+        self.youtube_ids = youtube_ids
+        self.task = task
+        self.to_skip = False
+        self.missing_videos = False
+
+    def parse_url_list(self):
+        """extract youtube ids from list"""
+        self.missing_videos = []
+        self.get_download()
+        self.get_indexed()
+        total = len(self.youtube_ids)
+        for idx, entry in enumerate(self.youtube_ids):
+            self._process_entry(entry)
+            if not self.task:
+                continue
+
+            self.task.send_progress(
+                message_lines=[f"Extracting items {idx + 1}/{total}"],
+                progress=(idx + 1) / total,
+            )
+
+    def _process_entry(self, entry):
+        """process single entry from url list"""
+        vid_type = self._get_vid_type(entry)
+        if entry["type"] == "video":
+            self._add_video(entry["url"], vid_type)
+        elif entry["type"] == "channel":
+            self._parse_channel(entry["url"], vid_type)
+        elif entry["type"] == "playlist":
+            self._parse_playlist(entry["url"])
+        else:
+            raise ValueError(f"invalid url_type: {entry}")
+
+    @staticmethod
+    def _get_vid_type(entry):
+        """add vid type enum if available"""
+        vid_type_str = entry.get("vid_type")
+        if not vid_type_str:
+            return VideoTypeEnum.UNKNOWN
+
+        return VideoTypeEnum(vid_type_str)
+
+    def _add_video(self, url, vid_type):
+        """add video to list"""
+        if url not in self.missing_videos and url not in self.to_skip:
+            self.missing_videos.append((url, vid_type))
+        else:
+            print(f"{url}: skipped adding already indexed video to download.")
+
+    def _parse_channel(self, url, vid_type):
+        """add all videos of channel to list"""
+        video_results = ChannelSubscription().get_last_youtube_videos(
+            url, limit=False, query_filter=vid_type
+        )
+        for video_id, _, vid_type in video_results:
+            self._add_video(video_id, vid_type)
+
+    def _parse_playlist(self, url):
+        """add all videos of playlist to list"""
+        playlist = YoutubePlaylist(url)
+        is_active = playlist.update_playlist()
+        if not is_active:
+            message = f"{playlist.youtube_id}: failed to extract metadata"
+            print(message)
+            raise ValueError(message)
+
+        entries = playlist.json_data["playlist_entries"]
+        to_add = [i["youtube_id"] for i in entries if not i["downloaded"]]
+        if not to_add:
+            return
+
+        for video_id in to_add:
+            # match vid_type later
+            self._add_video(video_id, VideoTypeEnum.UNKNOWN)
+
+    def add_to_pending(self, status="pending", auto_start=False):
+        """add missing videos to pending list"""
+        self.get_channels()
+        bulk_list = []
+
+        total = len(self.missing_videos)
+        videos_added = []
+        for idx, (youtube_id, vid_type) in enumerate(self.missing_videos):
+            if self.task and self.task.is_stopped():
+                break
+
+            print(f"{youtube_id}: [{idx + 1}/{total}]: add to queue")
+            self._notify_add(idx, total)
+            video_details = self.get_youtube_details(youtube_id, vid_type)
+            if not video_details:
+                continue
+
+            video_details.update(
+                {
+                    "status": status,
+                    "auto_start": auto_start,
+                }
+            )
+
+            action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(video_details))
+
+            url = video_details["vid_thumb_url"]
+            ThumbManager(youtube_id).download_video_thumb(url)
+            videos_added.append(youtube_id)
+
+            if len(bulk_list) >= 20:
+                self._ingest_bulk(bulk_list)
+                bulk_list = []
+
+        self._ingest_bulk(bulk_list)
+
+        return videos_added
+
+    def _ingest_bulk(self, bulk_list):
+        """add items to queue in bulk"""
+        if not bulk_list:
+            return
+
+        # add last newline
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+        _, _ = ElasticWrap("_bulk?refresh=true").post(query_str, ndjson=True)
+
+    def _notify_add(self, idx, total):
+        """send notification for adding videos to download queue"""
+        if not self.task:
+            return
+
+        self.task.send_progress(
+            message_lines=[
+                "Adding new videos to download queue.",
+                f"Extracting items {idx + 1}/{total}",
+            ],
+            progress=(idx + 1) / total,
+        )
+
+    def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEOS):
+        """get details from youtubedl for single pending video"""
+        vid = YtWrap(self.yt_obs, self.config).extract(youtube_id)
+        if not vid:
+            return False
+
+        if vid.get("id") != youtube_id:
+            # skip premium videos with different id
+            print(f"{youtube_id}: skipping premium video, id not matching")
+            return False
+        # stop if video is streaming live now
+        if vid["live_status"] in ["is_upcoming", "is_live"]:
+            print(f"{youtube_id}: skip is_upcoming or is_live")
+            return False
+
+        if vid["live_status"] == "was_live":
+            vid_type = VideoTypeEnum.STREAMS
+        else:
+            if self._check_shorts(vid):
+                vid_type = VideoTypeEnum.SHORTS
+            else:
+                vid_type = VideoTypeEnum.VIDEOS
+
+        if not vid.get("channel"):
+            print(f"{youtube_id}: skip video not part of channel")
+            return False
+
+        return self._parse_youtube_details(vid, vid_type)
+
+    @staticmethod
+    def _check_shorts(vid):
+        """check if vid is shorts video"""
+        if vid["width"] > vid["height"]:
+            return False
+
+        duration = vid.get("duration")
+        if duration and isinstance(duration, int):
+            if duration > 60:
+                return False
+
+        return is_shorts(vid["id"])
+
+    def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS):
+        """parse response"""
+        vid_id = vid.get("id")
+        published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
+            "%Y-%m-%d"
+        )
+
+        # build dict
+        youtube_details = {
+            "youtube_id": vid_id,
+            "channel_name": vid["channel"],
+            "vid_thumb_url": vid["thumbnail"],
+            "title": vid["title"],
+            "channel_id": vid["channel_id"],
+            "duration": get_duration_str(vid["duration"]),
+            "published": published,
+            "timestamp": int(datetime.now().timestamp()),
+            # Pulling enum value out so it is serializable
+            "vid_type": vid_type.value,
+        }
+        if self.all_channels:
+            youtube_details.update(
+                {"channel_indexed": vid["channel_id"] in self.all_channels}
+            )
+        return youtube_details

tubearchivist/home/src/download/subscriptions.py

@@ -0,0 +1,436 @@
+"""
+Functionality:
+- handle channel subscriptions
+- handle playlist subscriptions
+"""
+
+from home.src.download.thumbnails import ThumbManager
+from home.src.download.yt_dlp_base import YtWrap
+from home.src.es.connect import IndexPaginate
+from home.src.index.channel import YoutubeChannel
+from home.src.index.playlist import YoutubePlaylist
+from home.src.index.video import YoutubeVideo
+from home.src.index.video_constants import VideoTypeEnum
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import is_missing
+from home.src.ta.urlparser import Parser
+
+
+class ChannelSubscription:
+    """manage the list of channels subscribed"""
+
+    def __init__(self, task=False):
+        self.config = AppConfig().config
+        self.task = task
+
+    @staticmethod
+    def get_channels(subscribed_only=True):
+        """get a list of all channels subscribed to"""
+        data = {
+            "sort": [{"channel_name.keyword": {"order": "asc"}}],
+        }
+        if subscribed_only:
+            data["query"] = {"term": {"channel_subscribed": {"value": True}}}
+        else:
+            data["query"] = {"match_all": {}}
+
+        all_channels = IndexPaginate("ta_channel", data).get_results()
+
+        return all_channels
+
+    def get_last_youtube_videos(
+        self,
+        channel_id,
+        limit=True,
+        query_filter=None,
+        channel_overwrites=None,
+    ):
+        """get a list of last videos from channel"""
+        query_handler = VideoQueryBuilder(self.config, channel_overwrites)
+        queries = query_handler.build_queries(query_filter)
+        last_videos = []
+
+        for vid_type_enum, limit_amount in queries:
+            obs = {
+                "skip_download": True,
+                "extract_flat": True,
+            }
+            vid_type = vid_type_enum.value
+
+            if limit:
+                obs["playlistend"] = limit_amount
+
+            url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
+            channel_query = YtWrap(obs, self.config).extract(url)
+            if not channel_query:
+                continue
+
+            last_videos.extend(
+                [
+                    (i["id"], i["title"], vid_type)
+                    for i in channel_query["entries"]
+                ]
+            )
+
+        return last_videos
+
+    def find_missing(self):
+        """add missing videos from subscribed channels to pending"""
+        all_channels = self.get_channels()
+        if not all_channels:
+            return False
+
+        missing_videos = []
+
+        total = len(all_channels)
+        for idx, channel in enumerate(all_channels):
+            channel_id = channel["channel_id"]
+            print(f"{channel_id}: find missing videos.")
+            last_videos = self.get_last_youtube_videos(
+                channel_id,
+                channel_overwrites=channel.get("channel_overwrites"),
+            )
+
+            if last_videos:
+                ids_to_add = is_missing([i[0] for i in last_videos])
+                for video_id, _, vid_type in last_videos:
+                    if video_id in ids_to_add:
+                        missing_videos.append((video_id, vid_type))
+
+            if not self.task:
+                continue
+
+            if self.task.is_stopped():
+                self.task.send_progress(["Received Stop signal."])
+                break
+
+            self.task.send_progress(
+                message_lines=[f"Scanning Channel {idx + 1}/{total}"],
+                progress=(idx + 1) / total,
+            )
+
+        return missing_videos
+
+    @staticmethod
+    def change_subscribe(channel_id, channel_subscribed):
+        """subscribe or unsubscribe from channel and update"""
+        channel = YoutubeChannel(channel_id)
+        channel.build_json()
+        channel.json_data["channel_subscribed"] = channel_subscribed
+        channel.upload_to_es()
+        channel.sync_to_videos()
+
+
+class VideoQueryBuilder:
+    """Build queries for yt-dlp."""
+
+    def __init__(self, config: dict, channel_overwrites: dict | None = None):
+        self.config = config
+        self.channel_overwrites = channel_overwrites or {}
+
+    def build_queries(
+        self, video_type: VideoTypeEnum | None, limit: bool = True
+    ) -> list[tuple[VideoTypeEnum, int | None]]:
+        """Build queries for all or specific video type."""
+        query_methods = {
+            VideoTypeEnum.VIDEOS: self.videos_query,
+            VideoTypeEnum.STREAMS: self.streams_query,
+            VideoTypeEnum.SHORTS: self.shorts_query,
+        }
+
+        if video_type:
+            # build query for specific type
+            query_method = query_methods.get(video_type)
+            if query_method:
+                query = query_method(limit)
+                if query[1] != 0:
+                    return [query]
+                return []
+
+        # Build and return queries for all video types
+        queries = []
+        for build_query in query_methods.values():
+            query = build_query(limit)
+            if query[1] != 0:
+                queries.append(query)
+
+        return queries
+
+    def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
+        """Build query for videos."""
+        return self._build_generic_query(
+            video_type=VideoTypeEnum.VIDEOS,
+            overwrite_key="subscriptions_channel_size",
+            config_key="channel_size",
+            limit=limit,
+        )
+
+    def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
+        """Build query for streams."""
+        return self._build_generic_query(
+            video_type=VideoTypeEnum.STREAMS,
+            overwrite_key="subscriptions_live_channel_size",
+            config_key="live_channel_size",
+            limit=limit,
+        )
+
+    def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
+        """Build query for shorts."""
+        return self._build_generic_query(
+            video_type=VideoTypeEnum.SHORTS,
+            overwrite_key="subscriptions_shorts_channel_size",
+            config_key="shorts_channel_size",
+            limit=limit,
+        )
+
+    def _build_generic_query(
+        self,
+        video_type: VideoTypeEnum,
+        overwrite_key: str,
+        config_key: str,
+        limit: bool,
+    ) -> tuple[VideoTypeEnum, int | None]:
+        """Generic query for video page scraping."""
+        if not limit:
+            return (video_type, None)
+
+        if (
+            overwrite_key in self.channel_overwrites
+            and self.channel_overwrites[overwrite_key] is not None
+        ):
+            overwrite = self.channel_overwrites[overwrite_key]
+            return (video_type, overwrite)
+
+        if overwrite := self.config["subscriptions"].get(config_key):
+            return (video_type, overwrite)
+
+        return (video_type, 0)
+
+
+class PlaylistSubscription:
+    """manage the playlist download functionality"""
+
+    def __init__(self, task=False):
+        self.config = AppConfig().config
+        self.task = task
+
+    @staticmethod
+    def get_playlists(subscribed_only=True):
+        """get a list of all active playlists"""
+        data = {
+            "sort": [{"playlist_channel.keyword": {"order": "desc"}}],
+        }
+        data["query"] = {
+            "bool": {"must": [{"term": {"playlist_active": {"value": True}}}]}
+        }
+        if subscribed_only:
+            data["query"]["bool"]["must"].append(
+                {"term": {"playlist_subscribed": {"value": True}}}
+            )
+
+        all_playlists = IndexPaginate("ta_playlist", data).get_results()
+
+        return all_playlists
+
+    def process_url_str(self, new_playlists, subscribed=True):
+        """process playlist subscribe form url_str"""
+        for idx, playlist in enumerate(new_playlists):
+            playlist_id = playlist["url"]
+            if not playlist["type"] == "playlist":
+                print(f"{playlist_id} not a playlist, skipping...")
+                continue
+
+            playlist_h = YoutubePlaylist(playlist_id)
+            playlist_h.build_json()
+            if not playlist_h.json_data:
+                message = f"{playlist_h.youtube_id}: failed to extract data"
+                print(message)
+                raise ValueError(message)
+
+            playlist_h.json_data["playlist_subscribed"] = subscribed
+            playlist_h.upload_to_es()
+            playlist_h.add_vids_to_playlist()
+            self.channel_validate(playlist_h.json_data["playlist_channel_id"])
+
+            url = playlist_h.json_data["playlist_thumbnail"]
+            thumb = ThumbManager(playlist_id, item_type="playlist")
+            thumb.download_playlist_thumb(url)
+
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[
+                        f"Processing {idx + 1} of {len(new_playlists)}"
+                    ],
+                    progress=(idx + 1) / len(new_playlists),
+                )
+
+    @staticmethod
+    def channel_validate(channel_id):
+        """make sure channel of playlist is there"""
+        channel = YoutubeChannel(channel_id)
+        channel.build_json(upload=True)
+
+    @staticmethod
+    def change_subscribe(playlist_id, subscribe_status):
+        """change the subscribe status of a playlist"""
+        playlist = YoutubePlaylist(playlist_id)
+        playlist.build_json()
+        playlist.json_data["playlist_subscribed"] = subscribe_status
+        playlist.upload_to_es()
+
+    def find_missing(self):
+        """find videos in subscribed playlists not downloaded yet"""
+        all_playlists = [i["playlist_id"] for i in self.get_playlists()]
+        if not all_playlists:
+            return False
+
+        missing_videos = []
+        total = len(all_playlists)
+        for idx, playlist_id in enumerate(all_playlists):
+            playlist = YoutubePlaylist(playlist_id)
+            is_active = playlist.update_playlist()
+            if not is_active:
+                playlist.deactivate()
+                continue
+
+            playlist_entries = playlist.json_data["playlist_entries"]
+            size_limit = self.config["subscriptions"]["channel_size"]
+            if size_limit:
+                del playlist_entries[size_limit:]
+
+            to_check = [
+                i["youtube_id"]
+                for i in playlist_entries
+                if i["downloaded"] is False
+            ]
+            needs_downloading = is_missing(to_check)
+            missing_videos.extend(needs_downloading)
+
+            if not self.task:
+                continue
+
+            if self.task.is_stopped():
+                self.task.send_progress(["Received Stop signal."])
+                break
+
+            self.task.send_progress(
+                message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
+                progress=(idx + 1) / total,
+            )
+
+        return missing_videos
+
+
+class SubscriptionScanner:
+    """add missing videos to queue"""
+
+    def __init__(self, task=False):
+        self.task = task
+        self.missing_videos = False
+        self.auto_start = AppConfig().config["subscriptions"].get("auto_start")
+
+    def scan(self):
+        """scan channels and playlists"""
+        if self.task:
+            self.task.send_progress(["Rescanning channels and playlists."])
+
+        self.missing_videos = []
+        self.scan_channels()
+        if self.task and not self.task.is_stopped():
+            self.scan_playlists()
+
+        return self.missing_videos
+
+    def scan_channels(self):
+        """get missing from channels"""
+        channel_handler = ChannelSubscription(task=self.task)
+        missing = channel_handler.find_missing()
+        if not missing:
+            return
+
+        for vid_id, vid_type in missing:
+            self.missing_videos.append(
+                {"type": "video", "vid_type": vid_type, "url": vid_id}
+            )
+
+    def scan_playlists(self):
+        """get missing from playlists"""
+        playlist_handler = PlaylistSubscription(task=self.task)
+        missing = playlist_handler.find_missing()
+        if not missing:
+            return
+
+        for i in missing:
+            self.missing_videos.append(
+                {
+                    "type": "video",
+                    "vid_type": VideoTypeEnum.VIDEOS.value,
+                    "url": i,
+                }
+            )
+
+
+class SubscriptionHandler:
+    """subscribe to channels and playlists from url_str"""
+
+    def __init__(self, url_str, task=False):
+        self.url_str = url_str
+        self.task = task
+        self.to_subscribe = False
+
+    def subscribe(self, expected_type=False):
+        """subscribe to url_str items"""
+        if self.task:
+            self.task.send_progress(["Processing form content."])
+        self.to_subscribe = Parser(self.url_str).parse()
+
+        total = len(self.to_subscribe)
+        for idx, item in enumerate(self.to_subscribe):
+            if self.task:
+                self._notify(idx, item, total)
+
+            self.subscribe_type(item, expected_type=expected_type)
+
+    def subscribe_type(self, item, expected_type):
+        """process single item"""
+        if item["type"] == "playlist":
+            if expected_type and expected_type != "playlist":
+                raise TypeError(
+                    f"expected {expected_type} url but got {item.get('type')}"
+                )
+
+            PlaylistSubscription().process_url_str([item])
+            return
+
+        if item["type"] == "video":
+            # extract channel id from video
+            video = YoutubeVideo(item["url"])
+            video.get_from_youtube()
+            video.process_youtube_meta()
+            channel_id = video.channel_id
+        elif item["type"] == "channel":
+            channel_id = item["url"]
+        else:
+            raise ValueError("failed to subscribe to: " + item["url"])
+
+        if expected_type and expected_type != "channel":
+            raise TypeError(
+                f"expected {expected_type} url but got {item.get('type')}"
+            )
+
+        self._subscribe(channel_id)
+
+    def _subscribe(self, channel_id):
+        """subscribe to channel"""
+        ChannelSubscription().change_subscribe(
+            channel_id, channel_subscribed=True
+        )
+
+    def _notify(self, idx, item, total):
+        """send notification message to redis"""
+        subscribe_type = item["type"].title()
+        message_lines = [
+            f"Subscribe to {subscribe_type}",
+            f"Progress: {idx + 1}/{total}",
+        ]
+        self.task.send_progress(message_lines, progress=(idx + 1) / total)

tubearchivist/home/src/download/thumbnails.py

@@ -0,0 +1,505 @@
+"""
+functionality:
+- handle download and caching for thumbnails
+- check for missing thumbnails
+"""
+
+import base64
+import os
+from io import BytesIO
+from time import sleep
+
+import requests
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.ta.helper import is_missing
+from home.src.ta.settings import EnvironmentSettings
+from mutagen.mp4 import MP4, MP4Cover
+from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+
+class ThumbManagerBase:
+    """base class for thumbnail management"""
+
+    CACHE_DIR = EnvironmentSettings.CACHE_DIR
+    VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
+    CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
+    PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
+
+    def __init__(self, item_id, item_type, fallback=False):
+        self.item_id = item_id
+        self.item_type = item_type
+        self.fallback = fallback
+
+    def download_raw(self, url):
+        """download thumbnail for video"""
+        if not url:
+            return self.get_fallback()
+
+        for i in range(3):
+            try:
+                response = requests.get(url, stream=True, timeout=5)
+                if response.ok:
+                    try:
+                        img = Image.open(response.raw)
+                        if isinstance(img, Image.Image):
+                            return img
+                        return self.get_fallback()
+
+                    except (UnidentifiedImageError, OSError):
+                        print(f"failed to open thumbnail: {url}")
+                        return self.get_fallback()
+
+                if response.status_code == 404:
+                    return self.get_fallback()
+
+            except (
+                requests.exceptions.RequestException,
+                requests.exceptions.ReadTimeout,
+            ):
+                print(f"{self.item_id}: retry thumbnail download {url}")
+                sleep((i + 1) ** i)
+
+        return self.get_fallback()
+
+    def get_fallback(self):
+        """get fallback thumbnail if not available"""
+        print(f"{self.item_id}: failed to extract thumbnail, use fallback")
+        if self.fallback:
+            img_raw = Image.open(self.fallback)
+            return img_raw
+
+        app_root = EnvironmentSettings.APP_DIR
+        default_map = {
+            "video": os.path.join(
+                app_root, "static/img/default-video-thumb.jpg"
+            ),
+            "playlist": os.path.join(
+                app_root, "static/img/default-playlist-thumb.jpg"
+            ),
+            "icon": os.path.join(
+                app_root, "static/img/default-channel-icon.jpg"
+            ),
+            "banner": os.path.join(
+                app_root, "static/img/default-channel-banner.jpg"
+            ),
+            "tvart": os.path.join(
+                app_root, "static/img/default-channel-art.jpg"
+            ),
+        }
+
+        img_raw = Image.open(default_map[self.item_type])
+
+        return img_raw
+
+
+class ThumbManager(ThumbManagerBase):
+    """handle thumbnails related functions"""
+
+    def __init__(self, item_id, item_type="video", fallback=False):
+        super().__init__(item_id, item_type, fallback=fallback)
+
+    def download(self, url):
+        """download thumbnail"""
+        print(f"{self.item_id}: download {self.item_type} thumbnail")
+        if self.item_type == "video":
+            self.download_video_thumb(url)
+        elif self.item_type == "channel":
+            self.download_channel_art(url)
+        elif self.item_type == "playlist":
+            self.download_playlist_thumb(url)
+
+    def delete(self):
+        """delete thumbnail file"""
+        print(f"{self.item_id}: delete {self.item_type} thumbnail")
+        if self.item_type == "video":
+            self.delete_video_thumb()
+        elif self.item_type == "channel":
+            self.delete_channel_thumb()
+        elif self.item_type == "playlist":
+            self.delete_playlist_thumb()
+
+    def download_video_thumb(self, url, skip_existing=False):
+        """pass url for video thumbnail"""
+        folder_path = os.path.join(self.VIDEO_DIR, self.item_id[0].lower())
+        thumb_path = self.vid_thumb_path(absolute=True)
+
+        if skip_existing and os.path.exists(thumb_path):
+            return
+
+        os.makedirs(folder_path, exist_ok=True)
+        img_raw = self.download_raw(url)
+        width, height = img_raw.size
+
+        if not width / height == 16 / 9:
+            new_height = width / 16 * 9
+            offset = (height - new_height) / 2
+            img_raw = img_raw.crop((0, offset, width, height - offset))
+
+        img_raw.convert("RGB").save(thumb_path)
+
+    def vid_thumb_path(self, absolute=False, create_folder=False):
+        """build expected path for video thumbnail from youtube_id"""
+        folder_name = self.item_id[0].lower()
+        folder_path = os.path.join("videos", folder_name)
+        thumb_path = os.path.join(folder_path, f"{self.item_id}.jpg")
+        if absolute:
+            thumb_path = os.path.join(self.CACHE_DIR, thumb_path)
+
+        if create_folder:
+            folder_path = os.path.join(self.CACHE_DIR, folder_path)
+            os.makedirs(folder_path, exist_ok=True)
+
+        return thumb_path
+
+    def download_channel_art(self, urls, skip_existing=False):
+        """pass tuple of channel thumbnails"""
+        channel_thumb, channel_banner, channel_tv = urls
+        self._download_channel_thumb(channel_thumb, skip_existing)
+        self._download_channel_banner(channel_banner, skip_existing)
+        self._download_channel_tv(channel_tv, skip_existing)
+
+    def _download_channel_thumb(self, channel_thumb, skip_existing):
+        """download channel thumbnail"""
+
+        thumb_path = os.path.join(
+            self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg"
+        )
+        self.item_type = "icon"
+
+        if skip_existing and os.path.exists(thumb_path):
+            return
+
+        img_raw = self.download_raw(channel_thumb)
+        img_raw.convert("RGB").save(thumb_path)
+
+    def _download_channel_banner(self, channel_banner, skip_existing):
+        """download channel banner"""
+
+        banner_path = os.path.join(
+            self.CHANNEL_DIR, self.item_id + "_banner.jpg"
+        )
+        self.item_type = "banner"
+        if skip_existing and os.path.exists(banner_path):
+            return
+
+        img_raw = self.download_raw(channel_banner)
+        img_raw.convert("RGB").save(banner_path)
+
+    def _download_channel_tv(self, channel_tv, skip_existing):
+        """download channel tv art"""
+        art_path = os.path.join(self.CHANNEL_DIR, self.item_id + "_tvart.jpg")
+        self.item_type = "tvart"
+        if skip_existing and os.path.exists(art_path):
+            return
+
+        img_raw = self.download_raw(channel_tv)
+        img_raw.convert("RGB").save(art_path)
+
+    def download_playlist_thumb(self, url, skip_existing=False):
+        """pass thumbnail url"""
+        thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
+        if skip_existing and os.path.exists(thumb_path):
+            return
+
+        img_raw = (
+            self.download_raw(url)
+            if not isinstance(url, str) or url.startswith("http")
+            else Image.open(os.path.join(self.CACHE_DIR, url))
+        )
+        width, height = img_raw.size
+
+        if not width / height == 16 / 9:
+            new_height = width / 16 * 9
+            offset = (height - new_height) / 2
+            img_raw = img_raw.crop((0, offset, width, height - offset))
+        img_raw = img_raw.resize((336, 189))
+        img_raw.convert("RGB").save(thumb_path)
+
+    def delete_video_thumb(self):
+        """delete video thumbnail if exists"""
+        thumb_path = self.vid_thumb_path()
+        to_delete = os.path.join(self.CACHE_DIR, thumb_path)
+        if os.path.exists(to_delete):
+            os.remove(to_delete)
+
+    def delete_channel_thumb(self):
+        """delete all artwork of channel"""
+        thumb = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg")
+        banner = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_banner.jpg")
+        if os.path.exists(thumb):
+            os.remove(thumb)
+        if os.path.exists(banner):
+            os.remove(banner)
+
+    def delete_playlist_thumb(self):
+        """delete playlist thumbnail"""
+        thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
+        if os.path.exists(thumb_path):
+            os.remove(thumb_path)
+
+    def get_vid_base64_blur(self):
+        """return base64 encoded placeholder"""
+        file_path = os.path.join(self.CACHE_DIR, self.vid_thumb_path())
+        img_raw = Image.open(file_path)
+        img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20))
+        img_blur = img_raw.filter(ImageFilter.BLUR)
+        buffer = BytesIO()
+        img_blur.save(buffer, format="JPEG")
+        img_data = buffer.getvalue()
+        img_base64 = base64.b64encode(img_data).decode()
+        data_url = f"data:image/jpg;base64,{img_base64}"
+
+        return data_url
+
+
+class ValidatorCallback:
+    """handle callback validate thumbnails page by page"""
+
+    def __init__(self, source, index_name, counter=0):
+        self.source = source
+        self.index_name = index_name
+        self.counter = counter
+
+    def run(self):
+        """run the task for page"""
+        print(f"{self.index_name}: validate artwork")
+        if self.index_name == "ta_video":
+            self._validate_videos()
+        elif self.index_name == "ta_channel":
+            self._validate_channels()
+        elif self.index_name == "ta_playlist":
+            self._validate_playlists()
+
+    def _validate_videos(self):
+        """check if video thumbnails are correct"""
+        for video in self.source:
+            url = video["_source"]["vid_thumb_url"]
+            handler = ThumbManager(video["_source"]["youtube_id"])
+            handler.download_video_thumb(url, skip_existing=True)
+
+    def _validate_channels(self):
+        """check if all channel artwork is there"""
+        for channel in self.source:
+            urls = (
+                channel["_source"]["channel_thumb_url"],
+                channel["_source"]["channel_banner_url"],
+                channel["_source"].get("channel_tvart_url", False),
+            )
+            handler = ThumbManager(channel["_source"]["channel_id"])
+            handler.download_channel_art(urls, skip_existing=True)
+
+    def _validate_playlists(self):
+        """check if all playlist artwork is there"""
+        for playlist in self.source:
+            url = playlist["_source"]["playlist_thumbnail"]
+            handler = ThumbManager(playlist["_source"]["playlist_id"])
+            handler.download_playlist_thumb(url, skip_existing=True)
+
+
+class ThumbValidator:
+    """validate thumbnails"""
+
+    INDEX = [
+        {
+            "data": {
+                "query": {"term": {"active": {"value": True}}},
+                "_source": ["vid_thumb_url", "youtube_id"],
+            },
+            "name": "ta_video",
+        },
+        {
+            "data": {
+                "query": {"term": {"channel_active": {"value": True}}},
+                "_source": {
+                    "excludes": ["channel_description", "channel_overwrites"]
+                },
+            },
+            "name": "ta_channel",
+        },
+        {
+            "data": {
+                "query": {"term": {"playlist_active": {"value": True}}},
+                "_source": ["playlist_id", "playlist_thumbnail"],
+            },
+            "name": "ta_playlist",
+        },
+    ]
+
+    def __init__(self, task=False):
+        self.task = task
+
+    def validate(self):
+        """validate all indexes"""
+        for index in self.INDEX:
+            total = self._get_total(index["name"])
+            if not total:
+                continue
+
+            paginate = IndexPaginate(
+                index_name=index["name"],
+                data=index["data"],
+                size=1000,
+                callback=ValidatorCallback,
+                task=self.task,
+                total=total,
+            )
+            _ = paginate.get_results()
+
+    def clean_up(self):
+        """clean up all thumbs"""
+        self._clean_up_vids()
+        self._clean_up_channels()
+        self._clean_up_playlists()
+
+    def _clean_up_vids(self):
+        """clean unneeded vid thumbs"""
+        video_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "videos")
+        video_folders = os.listdir(video_dir)
+        for video_folder in video_folders:
+            folder_path = os.path.join(video_dir, video_folder)
+            thumbs_is = {i.split(".")[0] for i in os.listdir(folder_path)}
+            thumbs_should = self._get_vid_thumbs_should(video_folder)
+            to_delete = thumbs_is - thumbs_should
+            for thumb in to_delete:
+                delete_path = os.path.join(folder_path, f"{thumb}.jpg")
+                os.remove(delete_path)
+
+            if to_delete:
+                message = (
+                    f"[thumbs][video][{video_folder}] "
+                    + f"delete {len(to_delete)} unused thumbnails"
+                )
+                print(message)
+                if self.task:
+                    self.task.send_progress([message])
+
+    @staticmethod
+    def _get_vid_thumbs_should(video_folder: str) -> set[str]:
+        """get indexed"""
+        should_list = [
+            {"prefix": {"youtube_id": {"value": video_folder.lower()}}},
+            {"prefix": {"youtube_id": {"value": video_folder.upper()}}},
+        ]
+        data = {
+            "query": {"bool": {"should": should_list}},
+            "_source": ["youtube_id"],
+        }
+        result = IndexPaginate("ta_video,ta_download", data).get_results()
+        thumbs_should = {i["youtube_id"] for i in result}
+
+        return thumbs_should
+
+    def _clean_up_channels(self):
+        """clean unneeded channel thumbs"""
+        channel_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "channels")
+        channel_art = os.listdir(channel_dir)
+        thumbs_is = {"_".join(i.split("_")[:-1]) for i in channel_art}
+        to_delete = is_missing(list(thumbs_is), "ta_channel", "channel_id")
+        for channel_thumb in channel_art:
+            if channel_thumb[:24] in to_delete:
+                delete_path = os.path.join(channel_dir, channel_thumb)
+                os.remove(delete_path)
+
+        if to_delete:
+            message = (
+                "[thumbs][channel] "
+                + f"delete {len(to_delete)} unused channel art"
+            )
+            print(message)
+            if self.task:
+                self.task.send_progress([message])
+
+    def _clean_up_playlists(self):
+        """clean up unneeded playlist thumbs"""
+        playlist_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "playlists")
+        playlist_art = os.listdir(playlist_dir)
+        thumbs_is = {i.split(".")[0] for i in playlist_art}
+        to_delete = is_missing(list(thumbs_is), "ta_playlist", "playlist_id")
+        for playlist_id in to_delete:
+            delete_path = os.path.join(playlist_dir, f"{playlist_id}.jpg")
+            os.remove(delete_path)
+
+        if to_delete:
+            message = (
+                "[thumbs][playlist] "
+                + f"delete {len(to_delete)} unused playlist art"
+            )
+            print(message)
+            if self.task:
+                self.task.send_progress([message])
+
+    @staticmethod
+    def _get_total(index_name):
+        """get total documents in index"""
+        path = f"{index_name}/_count"
+        response, _ = ElasticWrap(path).get()
+
+        return response.get("count")
+
+
+class ThumbFilesystem:
+    """sync thumbnail files to media files"""
+
+    INDEX_NAME = "ta_video"
+
+    def __init__(self, task=False):
+        self.task = task
+
+    def embed(self):
+        """entry point"""
+        data = {
+            "query": {"match_all": {}},
+            "_source": ["media_url", "youtube_id"],
+        }
+        paginate = IndexPaginate(
+            index_name=self.INDEX_NAME,
+            data=data,
+            size=200,
+            callback=EmbedCallback,
+            task=self.task,
+            total=self._get_total(),
+        )
+        _ = paginate.get_results()
+
+    def _get_total(self):
+        """get total documents in index"""
+        path = f"{self.INDEX_NAME}/_count"
+        response, _ = ElasticWrap(path).get()
+
+        return response.get("count")
+
+
+class EmbedCallback:
+    """callback class to embed thumbnails"""
+
+    CACHE_DIR = EnvironmentSettings.CACHE_DIR
+    MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
+    FORMAT = MP4Cover.FORMAT_JPEG
+
+    def __init__(self, source, index_name, counter=0):
+        self.source = source
+        self.index_name = index_name
+        self.counter = counter
+
+    def run(self):
+        """run embed"""
+        for video in self.source:
+            video_id = video["_source"]["youtube_id"]
+            media_url = os.path.join(
+                self.MEDIA_DIR, video["_source"]["media_url"]
+            )
+            thumb_path = os.path.join(
+                self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path()
+            )
+            if os.path.exists(thumb_path):
+                self.embed(media_url, thumb_path)
+
+    def embed(self, media_url, thumb_path):
+        """embed thumb in single media file"""
+        video = MP4(media_url)
+        with open(thumb_path, "rb") as f:
+            video["covr"] = [MP4Cover(f.read(), imageformat=self.FORMAT)]
+
+        video.save()

tubearchivist/home/src/download/yt_dlp_base.py

@@ -0,0 +1,173 @@
+"""
+functionality:
+- base class to make all calls to yt-dlp
+- handle yt-dlp errors
+"""
+
+import os
+from datetime import datetime
+from http import cookiejar
+from io import StringIO
+
+import yt_dlp
+from home.src.ta.settings import EnvironmentSettings
+from home.src.ta.ta_redis import RedisArchivist
+
+
+class YtWrap:
+    """wrap calls to yt"""
+
+    OBS_BASE = {
+        "default_search": "ytsearch",
+        "quiet": True,
+        "check_formats": "selected",
+        "socket_timeout": 10,
+        "extractor_retries": 3,
+        "retries": 10,
+    }
+
+    def __init__(self, obs_request, config=False):
+        self.obs_request = obs_request
+        self.config = config
+        self.build_obs()
+
+    def build_obs(self):
+        """build yt-dlp obs"""
+        self.obs = self.OBS_BASE.copy()
+        self.obs.update(self.obs_request)
+        if self.config:
+            self.add_cookie()
+
+    def add_cookie(self):
+        """add cookie if enabled"""
+        if self.config["downloads"]["cookie_import"]:
+            cookie_io = CookieHandler(self.config).get()
+            self.obs["cookiefile"] = cookie_io
+
+    def download(self, url):
+        """make download request"""
+        with yt_dlp.YoutubeDL(self.obs) as ydl:
+            try:
+                ydl.download([url])
+            except yt_dlp.utils.DownloadError as err:
+                print(f"{url}: failed to download with message {err}")
+                if "Temporary failure in name resolution" in str(err):
+                    raise ConnectionError("lost the internet, abort!") from err
+
+                return False, str(err)
+
+        return True, True
+
+    def extract(self, url):
+        """make extract request"""
+        try:
+            response = yt_dlp.YoutubeDL(self.obs).extract_info(url)
+        except cookiejar.LoadError as err:
+            print(f"cookie file is invalid: {err}")
+            return False
+        except yt_dlp.utils.ExtractorError as err:
+            print(f"{url}: failed to extract with message: {err}, continue...")
+            return False
+        except yt_dlp.utils.DownloadError as err:
+            if "This channel does not have a" in str(err):
+                return False
+
+            print(f"{url}: failed to get info from youtube with message {err}")
+            if "Temporary failure in name resolution" in str(err):
+                raise ConnectionError("lost the internet, abort!") from err
+
+            return False
+
+        return response
+
+
+class CookieHandler:
+    """handle youtube cookie for yt-dlp"""
+
+    def __init__(self, config):
+        self.cookie_io = False
+        self.config = config
+        self.cache_dir = EnvironmentSettings.CACHE_DIR
+
+    def get(self):
+        """get cookie io stream"""
+        cookie = RedisArchivist().get_message("cookie")
+        self.cookie_io = StringIO(cookie)
+        return self.cookie_io
+
+    def import_cookie(self):
+        """import cookie from file"""
+        import_path = os.path.join(
+            self.cache_dir, "import", "cookies.google.txt"
+        )
+
+        try:
+            with open(import_path, encoding="utf-8") as cookie_file:
+                cookie = cookie_file.read()
+        except FileNotFoundError as err:
+            print(f"cookie: {import_path} file not found")
+            raise err
+
+        self.set_cookie(cookie)
+
+        os.remove(import_path)
+        print("cookie: import successful")
+
+    def set_cookie(self, cookie):
+        """set cookie str and activate in config"""
+        RedisArchivist().set_message("cookie", cookie, save=True)
+        path = ".downloads.cookie_import"
+        RedisArchivist().set_message("config", True, path=path, save=True)
+        self.config["downloads"]["cookie_import"] = True
+        print("cookie: activated and stored in Redis")
+
+    @staticmethod
+    def revoke():
+        """revoke cookie"""
+        RedisArchivist().del_message("cookie")
+        RedisArchivist().del_message("cookie:valid")
+        RedisArchivist().set_message(
+            "config", False, path=".downloads.cookie_import"
+        )
+        print("cookie: revoked")
+
+    def validate(self):
+        """validate cookie using the liked videos playlist"""
+        print("validating cookie")
+        obs_request = {
+            "skip_download": True,
+            "extract_flat": True,
+        }
+        validator = YtWrap(obs_request, self.config)
+        response = bool(validator.extract("LL"))
+        self.store_validation(response)
+
+        # update in redis to avoid expiring
+        modified = validator.obs["cookiefile"].getvalue()
+        if modified:
+            RedisArchivist().set_message("cookie", modified)
+
+        if not response:
+            mess_dict = {
+                "status": "message:download",
+                "level": "error",
+                "title": "Cookie validation failed, exiting...",
+                "message": "",
+            }
+            RedisArchivist().set_message(
+                "message:download", mess_dict, expire=4
+            )
+            print("cookie validation failed, exiting...")
+
+        return response
+
+    @staticmethod
+    def store_validation(response):
+        """remember last validation"""
+        now = datetime.now()
+        message = {
+            "status": response,
+            "validated": int(now.timestamp()),
+            "validated_str": now.strftime("%Y-%m-%d %H:%M"),
+        }
+        RedisArchivist().set_message("cookie:valid", message)

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -0,0 +1,451 @@
+"""
+functionality:
+- handle yt_dlp
+- build options and post processor
+- download video files
+- move to archive
+"""
+
+import os
+import shutil
+from datetime import datetime
+
+from home.src.download.queue import PendingList
+from home.src.download.subscriptions import PlaylistSubscription
+from home.src.download.yt_dlp_base import YtWrap
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.index.channel import YoutubeChannel
+from home.src.index.comments import CommentList
+from home.src.index.playlist import YoutubePlaylist
+from home.src.index.video import YoutubeVideo, index_new_video
+from home.src.index.video_constants import VideoTypeEnum
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import get_channel_overwrites, ignore_filelist
+from home.src.ta.settings import EnvironmentSettings
+from home.src.ta.ta_redis import RedisQueue
+
+
+class DownloaderBase:
+    """base class for shared config"""
+
+    CACHE_DIR = EnvironmentSettings.CACHE_DIR
+    MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
+    CHANNEL_QUEUE = "download:channel"
+    PLAYLIST_QUEUE = "download:playlist:full"
+    PLAYLIST_QUICK = "download:playlist:quick"
+    VIDEO_QUEUE = "download:video"
+
+    def __init__(self, task):
+        self.task = task
+        self.config = AppConfig().config
+        self.channel_overwrites = get_channel_overwrites()
+        self.now = int(datetime.now().timestamp())
+
+
+class VideoDownloader(DownloaderBase):
+    """handle the video download functionality"""
+
+    def __init__(self, task=False):
+        super().__init__(task)
+        self.obs = False
+        self._build_obs()
+
+    def run_queue(self, auto_only=False) -> tuple[int, int]:
+        """setup download queue in redis loop until no more items"""
+        downloaded = 0
+        failed = 0
+        while True:
+            video_data = self._get_next(auto_only)
+            if self.task.is_stopped() or not video_data:
+                self._reset_auto()
+                break
+
+            youtube_id = video_data["youtube_id"]
+            channel_id = video_data["channel_id"]
+            print(f"{youtube_id}: Downloading video")
+            self._notify(video_data, "Validate download format")
+
+            success = self._dl_single_vid(youtube_id, channel_id)
+            if not success:
+                failed += 1
+                continue
+
+            self._notify(video_data, "Add video metadata to index", progress=1)
+            video_type = VideoTypeEnum(video_data["vid_type"])
+            vid_dict = index_new_video(youtube_id, video_type=video_type)
+            RedisQueue(self.CHANNEL_QUEUE).add(channel_id)
+            RedisQueue(self.VIDEO_QUEUE).add(youtube_id)
+
+            self._notify(video_data, "Move downloaded file to archive")
+            self.move_to_archive(vid_dict)
+            self._delete_from_pending(youtube_id)
+            downloaded += 1
+
+        # post processing
+        DownloadPostProcess(self.task).run()
+
+        return downloaded, failed
+
+    def _notify(self, video_data, message, progress=False):
+        """send progress notification to task"""
+        if not self.task:
+            return
+
+        typ = VideoTypeEnum(video_data["vid_type"]).value.rstrip("s").title()
+        title = video_data.get("title")
+        self.task.send_progress(
+            [f"Processing {typ}: {title}", message], progress=progress
+        )
+
+    def _get_next(self, auto_only):
+        """get next item in queue"""
+        must_list = [{"term": {"status": {"value": "pending"}}}]
+        must_not_list = [{"exists": {"field": "message"}}]
+        if auto_only:
+            must_list.append({"term": {"auto_start": {"value": True}}})
+
+        data = {
+            "size": 1,
+            "query": {"bool": {"must": must_list, "must_not": must_not_list}},
+            "sort": [
+                {"auto_start": {"order": "desc"}},
+                {"timestamp": {"order": "asc"}},
+            ],
+        }
+        path = "ta_download/_search"
+        response, _ = ElasticWrap(path).get(data=data)
+        if not response["hits"]["hits"]:
+            return False
+
+        return response["hits"]["hits"][0]["_source"]
+
+    def _progress_hook(self, response):
+        """process the progress_hooks from yt_dlp"""
+        progress = False
+        try:
+            size = response.get("_total_bytes_str")
+            if size.strip() == "N/A":
+                size = response.get("_total_bytes_estimate_str", "N/A")
+
+            percent = response["_percent_str"]
+            progress = float(percent.strip("%")) / 100
+            speed = response["_speed_str"]
+            eta = response["_eta_str"]
+            message = f"{percent} of {size} at {speed} - time left: {eta}"
+        except KeyError:
+            message = "processing"
+
+        if self.task:
+            title = response["info_dict"]["title"]
+            self.task.send_progress([title, message], progress=progress)
+
+    def _build_obs(self):
+        """collection to build all obs passed to yt-dlp"""
+        self._build_obs_basic()
+        self._build_obs_user()
+        self._build_obs_postprocessors()
+
+    def _build_obs_basic(self):
+        """initial obs"""
+        self.obs = {
+            "merge_output_format": "mp4",
+            "outtmpl": (self.CACHE_DIR + "/download/%(id)s.mp4"),
+            "progress_hooks": [self._progress_hook],
+            "noprogress": True,
+            "continuedl": True,
+            "writethumbnail": False,
+            "noplaylist": True,
+            "color": "no_color",
+        }
+
+    def _build_obs_user(self):
+        """build user customized options"""
+        if self.config["downloads"]["format"]:
+            self.obs["format"] = self.config["downloads"]["format"]
+        if self.config["downloads"]["format_sort"]:
+            format_sort = self.config["downloads"]["format_sort"]
+            format_sort_list = [i.strip() for i in format_sort.split(",")]
+            self.obs["format_sort"] = format_sort_list
+        if self.config["downloads"]["limit_speed"]:
+            self.obs["ratelimit"] = (
+                self.config["downloads"]["limit_speed"] * 1024
+            )
+
+        throttle = self.config["downloads"]["throttledratelimit"]
+        if throttle:
+            self.obs["throttledratelimit"] = throttle * 1024
+
+    def _build_obs_postprocessors(self):
+        """add postprocessor to obs"""
+        postprocessors = []
+
+        if self.config["downloads"]["add_metadata"]:
+            postprocessors.append(
+                {
+                    "key": "FFmpegMetadata",
+                    "add_chapters": True,
+                    "add_metadata": True,
+                }
+            )
+            postprocessors.append(
+                {
+                    "key": "MetadataFromField",
+                    "formats": [
+                        "%(title)s:%(meta_title)s",
+                        "%(uploader)s:%(meta_artist)s",
+                        ":(?P<album>)",
+                    ],
+                    "when": "pre_process",
+                }
+            )
+
+        if self.config["downloads"]["add_thumbnail"]:
+            postprocessors.append(
+                {
+                    "key": "EmbedThumbnail",
+                    "already_have_thumbnail": True,
+                }
+            )
+            self.obs["writethumbnail"] = True
+
+        self.obs["postprocessors"] = postprocessors
+
+    def _set_overwrites(self, obs: dict, channel_id: str) -> None:
+        """add overwrites to obs"""
+        overwrites = self.channel_overwrites.get(channel_id)
+        if overwrites and overwrites.get("download_format"):
+            obs["format"] = overwrites.get("download_format")
+
+    def _dl_single_vid(self, youtube_id: str, channel_id: str) -> bool:
+        """download single video"""
+        obs = self.obs.copy()
+        self._set_overwrites(obs, channel_id)
+        dl_cache = os.path.join(self.CACHE_DIR, "download")
+
+        success, message = YtWrap(obs, self.config).download(youtube_id)
+        if not success:
+            self._handle_error(youtube_id, message)
+
+        if self.obs["writethumbnail"]:
+            # webp files don't get cleaned up automatically
+            all_cached = ignore_filelist(os.listdir(dl_cache))
+            to_clean = [i for i in all_cached if not i.endswith(".mp4")]
+            for file_name in to_clean:
+                file_path = os.path.join(dl_cache, file_name)
+                os.remove(file_path)
+
+        return success
+
+    @staticmethod
+    def _handle_error(youtube_id, message):
+        """store error message"""
+        data = {"doc": {"message": message}}
+        _, _ = ElasticWrap(f"ta_download/_update/{youtube_id}").post(data=data)
+
+    def move_to_archive(self, vid_dict):
+        """move downloaded video from cache to archive"""
+        host_uid = EnvironmentSettings.HOST_UID
+        host_gid = EnvironmentSettings.HOST_GID
+        # make folder
+        folder = os.path.join(
+            self.MEDIA_DIR, vid_dict["channel"]["channel_id"]
+        )
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+            if host_uid and host_gid:
+                os.chown(folder, host_uid, host_gid)
+        # move media file
+        media_file = vid_dict["youtube_id"] + ".mp4"
+        old_path = os.path.join(self.CACHE_DIR, "download", media_file)
+        new_path = os.path.join(self.MEDIA_DIR, vid_dict["media_url"])
+        # move media file and fix permission
+        shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+        if host_uid and host_gid:
+            os.chown(new_path, host_uid, host_gid)
+
+    @staticmethod
+    def _delete_from_pending(youtube_id):
+        """delete downloaded video from pending index if its there"""
+        path = f"ta_download/_doc/{youtube_id}?refresh=true"
+        _, _ = ElasticWrap(path).delete()
+
+    def _reset_auto(self):
+        """reset autostart to defaults after queue stop"""
+        path = "ta_download/_update_by_query"
+        data = {
+            "query": {"term": {"auto_start": {"value": True}}},
+            "script": {
+                "source": "ctx._source.auto_start = false",
+                "lang": "painless",
+            },
+        }
+        response, _ = ElasticWrap(path).post(data=data)
+        updated = response.get("updated")
+        if updated:
+            print(f"[download] reset auto start on {updated} videos.")
+
+
+class DownloadPostProcess(DownloaderBase):
+    """handle task to run after download queue finishes"""
+
+    def run(self):
+        """run all functions"""
+        self.auto_delete_all()
+        self.auto_delete_overwrites()
+        self.refresh_playlist()
+        self.match_videos()
+        self.get_comments()
+
+    def auto_delete_all(self):
+        """handle auto delete"""
+        autodelete_days = self.config["downloads"]["autodelete_days"]
+        if not autodelete_days:
+            return
+
+        print(f"auto delete older than {autodelete_days} days")
+        now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
+        data = {
+            "query": {"range": {"player.watched_date": {"lte": now_lte}}},
+            "sort": [{"player.watched_date": {"order": "asc"}}],
+        }
+        self._auto_delete_watched(data)
+
+    def auto_delete_overwrites(self):
+        """handle per channel auto delete from overwrites"""
+        for channel_id, value in self.channel_overwrites.items():
+            if "autodelete_days" in value:
+                autodelete_days = value.get("autodelete_days")
+                print(f"{channel_id}: delete older than {autodelete_days}d")
+                now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
+                must_list = [
+                    {"range": {"player.watched_date": {"lte": now_lte}}},
+                    {"term": {"channel.channel_id": {"value": channel_id}}},
+                ]
+                data = {
+                    "query": {"bool": {"must": must_list}},
+                    "sort": [{"player.watched_date": {"order": "desc"}}],
+                }
+                self._auto_delete_watched(data)
+
+    @staticmethod
+    def _auto_delete_watched(data):
+        """delete watched videos after x days"""
+        to_delete = IndexPaginate("ta_video", data).get_results()
+        if not to_delete:
+            return
+
+        for video in to_delete:
+            youtube_id = video["youtube_id"]
+            print(f"{youtube_id}: auto delete video")
+            YoutubeVideo(youtube_id).delete_media_file()
+
+        print("add deleted to ignore list")
+        vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
+        pending = PendingList(youtube_ids=vids)
+        pending.parse_url_list()
+        _ = pending.add_to_pending(status="ignore")
+
+    def refresh_playlist(self) -> None:
+        """match videos with playlists"""
+        self.add_playlists_to_refresh()
+
+        queue = RedisQueue(self.PLAYLIST_QUEUE)
+        while True:
+            total = queue.max_score()
+            playlist_id, idx = queue.get_next()
+            if not playlist_id or not idx or not total:
+                break
+
+            playlist = YoutubePlaylist(playlist_id)
+            playlist.update_playlist(skip_on_empty=True)
+
+            if not self.task:
+                continue
+
+            channel_name = playlist.json_data["playlist_channel"]
+            playlist_title = playlist.json_data["playlist_name"]
+            message = [
+                f"Post Processing Playlists for: {channel_name}",
+                f"{playlist_title} [{idx}/{total}]",
+            ]
+            progress = idx / total
+            self.task.send_progress(message, progress=progress)
+
+    def add_playlists_to_refresh(self) -> None:
+        """add playlists to refresh"""
+        if self.task:
+            message = ["Post Processing Playlists", "Scanning for Playlists"]
+            self.task.send_progress(message)
+
+        self._add_playlist_sub()
+        self._add_channel_playlists()
+        self._add_video_playlists()
+
+    def _add_playlist_sub(self):
+        """add subscribed playlists to refresh"""
+        subs = PlaylistSubscription().get_playlists()
+        to_add = [i["playlist_id"] for i in subs]
+        RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
+
+    def _add_channel_playlists(self):
+        """add playlists from channels to refresh"""
+        queue = RedisQueue(self.CHANNEL_QUEUE)
+        while True:
+            channel_id, _ = queue.get_next()
+            if not channel_id:
+                break
+
+            channel = YoutubeChannel(channel_id)
+            channel.get_from_es()
+            overwrites = channel.get_overwrites()
+            if "index_playlists" in overwrites:
+                channel.get_all_playlists()
+                to_add = [i[0] for i in channel.all_playlists]
+                RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
+
+    def _add_video_playlists(self):
+        """add other playlists for quick sync"""
+        all_playlists = RedisQueue(self.PLAYLIST_QUEUE).get_all()
+        must_not = [{"terms": {"playlist_id": all_playlists}}]
+        video_ids = RedisQueue(self.VIDEO_QUEUE).get_all()
+        must = [{"terms": {"playlist_entries.youtube_id": video_ids}}]
+        data = {
+            "query": {"bool": {"must_not": must_not, "must": must}},
+            "_source": ["playlist_id"],
+        }
+        playlists = IndexPaginate("ta_playlist", data).get_results()
+        to_add = [i["playlist_id"] for i in playlists]
+        RedisQueue(self.PLAYLIST_QUICK).add_list(to_add)
+
+    def match_videos(self) -> None:
+        """scan rest of indexed playlists to match videos"""
+        queue = RedisQueue(self.PLAYLIST_QUICK)
+        while True:
+            total = queue.max_score()
+            playlist_id, idx = queue.get_next()
+            if not playlist_id or not idx or not total:
+                break
+
+            playlist = YoutubePlaylist(playlist_id)
+            playlist.get_from_es()
+            playlist.add_vids_to_playlist()
+            playlist.remove_vids_from_playlist()
+
+            if not self.task:
+                continue
+
+            message = [
+                "Post Processing Playlists.",
+                f"Validate Playlists: - {idx}/{total}",
+            ]
+            progress = idx / total
+            self.task.send_progress(message, progress=progress)
+
+    def get_comments(self):
+        """get comments from youtube"""
+        video_queue = RedisQueue(self.VIDEO_QUEUE)
+        comment_list = CommentList(task=self.task)
+        comment_list.add(video_ids=video_queue.get_all())
+
+        video_queue.clear()
+        comment_list.index()

tubearchivist/home/src/es/backup.py

@@ -0,0 +1,273 @@
+"""
+Functionality:
+- Handle json zip file based backup
+- create backup
+- restore backup
+"""
+
+import json
+import os
+import zipfile
+from datetime import datetime
+
+from home.models import CustomPeriodicTask
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import get_mapping, ignore_filelist
+from home.src.ta.settings import EnvironmentSettings
+
+
+class ElasticBackup:
+    """dump index to nd-json files for later bulk import"""
+
+    INDEX_SPLIT = ["comment"]
+    CACHE_DIR = EnvironmentSettings.CACHE_DIR
+    BACKUP_DIR = os.path.join(CACHE_DIR, "backup")
+
+    def __init__(self, reason=False, task=False):
+        self.config = AppConfig().config
+        self.timestamp = datetime.now().strftime("%Y%m%d")
+        self.index_config = get_mapping()
+        self.reason = reason
+        self.task = task
+
+    def backup_all_indexes(self):
+        """backup all indexes, add reason to init"""
+        print("backup all indexes")
+        if not self.reason:
+            raise ValueError("missing backup reason in ElasticBackup")
+
+        if self.task:
+            self.task.send_progress(["Scanning your index."])
+        for index in self.index_config:
+            index_name = index["index_name"]
+            print(f"backup: export in progress for {index_name}")
+            if not self.index_exists(index_name):
+                print(f"skip backup for not yet existing index {index_name}")
+                continue
+
+            self.backup_index(index_name)
+
+        if self.task:
+            self.task.send_progress(["Compress files to zip archive."])
+        self.zip_it()
+        if self.reason == "auto":
+            self.rotate_backup()
+
+    def backup_index(self, index_name):
+        """export all documents of a single index"""
+        paginate_kwargs = {
+            "data": {"query": {"match_all": {}}},
+            "keep_source": True,
+            "callback": BackupCallback,
+            "task": self.task,
+            "total": self._get_total(index_name),
+        }
+
+        if index_name in self.INDEX_SPLIT:
+            paginate_kwargs.update({"size": 200})
+
+        paginate = IndexPaginate(f"ta_{index_name}", **paginate_kwargs)
+        _ = paginate.get_results()
+
+    @staticmethod
+    def _get_total(index_name):
+        """get total documents in index"""
+        path = f"ta_{index_name}/_count"
+        response, _ = ElasticWrap(path).get()
+
+        return response.get("count")
+
+    def zip_it(self):
+        """pack it up into single zip file"""
+        file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
+
+        to_backup = []
+        for file in os.listdir(self.BACKUP_DIR):
+            if file.endswith(".json"):
+                to_backup.append(os.path.join(self.BACKUP_DIR, file))
+
+        backup_file = os.path.join(self.BACKUP_DIR, file_name)
+
+        comp = zipfile.ZIP_DEFLATED
+        with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
+            for backup_file in to_backup:
+                zip_f.write(backup_file, os.path.basename(backup_file))
+
+        # cleanup
+        for backup_file in to_backup:
+            os.remove(backup_file)
+
+    def post_bulk_restore(self, file_name):
+        """send bulk to es"""
+        file_path = os.path.join(self.CACHE_DIR, file_name)
+        with open(file_path, "r", encoding="utf-8") as f:
+            data = f.read()
+
+        if not data.strip():
+            return
+
+        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
+
+    def get_all_backup_files(self):
+        """build all available backup files for view"""
+        all_backup_files = ignore_filelist(os.listdir(self.BACKUP_DIR))
+        all_available_backups = [
+            i
+            for i in all_backup_files
+            if i.startswith("ta_") and i.endswith(".zip")
+        ]
+        all_available_backups.sort(reverse=True)
+
+        backup_dicts = []
+        for filename in all_available_backups:
+            data = self.build_backup_file_data(filename)
+            backup_dicts.append(data)
+
+        return backup_dicts
+
+    def build_backup_file_data(self, filename):
+        """build metadata of single backup file"""
+        file_path = os.path.join(self.BACKUP_DIR, filename)
+        if not os.path.exists(file_path):
+            return False
+
+        file_split = filename.split("-")
+        if len(file_split) == 2:
+            timestamp = file_split[1].strip(".zip")
+            reason = False
+        elif len(file_split) == 3:
+            timestamp = file_split[1]
+            reason = file_split[2].strip(".zip")
+
+        data = {
+            "filename": filename,
+            "file_path": file_path,
+            "file_size": os.path.getsize(file_path),
+            "timestamp": timestamp,
+            "reason": reason,
+        }
+
+        return data
+
+    def restore(self, filename):
+        """
+        restore from backup zip file
+        call reset from ElasitIndexWrap first to start blank
+        """
+        zip_content = self._unpack_zip_backup(filename)
+        self._restore_json_files(zip_content)
+
+    def _unpack_zip_backup(self, filename):
+        """extract backup zip and return filelist"""
+        file_path = os.path.join(self.BACKUP_DIR, filename)
+
+        with zipfile.ZipFile(file_path, "r") as z:
+            zip_content = z.namelist()
+            z.extractall(self.BACKUP_DIR)
+
+        return zip_content
+
+    def _restore_json_files(self, zip_content):
+        """go through the unpacked files and restore"""
+        for idx, json_f in enumerate(zip_content):
+            self._notify_restore(idx, json_f, len(zip_content))
+            file_name = os.path.join(self.BACKUP_DIR, json_f)
+
+            if not json_f.startswith("es_") or not json_f.endswith(".json"):
+                os.remove(file_name)
+                continue
+
+            print("restoring: " + json_f)
+            self.post_bulk_restore(file_name)
+            os.remove(file_name)
+
+    def _notify_restore(self, idx, json_f, total_files):
+        """notify restore progress"""
+        message = [f"Restore index from json backup file {json_f}."]
+        progress = (idx + 1) / total_files
+        self.task.send_progress(message_lines=message, progress=progress)
+
+    @staticmethod
+    def index_exists(index_name):
+        """check if index already exists to skip"""
+        _, status_code = ElasticWrap(f"ta_{index_name}").get()
+        exists = status_code == 200
+
+        return exists
+
+    def rotate_backup(self):
+        """delete old backups if needed"""
+        try:
+            task = CustomPeriodicTask.objects.get(name="run_backup")
+        except CustomPeriodicTask.DoesNotExist:
+            return
+
+        rotate = task.task_config.get("rotate")
+        if not rotate:
+            return
+
+        all_backup_files = self.get_all_backup_files()
+        auto = [i for i in all_backup_files if i["reason"] == "auto"]
+
+        if len(auto) <= rotate:
+            print("no backup files to rotate")
+            return
+
+        all_to_delete = auto[rotate:]
+        for to_delete in all_to_delete:
+            self.delete_file(to_delete["filename"])
+
+    def delete_file(self, filename):
+        """delete backup file"""
+        file_path = os.path.join(self.BACKUP_DIR, filename)
+        if not os.path.exists(file_path):
+            print(f"backup file not found: {filename}")
+            return False
+
+        print(f"remove old backup file: {file_path}")
+        os.remove(file_path)
+
+        return file_path
+
+
+class BackupCallback:
+    """handle backup ndjson writer as callback for IndexPaginate"""
+
+    def __init__(self, source, index_name, counter=0):
+        self.source = source
+        self.index_name = index_name
+        self.counter = counter
+        self.timestamp = datetime.now().strftime("%Y%m%d")
+        self.cache_dir = EnvironmentSettings.CACHE_DIR
+
+    def run(self):
+        """run the junk task"""
+        file_content = self._build_bulk()
+        self._write_es_json(file_content)
+
+    def _build_bulk(self):
+        """build bulk query data from all_results"""
+        bulk_list = []
+
+        for document in self.source:
+            document_id = document["_id"]
+            es_index = document["_index"]
+            action = {"index": {"_index": es_index, "_id": document_id}}
+            source = document["_source"]
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(source))
+
+        # add last newline
+        bulk_list.append("\n")
+        file_content = "\n".join(bulk_list)
+
+        return file_content
+
+    def _write_es_json(self, file_content):
+        """write nd-json file for es _bulk API to disk"""
+        index = self.index_name.lstrip("ta_")
+        file_name = f"es_{index}-{self.timestamp}-{self.counter}.json"
+        file_path = os.path.join(self.cache_dir, "backup", file_name)
+        with open(file_path, "a+", encoding="utf-8") as f:
+            f.write(file_content)

tubearchivist/home/src/es/connect.py

@@ -0,0 +1,231 @@
+"""
+functionality:
+- wrapper around requests to call elastic search
+- reusable search_after to extract total index
+"""
+
+# pylint: disable=missing-timeout
+
+import json
+from typing import Any
+
+import requests
+import urllib3
+from home.src.ta.settings import EnvironmentSettings
+
+
+class ElasticWrap:
+    """makes all calls to elastic search
+    returns response json and status code tuple
+    """
+
+    def __init__(self, path: str):
+        self.url: str = f"{EnvironmentSettings.ES_URL}/{path}"
+        self.auth: tuple[str, str] = (
+            EnvironmentSettings.ES_USER,
+            EnvironmentSettings.ES_PASS,
+        )
+
+        if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
+            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    def get(
+        self,
+        data: bool | dict = False,
+        timeout: int = 10,
+        print_error: bool = True,
+    ) -> tuple[dict, int]:
+        """get data from es"""
+
+        kwargs: dict[str, Any] = {
+            "auth": self.auth,
+            "timeout": timeout,
+        }
+
+        if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
+            kwargs["verify"] = False
+
+        if data:
+            kwargs["json"] = data
+
+        response = requests.get(self.url, **kwargs)
+
+        if print_error and not response.ok:
+            print(response.text)
+
+        return response.json(), response.status_code
+
+    def post(
+        self, data: bool | dict = False, ndjson: bool = False
+    ) -> tuple[dict, int]:
+        """post data to es"""
+
+        kwargs: dict[str, Any] = {"auth": self.auth}
+
+        if ndjson and data:
+            kwargs.update(
+                {
+                    "headers": {"Content-type": "application/x-ndjson"},
+                    "data": data,
+                }
+            )
+        elif data:
+            kwargs.update(
+                {
+                    "headers": {"Content-type": "application/json"},
+                    "data": json.dumps(data),
+                }
+            )
+
+        if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
+            kwargs["verify"] = False
+
+        response = requests.post(self.url, **kwargs)
+
+        if not response.ok:
+            print(response.text)
+
+        return response.json(), response.status_code
+
+    def put(
+        self,
+        data: bool | dict = False,
+        refresh: bool = False,
+    ) -> tuple[dict, Any]:
+        """put data to es"""
+
+        if refresh:
+            self.url = f"{self.url}/?refresh=true"
+
+        kwargs: dict[str, Any] = {
+            "json": data,
+            "auth": self.auth,
+        }
+
+        if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
+            kwargs["verify"] = False
+
+        response = requests.put(self.url, **kwargs)
+
+        if not response.ok:
+            print(response.text)
+            print(data)
+            raise ValueError("failed to add item to index")
+
+        return response.json(), response.status_code
+
+    def delete(
+        self,
+        data: bool | dict = False,
+        refresh: bool = False,
+    ) -> tuple[dict, Any]:
+        """delete document from es"""
+
+        if refresh:
+            self.url = f"{self.url}/?refresh=true"
+
+        kwargs: dict[str, Any] = {"auth": self.auth}
+
+        if data:
+            kwargs["json"] = data
+
+        if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
+            kwargs["verify"] = False
+
+        response = requests.delete(self.url, **kwargs)
+
+        if not response.ok:
+            print(response.text)
+
+        return response.json(), response.status_code
+
+
+class IndexPaginate:
+    """use search_after to go through whole index
+    kwargs:
+    - size: int, overwrite DEFAULT_SIZE
+    - keep_source: bool, keep _source key from es results
+    - callback: obj, Class implementing run method callback for every loop
+    - task: task object to send notification
+    - total: int, total items in index for progress message
+    """
+
+    DEFAULT_SIZE = 500
+
+    def __init__(self, index_name, data, **kwargs):
+        self.index_name = index_name
+        self.data = data
+        self.pit_id = False
+        self.kwargs = kwargs
+
+    def get_results(self):
+        """get all results, add task and total for notifications"""
+        self.get_pit()
+        self.validate_data()
+        all_results = self.run_loop()
+        self.clean_pit()
+        return all_results
+
+    def get_pit(self):
+        """get pit for index"""
+        path = f"{self.index_name}/_pit?keep_alive=10m"
+        response, _ = ElasticWrap(path).post()
+        self.pit_id = response["id"]
+
+    def validate_data(self):
+        """add pit and size to data"""
+        if not self.data:
+            self.data = {}
+
+        if "query" not in self.data.keys():
+            self.data.update({"query": {"match_all": {}}})
+
+        if "sort" not in self.data.keys():
+            self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
+
+        self.data["size"] = self.kwargs.get("size") or self.DEFAULT_SIZE
+        self.data["pit"] = {"id": self.pit_id, "keep_alive": "10m"}
+
+    def run_loop(self):
+        """loop through results until last hit"""
+        all_results = []
+        counter = 0
+        while True:
+            response, _ = ElasticWrap("_search").get(data=self.data)
+            all_hits = response["hits"]["hits"]
+            if not all_hits:
+                break
+
+            for hit in all_hits:
+                if self.kwargs.get("keep_source"):
+                    all_results.append(hit)
+                else:
+                    all_results.append(hit["_source"])
+
+            if self.kwargs.get("callback"):
+                self.kwargs.get("callback")(
+                    all_hits, self.index_name, counter=counter
+                ).run()
+
+            if self.kwargs.get("task"):
+                print(f"{self.index_name}: processing page {counter}")
+                self._notify(len(all_results))
+
+            counter += 1
+
+            # update search_after with last hit data
+            self.data["search_after"] = all_hits[-1]["sort"]
+
+        return all_results
+
+    def _notify(self, processed):
+        """send notification on task"""
+        total = self.kwargs.get("total")
+        progress = processed / total
+        index_clean = self.index_name.lstrip("ta_").title()
+        message = [f"Processing {index_clean}s {processed}/{total}"]
+        self.kwargs.get("task").send_progress(message, progress=progress)
+
+    def clean_pit(self):
+        """delete pit from elastic search"""
+        ElasticWrap("_pit").delete(data={"id": self.pit_id})

tubearchivist/home/src/es/index_mapping.json

@@ -0,0 +1,687 @@
+{
+    "index_config": [{
+            "index_name": "config",
+            "expected_map": {
+                "config": {
+                    "type": "object",
+                    "enabled": false
+                }
+            },
+            "expected_set": {
+                "number_of_replicas": "0"
+            }
+        },
+        {
+            "index_name": "channel",
+            "expected_map": {
+                "channel_id": {
+                    "type": "keyword"
+                },
+                "channel_name": {
+                    "type": "text",
+                    "analyzer": "english",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256,
+                            "normalizer": "to_lower"
+                        },
+                        "search_as_you_type": {
+                            "type": "search_as_you_type",
+                            "doc_values": false,
+                            "max_shingle_size": 3
+                        }
+                    }
+                },
+                "channel_banner_url": {
+                    "type": "keyword",
+                    "index": false
+                },
+                "channel_tvart_url": {
+                    "type": "keyword",
+                    "index": false
+                },
+                "channel_thumb_url": {
+                    "type": "keyword",
+                    "index": false
+                },
+                "channel_description": {
+                    "type": "text"
+                },
+                "channel_last_refresh": {
+                    "type": "date",
+                    "format": "epoch_second"
+                },
+                "channel_tags": {
+                    "type": "text",
+                    "analyzer": "english",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+                },
+                "channel_overwrites": {
+                    "properties": {
+                        "download_format": {
+                            "type": "text"
+                        },
+                        "autodelete_days": {
+                            "type": "long"
+                        },
+                        "index_playlists": {
+                            "type": "boolean"
+                        },
+                        "integrate_sponsorblock": {
+                            "type": "boolean"
+                        },
+                        "subscriptions_channel_size": {
+                            "type": "long"
+                        },
+                        "subscriptions_live_channel_size": {
+                            "type": "long"
+                        },
+                        "subscriptions_shorts_channel_size": {
+                            "type": "long"
+                        }
+                    }
+                }
+            },
+            "expected_set": {
+                "analysis": {
+                    "normalizer": {
+                        "to_lower": {
+                            "type": "custom",
+                            "filter": ["lowercase"]
+                        }
+                    }
+                },
+                "number_of_replicas": "0"
+            }
+        },
+        {
+            "index_name": "video",
+            "expected_map": {
+                "vid_thumb_url": {
+                    "type": "text",
+                    "index": false
+                },
+                "vid_thumb_base64": {
+                    "type": "text",
+                    "index": false
+                },
+                "date_downloaded": {
+                    "type": "date",
+                    "format": "epoch_second"
+                },
+                "channel": {
+                    "properties": {
+                        "channel_id": {
+                            "type": "keyword"
+                        },
+                        "channel_name": {
+                            "type": "text",
+                            "analyzer": "english",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "ignore_above": 256,
+                                    "normalizer": "to_lower"
+                                },
+                                "search_as_you_type": {
+                                    "type": "search_as_you_type",
+                                    "doc_values": false,
+                                    "max_shingle_size": 3
+                                }
+                            }
+                        },
+                        "channel_banner_url": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "channel_tvart_url": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "channel_thumb_url": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "channel_description": {
+                            "type": "text"
+                        },
+                        "channel_last_refresh": {
+                            "type": "date",
+                            "format": "epoch_second"
+                        },
+                        "channel_tags": {
+                            "type": "text",
+                            "analyzer": "english",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "ignore_above": 256
+                                }
+                            }
+                        },
+                        "channel_overwrites": {
+                            "properties": {
+                                "download_format": {
+                                    "type": "text"
+                                },
+                                "autodelete_days": {
+                                    "type": "long"
+                                },
+                                "index_playlists": {
+                                    "type": "boolean"
+                                },
+                                "integrate_sponsorblock": {
+                                    "type": "boolean"
+                                },
+                                "subscriptions_channel_size": {
+                                    "type": "long"
+                                },
+                                "subscriptions_live_channel_size": {
+                                    "type": "long"
+                                },
+                                "subscriptions_shorts_channel_size": {
+                                    "type": "long"
+                                }
+                            }
+                        }
+                    }
+                },
+                "description": {
+                    "type": "text"
+                },
+                "media_url": {
+                    "type": "keyword",
+                    "index": false
+                },
+                "media_size": {
+                    "type": "long"
+                },
+                "tags": {
+                    "type": "text",
+                    "analyzer": "english",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+                },
+                "title": {
+                    "type": "text",
+                    "analyzer": "english",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256,
+                            "normalizer": "to_lower"
+                        },
+                        "search_as_you_type": {
+                            "type": "search_as_you_type",
+                            "doc_values": false,
+                            "max_shingle_size": 3
+                        }
+                    }
+                },
+                "vid_last_refresh": {
+                    "type": "date",
+                    "format": "epoch_second"
+                },
+                "youtube_id": {
+                    "type": "keyword"
+                },
+                "vid_type": {
+                    "type": "keyword"
+                },
+                "published": {
+                    "type": "date"
+                },
+                "playlist": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256,
+                            "normalizer": "to_lower"
+                        }
+                    }
+                },
+                "comment_count": {
+                    "type": "long"
+                },
+                "stats": {
+                    "properties": {
+                        "average_rating": {
+                            "type": "float"
+                        },
+                        "dislike_count": {
+                            "type": "long"
+                        },
+                        "like_count": {
+                            "type": "long"
+                        },
+                        "view_count": {
+                            "type": "long"
+                        }
+                    }
+                },
+                "player": {
+                    "properties": {
+                        "duration": {
+                            "type": "long"
+                        },
+                        "duration_str": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "watched": {
+                            "type": "boolean"
+                        },
+                        "watched_date": {
+                            "type": "date",
+                            "format": "epoch_second"
+                        }
+                    }
+                },
+                "subtitles": {
+                    "properties": {
+                        "ext": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "lang": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "media_url": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "name": {
+                            "type": "keyword"
+                        },
+                        "source": {
+                            "type": "keyword"
+                        },
+                        "url": {
+                            "type": "keyword",
+                            "index": false
+                        }
+                    }
+                },
+                "streams": {
+                    "properties": {
+                        "type": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "index": {
+                            "type": "short",
+                            "index": false
+                        },
+                        "codec": {
+                            "type": "text"
+                        },
+                        "width": {
+                            "type": "short"
+                        },
+                        "height": {
+                            "type": "short"
+                        },
+                        "bitrate": {
+                            "type": "integer"
+                        }
+                    }
+                },
+                "sponsorblock": {
+                    "properties": {
+                        "last_refresh": {
+                            "type": "date",
+                            "format": "epoch_second"
+                        },
+                        "has_unlocked": {
+                            "type": "boolean"
+                        },
+                        "is_enabled": {
+                            "type": "boolean"
+                        },
+                        "segments": {
+                            "properties": {
+                                "UUID": {
+                                    "type": "keyword"
+                                },
+                                "actionType": {
+                                    "type": "keyword"
+                                },
+                                "category": {
+                                    "type": "keyword"
+                                },
+                                "locked": {
+                                    "type": "short"
+                                },
+                                "segment": {
+                                    "type": "float"
+                                },
+                                "videoDuration": {
+                                    "type": "float"
+                                },
+                                "votes": {
+                                    "type": "long"
+                                }
+                            }
+                        }
+                    }
+                }
+            },
+            "expected_set": {
+                "analysis": {
+                    "normalizer": {
+                        "to_lower": {
+                            "type": "custom",
+                            "filter": ["lowercase"]
+                        }
+                    }
+                },
+                "number_of_replicas": "0"
+            }
+        },
+        {
+            "index_name": "download",
+            "expected_map": {
+                "timestamp": {
+                    "type": "date",
+                    "format": "epoch_second"
+                },
+                "channel_id": {
+                    "type": "keyword"
+                },
+                "channel_name": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256,
+                            "normalizer": "to_lower"
+                        }
+                    }
+                },
+                "status": {
+                    "type": "keyword"
+                },
+                "title": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256,
+                            "normalizer": "to_lower"
+                        }
+                    }
+                },
+                "vid_thumb_url": {
+                    "type": "keyword"
+                },
+                "youtube_id": {
+                    "type": "keyword"
+                },
+                "vid_type": {
+                    "type": "keyword"
+                },
+                "auto_start": {
+                    "type": "boolean"
+                },
+                "message": {
+                    "type": "text"
+                }
+            },
+            "expected_set": {
+                "analysis": {
+                    "normalizer": {
+                        "to_lower": {
+                            "type": "custom",
+                            "filter": ["lowercase"]
+                        }
+                    }
+                },
+                "number_of_replicas": "0"
+            }
+        },
+        {
+            "index_name": "playlist",
+            "expected_map": {
+                "playlist_id": {
+                    "type": "keyword"
+                },
+                "playlist_description": {
+                    "type": "text"
+                },
+                "playlist_name": {
+                    "type": "text",
+                    "analyzer": "english",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256,
+                            "normalizer": "to_lower"
+                        },
+                        "search_as_you_type": {
+                            "type": "search_as_you_type",
+                            "doc_values": false,
+                            "max_shingle_size": 3
+                        }
+                    }
+                },
+                "playlist_channel": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256,
+                            "normalizer": "to_lower"
+                        }
+                    }
+                },
+                "playlist_channel_id": {
+                    "type": "keyword"
+                },
+                "playlist_thumbnail": {
+                    "type": "keyword"
+                },
+                "playlist_last_refresh": {
+                    "type": "date",
+                    "format": "epoch_second"
+                },
+                "playlist_entries": {
+                    "properties": {
+                        "downloaded": {
+                            "type": "boolean"
+                        },
+                        "idx": {
+                            "type": "long"
+                        },
+                        "title": {
+                            "type": "text",
+                            "analyzer": "english",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "ignore_above": 256,
+                                    "normalizer": "to_lower"
+                                }
+                            }
+                        },
+                        "uploader": {
+                            "type": "text",
+                            "analyzer": "english",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "ignore_above": 256,
+                                    "normalizer": "to_lower"
+                                }
+                            }
+                        },
+                        "youtube_id": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            },
+            "expected_set": {
+                "analysis": {
+                    "normalizer": {
+                        "to_lower": {
+                            "type": "custom",
+                            "filter": ["lowercase"]
+                        }
+                    }
+                },
+                "number_of_replicas": "0"
+            }
+        },
+        {
+            "index_name": "subtitle",
+            "expected_map": {
+                "youtube_id": {
+                    "type": "keyword"
+                },
+                "title": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256,
+                            "normalizer": "to_lower"
+                        }
+                    }
+                },
+                "subtitle_fragment_id": {
+                    "type": "keyword"
+                },
+                "subtitle_channel": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256,
+                            "normalizer": "to_lower"
+                        }
+                    }
+                },
+                "subtitle_channel_id": {
+                    "type": "keyword"
+                },
+                "subtitle_start": {
+                    "type": "text"
+                },
+                "subtitle_end": {
+                    "type": "text"
+                },
+                "subtitle_last_refresh": {
+                    "type": "date",
+                    "format": "epoch_second"
+                },
+                "subtitle_index": {
+                    "type": "long"
+                },
+                "subtitle_lang": {
+                    "type": "keyword"
+                },
+                "subtitle_source": {
+                    "type": "keyword"
+                },
+                "subtitle_line": {
+                    "type": "text",
+                    "analyzer": "english"
+                }
+            },
+            "expected_set": {
+                "analysis": {
+                    "normalizer": {
+                        "to_lower": {
+                            "type": "custom",
+                            "filter": ["lowercase"]
+                        }
+                    }
+                },
+                "number_of_replicas": "0"
+            }
+        },
+        {
+            "index_name": "comment",
+            "expected_map": {
+                "youtube_id": {
+                    "type": "keyword"
+                },
+                "comment_last_refresh": {
+                    "type": "date",
+                    "format": "epoch_second"
+                },
+                "comment_channel_id": {
+                    "type": "keyword"
+                },
+                "comment_comments": {
+                    "properties": {
+                        "comment_id": {
+                            "type": "keyword"
+                        },
+                        "comment_text": {
+                            "type": "text"
+                        },
+                        "comment_timestamp": {
+                            "type": "date",
+                            "format": "epoch_second"
+                        },
+                        "comment_time_text": {
+                            "type": "text"
+                        },
+                        "comment_likecount": {
+                            "type": "long"
+                        },
+                        "comment_is_favorited": {
+                            "type": "boolean"
+                        },
+                        "comment_author": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "ignore_above": 256,
+                                    "normalizer": "to_lower"
+                                }
+                            }
+                        },
+                        "comment_author_id": {
+                            "type": "keyword"
+                        },
+                        "comment_author_thumbnail": {
+                            "type": "keyword"
+                        },
+                        "comment_author_is_uploader": {
+                            "type": "boolean"
+                        },
+                        "comment_parent": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            },
+            "expected_set": {
+                "analysis": {
+                    "normalizer": {
+                        "to_lower": {
+                            "type": "custom",
+                            "filter": ["lowercase"]
+                        }
+                    }
+                },
+                "number_of_replicas": "0"
+            }
+        }
+    ]
+}

tubearchivist/home/src/es/index_setup.py

@@ -0,0 +1,210 @@
+"""
+functionality:
+- setup elastic index at first start
+- verify and update index mapping and settings if needed
+- backup and restore metadata
+"""
+
+from home.src.es.backup import ElasticBackup
+from home.src.es.connect import ElasticWrap
+from home.src.es.snapshot import ElasticSnapshot
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import get_mapping
+
+
+class ElasticIndex:
+    """interact with a single index"""
+
+    def __init__(self, index_name, expected_map=False, expected_set=False):
+        self.index_name = index_name
+        self.expected_map = expected_map
+        self.expected_set = expected_set
+        self.exists, self.details = self.index_exists()
+
+    def index_exists(self):
+        """check if index already exists and return mapping if it does"""
+        response, status_code = ElasticWrap(f"ta_{self.index_name}").get()
+        exists = status_code == 200
+        details = response.get(f"ta_{self.index_name}", False)
+
+        return exists, details
+
+    def validate(self):
+        """
+        check if all expected mappings and settings match
+        returns True when rebuild is needed
+        """
+
+        if self.expected_map:
+            rebuild = self.validate_mappings()
+            if rebuild:
+                return rebuild
+
+        if self.expected_set:
+            rebuild = self.validate_settings()
+            if rebuild:
+                return rebuild
+
+        return False
+
+    def validate_mappings(self):
+        """check if all mappings are as expected"""
+        now_map = self.details["mappings"]["properties"]
+
+        for key, value in self.expected_map.items():
+            # nested
+            if list(value.keys()) == ["properties"]:
+                for key_n, value_n in value["properties"].items():
+                    if key not in now_map:
+                        print(f"detected mapping change: {key_n}, {value_n}")
+                        return True
+                    if key_n not in now_map[key]["properties"].keys():
+                        print(f"detected mapping change: {key_n}, {value_n}")
+                        return True
+                    if not value_n == now_map[key]["properties"][key_n]:
+                        print(f"detected mapping change: {key_n}, {value_n}")
+                        return True
+
+                continue
+
+            # not nested
+            if key not in now_map.keys():
+                print(f"detected mapping change: {key}, {value}")
+                return True
+            if not value == now_map[key]:
+                print(f"detected mapping change: {key}, {value}")
+                return True
+
+        return False
+
+    def validate_settings(self):
+        """check if all settings are as expected"""
+
+        now_set = self.details["settings"]["index"]
+
+        for key, value in self.expected_set.items():
+            if key not in now_set.keys():
+                print(key, value)
+                return True
+
+            if not value == now_set[key]:
+                print(key, value)
+                return True
+
+        return False
+
+    def rebuild_index(self):
+        """rebuild with new mapping"""
+        print(f"applying new mappings to index ta_{self.index_name}...")
+        self.create_blank(for_backup=True)
+        self.reindex("backup")
+        self.delete_index(backup=False)
+        self.create_blank()
+        self.reindex("restore")
+        self.delete_index()
+
+    def reindex(self, method):
+        """create on elastic search"""
+        if method == "backup":
+            source = f"ta_{self.index_name}"
+            destination = f"ta_{self.index_name}_backup"
+        elif method == "restore":
+            source = f"ta_{self.index_name}_backup"
+            destination = f"ta_{self.index_name}"
+
+        data = {"source": {"index": source}, "dest": {"index": destination}}
+        _, _ = ElasticWrap("_reindex?refresh=true").post(data=data)
+
+    def delete_index(self, backup=True):
+        """delete index passed as argument"""
+        path = f"ta_{self.index_name}"
+        if backup:
+            path = path + "_backup"
+
+        _, _ = ElasticWrap(path).delete()
+
+    def create_blank(self, for_backup=False):
+        """apply new mapping and settings for blank new index"""
+        print(f"create new blank index with name ta_{self.index_name}...")
+        path = f"ta_{self.index_name}"
+        if for_backup:
+            path = f"{path}_backup"
+
+        data = {}
+        if self.expected_set:
+            data.update({"settings": self.expected_set})
+        if self.expected_map:
+            data.update({"mappings": {"properties": self.expected_map}})
+
+        _, _ = ElasticWrap(path).put(data)
+
+
+class ElasitIndexWrap:
+    """interact with all index mapping and setup"""
+
+    def __init__(self):
+        self.index_config = get_mapping()
+        self.backup_run = False
+
+    def setup(self):
+        """setup elastic index, run at startup"""
+        for index in self.index_config:
+            index_name, expected_map, expected_set = self._config_split(index)
+            handler = ElasticIndex(index_name, expected_map, expected_set)
+            if not handler.exists:
+                handler.create_blank()
+                continue
+
+            rebuild = handler.validate()
+            if rebuild:
+                self._check_backup()
+                handler.rebuild_index()
+                continue
+
+            # else all good
+            print(f"ta_{index_name} index is created and up to date...")
+
+    def reset(self):
+        """reset all indexes to blank"""
+        self.delete_all()
+        self.create_all_blank()
+
+    def delete_all(self):
+        """delete all indexes"""
+        print("reset elastic index")
+        for index in self.index_config:
+            index_name, _, _ = self._config_split(index)
+            handler = ElasticIndex(index_name)
+            handler.delete_index(backup=False)
+
+    def create_all_blank(self):
+        """create all blank indexes"""
+        print("create all new indexes in elastic from template")
+        for index in self.index_config:
+            index_name, expected_map, expected_set = self._config_split(index)
+            handler = ElasticIndex(index_name, expected_map, expected_set)
+            handler.create_blank()
+
+    @staticmethod
+    def _config_split(index):
+        """split index config keys"""
+        index_name = index["index_name"]
+        expected_map = index["expected_map"]
+        expected_set = index["expected_set"]
+
+        return index_name, expected_map, expected_set
+
+    def _check_backup(self):
+        """create backup if needed"""
+        if self.backup_run:
+            return
+
+        config = AppConfig().config
+        if config["application"]["enable_snapshot"]:
+            # take snapshot if enabled
+            ElasticSnapshot().take_snapshot_now(wait=True)
+        else:
+            # fallback to json backup
+            ElasticBackup(reason="update").backup_all_indexes()
+
+        self.backup_run = True

tubearchivist/home/src/es/snapshot.py

@@ -0,0 +1,286 @@
+"""
+functionality:
+- handle snapshots in ES
+"""
+
+from datetime import datetime
+from time import sleep
+from zoneinfo import ZoneInfo
+
+from home.src.es.connect import ElasticWrap
+from home.src.ta.helper import get_mapping
+from home.src.ta.settings import EnvironmentSettings
+
+
+class ElasticSnapshot:
+    """interact with snapshots on ES"""
+
+    REPO = "ta_snapshot"
+    REPO_SETTINGS = {
+        "compress": "true",
+        "chunk_size": "1g",
+        "location": EnvironmentSettings.ES_SNAPSHOT_DIR,
+    }
+    POLICY = "ta_daily"
+
+    def __init__(self):
+        self.all_indices = self._get_all_indices()
+
+    def _get_all_indices(self):
+        """return all indices names managed by TA"""
+        mapping = get_mapping()
+        all_indices = [f"ta_{i['index_name']}" for i in mapping]
+
+        return all_indices
+
+    def setup(self):
+        """setup the snapshot in ES, create or update if needed"""
+        print("snapshot: run setup")
+        repo_exists = self._check_repo_exists()
+        if not repo_exists:
+            self.create_repo()
+
+        policy_exists = self._check_policy_exists()
+        if not policy_exists:
+            self.create_policy()
+
+        is_outdated = self._needs_startup_snapshot()
+        if is_outdated:
+            _ = self.take_snapshot_now()
+
+    def _check_repo_exists(self):
+        """check if expected repo already exists"""
+        path = f"_snapshot/{self.REPO}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 200:
+            print(f"snapshot: repo {self.REPO} already created")
+            matching = response[self.REPO]["settings"] == self.REPO_SETTINGS
+            if not matching:
+                print(f"snapshot: update repo settings {self.REPO_SETTINGS}")
+
+            return matching
+
+        print(f"snapshot: setup repo {self.REPO} config {self.REPO_SETTINGS}")
+        return False
+
+    def create_repo(self):
+        """create filesystem repo"""
+        path = f"_snapshot/{self.REPO}"
+        data = {
+            "type": "fs",
+            "settings": self.REPO_SETTINGS,
+        }
+        response, statuscode = ElasticWrap(path).post(data=data)
+        if statuscode == 200:
+            print(f"snapshot: repo setup correctly: {response}")
+
+    def _check_policy_exists(self):
+        """check if snapshot policy is set correctly"""
+        policy = self._get_policy()
+        expected_policy = self._build_policy_data()
+        if not policy:
+            print(f"snapshot: create policy {self.POLICY} {expected_policy}")
+            return False
+
+        if policy["policy"] != expected_policy:
+            print(f"snapshot: update policy settings {expected_policy}")
+            return False
+
+        print("snapshot: policy is set.")
+        return True
+
+    def _get_policy(self):
+        """get policy from es"""
+        path = f"_slm/policy/{self.POLICY}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode != 200:
+            return False
+
+        return response[self.POLICY]
+
+    def create_policy(self):
+        """create snapshot lifetime policy"""
+        path = f"_slm/policy/{self.POLICY}"
+        data = self._build_policy_data()
+        response, statuscode = ElasticWrap(path).put(data)
+        if statuscode == 200:
+            print(f"snapshot: policy setup correctly: {response}")
+
+    def _build_policy_data(self):
+        """build policy dict from config"""
+        at_12 = datetime.now().replace(hour=12, minute=0, second=0)
+        hour = at_12.astimezone(ZoneInfo("UTC")).hour
+
+        return {
+            "schedule": f"0 0 {hour} * * ?",
+            "name": f"<{self.POLICY}_>",
+            "repository": self.REPO,
+            "config": {
+                "indices": self.all_indices,
+                "ignore_unavailable": True,
+                "include_global_state": True,
+            },
+            "retention": {
+                "expire_after": "30d",
+                "min_count": 5,
+                "max_count": 50,
+            },
+        }
+
+    def _needs_startup_snapshot(self):
+        """check if last snapshot is expired"""
+        snap_dicts = self._get_all_snapshots()
+        if not snap_dicts:
+            print("snapshot: create initial snapshot")
+            return True
+
+        last_stamp = snap_dicts[0]["end_stamp"]
+        now = int(datetime.now().timestamp())
+        outdated = (now - last_stamp) / 60 / 60 > 24
+        if outdated:
+            print("snapshot: is outdated, create new now")
+
+        print("snapshot: last snapshot is up-to-date")
+        return outdated
+
+    def take_snapshot_now(self, wait=False):
+        """execute daily snapshot now"""
+        path = f"_slm/policy/{self.POLICY}/_execute"
+        response, statuscode = ElasticWrap(path).post()
+        if statuscode == 200:
+            print(f"snapshot: executing now: {response}")
+
+        if wait:
+            self._wait_for_snapshot(response["snapshot_name"])
+
+        return response
+
+    def _wait_for_snapshot(self, snapshot_name):
+        """return after snapshot_name completes"""
+        path = f"_snapshot/{self.REPO}/{snapshot_name}"
+
+        while True:
+            # wait for task to be created
+            sleep(1)
+            _, statuscode = ElasticWrap(path).get()
+            if statuscode == 200:
+                break
+
+        while True:
+            # wait for snapshot success
+            response, statuscode = ElasticWrap(path).get()
+            snapshot_state = response["snapshots"][0]["state"]
+            if snapshot_state == "SUCCESS":
+                break
+
+            print(f"snapshot: {snapshot_name} in state {snapshot_state}")
+            print("snapshot: wait to complete")
+            sleep(5)
+
+        print(f"snapshot: completed - {response}")
+
+    def get_snapshot_stats(self):
+        """get snapshot info for frontend"""
+        snapshot_info = self._build_policy_details()
+        if snapshot_info:
+            snapshot_info.update({"snapshots": self._get_all_snapshots()})
+
+        return snapshot_info
+
+    def get_single_snapshot(self, snapshot_id):
+        """get single snapshot metadata"""
+        path = f"_snapshot/{self.REPO}/{snapshot_id}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 404:
+            print(f"snapshots: not found: {snapshot_id}")
+            return False
+
+        snapshot = response["snapshots"][0]
+        return self._parse_single_snapshot(snapshot)
+
+    def _get_all_snapshots(self):
+        """get a list of all registered snapshots"""
+        path = f"_snapshot/{self.REPO}/*?sort=start_time&order=desc"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 404:
+            print("snapshots: not configured")
+            return False
+
+        all_snapshots = response["snapshots"]
+        if not all_snapshots:
+            print("snapshots: no snapshots found")
+            return False
+
+        snap_dicts = []
+        for snapshot in all_snapshots:
+            snap_dict = self._parse_single_snapshot(snapshot)
+            snap_dicts.append(snap_dict)
+
+        return snap_dicts
+
+    def _parse_single_snapshot(self, snapshot):
+        """extract relevant metadata from single snapshot"""
+        snap_dict = {
+            "id": snapshot["snapshot"],
+            "state": snapshot["state"],
+            "es_version": snapshot["version"],
+            "start_date": self._date_converter(snapshot["start_time"]),
+            "end_date": self._date_converter(snapshot["end_time"]),
+            "end_stamp": snapshot["end_time_in_millis"] // 1000,
+            "duration_s": snapshot["duration_in_millis"] // 1000,
+        }
+        return snap_dict
+
+    def _build_policy_details(self):
+        """get additional policy details"""
+        policy = self._get_policy()
+        if not policy:
+            return False
+
+        next_exec = policy["next_execution_millis"] // 1000
+        next_exec_date = datetime.fromtimestamp(next_exec)
+        next_exec_str = next_exec_date.strftime("%Y-%m-%d %H:%M")
+        expire_after = policy["policy"]["retention"]["expire_after"]
+        policy_metadata = {
+            "next_exec": next_exec,
+            "next_exec_str": next_exec_str,
+            "expire_after": expire_after,
+        }
+        return policy_metadata
+
+    @staticmethod
+    def _date_converter(date_utc):
+        """convert datetime string"""
+        expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
+        date = datetime.strptime(date_utc, expected_format)
+        local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
+        converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ))
+        converted_str = converted.strftime("%Y-%m-%d %H:%M")
+
+        return converted_str
+
+    def restore_all(self, snapshot_name):
+        """restore snapshot by name"""
+        for index in self.all_indices:
+            _, _ = ElasticWrap(index).delete()
+
+        path = f"_snapshot/{self.REPO}/{snapshot_name}/_restore"
+        data = {"indices": "*"}
+        response, statuscode = ElasticWrap(path).post(data=data)
+        if statuscode == 200:
+            print(f"snapshot: executing now: {response}")
+            return response
+
+        print(f"snapshot: failed to restore, {statuscode} {response}")
+        return False
+
+    def delete_single_snapshot(self, snapshot_id):
+        """delete single snapshot from index"""
+        path = f"_snapshot/{self.REPO}/{snapshot_id}"
+        response, statuscode = ElasticWrap(path).delete()
+        if statuscode == 200:
+            print(f"snapshot: deleting {snapshot_id} {response}")
+            return response
+
+        print(f"snapshot: failed to delete, {statuscode} {response}")
+        return False

tubearchivist/home/src/frontend/forms.py

@@ -0,0 +1,270 @@
+"""functionality:
+- hold all form classes used in the views
+"""
+
+import os
+
+from django import forms
+from django.contrib.auth.forms import AuthenticationForm
+from django.forms.widgets import PasswordInput, TextInput
+from home.src.ta.helper import get_stylesheets
+
+
+class CustomAuthForm(AuthenticationForm):
+    """better styled login form"""
+
+    username = forms.CharField(
+        widget=TextInput(
+            attrs={
+                "placeholder": "Username",
+                "autofocus": True,
+                "autocomplete": True,
+            }
+        ),
+        label=False,
+    )
+    password = forms.CharField(
+        widget=PasswordInput(attrs={"placeholder": "Password"}), label=False
+    )
+    remember_me = forms.BooleanField(required=False)
+
+
+class UserSettingsForm(forms.Form):
+    """user configurations values"""
+
+    STYLESHEET_CHOICES = [("", "-- change stylesheet --")]
+    STYLESHEET_CHOICES.extend(
+        [
+            (stylesheet, os.path.splitext(stylesheet)[0].title())
+            for stylesheet in get_stylesheets()
+        ]
+    )
+
+    stylesheet = forms.ChoiceField(
+        widget=forms.Select, choices=STYLESHEET_CHOICES, required=False
+    )
+    page_size = forms.IntegerField(required=False)
+
+
+class ApplicationSettingsForm(forms.Form):
+    """handle all application settings"""
+
+    AUTOSTART_CHOICES = [
+        ("", "-- change subscription autostart --"),
+        ("0", "disable auto start"),
+        ("1", "enable auto start"),
+    ]
+
+    METADATA_CHOICES = [
+        ("", "-- change metadata embed --"),
+        ("0", "don't embed metadata"),
+        ("1", "embed metadata"),
+    ]
+
+    THUMBNAIL_CHOICES = [
+        ("", "-- change thumbnail embed --"),
+        ("0", "don't embed thumbnail"),
+        ("1", "embed thumbnail"),
+    ]
+
+    RYD_CHOICES = [
+        ("", "-- change ryd integrations"),
+        ("0", "disable ryd integration"),
+        ("1", "enable ryd integration"),
+    ]
+
+    SP_CHOICES = [
+        ("", "-- change sponsorblock integrations"),
+        ("0", "disable sponsorblock integration"),
+        ("1", "enable sponsorblock integration"),
+    ]
+
+    SNAPSHOT_CHOICES = [
+        ("", "-- change snapshot settings --"),
+        ("0", "disable system snapshots"),
+        ("1", "enable system snapshots"),
+    ]
+
+    SUBTITLE_SOURCE_CHOICES = [
+        ("", "-- change subtitle source settings"),
+        ("user", "only download user created"),
+        ("auto", "also download auto generated"),
+    ]
+
+    SUBTITLE_INDEX_CHOICES = [
+        ("", "-- change subtitle index settings --"),
+        ("0", "disable subtitle index"),
+        ("1", "enable subtitle index"),
+    ]
+
+    COMMENT_SORT_CHOICES = [
+        ("", "-- change comments sort settings --"),
+        ("top", "sort comments by top"),
+        ("new", "sort comments by new"),
+    ]
+
+    COOKIE_IMPORT_CHOICES = [
+        ("", "-- change cookie settings"),
+        ("0", "remove cookie"),
+        ("1", "import cookie"),
+    ]
+
+    subscriptions_channel_size = forms.IntegerField(
+        required=False, min_value=1
+    )
+    subscriptions_live_channel_size = forms.IntegerField(
+        required=False, min_value=0
+    )
+    subscriptions_shorts_channel_size = forms.IntegerField(
+        required=False, min_value=0
+    )
+    subscriptions_auto_start = forms.ChoiceField(
+        widget=forms.Select, choices=AUTOSTART_CHOICES, required=False
+    )
+    downloads_limit_speed = forms.IntegerField(required=False)
+    downloads_throttledratelimit = forms.IntegerField(required=False)
+    downloads_sleep_interval = forms.IntegerField(required=False)
+    downloads_autodelete_days = forms.IntegerField(required=False)
+    downloads_format = forms.CharField(required=False)
+    downloads_format_sort = forms.CharField(required=False)
+    downloads_extractor_lang = forms.CharField(required=False)
+    downloads_add_metadata = forms.ChoiceField(
+        widget=forms.Select, choices=METADATA_CHOICES, required=False
+    )
+    downloads_add_thumbnail = forms.ChoiceField(
+        widget=forms.Select, choices=THUMBNAIL_CHOICES, required=False
+    )
+    downloads_subtitle = forms.CharField(required=False)
+    downloads_subtitle_source = forms.ChoiceField(
+        widget=forms.Select, choices=SUBTITLE_SOURCE_CHOICES, required=False
+    )
+    downloads_subtitle_index = forms.ChoiceField(
+        widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False
+    )
+    downloads_comment_max = forms.CharField(required=False)
+    downloads_comment_sort = forms.ChoiceField(
+        widget=forms.Select, choices=COMMENT_SORT_CHOICES, required=False
+    )
+    downloads_cookie_import = forms.ChoiceField(
+        widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False
+    )
+    downloads_integrate_ryd = forms.ChoiceField(
+        widget=forms.Select, choices=RYD_CHOICES, required=False
+    )
+    downloads_integrate_sponsorblock = forms.ChoiceField(
+        widget=forms.Select, choices=SP_CHOICES, required=False
+    )
+    application_enable_snapshot = forms.ChoiceField(
+        widget=forms.Select, choices=SNAPSHOT_CHOICES, required=False
+    )
+
+
+class MultiSearchForm(forms.Form):
+    """multi search form for /search/"""
+
+    searchInput = forms.CharField(
+        label="",
+        widget=forms.TextInput(
+            attrs={
+                "autocomplete": "off",
+                "oninput": "searchMulti(this.value)",
+                "autofocus": True,
+            }
+        ),
+    )
+    home = forms.CharField(widget=forms.HiddenInput())
+    channel = forms.CharField(widget=forms.HiddenInput())
+    playlist = forms.CharField(widget=forms.HiddenInput())
+
+
+class AddToQueueForm(forms.Form):
+    """text area form to add to downloads"""
+
+    HELP_TEXT = "Enter at least one video, channel or playlist id/URL here..."
+
+    vid_url = forms.CharField(
+        label=False,
+        widget=forms.Textarea(
+            attrs={
+                "rows": 4,
+                "placeholder": HELP_TEXT,
+            }
+        ),
+    )
+
+
+class SubscribeToChannelForm(forms.Form):
+    """text area form to subscribe to multiple channels"""
+
+    subscribe = forms.CharField(
+        label="Subscribe to channels",
+        widget=forms.Textarea(
+            attrs={
+                "rows": 3,
+                "placeholder": "Input channel ID, URL or Video of a channel",
+            }
+        ),
+    )
+
+
+class SubscribeToPlaylistForm(forms.Form):
+    """text area form to subscribe to multiple playlists"""
+
+    subscribe = forms.CharField(
+        label="Subscribe to playlists",
+        widget=forms.Textarea(
+            attrs={
+                "rows": 3,
+                "placeholder": "Input playlist IDs or URLs",
+            }
+        ),
+    )
+
+
+class CreatePlaylistForm(forms.Form):
+    """text area form to create a single custom playlist"""
+
+    create = forms.CharField(
+        label="Or create custom playlist",
+        widget=forms.Textarea(
+            attrs={
+                "rows": 1,
+                "placeholder": "Input playlist name",
+            }
+        ),
+    )
+
+
+class ChannelOverwriteForm(forms.Form):
+    """custom overwrites for channel settings"""
+
+    PLAYLIST_INDEX = [
+        ("", "-- change playlist index --"),
+        ("0", "Disable playlist index"),
+        ("1", "Enable playlist index"),
+    ]
+
+    SP_CHOICES = [
+        ("", "-- change sponsorblock integrations"),
+        ("disable", "disable sponsorblock integration"),
+        ("1", "enable sponsorblock integration"),
+        ("0", "unset sponsorblock integration"),
+    ]
+
+    download_format = forms.CharField(label=False, required=False)
+    autodelete_days = forms.IntegerField(label=False, required=False)
+    index_playlists = forms.ChoiceField(
+        widget=forms.Select, choices=PLAYLIST_INDEX, required=False
+    )
+    integrate_sponsorblock = forms.ChoiceField(
+        widget=forms.Select, choices=SP_CHOICES, required=False
+    )
+    subscriptions_channel_size = forms.IntegerField(
+        label=False, required=False
+    )
+    subscriptions_live_channel_size = forms.IntegerField(
+        label=False, required=False
+    )
+    subscriptions_shorts_channel_size = forms.IntegerField(
+        label=False, required=False
+    )

tubearchivist/home/src/frontend/forms_schedule.py

@@ -0,0 +1,101 @@
+"""
+Functionality:
+- handle schedule forms
+- implement form validation
+"""
+
+from celery.schedules import crontab
+from django import forms
+from home.src.ta.task_config import TASK_CONFIG
+
+
+class CrontabValidator:
+    """validate crontab"""
+
+    @staticmethod
+    def validate_fields(cron_fields):
+        """expect 3 cron fields"""
+        if not len(cron_fields) == 3:
+            raise forms.ValidationError("expected three cron schedule fields")
+
+    @staticmethod
+    def validate_minute(minute_field):
+        """expect minute int"""
+        try:
+            minute_value = int(minute_field)
+            if not 0 <= minute_value <= 59:
+                raise forms.ValidationError(
+                    "Invalid value for minutes. Must be between 0 and 59."
+                )
+        except ValueError as err:
+            raise forms.ValidationError(
+                "Invalid value for minutes. Must be an integer."
+            ) from err
+
+    @staticmethod
+    def validate_cron_tab(minute, hour, day_of_week):
+        """check if crontab can be created"""
+        try:
+            crontab(minute=minute, hour=hour, day_of_week=day_of_week)
+        except ValueError as err:
+            raise forms.ValidationError(f"invalid crontab: {err}") from err
+
+    def validate(self, cron_expression):
+        """create crontab schedule"""
+        if cron_expression == "auto":
+            return
+
+        cron_fields = cron_expression.split()
+        self.validate_fields(cron_fields)
+
+        minute, hour, day_of_week = cron_fields
+        self.validate_minute(minute)
+        self.validate_cron_tab(minute, hour, day_of_week)
+
+
+def validate_cron(cron_expression):
+    """callable for field"""
+    CrontabValidator().validate(cron_expression)
+
+
+class SchedulerSettingsForm(forms.Form):
+    """handle scheduler settings"""
+
+    update_subscribed = forms.CharField(
+        required=False, validators=[validate_cron]
+    )
+    download_pending = forms.CharField(
+        required=False, validators=[validate_cron]
+    )
+    check_reindex = forms.CharField(required=False, validators=[validate_cron])
+    check_reindex_days = forms.IntegerField(required=False)
+    thumbnail_check = forms.CharField(
+        required=False, validators=[validate_cron]
+    )
+    run_backup = forms.CharField(required=False, validators=[validate_cron])
+    run_backup_rotate = forms.IntegerField(required=False)
+
+
+class NotificationSettingsForm(forms.Form):
+    """add notification URL"""
+
+    SUPPORTED_TASKS = [
+        "update_subscribed",
+        "extract_download",
+        "download_pending",
+        "check_reindex",
+    ]
+    TASK_LIST = [(i, TASK_CONFIG[i]["title"]) for i in SUPPORTED_TASKS]
+
+    TASK_CHOICES = [("", "-- select task --")]
+    TASK_CHOICES.extend(TASK_LIST)
+
+    PLACEHOLDER = "Apprise notification URL"
+
+    task = forms.ChoiceField(
+        widget=forms.Select, choices=TASK_CHOICES, required=False
+    )
+    notification_url = forms.CharField(
+        required=False,
+        widget=forms.TextInput(attrs={"placeholder": PLACEHOLDER}),
+    )

tubearchivist/home/src/frontend/searching.py

@@ -0,0 +1,382 @@
+"""
+Functionality:
+- handle search to populate results to view
+- cache youtube video thumbnails and channel artwork
+- parse values in hit_cleanup for frontend
+- calculate pagination values
+"""
+
+from api.src.search_processor import SearchProcess
+from home.src.es.connect import ElasticWrap
+
+
+class SearchForm:
+    """build query from search form data"""
+
+    def multi_search(self, search_query):
+        """searching through index"""
+        path, query, query_type = SearchParser(search_query).run()
+        response, _ = ElasticWrap(path).get(data=query)
+        search_results = SearchProcess(response).process()
+        all_results = self.build_results(search_results)
+
+        return {"results": all_results, "queryType": query_type}
+
+    @staticmethod
+    def build_results(search_results):
+        """build the all_results dict"""
+        video_results = []
+        channel_results = []
+        playlist_results = []
+        fulltext_results = []
+        if search_results:
+            for result in search_results:
+                if result["_index"] == "ta_video":
+                    video_results.append(result)
+                elif result["_index"] == "ta_channel":
+                    channel_results.append(result)
+                elif result["_index"] == "ta_playlist":
+                    playlist_results.append(result)
+                elif result["_index"] == "ta_subtitle":
+                    fulltext_results.append(result)
+
+        all_results = {
+            "video_results": video_results,
+            "channel_results": channel_results,
+            "playlist_results": playlist_results,
+            "fulltext_results": fulltext_results,
+        }
+
+        return all_results
+
+
+class SearchParser:
+    """handle structured searches"""
+
+    def __init__(self, search_query):
+        self.query_words = search_query.lower().split()
+        self.query_map = {"term": [], "fuzzy": []}
+        self.append_to = "term"
+
+    def run(self):
+        """collection, return path and query dict for es"""
+        print(f"query words: {self.query_words}")
+        query_type = self._find_map()
+        self._run_words()
+        self._delete_unset()
+        self._match_data_types()
+
+        path, query = QueryBuilder(self.query_map, query_type).run()
+
+        return path, query, query_type
+
+    def _find_map(self):
+        """find query in keyword map"""
+        first_word = self.query_words[0]
+        key_word_map = self._get_map()
+
+        if ":" in first_word:
+            index_match, query_string = first_word.split(":")
+            if index_match in key_word_map:
+                self.query_map.update(key_word_map.get(index_match))
+                self.query_words[0] = query_string
+                return index_match
+
+        self.query_map.update(key_word_map.get("simple"))
+        print(f"query_map: {self.query_map}")
+
+        return "simple"
+
+    @staticmethod
+    def _get_map():
+        """return map to build on"""
+        return {
+            "simple": {
+                "index": "ta_video,ta_channel,ta_playlist",
+            },
+            "video": {
+                "index": "ta_video",
+                "channel": [],
+                "active": [],
+            },
+            "channel": {
+                "index": "ta_channel",
+                "active": [],
+                "subscribed": [],
+            },
+            "playlist": {
+                "index": "ta_playlist",
+                "active": [],
+                "subscribed": [],
+            },
+            "full": {
+                "index": "ta_subtitle",
+                "lang": [],
+                "source": [],
+            },
+        }
+
+    def _run_words(self):
+        """append word by word"""
+        for word in self.query_words:
+            if ":" in word:
+                keyword, search_string = word.split(":")
+                if keyword in self.query_map:
+                    self.append_to = keyword
+                    word = search_string
+
+            if word:
+                self.query_map[self.append_to].append(word)
+
+    def _delete_unset(self):
+        """delete unset keys"""
+        new_query_map = {}
+        for key, value in self.query_map.items():
+            if value:
+                new_query_map.update({key: value})
+        self.query_map = new_query_map
+
+    def _match_data_types(self):
+        """match values with data types"""
+        for key, value in self.query_map.items():
+            if key in ["term", "channel"]:
+                self.query_map[key] = " ".join(self.query_map[key])
+            if key in ["active", "subscribed"]:
+                self.query_map[key] = "yes" in value
+
+
+class QueryBuilder:
+    """build query for ES from form data"""
+
+    def __init__(self, query_map, query_type):
+        self.query_map = query_map
+        self.query_type = query_type
+
+    def run(self):
+        """build query"""
+        path = self._build_path()
+        query = self.build_query()
+        print(f"es path: {path}")
+        print(f"query: {query}")
+
+        return path, query
+
+    def _build_path(self):
+        """build es index search path"""
+        return f"{self.query_map.get('index')}/_search"
+
+    def build_query(self):
+        """build query based on query_type"""
+
+        exec_map = {
+            "simple": self._build_simple,
+            "video": self._build_video,
+            "channel": self._build_channel,
+            "playlist": self._build_playlist,
+            "full": self._build_fulltext,
+        }
+
+        build_must_list = exec_map[self.query_type]
+
+        if self.query_type == "full":
+            query = build_must_list()
+        else:
+            query = {
+                "size": 30,
+                "query": {"bool": {"must": build_must_list()}},
+            }
+
+        return query
+
+    def _get_fuzzy(self):
+        """return fuziness valuee"""
+        fuzzy_value = self.query_map.get("fuzzy", ["auto"])[0]
+        if fuzzy_value == "no":
+            return 0
+
+        if not fuzzy_value.isdigit():
+            return "auto"
+
+        if int(fuzzy_value) > 2:
+            return "2"
+
+        return fuzzy_value
+
+    def _build_simple(self):
+        """build simple cross index query"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "channel_name._2gram",
+                            "channel_name._3gram",
+                            "channel_name.search_as_you_type",
+                            "playlist_name._2gram",
+                            "playlist_name._3gram",
+                            "playlist_name.search_as_you_type",
+                            "title._2gram",
+                            "title._3gram",
+                            "title.search_as_you_type",
+                        ],
+                    }
+                }
+            )
+
+        return must_list
+
+    def _build_video(self):
+        """build video query"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "title._2gram^2",
+                            "title._3gram^2",
+                            "title.search_as_you_type^2",
+                            "tags",
+                            "category",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"active": {"value": active}}})
+
+        if (channel := self.query_map.get("channel")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": channel,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "channel.channel_name._2gram",
+                            "channel.channel_name._3gram",
+                            "channel.channel_name.search_as_you_type",
+                        ],
+                    }
+                }
+            )
+
+        return must_list
+
+    def _build_channel(self):
+        """build query for channel"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "channel_description",
+                            "channel_name._2gram^2",
+                            "channel_name._3gram^2",
+                            "channel_name.search_as_you_type^2",
+                            "channel_tags",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"channel_active": {"value": active}}})
+
+        if (subscribed := self.query_map.get("subscribed")) is not None:
+            must_list.append(
+                {"term": {"channel_subscribed": {"value": subscribed}}}
+            )
+
+        return must_list
+
+    def _build_playlist(self):
+        """build query for playlist"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "playlist_description",
+                            "playlist_name._2gram^2",
+                            "playlist_name._3gram^2",
+                            "playlist_name.search_as_you_type^2",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"playlist_active": {"value": active}}})
+
+        if (subscribed := self.query_map.get("subscribed")) is not None:
+            must_list.append(
+                {"term": {"playlist_subscribed": {"value": subscribed}}}
+            )
+
+        return must_list
+
+    def _build_fulltext(self):
+        """build query for fulltext search"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "match": {
+                        "subtitle_line": {
+                            "query": term,
+                            "fuzziness": self._get_fuzzy(),
+                        }
+                    }
+                }
+            )
+
+        if (lang := self.query_map.get("lang")) is not None:
+            must_list.append({"term": {"subtitle_lang": {"value": lang[0]}}})
+
+        if (source := self.query_map.get("source")) is not None:
+            must_list.append(
+                {"term": {"subtitle_source": {"value": source[0]}}}
+            )
+
+        query = {
+            "size": 30,
+            "query": {"bool": {"must": must_list}},
+            "highlight": {
+                "fields": {
+                    "subtitle_line": {
+                        "number_of_fragments": 0,
+                        "pre_tags": ['<span class="settings-current">'],
+                        "post_tags": ["</span>"],
+                    }
+                }
+            },
+        }
+
+        return query

tubearchivist/home/src/frontend/watched.py

@@ -0,0 +1,105 @@
+"""
+functionality:
+- handle watched state for videos, channels and playlists
+"""
+
+from datetime import datetime
+
+from home.src.es.connect import ElasticWrap
+from home.src.ta.urlparser import Parser
+
+
+class WatchState:
+    """handle watched checkbox for videos and channels"""
+
+    def __init__(self, youtube_id, is_watched):
+        self.youtube_id = youtube_id
+        self.is_watched = is_watched
+        self.stamp = int(datetime.now().timestamp())
+        self.pipeline = f"_ingest/pipeline/watch_{youtube_id}"
+
+    def change(self):
+        """change watched state of item(s)"""
+        print(f"{self.youtube_id}: change watched state to {self.is_watched}")
+        url_type = self._dedect_type()
+        if url_type == "video":
+            self.change_vid_state()
+            return
+
+        self._add_pipeline()
+        path = f"ta_video/_update_by_query?pipeline=watch_{self.youtube_id}"
+        data = self._build_update_data(url_type)
+        _, _ = ElasticWrap(path).post(data)
+        self._delete_pipeline()
+
+    def _dedect_type(self):
+        """find youtube id type"""
+        url_process = Parser(self.youtube_id).parse()
+        url_type = url_process[0]["type"]
+        return url_type
+
+    def change_vid_state(self):
+        """change watched state of video"""
+        path = f"ta_video/_update/{self.youtube_id}"
+        data = {
+            "doc": {
+                "player": {
+                    "watched": self.is_watched,
+                    "watched_date": self.stamp,
+                }
+            }
+        }
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code != 200:
+            print(response)
+            raise ValueError("failed to mark video as watched")
+
+    def _build_update_data(self, url_type):
+        """build update by query data based on url_type"""
+        term_key_map = {
+            "channel": "channel.channel_id",
+            "playlist": "playlist.keyword",
+        }
+        term_key = term_key_map.get(url_type)
+
+        return {
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {term_key: {"value": self.youtube_id}}},
+                        {
+                            "term": {
+                                "player.watched": {
+                                    "value": not self.is_watched
+                                }
+                            }
+                        },
+                    ],
+                }
+            }
+        }
+
+    def _add_pipeline(self):
+        """add ingest pipeline"""
+        data = {
+            "description": f"{self.youtube_id}: watched {self.is_watched}",
+            "processors": [
+                {
+                    "set": {
+                        "field": "player.watched",
+                        "value": self.is_watched,
+                    }
+                },
+                {
+                    "set": {
+                        "field": "player.watched_date",
+                        "value": self.stamp,
+                    }
+                },
+            ],
+        }
+        _, _ = ElasticWrap(self.pipeline).put(data)
+
+    def _delete_pipeline(self):
+        """delete pipeline"""
+        ElasticWrap(self.pipeline).delete()

tubearchivist/home/src/helper.py

@@ -1,159 +0,0 @@
-"""
-Loose collection of helper functions
-- don't import AppConfig class here to avoid circular imports
-"""
-
-import json
-import os
-import re
-import string
-import subprocess
-import unicodedata
-
-import requests
-import redis
-
-REDIS_HOST = os.environ.get('REDIS_HOST')
-
-
-def get_total_hits(index, es_url, match_field):
-    """ get total hits from index """
-    headers = {'Content-type': 'application/json'}
-    data = {"query": {"match": {match_field: True}}}
-    payload = json.dumps(data)
-    url = f'{es_url}/{index}/_search?filter_path=hits.total'
-    request = requests.post(url, data=payload, headers=headers)
-    if not request.ok:
-        print(request.text)
-    total_json = json.loads(request.text)
-    total_hits = total_json['hits']['total']['value']
-    return total_hits
-
-
-def clean_string(file_name):
-    """ clean string to only asci characters """
-    whitelist = "-_.() " + string.ascii_letters + string.digits
-    normalized = unicodedata.normalize('NFKD', file_name)
-    ascii_only = normalized.encode('ASCII', 'ignore').decode().strip()
-    white_listed = ''.join(c for c in ascii_only if c in whitelist)
-    cleaned = re.sub(r'[ ]{2,}', ' ', white_listed)
-    return cleaned
-
-
-def process_url_list(url_str):
-    """ parse url_list to find valid youtube video or channel ids """
-    to_replace = ['watch?v=', 'playlist?list=']
-    url_list = re.split('\n+', url_str[0])
-    youtube_ids = []
-    for url in url_list:
-        url_clean = url.strip().strip('/').split('/')[-1]
-        for i in to_replace:
-            url_clean = url_clean.replace(i, '')
-        url_no_param = url_clean.split('&')[0]
-        str_len = len(url_no_param)
-        if str_len == 11:
-            link_type = 'video'
-        elif str_len == 24:
-            link_type = 'channel'
-        elif str_len == 34:
-            link_type = 'playlist'
-        else:
-            # unable to parse
-            raise ValueError('not a valid url: ' + url)
-
-        youtube_ids.append({"url": url_no_param, "type": link_type})
-
-    return youtube_ids
-
-
-def set_message(key, message, expire=True):
-    """ write new message to redis """
-    redis_connection = redis.Redis(host=REDIS_HOST)
-    redis_connection.execute_command(
-        'JSON.SET', key, '.', json.dumps(message)
-    )
-    if expire:
-        redis_connection.execute_command('EXPIRE', key, 20)
-
-
-def get_message(key):
-    """ get any message from JSON key """
-    redis_connection = redis.Redis(host=REDIS_HOST)
-    reply = redis_connection.execute_command('JSON.GET', key)
-    if reply:
-        json_str = json.loads(reply)
-    else:
-        json_str = {"status": False}
-    return json_str
-
-
-def get_dl_message(cache_dir):
-    """ get latest message if available """
-    redis_connection = redis.Redis(host=REDIS_HOST)
-    reply = redis_connection.execute_command('JSON.GET', 'progress:download')
-    if reply:
-        json_str = json.loads(reply)
-    elif json_str := monitor_cache_dir(cache_dir):
-        json_str = monitor_cache_dir(cache_dir)
-    else:
-        json_str = {"status": False}
-    return json_str
-
-
-def get_lock(lock_key):
-    """ handle lock for task management """
-    redis_lock = redis.Redis(host=REDIS_HOST).lock(lock_key)
-    return redis_lock
-
-
-def monitor_cache_dir(cache_dir):
-    """
-    look at download cache dir directly as alterative progress info
-    """
-    dl_cache = os.path.join(cache_dir, 'download')
-    cache_file = os.listdir(dl_cache)
-    if cache_file:
-        filename = cache_file[0][12:].replace('_', ' ').split('.')[0]
-        mess_dict = {
-            "status": "downloading",
-            "level": "info",
-            "title": "Downloading: " + filename,
-            "message": ""
-        }
-    else:
-        return False
-
-    return mess_dict
-
-
-class DurationConverter:
-    """
-    using ffmpeg to get and parse duration from filepath
-    """
-
-    @staticmethod
-    def get_sec(file_path):
-        """ read duration from file """
-        duration = subprocess.run([
-            "ffprobe", "-v", "error", "-show_entries", "format=duration",
-            "-of", "default=noprint_wrappers=1:nokey=1", file_path
-        ], capture_output=True, check=True)
-        duration_sec = int(float(duration.stdout.decode().strip()))
-        return duration_sec
-
-    @staticmethod
-    def get_str(duration_sec):
-        """ takes duration in sec and returns clean string """
-        hours = duration_sec // 3600
-        minutes = (duration_sec - (hours * 3600)) // 60
-        secs = duration_sec - (hours * 3600) - (minutes * 60)
-
-        duration_str = str()
-        if hours:
-            duration_str = str(hours).zfill(2) + ':'
-        if minutes:
-            duration_str = duration_str + str(minutes).zfill(2) + ':'
-        else:
-            duration_str = duration_str + '00:'
-        duration_str = duration_str + str(secs).zfill(2)
-        return duration_str

tubearchivist/home/src/index.py

@@ -1,395 +0,0 @@
-"""
-Functionality:
-- index new videos into elastisearch
-- extract video info with yt_dlp
-- scrape youtube channel page if needed
-"""
-
-import json
-import re
-
-from datetime import datetime
-from time import sleep
-import os
-
-import requests
-import yt_dlp as youtube_dl
-
-from bs4 import BeautifulSoup
-
-from home.src.config import AppConfig
-from home.src.helper import clean_string, DurationConverter
-
-
-class YoutubeChannel:
-    """ represents a single youtube channel """
-
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG['application']['es_url']
-    CACHE_DIR = CONFIG['application']['cache_dir']
-
-    def __init__(self, channel_id):
-        self.channel_id = channel_id
-        self.json_data = None
-        self.source = None
-        self.channel_dict = self.build_channel_dict()
-
-    def build_channel_dict(self, scrape=False):
-        """ combine the dicts build from extracted json payload """
-        if scrape:
-            channel_dict = False
-        else:
-            channel_dict = self.get_es_channel()
-        if not channel_dict:
-            print('scrape data from youtube')
-            self.scrape_channel()
-            channel_dict = self.parse_channel_main()
-            channel_dict.update(self.parse_channel_meta())
-            self.source = 'scraped'
-        return channel_dict
-
-    def get_es_channel(self):
-        """ get from elastic search first if possible """
-        channel_id = self.channel_id
-        url = f'{self.ES_URL}/ta_channel/_doc/{channel_id}'
-        response = requests.get(url)
-        if response.ok:
-            channel_source = response.json()['_source']
-            self.source = 'elastic'
-            return channel_source
-        return False
-
-    def scrape_channel(self):
-        """ scrape channel page for additional infos """
-        channel_id = self.channel_id
-        url = f'https://www.youtube.com/channel/{channel_id}/about?hl=en'
-        cookies = {
-            'CONSENT': 'YES+xxxxxxxxxxxxxxxxxxxxxxxxxxx'
-        }
-        response = requests.get(url, cookies=cookies)
-        if response.ok:
-            channel_page = response.text
-        else:
-            print(f'failed to extract channel info for: {channel_id}')
-            raise ConnectionError
-        soup = BeautifulSoup(channel_page, 'html.parser')
-        # load script into json
-        all_scripts = soup.find('body').find_all('script')
-        for script in all_scripts:
-            if 'var ytInitialData = ' in str(script):
-                script_content = str(script)
-                break
-        # extract payload
-        script_content = script_content.split('var ytInitialData = ')[1]
-        json_raw = script_content.rstrip(';</script>')
-        json_data = json.loads(json_raw)
-        # add to self
-        self.json_data = json_data
-
-    def parse_channel_main(self):
-        """ extract maintab values from scraped channel json data """
-        main_tab = self.json_data['header']['c4TabbedHeaderRenderer']
-        channel_name = main_tab['title']
-        last_refresh = int(datetime.now().strftime("%s"))
-        # channel_subs
-        try:
-            sub_text_simple = main_tab['subscriberCountText']['simpleText']
-            sub_text = sub_text_simple.split(' ')[0]
-            if sub_text[-1] == 'K':
-                channel_subs = int(float(sub_text.replace('K', ''))*1000)
-            elif sub_text[-1] == 'M':
-                channel_subs = int(float(sub_text.replace('M', ''))*1000000)
-            elif int(sub_text) >= 0:
-                channel_subs = int(sub_text)
-            else:
-                message = f'{sub_text} not dealt with'
-                print(message)
-        except KeyError:
-            channel_subs = 0
-        # banner
-        try:
-            all_banners = main_tab['banner']['thumbnails']
-            banner = sorted(all_banners, key=lambda k: k['width'])[-1]['url']
-        except KeyError:
-            banner = False
-        # build and return dict
-        main_channel_dict = {
-            'channel_active': True,
-            'channel_last_refresh': last_refresh,
-            'channel_subs': channel_subs,
-            'channel_banner_url': banner,
-            'channel_name': channel_name,
-            'channel_id': self.channel_id
-        }
-        return main_channel_dict
-
-    def parse_channel_meta(self):
-        """ extract meta tab values from channel payload """
-        # meta tab
-        json_data = self.json_data
-        meta_tab = json_data['metadata']['channelMetadataRenderer']
-        description = meta_tab['description']
-        all_thumbs = meta_tab['avatar']['thumbnails']
-        thumb_url = sorted(all_thumbs, key=lambda k: k['width'])[-1]['url']
-        # stats tab
-        renderer = 'twoColumnBrowseResultsRenderer'
-        all_tabs = json_data['contents'][renderer]['tabs']
-        for tab in all_tabs:
-            if 'tabRenderer' in tab.keys():
-                if tab['tabRenderer']['title'] == 'About':
-                    about_tab = (tab['tabRenderer']['content']
-                                 ['sectionListRenderer']['contents'][0]
-                                 ['itemSectionRenderer']['contents'][0]
-                                 ['channelAboutFullMetadataRenderer'])
-                    break
-        try:
-            channel_views_text = about_tab['viewCountText']['simpleText']
-            channel_views = int(re.sub(r"\D", "", channel_views_text))
-        except KeyError:
-            channel_views = 0
-
-        meta_channel_dict = {
-            'channel_description': description,
-            'channel_thumb_url': thumb_url,
-            'channel_views': channel_views
-        }
-
-        return meta_channel_dict
-
-    def upload_to_es(self):
-        """ upload channel data to elastic search """
-        url = f'{self.ES_URL}/ta_channel/_doc/{self.channel_id}'
-        response = requests.put(url, json=self.channel_dict)
-        print(f'added {self.channel_id} to es')
-        if not response.ok:
-            print(response.text)
-
-    def clear_cache(self):
-        """ delete banner and thumb from cache if available """
-        channel_cache = os.path.join(self.CACHE_DIR, 'channels')
-        thumb = os.path.join(channel_cache, self.channel_id + '_thumb.jpg')
-        banner = os.path.join(channel_cache, self.channel_id + '_banner.jpg')
-        if os.path.exists(thumb):
-            os.remove(thumb)
-        if os.path.exists(banner):
-            os.remove(banner)
-
-    def sync_to_videos(self):
-        """ sync new channel_dict to all videos of channel """
-        headers = {'Content-type': 'application/json'}
-        channel_id = self.channel_id
-        # add ingest pipeline
-        processors = []
-        for field, value in self.channel_dict.items():
-            line = {"set": {"field": "channel." + field, "value": value}}
-            processors.append(line)
-        data = {
-            "description": channel_id,
-            "processors": processors
-        }
-        payload = json.dumps(data)
-        url = self.ES_URL + '/_ingest/pipeline/' + channel_id
-        request = requests.put(url, data=payload, headers=headers)
-        if not request.ok:
-            print(request.text)
-        # apply pipeline
-        data = {
-            "query": {"match": {"channel.channel_id": channel_id}}
-        }
-        payload = json.dumps(data)
-        url = self.ES_URL + '/ta_video/_update_by_query?pipeline=' + channel_id
-        request = requests.post(url, data=payload, headers=headers)
-        if not request.ok:
-            print(request.text)
-
-    def get_total_hits(self):
-        """ get total channels indexed """
-        headers = {'Content-type': 'application/json'}
-        data = {"query": {"match_all": {}}}
-        payload = json.dumps(data)
-        url = f'{self.ES_URL}/ta_channel/_search?filter_path=hits.total'
-        request = requests.post(url, data=payload, headers=headers)
-        if not request.ok:
-            print(request.text)
-        total_hits = json.loads(request.text)['hits']['total']['value']
-        return total_hits
-
-
-class YoutubeVideo:
-    """ represents a signle youtube video """
-
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG['application']['es_url']
-    CACHE_DIR = CONFIG['application']['cache_dir']
-    VIDEOS = CONFIG['application']['videos']
-
-    def __init__(self, youtube_id):
-        self.youtube_id = youtube_id
-        self.channel_id = None
-        self.vid_dict = self.get_wrapper()
-
-    def get_wrapper(self):
-        """ wrapper to loop around youtube_dl to retry on failure """
-        print(f'get video data for {self.youtube_id}')
-        for i in range(3):
-            try:
-                vid_dict = self.get_youtubedl_vid_data()
-            except KeyError as e:
-                print(e)
-                sleep((i + 1) ** 2)
-                continue
-            else:
-                break
-
-        return vid_dict
-
-    def get_youtubedl_vid_data(self):
-        """ parse youtubedl extract info """
-        youtube_id = self.youtube_id
-        obs = {
-            'quiet': True,
-            'default_search': 'ytsearch',
-            'skip_download': True
-        }
-        try:
-            vid = youtube_dl.YoutubeDL(obs).extract_info(youtube_id)
-        except (
-                youtube_dl.utils.ExtractorError,
-                youtube_dl.utils.DownloadError
-                ):
-            print('failed to get info for ' + youtube_id)
-            return False
-        # extract
-        self.channel_id = vid['channel_id']
-        upload_date = vid['upload_date']
-        upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
-        published = upload_date_time.strftime("%Y-%m-%d")
-        last_refresh = int(datetime.now().strftime("%s"))
-        # likes
-        try:
-            like_count = vid['like_count']
-        except KeyError:
-            like_count = 0
-        try:
-            dislike_count = vid['dislike_count']
-        except KeyError:
-            dislike_count = 0
-        # build dicts
-        stats = {
-            "view_count": vid['view_count'],
-            "like_count": like_count,
-            "dislike_count": dislike_count,
-            "average_rating": vid['average_rating']
-        }
-        vid_basic = {
-            "title": vid['title'],
-            "description": vid['description'],
-            "category": vid['categories'],
-            "vid_thumb_url": vid['thumbnail'],
-            "tags": vid['tags'],
-            "published": published,
-            "stats": stats,
-            "vid_last_refresh": last_refresh,
-            "date_downloaded": last_refresh,
-            "youtube_id": youtube_id,
-            "active": True,
-            "channel": False
-        }
-
-        return vid_basic
-
-    def add_player(self, missing_vid):
-        """ add player information for new videos """
-        cache_path = self.CACHE_DIR + '/download/'
-        videos = self.VIDEOS
-
-        if missing_vid:
-            # coming from scan_filesystem
-            channel_name, file_name, _ = missing_vid
-            vid_path = os.path.join(videos, channel_name, file_name)
-        else:
-            # coming from VideoDownload
-            all_cached = os.listdir(cache_path)
-            for file_cached in all_cached:
-                if self.youtube_id in file_cached:
-                    vid_path = os.path.join(cache_path, file_cached)
-                    break
-
-        duration_handler = DurationConverter()
-        duration = duration_handler.get_sec(vid_path)
-        duration_str = duration_handler.get_str(duration)
-        player = {
-            "watched": False,
-            "duration": duration,
-            "duration_str": duration_str
-        }
-        self.vid_dict['player'] = player
-
-    def build_file_path(self, channel_name):
-        """ build media_url from where file will be located """
-        clean_channel_name = clean_string(channel_name)
-        timestamp = self.vid_dict['published'].replace('-', '')
-        youtube_id = self.vid_dict['youtube_id']
-        title = self.vid_dict['title']
-        clean_title = clean_string(title)
-        filename = f'{timestamp}_{youtube_id}_{clean_title}.mp4'
-        media_url = os.path.join(clean_channel_name, filename)
-        self.vid_dict['media_url'] = media_url
-
-    def get_es_data(self):
-        """ get current data from elastic search """
-        url = self.ES_URL + '/ta_video/_doc/' + self.youtube_id
-        response = requests.get(url)
-        if not response.ok:
-            print(response.text)
-        es_vid_dict = json.loads(response.text)
-        return es_vid_dict
-
-    def upload_to_es(self):
-        """ upload channel data to elastic search """
-        url = f'{self.ES_URL}/ta_video/_doc/{self.youtube_id}'
-        response = requests.put(url, json=self.vid_dict)
-        if not response.ok:
-            print(response.text)
-
-    def delete_cache(self):
-        """ delete thumbnail from cache if exist """
-        video_cache = os.path.join(self.CACHE_DIR, 'videos')
-        thumb = os.path.join(video_cache, self.youtube_id + '.jpg')
-        if os.path.exists(thumb):
-            os.remove(thumb)
-
-    def deactivate(self):
-        """ deactivate document on extractor error """
-        youtube_id = self.youtube_id
-        headers = {'Content-type': 'application/json'}
-        url = f'{self.ES_URL}/ta_video/_update/{youtube_id}'
-        data = {"script": "ctx._source.active = false"}
-        json_str = json.dumps(data)
-        response = requests.post(url, data=json_str, headers=headers)
-        print(f'deactivated {youtube_id}')
-        if not response.ok:
-            print(response.text)
-
-
-def index_new_video(youtube_id, missing_vid=False):
-    """ combine video and channel classes for new video index """
-    vid_handler = YoutubeVideo(youtube_id)
-    if not vid_handler.vid_dict:
-        raise ValueError('failed to get metadata for ' + youtube_id)
-
-    channel_handler = YoutubeChannel(vid_handler.channel_id)
-    # add filepath to vid_dict
-    channel_name = channel_handler.channel_dict['channel_name']
-    vid_handler.build_file_path(channel_name)
-    # add channel and player to video
-    vid_handler.add_player(missing_vid)
-    vid_handler.vid_dict['channel'] = channel_handler.channel_dict
-    # add new channel to es
-    if channel_handler.source == 'scraped':
-        channel_handler.channel_dict['channel_subscribed'] = False
-        channel_handler.upload_to_es()
-    # upload video to es
-    vid_handler.upload_to_es()
-    # return vid_dict for further processing
-    return vid_handler.vid_dict

tubearchivist/home/src/index/channel.py

@@ -0,0 +1,355 @@
+"""
+functionality:
+- get metadata from youtube for a channel
+- index and update in es
+"""
+
+import json
+import os
+from datetime import datetime
+
+from home.src.download.thumbnails import ThumbManager
+from home.src.download.yt_dlp_base import YtWrap
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.index.generic import YouTubeItem
+from home.src.index.playlist import YoutubePlaylist
+from home.src.ta.settings import EnvironmentSettings
+
+
+class YoutubeChannel(YouTubeItem):
+    """represents a single youtube channel"""
+
+    es_path = False
+    index_name = "ta_channel"
+    yt_base = "https://www.youtube.com/channel/"
+    yt_obs = {
+        "playlist_items": "1,0",
+        "skip_download": True,
+    }
+
+    def __init__(self, youtube_id, task=False):
+        super().__init__(youtube_id)
+        self.all_playlists = False
+        self.task = task
+
+    def build_json(self, upload=False, fallback=False):
+        """get from es or from youtube"""
+        self.get_from_es()
+        if self.json_data:
+            return
+
+        self.get_from_youtube()
+        if not self.youtube_meta and fallback:
+            self._video_fallback(fallback)
+        else:
+            self.process_youtube_meta()
+            self.get_channel_art()
+
+        if upload:
+            self.upload_to_es()
+
+    def process_youtube_meta(self):
+        """extract relevant fields"""
+        self.youtube_meta["thumbnails"].reverse()
+        self.json_data = {
+            "channel_active": True,
+            "channel_description": self.youtube_meta.get("description", False),
+            "channel_id": self.youtube_id,
+            "channel_last_refresh": int(datetime.now().timestamp()),
+            "channel_name": self.youtube_meta["uploader"],
+            "channel_subs": self.youtube_meta.get("channel_follower_count", 0),
+            "channel_subscribed": False,
+            "channel_tags": self._parse_tags(self.youtube_meta.get("tags")),
+            "channel_banner_url": self._get_banner_art(),
+            "channel_thumb_url": self._get_thumb_art(),
+            "channel_tvart_url": self._get_tv_art(),
+            "channel_views": self.youtube_meta.get("view_count") or 0,
+        }
+
+    def _parse_tags(self, tags):
+        """parse channel tags"""
+        if not tags:
+            return False
+
+        joined = " ".join(tags)
+        return [i.strip() for i in joined.split('"') if i and not i == " "]
+
+    def _get_thumb_art(self):
+        """extract thumb art"""
+        for i in self.youtube_meta["thumbnails"]:
+            if not i.get("width"):
+                continue
+            if i.get("width") == i.get("height"):
+                return i["url"]
+
+        return False
+
+    def _get_tv_art(self):
+        """extract tv artwork"""
+        for i in self.youtube_meta["thumbnails"]:
+            if i.get("id") == "banner_uncropped":
+                return i["url"]
+        for i in self.youtube_meta["thumbnails"]:
+            if not i.get("width"):
+                continue
+            if i["width"] // i["height"] < 2 and not i["width"] == i["height"]:
+                return i["url"]
+
+        return False
+
+    def _get_banner_art(self):
+        """extract banner artwork"""
+        for i in self.youtube_meta["thumbnails"]:
+            if not i.get("width"):
+                continue
+            if i["width"] // i["height"] > 5:
+                return i["url"]
+
+        return False
+
+    def _video_fallback(self, fallback):
+        """use video metadata as fallback"""
+        print(f"{self.youtube_id}: fallback to video metadata")
+        self.json_data = {
+            "channel_active": False,
+            "channel_last_refresh": int(datetime.now().timestamp()),
+            "channel_subs": fallback.get("channel_follower_count", 0),
+            "channel_name": fallback["uploader"],
+            "channel_banner_url": False,
+            "channel_tvart_url": False,
+            "channel_id": self.youtube_id,
+            "channel_subscribed": False,
+            "channel_tags": False,
+            "channel_description": False,
+            "channel_thumb_url": False,
+            "channel_views": 0,
+        }
+        self._info_json_fallback()
+
+    def _info_json_fallback(self):
+        """read channel info.json for additional metadata"""
+        info_json = os.path.join(
+            EnvironmentSettings.CACHE_DIR,
+            "import",
+            f"{self.youtube_id}.info.json",
+        )
+        if os.path.exists(info_json):
+            print(f"{self.youtube_id}: read info.json file")
+            with open(info_json, "r", encoding="utf-8") as f:
+                content = json.loads(f.read())
+
+            self.json_data.update(
+                {
+                    "channel_subs": content.get("channel_follower_count", 0),
+                    "channel_description": content.get("description", False),
+                }
+            )
+            os.remove(info_json)
+
+    def get_channel_art(self):
+        """download channel art for new channels"""
+        urls = (
+            self.json_data["channel_thumb_url"],
+            self.json_data["channel_banner_url"],
+            self.json_data["channel_tvart_url"],
+        )
+        ThumbManager(self.youtube_id, item_type="channel").download(urls)
+
+    def sync_to_videos(self):
+        """sync new channel_dict to all videos of channel"""
+        # add ingest pipeline
+        processors = []
+        for field, value in self.json_data.items():
+            line = {"set": {"field": "channel." + field, "value": value}}
+            processors.append(line)
+        data = {"description": self.youtube_id, "processors": processors}
+        ingest_path = f"_ingest/pipeline/{self.youtube_id}"
+        _, _ = ElasticWrap(ingest_path).put(data)
+        # apply pipeline
+        data = {"query": {"match": {"channel.channel_id": self.youtube_id}}}
+        update_path = f"ta_video/_update_by_query?pipeline={self.youtube_id}"
+        _, _ = ElasticWrap(update_path).post(data)
+
+    def get_folder_path(self):
+        """get folder where media files get stored"""
+        folder_path = os.path.join(
+            EnvironmentSettings.MEDIA_DIR,
+            self.json_data["channel_id"],
+        )
+        return folder_path
+
+    def delete_es_videos(self):
+        """delete all channel documents from elasticsearch"""
+        data = {
+            "query": {
+                "term": {"channel.channel_id": {"value": self.youtube_id}}
+            }
+        }
+        _, _ = ElasticWrap("ta_video/_delete_by_query").post(data)
+
+    def delete_es_comments(self):
+        """delete all comments from this channel"""
+        data = {
+            "query": {
+                "term": {"comment_channel_id": {"value": self.youtube_id}}
+            }
+        }
+        _, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
+
+    def delete_es_subtitles(self):
+        """delete all subtitles from this channel"""
+        data = {
+            "query": {
+                "term": {"subtitle_channel_id": {"value": self.youtube_id}}
+            }
+        }
+        _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data)
+
+    def delete_playlists(self):
+        """delete all indexed playlist from es"""
+        all_playlists = self.get_indexed_playlists()
+        for playlist in all_playlists:
+            YoutubePlaylist(playlist["playlist_id"]).delete_metadata()
+
+    def delete_channel(self):
+        """delete channel and all videos"""
+        print(f"{self.youtube_id}: delete channel")
+        self.get_from_es()
+        if not self.json_data:
+            raise FileNotFoundError
+
+        folder_path = self.get_folder_path()
+        print(f"{self.youtube_id}: delete all media files")
+        try:
+            all_videos = os.listdir(folder_path)
+            for video in all_videos:
+                video_path = os.path.join(folder_path, video)
+                os.remove(video_path)
+            os.rmdir(folder_path)
+        except FileNotFoundError:
+            print(f"no videos found for {folder_path}")
+
+        print(f"{self.youtube_id}: delete indexed playlists")
+        self.delete_playlists()
+        print(f"{self.youtube_id}: delete indexed videos")
+        self.delete_es_videos()
+        self.delete_es_comments()
+        self.delete_es_subtitles()
+        self.del_in_es()
+
+    def index_channel_playlists(self):
+        """add all playlists of channel to index"""
+        print(f"{self.youtube_id}: index all playlists")
+        self.get_from_es()
+        channel_name = self.json_data["channel_name"]
+        self.task.send_progress([f"{channel_name}: Looking for Playlists"])
+        self.get_all_playlists()
+        if not self.all_playlists:
+            print(f"{self.youtube_id}: no playlists found.")
+            return
+
+        total = len(self.all_playlists)
+        for idx, playlist in enumerate(self.all_playlists):
+            if self.task:
+                self._notify_single_playlist(idx, total)
+
+            self._index_single_playlist(playlist)
+            print("add playlist: " + playlist[1])
+
+    def _notify_single_playlist(self, idx, total):
+        """send notification"""
+        channel_name = self.json_data["channel_name"]
+        message = [
+            f"{channel_name}: Scanning channel for playlists",
+            f"Progress: {idx + 1}/{total}",
+        ]
+        self.task.send_progress(message, progress=(idx + 1) / total)
+
+    @staticmethod
+    def _index_single_playlist(playlist):
+        """add single playlist if needed"""
+        playlist = YoutubePlaylist(playlist[0])
+        playlist.update_playlist(skip_on_empty=True)
+
+    def get_channel_videos(self):
+        """get all videos from channel"""
+        data = {
+            "query": {
+                "term": {"channel.channel_id": {"value": self.youtube_id}}
+            },
+            "_source": ["youtube_id", "vid_type"],
+        }
+        all_videos = IndexPaginate("ta_video", data).get_results()
+        return all_videos
+
+    def get_all_playlists(self):
+        """get all playlists owned by this channel"""
+        url = (
+            f"https://www.youtube.com/channel/{self.youtube_id}"
+            + "/playlists?view=1&sort=dd&shelf_id=0"
+        )
+        obs = {"skip_download": True, "extract_flat": True}
+        playlists = YtWrap(obs, self.config).extract(url)
+        all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
+        self.all_playlists = all_entries
+
+    def get_indexed_playlists(self, active_only=False):
+        """get all indexed playlists from channel"""
+        must_list = [
+            {"term": {"playlist_channel_id": {"value": self.youtube_id}}}
+        ]
+        if active_only:
+            must_list.append({"term": {"playlist_active": {"value": True}}})
+
+        data = {"query": {"bool": {"must": must_list}}}
+
+        all_playlists = IndexPaginate("ta_playlist", data).get_results()
+        return all_playlists
+
+    def get_overwrites(self) -> dict:
+        """get all per channel overwrites"""
+        return self.json_data.get("channel_overwrites", {})
+
+    def set_overwrites(self, overwrites):
+        """set per channel overwrites"""
+        valid_keys = [
+            "download_format",
+            "autodelete_days",
+            "index_playlists",
+            "integrate_sponsorblock",
+            "subscriptions_channel_size",
+            "subscriptions_live_channel_size",
+            "subscriptions_shorts_channel_size",
+        ]
+
+        to_write = self.json_data.get("channel_overwrites", {})
+        for key, value in overwrites.items():
+            if key not in valid_keys:
+                raise ValueError(f"invalid overwrite key: {key}")
+            elif value == "disable":
+                to_write[key] = False
+                continue
+            elif value == "0":
+                if key in to_write:
+                    del to_write[key]
+                continue
+            elif value == "1":
+                to_write[key] = True
+                continue
+            elif isinstance(value, int) and int(value) < 0:
+                if key in to_write:
+                    del to_write[key]
+                continue
+            elif value is not None and value != "":
+                to_write.update({key: value})
+
+        self.json_data["channel_overwrites"] = to_write
+
+
+def channel_overwrites(channel_id, overwrites):
+    """collection to overwrite settings per channel"""
+    channel = YoutubeChannel(channel_id)
+    channel.build_json()
+    channel.set_overwrites(overwrites)
+    channel.upload_to_es()
+    channel.sync_to_videos()

tubearchivist/home/src/index/comments.py

@@ -0,0 +1,227 @@
+"""
+Functionality:
+- Download comments
+- Index comments in ES
+- Retrieve comments from ES
+"""
+
+from datetime import datetime
+
+from home.src.download.yt_dlp_base import YtWrap
+from home.src.es.connect import ElasticWrap
+from home.src.ta.config import AppConfig
+from home.src.ta.ta_redis import RedisQueue
+
+
+class Comments:
+    """interact with comments per video"""
+
+    def __init__(self, youtube_id, config=False):
+        self.youtube_id = youtube_id
+        self.es_path = f"ta_comment/_doc/{youtube_id}"
+        self.json_data = False
+        self.config = config
+        self.is_activated = False
+        self.comments_format = False
+
+    def build_json(self):
+        """build json document for es"""
+        print(f"{self.youtube_id}: get comments")
+        self.check_config()
+        if not self.is_activated:
+            return
+
+        comments_raw, channel_id = self.get_yt_comments()
+        if not comments_raw and not channel_id:
+            return
+
+        self.format_comments(comments_raw)
+
+        self.json_data = {
+            "youtube_id": self.youtube_id,
+            "comment_last_refresh": int(datetime.now().timestamp()),
+            "comment_channel_id": channel_id,
+            "comment_comments": self.comments_format,
+        }
+
+    def check_config(self):
+        """read config if not attached"""
+        if not self.config:
+            self.config = AppConfig().config
+
+        self.is_activated = bool(self.config["downloads"]["comment_max"])
+
+    def build_yt_obs(self):
+        """
+        get extractor config
+        max-comments,max-parents,max-replies,max-replies-per-thread
+        """
+        max_comments = self.config["downloads"]["comment_max"]
+        max_comments_list = [i.strip() for i in max_comments.split(",")]
+        comment_sort = self.config["downloads"]["comment_sort"]
+
+        yt_obs = {
+            "check_formats": None,
+            "skip_download": True,
+            "getcomments": True,
+            "ignoreerrors": True,
+            "extractor_args": {
+                "youtube": {
+                    "max_comments": max_comments_list,
+                    "comment_sort": [comment_sort],
+                    "player_client": ["ios", "web"],  # workaround yt-dlp #9554
+                }
+            },
+        }
+
+        return yt_obs
+
+    def get_yt_comments(self):
+        """get comments from youtube"""
+        yt_obs = self.build_yt_obs()
+        info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id)
+        if not info_json:
+            return False, False
+
+        comments_raw = info_json.get("comments")
+        channel_id = info_json.get("channel_id")
+        return comments_raw, channel_id
+
+    def format_comments(self, comments_raw):
+        """process comments to match format"""
+        comments = []
+
+        if comments_raw:
+            for comment in comments_raw:
+                cleaned_comment = self.clean_comment(comment)
+                if not cleaned_comment:
+                    continue
+
+                comments.append(cleaned_comment)
+
+        self.comments_format = comments
+
+    def clean_comment(self, comment):
+        """parse metadata from comment for indexing"""
+        if not comment.get("text"):
+            # comment text can be empty
+            print(f"{self.youtube_id}: Failed to extract text, {comment}")
+            return False
+
+        time_text_datetime = datetime.utcfromtimestamp(comment["timestamp"])
+
+        if time_text_datetime.hour == 0 and time_text_datetime.minute == 0:
+            format_string = "%Y-%m-%d"
+        else:
+            format_string = "%Y-%m-%d %H:%M"
+
+        time_text = time_text_datetime.strftime(format_string)
+
+        if not comment.get("author"):
+            comment["author"] = comment.get("author_id", "Unknown")
+
+        cleaned_comment = {
+            "comment_id": comment["id"],
+            "comment_text": comment["text"].replace("\xa0", ""),
+            "comment_timestamp": comment["timestamp"],
+            "comment_time_text": time_text,
+            "comment_likecount": comment.get("like_count", None),
+            "comment_is_favorited": comment.get("is_favorited", False),
+            "comment_author": comment["author"],
+            "comment_author_id": comment["author_id"],
+            "comment_author_thumbnail": comment["author_thumbnail"],
+            "comment_author_is_uploader": comment.get(
+                "author_is_uploader", False
+            ),
+            "comment_parent": comment["parent"],
+        }
+
+        return cleaned_comment
+
+    def upload_comments(self):
+        """upload comments to es"""
+        if not self.is_activated:
+            return
+
+        print(f"{self.youtube_id}: upload comments")
+        _, _ = ElasticWrap(self.es_path).put(self.json_data)
+
+        vid_path = f"ta_video/_update/{self.youtube_id}"
+        data = {"doc": {"comment_count": len(self.comments_format)}}
+        _, _ = ElasticWrap(vid_path).post(data=data)
+
+    def delete_comments(self):
+        """delete comments from es"""
+        print(f"{self.youtube_id}: delete comments")
+        _, _ = ElasticWrap(self.es_path).delete(refresh=True)
+
+    def get_es_comments(self):
+        """get comments from ES"""
+        response, statuscode = ElasticWrap(self.es_path).get()
+        if statuscode == 404:
+            print(f"comments: not found {self.youtube_id}")
+            return False
+
+        return response.get("_source")
+
+    def reindex_comments(self):
+        """update comments from youtube"""
+        self.check_config()
+        if not self.is_activated:
+            return
+
+        self.build_json()
+        if not self.json_data:
+            return
+
+        es_comments = self.get_es_comments()
+
+        if not self.comments_format:
+            return
+
+        if not self.comments_format and es_comments["comment_comments"]:
+            # don't overwrite comments in es
+            return
+
+        self.delete_comments()
+        self.upload_comments()
+
+
+class CommentList:
+    """interact with comments in group"""
+
+    COMMENT_QUEUE = "index:comment"
+
+    def __init__(self, task=False):
+        self.task = task
+        self.config = AppConfig().config
+
+    def add(self, video_ids: list[str]) -> None:
+        """add list of videos to get comments, if enabled in config"""
+        if not self.config["downloads"].get("comment_max"):
+            return
+
+        RedisQueue(self.COMMENT_QUEUE).add_list(video_ids)
+
+    def index(self):
+        """run comment index"""
+        queue = RedisQueue(self.COMMENT_QUEUE)
+        while True:
+            total = queue.max_score()
+            youtube_id, idx = queue.get_next()
+            if not youtube_id or not idx or not total:
+                break
+
+            if self.task:
+                self.notify(idx, total)
+
+            comment = Comments(youtube_id, config=self.config)
+            comment.build_json()
+            if comment.json_data:
+                comment.upload_comments()
+
+    def notify(self, idx, total_videos):
+        """send notification on task"""
+        message = [f"Add comments for new videos {idx}/{total_videos}"]
+        progress = idx / total_videos
+        self.task.send_progress(message, progress=progress)

tubearchivist/home/src/index/filesystem.py

@@ -0,0 +1,126 @@
+"""
+Functionality:
+- scan the filesystem to delete or index
+"""
+
+import os
+
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.index.comments import CommentList
+from home.src.index.video import YoutubeVideo, index_new_video
+from home.src.ta.helper import ignore_filelist
+from home.src.ta.settings import EnvironmentSettings
+
+
+class Scanner:
+    """scan index and filesystem"""
+
+    VIDEOS: str = EnvironmentSettings.MEDIA_DIR
+
+    def __init__(self, task=False) -> None:
+        self.task = task
+        self.to_delete: set[str] = set()
+        self.to_index: set[str] = set()
+
+    def scan(self) -> None:
+        """scan the filesystem"""
+        downloaded: set[str] = self._get_downloaded()
+        indexed: set[str] = self._get_indexed()
+        self.to_index = downloaded - indexed
+        self.to_delete = indexed - downloaded
+
+    def _get_downloaded(self) -> set[str]:
+        """get downloaded ids"""
+        if self.task:
+            self.task.send_progress(["Scan your filesystem for videos."])
+
+        downloaded: set = set()
+        channels = ignore_filelist(os.listdir(self.VIDEOS))
+        for channel in channels:
+            folder = os.path.join(self.VIDEOS, channel)
+            files = ignore_filelist(os.listdir(folder))
+            downloaded.update({i.split(".")[0] for i in files})
+
+        return downloaded
+
+    def _get_indexed(self) -> set:
+        """get all indexed ids"""
+        if self.task:
+            self.task.send_progress(["Get all videos indexed."])
+
+        data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
+        response = IndexPaginate("ta_video", data).get_results()
+        return {i["youtube_id"] for i in response}
+
+    def apply(self) -> None:
+        """apply all changes"""
+        self.delete()
+        self.index()
+        self.url_fix()
+
+    def delete(self) -> None:
+        """delete videos from index"""
+        if not self.to_delete:
+            print("nothing to delete")
+            return
+
+        if self.task:
+            self.task.send_progress(
+                [f"Remove {len(self.to_delete)} videos from index."]
+            )
+
+        for youtube_id in self.to_delete:
+            YoutubeVideo(youtube_id).delete_media_file()
+
+    def index(self) -> None:
+        """index new"""
+        if not self.to_index:
+            print("nothing to index")
+            return
+
+        total = len(self.to_index)
+        for idx, youtube_id in enumerate(self.to_index):
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[
+                        f"Index missing video {youtube_id}, {idx + 1}/{total}"
+                    ],
+                    progress=(idx + 1) / total,
+                )
+            index_new_video(youtube_id)
+
+        comment_list = CommentList(task=self.task)
+        comment_list.add(video_ids=list(self.to_index))
+        comment_list.index()
+
+    def url_fix(self) -> None:
+        """
+        update path v0.3.6 to v0.3.7
+        fix url not matching channel-videoid pattern
+        """
+        bool_must = (
+            "doc['media_url'].value == "
+            + "(doc['channel.channel_id'].value + '/' + "
+            + "doc['youtube_id'].value) + '.mp4'"
+        )
+        to_update = (
+            "ctx._source['media_url'] = "
+            + "ctx._source.channel['channel_id'] + '/' + "
+            + "ctx._source['youtube_id'] + '.mp4'"
+        )
+        data = {
+            "query": {
+                "bool": {
+                    "must_not": [{"script": {"script": {"source": bool_must}}}]
+                }
+            },
+            "script": {"source": to_update},
+        }
+        response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data)
+        updated = response.get("updates")
+        if updated:
+            print(f"updated {updated} bad media_url")
+            if self.task:
+                self.task.send_progress(
+                    [f"Updated {updated} wrong media urls."]
+                )