Merge branch 'testing' to master

docs: typo in readme

.dockerignore

@@ -17,8 +17,5 @@ venv/
 # Unneeded graphics
 assets/*
 
-# Unneeded docs
-docs/*
-
 # for local testing only
 testing.sh

.github/workflows/lint_python.yml

@@ -8,20 +8,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-      # note: this logic is duplicated in the `validate` function in ./deploy.sh
-      # if you update this file, you should update that as well
       - run: pip install --upgrade pip wheel
       - run: pip install bandit black codespell flake8 flake8-bugbear
                          flake8-comprehensions isort requests
       - run: ./deploy.sh validate
-      # - run: black --check --diff --line-length 79 .
-      # - run: codespell --skip="./.git,./package.json,./package-lock.json,./node_modules"
-      # - run: flake8 . --count --max-complexity=10 --max-line-length=79
-      #                 --show-source --statistics
-      # - run: isort --check-only --line-length 79 --profile black .
-      # - run: pip install -r tubearchivist/requirements.txt
-      # - run: mkdir --parents --verbose .mypy_cache
-      # - run: mypy --ignore-missing-imports --install-types --non-interactive .
-      # - run: python3 tubearchivist/manage.py test || true
-      # - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true
-      # - run: safety check

.gitignore

@@ -1,5 +1,6 @@
 # python testing cache
 __pycache__
+.venv
 
 # django testing db
 db.sqlite3

CONTRIBUTING.md

@@ -8,6 +8,8 @@ Welcome, and thanks for showing interest in improving Tube Archivist!
   - [Feature Request](#feature-request)
   - [Installation Help](#installation-help)
 - [How to make a Pull Request](#how-to-make-a-pull-request)
+- [Contributions beyond the scope](#contributions-beyond-the-scope)
+- [User Scripts](#user-scripts)
 - [Improve to the Documentation](#improve-to-the-documentation)
 - [Development Environment](#development-environment)
 ---
@@ -87,6 +89,33 @@ to validate your changes. If you omit the path, all the project files will get c
 
 ---
 
+## Contributions beyond the scope
+
+As you have read the [FAQ](https://docs.tubearchivist.com/faq/) and the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and have gotten an idea what this project tries to do, there will be some obvious shortcomings that stand out, that have been explicitly excluded from the scope of this project, at least for the time being.
+
+Extending the scope of this project will only be feasible with more [regular contributors](https://github.com/tubearchivist/tubearchivist/graphs/contributors) that are willing to help improve this project in the long run. Contributors that have an overall improvement of the project in mind and not just about implementing this *one* thing.  
+
+Small minor additions, or making a PR for a documented feature request or bug, even if that was and will be your only contribution to this project, are always welcome and is *not* what this is about.
+
+Beyond that, general rules to consider:
+
+- Maintainability is key: It's not just about implementing something and being done with it, it's about maintaining it, fixing bugs as they occur, improving on it and supporting it in the long run.
+- Others can do it better: Some problems have been solved by very talented developers. These things don't need to be reinvented again here in this project.
+- Develop for the 80%: New features and additions *should* be beneficial for 80% of the users. If you are trying to solve your own problem that only applies to you, maybe that would be better to do in your own fork or if possible by a standalone implementation using the API.
+- If all of that sounds too strict for you, as stated above, start becoming a regular contributor to this project.
+
+---
+
+## User Scripts
+Some of you might have created useful scripts or API integrations around this project. Sharing is caring! Please add a link to your script to the Readme [here](https://github.com/tubearchivist/tubearchivist#user-scripts).
+- Your repo should have a `LICENSE` file with one of the common open source licenses. People are expected to fork, adapt and build upon your great work.
+- Your script should not modify the *official* files of Tube Archivist. E.g. your symlink script should build links *outside* of your `/youtube` folder. Or your fancy script that creates a beautiful artwork gallery should do that *outside* of the `/cache` folder. Modifying the *official* files and folders of TA are probably not supported.
+- On the top of the repo you should have a mention and a link back to the Tube Archivist repo. Clearly state to **not** to open any issues on the main TA repo regarding your script.
+- Example template:
+  - `[<user>/<repo>](https://linktoyourrepo.com)`: A short one line description.
+
+---
+
 ## Improve to the Documentation
 
 The documentation available at [docs.tubearchivist.com](https://docs.tubearchivist.com/) and is build from a separate repo [tubearchivist/docs](https://github.com/tubearchivist/docs). The Readme has additional instructions on how to make changes.
@@ -136,14 +165,15 @@ bin/elasticsearch-service-tokens create elastic/kibana kibana
 
 Example docker compose, use same version as for Elasticsearch:
 ```yml
-kibana:
-  image: docker.elastic.co/kibana/kibana:0.0.0
-  container_name: kibana
-  environment:
-	- "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
-	- "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
-  ports:
-	- "5601:5601"
+services:
+  kibana:
+    image: docker.elastic.co/kibana/kibana:0.0.0
+    container_name: kibana
+    environment:
+    - "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
+    - "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
+    ports:
+    - "5601:5601"
 ```
 
 If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.

Dockerfile

@@ -3,18 +3,18 @@
 
 
 # First stage to build python wheel
-FROM python:3.10.9-slim-bullseye AS builder
+FROM python:3.11.3-slim-bullseye AS builder
 ARG TARGETPLATFORM
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev git
 
 # install requirements
 COPY ./tubearchivist/requirements.txt /requirements.txt
 RUN pip install --user -r requirements.txt
 
 # build final image
-FROM python:3.10.9-slim-bullseye as tubearchivist
+FROM python:3.11.3-slim-bullseye as tubearchivist
 
 ARG TARGETPLATFORM
 ARG INSTALL_DEBUG

README.md

@@ -1,6 +1,6 @@
-![Tube Archivist](assets/tube-archivist-banner.jpg?raw=true "Tube Archivist Banner")  
+![Tube Archivist](assets/tube-archivist-front.jpg?raw=true "Tube Archivist Banner")  
+[*more screenshots and video*](SHOWCASE.MD)
 
-<h1 align="center">Your self hosted YouTube media server</h1>
 <div align="center">
 <a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-docker.png" alt="tubearchivist-docker" title="Tube Archivist Docker Pulls" height="50" width="190"/></a>
 <a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-star.png" alt="tubearchivist-github-star" title="Tube Archivist GitHub Stars" height="50" width="190"/></a>
@@ -8,8 +8,6 @@
 <a href="https://www.tubearchivist.com/discord" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-discord.png" alt="tubearchivist-discord" title="TA Discord Server Members" height="50" width="190"/></a>
 </div>
 
-![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")
-
 ## Table of contents:
 * [Docs](https://docs.tubearchivist.com/) with [FAQ](https://docs.tubearchivist.com/faq/), and API documentation
 * [Core functionality](#core-functionality)
@@ -25,7 +23,7 @@
 ------------------------
 
 ## Core functionality
-Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. THis includes:  
+Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. This includes:  
 * Subscribe to your favorite YouTube channels
 * Download Videos using **yt-dlp**
 * Index and make videos searchable
@@ -36,8 +34,8 @@ Once your YouTube video collection grows, it becomes hard to search and find a s
 - [Discord](https://www.tubearchivist.com/discord): Connect with us on our Discord server.
 - [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit.
 - [Browser Extension](https://github.com/tubearchivist/browser-extension) Tube Archivist Companion, for [Firefox](https://addons.mozilla.org/addon/tubearchivist-companion/) and [Chrome](https://chrome.google.com/webstore/detail/tubearchivist-companion/jjnkmicfnfojkkgobdfeieblocadmcie)
+- [Jellyfin Integration](https://github.com/tubearchivist/tubearchivist-jf): Add your videos to Jellyfin.
 - [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.  
-- [Showcase](SHOWCASE.MD) To see more screenshots and Youtube videos showing off TubeArchvist.
 
 ## Installing
 For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. Minimal with dual core with 4 threads, better quad core plus. 
@@ -60,7 +58,12 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
 | TZ | Set your timezone for the scheduler | Required |
 | TA_PORT | Overwrite Nginx port | Optional |
 | TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
+| TA_ENABLE_AUTH_PROXY | Enables support for forwarding auth in reverse proxies | [Read more](https://docs.tubearchivist.com/configuration/forward-auth/) |
+| TA_AUTH_PROXY_USERNAME_HEADER | Header containing username to log in | Optional |
+| TA_AUTH_PROXY_LOGOUT_URL | Logout URL for forwarded auth | Optional |
 | ES_URL | URL That ElasticSearch runs on | Optional |
+| ES_DISABLE_VERIFY_SSL | Disable ElasticSearch SSL certificate verification | Optional |
+| ES_SNAPSHOT_DIR | Custom path where elastic search stores snapshots for master/data nodes | Optional |
 | HOST_GID | Allow TA to own the video files instead of container user | Optional |
 | HOST_UID | Allow TA to own the video files instead of container user | Optional |
 | ELASTIC_USER | Change the default ElasticSearch user | Optional |
@@ -79,7 +82,7 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
 ## Update
 Always use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tags are only for your testing environment, there might not be an update path for these testing builds. 
 
-You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. To take advantage of the latest fixes and improvements, make sure you are running the *latest and greatest*. After updating, check the footer to verify you are running the expected version.  
+You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. To update, you need to update the docker images, the method for which will depend on your platform. For example, if you're using `docker-compose`, run `docker-compose pull` and then restart with `docker-compose up -d`. After updating, check the footer to verify you are running the expected version.  
 
   - This project is tested for updates between one or two releases maximum. Further updates back may or may not be supported and you might have to reset your index and configurations to update. Ideally apply new updates at least once per month.  
   - There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.  
@@ -135,25 +138,25 @@ Similar to that, TubeArchivist will become all sorts of messed up when running o
 ## Known limitations
 - Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
 - Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
-- There is currently no flexibility in naming of the media files.
+- There is no flexibility in naming of the media files.
 
 ## Roadmap
 We have come far, nonetheless we are not short of ideas on how to improve and extend this project. Issues waiting for you to be tackled in no particular order:
 
 - [ ] User roles
 - [ ] Podcast mode to serve channel as mp3
-- [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage
-- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/tubearchivist/tubearchivist/issues/97))
 - [ ] User created playlists, random and repeat controls ([#108](https://github.com/tubearchivist/tubearchivist/issues/108), [#220](https://github.com/tubearchivist/tubearchivist/issues/220))
 - [ ] Auto play or play next link ([#226](https://github.com/tubearchivist/tubearchivist/issues/226))
 - [ ] Multi language support
 - [ ] Show total video downloaded vs total videos available in channel
-- [ ] Add statistics of index
-- [ ] Download speed schedule ([#198](https://github.com/tubearchivist/tubearchivist/issues/198))
 - [ ] Download or Ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163))
 - [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144))
+- [ ] Search comments
+- [ ] Search download queue
 
 Implemented:
+- [X] Add statistics of index [2023-09-03]
+- [X] Implement [Apprise](https://github.com/caronc/apprise) for notifications [2023-08-05]
 - [X] Download video comments [2022-11-30]
 - [X] Show similar videos on video page [2022-11-30]
 - [X] Implement complete offline media file import from json file [2022-08-20]
@@ -177,6 +180,11 @@ Implemented:
 - [X] Backup and restore [2021-09-22]
 - [X] Scan your file system to index already downloaded videos [2021-09-14]
 
+## User Scripts
+This is a list of useful user scripts, generously created from folks like you to extend this project and its functionality. This is your time to shine, [read this](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#user-scripts) then open a PR to add your script here.
+
+- Example 1
+
 ## Donate
 The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.  
 Second best way to support the development is to provide for caffeinated beverages:
@@ -187,6 +195,8 @@ Second best way to support the development is to provide for caffeinated beverag
 
 ## Notable mentions
 This is a selection of places where this project has been featured on reddit, in the news, blogs or any other online media, newest on top.
+* **ycombinator**: Tube Archivist on Hackernews front page, [2023-07-16][[link](https://news.ycombinator.com/item?id=36744395)]
+* **linux-community.de**: Tube Archivist bringt Ordnung in die Youtube-Sammlung, [German][2023-05-01][[link](https://www.linux-community.de/ausgaben/linuxuser/2023/05/tube-archivist-bringt-ordnung-in-die-youtube-sammlung/)]
 * **noted.lol**: Dev Debrief, An Interview With the Developer of Tube Archivist, [2023-03-30] [[link](https://noted.lol/dev-debrief-tube-archivist/)]
 * **console.substack.com**: Interview With Simon of Tube Archivist, [2023-01-29] [[link](https://console.substack.com/p/console-142#%C2%A7interview-with-simon-of-tube-archivist)]
 * **reddit.com**: Tube Archivist v0.3.0 - Now Archiving Comments, [2022-12-02] [[link](https://www.reddit.com/r/selfhosted/comments/zaonzp/tube_archivist_v030_now_archiving_comments/)]

SHOWCASE.MD

@@ -3,17 +3,23 @@
 Video featuring Tube Archivist generously created by [IBRACORP](https://www.youtube.com/@IBRACORP).
 
 ## Screenshots
-![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")  
+![login screenshot](assets/tube-archivist-login.png?raw=true "Tube Archivist Login")  
+*Login Page*: Secure way to access your media collection.
+
+![home screenshot](assets/tube-archivist-home.png?raw=true "Tube Archivist Home")  
 *Home Page*: Your recent videos, continue watching incomplete videos.
 
-![channels screenshot](assets/tube-archivist-screenshot-channels.png?raw=true "Tube Archivist Channels")  
+![channels screenshot](assets/tube-archivist-channels.png?raw=true "Tube Archivist Channels")  
 *All Channels*: A list of all your indexed channels, filtered by subscribed only.
 
-![single channel screenshot](assets/tube-archivist-screenshot-single-channel.png?raw=true "Tube Archivist Single Channel")  
+![single channel screenshot](assets/tube-archivist-single-channel.png?raw=true "Tube Archivist Single Channel")  
 *Single Channel*: Single channel page with additional metadata and sub pages.
 
-![video page screenshot](assets/tube-archivist-screenshot-video.png?raw=true "Tube Archivist Video Page")  
+![video page screenshot](assets/tube-archivist-video.png?raw=true "Tube Archivist Video Page")  
 *Video Page*: Stream your video directly from the interface.
 
-![video page screenshot](assets/tube-archivist-screenshot-download.png?raw=true "Tube Archivist Video Page")  
-*Downloads Page*: Add, control and monitor your download queue.
+![video page screenshot](assets/tube-archivist-download.png?raw=true "Tube Archivist Video Page")  
+*Downloads Page*: Add, control, and monitor your download queue.
+
+![search page screenshot](assets/tube-archivist-search.png?raw=true "Tube Archivist Search Page")  
+*Search Page*. Use expressions to quickly search through your collection.

deploy.sh

@@ -25,6 +25,7 @@ function sync_blackhole {
         --exclude ".gitignore" \
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
+        --exclude ".venv" \
         --exclude "db.sqlite3" \
         --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
@@ -49,6 +50,7 @@ function sync_test {
         --exclude ".gitignore" \
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
+        --exclude ".venv" \
         --exclude "db.sqlite3" \
         --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
@@ -87,14 +89,14 @@ function validate {
     # note: this logic is duplicated in the `./github/workflows/lint_python.yml` config
     # if you update this file, you should update that as well
     echo "running black"
-    black --exclude "migrations/*" --diff --color --check -l 79 "$check_path"
+    black --force-exclude "migrations/*" --diff --color --check -l 79 "$check_path"
     echo "running codespell"
-    codespell --skip="./.git,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
+    codespell --skip="./.git,./.venv,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
     echo "running flake8"
-    flake8 "$check_path" --exclude "migrations" --count --max-complexity=10 \
+    flake8 "$check_path" --exclude "migrations,.venv" --count --max-complexity=10 \
         --max-line-length=79 --show-source --statistics
     echo "running isort"
-    isort --skip "migrations" --check-only --diff --profile black -l 79 "$check_path"
+    isort --skip "migrations" --skip ".venv" --check-only --diff --profile black -l 79 "$check_path"
     printf "    \n> all validations passed\n"
 
 }

docker-compose.yml

@@ -20,6 +20,12 @@ services:
       - TA_PASSWORD=verysecret              # your initial TA credentials
       - ELASTIC_PASSWORD=verysecret         # set password for Elasticsearch
       - TZ=America/New_York                 # set your time zone
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 2m
+      timeout: 10s
+      retries: 3
+      start_period: 30s
     depends_on:
       - archivist-es
       - archivist-redis
@@ -34,7 +40,7 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.7.0
+    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.9.0
     container_name: archivist-es
     restart: unless-stopped
     environment:

docker_assets/run.sh

@@ -14,6 +14,7 @@ fi
 python manage.py ta_envcheck
 python manage.py ta_connection
 python manage.py ta_startup
+python manage.py ta_migpath
 
 # start all tasks
 nginx &

docs/Channels.md

@@ -1,3 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
-
-Equivalent Channels page is: [here](https://docs.tubearchivist.com/channels/).

docs/Downloads.md

@@ -1,3 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
-
-Equivalent Downloads page is: [here](https://docs.tubearchivist.com/downloads/).

docs/FAQ.md

@@ -1,3 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
-
-Equivalent FAQ page is: [here](https://docs.tubearchivist.com/faq/).

docs/Home.md

@@ -1 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.

docs/Installation.md

@@ -1,3 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
-
-Equivalent pages are located under *installation* on the left.

docs/Playlists.md

@@ -1,3 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
-
-Equivalent Playlist page is: [here](https://docs.tubearchivist.com/playlists/).

docs/Search.md

@@ -1,3 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
-
-Equivalent Search page is: [here](https://docs.tubearchivist.com/search/).

docs/Settings.md

@@ -1,3 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
-
-Equivalent Settings page is: [here](https://docs.tubearchivist.com/settings/).

docs/Users.md

@@ -1,3 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
-
-Equivalent Users page is: [here](https://docs.tubearchivist.com/users/).

docs/Video.md

@@ -1,3 +0,0 @@
-All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
-
-Equivalent Video page is: [here](https://docs.tubearchivist.com/video/).

tubearchivist/api/src/aggs.py

@@ -0,0 +1,243 @@
+"""aggregations"""
+
+from home.src.es.connect import ElasticWrap
+from home.src.ta.helper import get_duration_str
+
+
+class AggBase:
+    """base class for aggregation calls"""
+
+    path: str = ""
+    data: dict = {}
+    name: str = ""
+
+    def get(self):
+        """make get call"""
+        response, _ = ElasticWrap(self.path).get(self.data)
+        print(f"[agg][{self.name}] took {response.get('took')} ms to process")
+
+        return response.get("aggregations")
+
+    def process(self):
+        """implement in subclassess"""
+        raise NotImplementedError
+
+
+class Primary(AggBase):
+    """primary aggregation for total documents indexed"""
+
+    name = "primary"
+    path = "ta_video,ta_channel,ta_playlist,ta_subtitle,ta_download/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "video_type": {
+                "filter": {"exists": {"field": "active"}},
+                "aggs": {"filtered": {"terms": {"field": "vid_type"}}},
+            },
+            "channel_total": {"value_count": {"field": "channel_active"}},
+            "channel_sub": {"terms": {"field": "channel_subscribed"}},
+            "playlist_total": {"value_count": {"field": "playlist_active"}},
+            "playlist_sub": {"terms": {"field": "playlist_subscribed"}},
+            "download": {"terms": {"field": "status"}},
+        },
+    }
+
+    def process(self):
+        """make the call"""
+        aggregations = self.get()
+
+        videos = {"total": aggregations["video_type"].get("doc_count")}
+        videos.update(
+            {
+                i.get("key"): i.get("doc_count")
+                for i in aggregations["video_type"]["filtered"]["buckets"]
+            }
+        )
+        channels = {"total": aggregations["channel_total"].get("value")}
+        channels.update(
+            {
+                "sub_" + i.get("key_as_string"): i.get("doc_count")
+                for i in aggregations["channel_sub"]["buckets"]
+            }
+        )
+        playlists = {"total": aggregations["playlist_total"].get("value")}
+        playlists.update(
+            {
+                "sub_" + i.get("key_as_string"): i.get("doc_count")
+                for i in aggregations["playlist_sub"]["buckets"]
+            }
+        )
+        downloads = {
+            i.get("key"): i.get("doc_count")
+            for i in aggregations["download"]["buckets"]
+        }
+
+        response = {
+            "videos": videos,
+            "channels": channels,
+            "playlists": playlists,
+            "downloads": downloads,
+        }
+
+        return response
+
+
+class WatchProgress(AggBase):
+    """get watch progress"""
+
+    name = "watch_progress"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            name: {
+                "terms": {"field": "player.watched"},
+                "aggs": {
+                    "watch_docs": {
+                        "filter": {"terms": {"player.watched": [True, False]}},
+                        "aggs": {
+                            "true_count": {"value_count": {"field": "_index"}},
+                            "duration": {"sum": {"field": "player.duration"}},
+                        },
+                    },
+                },
+            },
+            "total_duration": {"sum": {"field": "player.duration"}},
+            "total_vids": {"value_count": {"field": "_index"}},
+        },
+    }
+
+    def process(self):
+        """make the call"""
+        aggregations = self.get()
+        buckets = aggregations[self.name]["buckets"]
+
+        response = {}
+        all_duration = int(aggregations["total_duration"].get("value"))
+        response.update(
+            {
+                "all": {
+                    "duration": all_duration,
+                    "duration_str": get_duration_str(all_duration),
+                    "items": aggregations["total_vids"].get("value"),
+                }
+            }
+        )
+
+        for bucket in buckets:
+            response.update(self._build_bucket(bucket, all_duration))
+
+        return response
+
+    @staticmethod
+    def _build_bucket(bucket, all_duration):
+        """parse bucket"""
+
+        duration = int(bucket["watch_docs"]["duration"]["value"])
+        duration_str = get_duration_str(duration)
+        items = bucket["watch_docs"]["true_count"]["value"]
+        if bucket["key_as_string"] == "false":
+            key = "unwatched"
+        else:
+            key = "watched"
+
+        bucket_parsed = {
+            key: {
+                "duration": duration,
+                "duration_str": duration_str,
+                "progress": duration / all_duration if all_duration else 0,
+                "items": items,
+            }
+        }
+
+        return bucket_parsed
+
+
+class DownloadHist(AggBase):
+    """get downloads histogram last week"""
+
+    name = "videos_last_week"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            name: {
+                "date_histogram": {
+                    "field": "date_downloaded",
+                    "calendar_interval": "day",
+                    "format": "yyyy-MM-dd",
+                    "order": {"_key": "desc"},
+                },
+                "aggs": {
+                    "total_videos": {"value_count": {"field": "youtube_id"}}
+                },
+            }
+        },
+        "query": {"range": {"date_downloaded": {"gte": "now-7d/d"}}},
+    }
+
+    def process(self):
+        """process query"""
+        aggregations = self.get()
+        buckets = aggregations[self.name]["buckets"]
+
+        response = [
+            {
+                "date": i.get("key_as_string"),
+                "count": i.get("doc_count"),
+            }
+            for i in buckets
+        ]
+
+        return response
+
+
+class BiggestChannel(AggBase):
+    """get channel aggregations"""
+
+    def __init__(self, order):
+        self.data["aggs"][self.name]["multi_terms"]["order"] = {order: "desc"}
+
+    name = "channel_stats"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            name: {
+                "multi_terms": {
+                    "terms": [
+                        {"field": "channel.channel_name.keyword"},
+                        {"field": "channel.channel_id"},
+                    ],
+                    "order": {"doc_count": "desc"},
+                },
+                "aggs": {
+                    "doc_count": {"value_count": {"field": "_index"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                    "media_size": {"sum": {"field": "media_size"}},
+                },
+            },
+        },
+    }
+    order_choices = ["doc_count", "duration", "media_size"]
+
+    def process(self):
+        """process aggregation, order_by validated in the view"""
+
+        aggregations = self.get()
+        buckets = aggregations[self.name]["buckets"]
+
+        response = [
+            {
+                "id": i["key"][1],
+                "name": i["key"][0].title(),
+                "doc_count": i["doc_count"]["value"],
+                "duration": i["duration"]["value"],
+                "duration_str": get_duration_str(int(i["duration"]["value"])),
+                "media_size": i["media_size"]["value"],
+            }
+            for i in buckets
+        ]
+
+        return response

tubearchivist/api/src/search_processor.py

@@ -8,7 +8,7 @@ import urllib.parse
 
 from home.src.download.thumbnails import ThumbManager
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import date_praser
+from home.src.ta.helper import date_praser, get_duration_str
 
 
 class SearchProcess:
@@ -50,6 +50,16 @@ class SearchProcess:
             processed = self._process_download(result["_source"])
         if index == "ta_comment":
             processed = self._process_comment(result["_source"])
+        if index == "ta_subtitle":
+            processed = self._process_subtitle(result)
+
+        if isinstance(processed, dict):
+            processed.update(
+                {
+                    "_index": index,
+                    "_score": round(result.get("_score") or 0, 2),
+                }
+            )
 
         return processed
 
@@ -139,3 +149,29 @@ class SearchProcess:
                 processed_comments[-1]["comment_replies"].append(comment)
 
         return processed_comments
+
+    def _process_subtitle(self, result):
+        """take complete result dict to extract highlight"""
+        subtitle_dict = result["_source"]
+        highlight = result.get("highlight")
+        if highlight:
+            # replace lines with the highlighted markdown
+            subtitle_line = highlight.get("subtitle_line")[0]
+            subtitle_dict.update({"subtitle_line": subtitle_line})
+
+        thumb_path = ThumbManager(subtitle_dict["youtube_id"]).vid_thumb_path()
+        subtitle_dict.update({"vid_thumb_url": f"/cache/{thumb_path}"})
+
+        return subtitle_dict
+
+
+def process_aggs(response):
+    """convert aggs duration to str"""
+
+    if response.get("aggregations"):
+        aggs = response["aggregations"]
+        if "total_duration" in aggs:
+            duration_sec = int(aggs["total_duration"]["value"])
+            aggs["total_duration"].update(
+                {"value_str": get_duration_str(duration_sec)}
+            )

tubearchivist/api/urls.py

@@ -41,6 +41,11 @@ urlpatterns = [
         views.ChannelApiListView.as_view(),
         name="api-channel-list",
     ),
+    path(
+        "channel/search/",
+        views.ChannelApiSearchView.as_view(),
+        name="api-channel-search",
+    ),
     path(
         "channel/<slug:channel_id>/",
         views.ChannelApiView.as_view(),
@@ -131,4 +136,24 @@ urlpatterns = [
         views.NotificationView.as_view(),
         name="api-notification",
     ),
+    path(
+        "stats/primary/",
+        views.StatPrimaryView.as_view(),
+        name="api-stats-primary",
+    ),
+    path(
+        "stats/watch/",
+        views.StatWatchProgress.as_view(),
+        name="api-stats-watch",
+    ),
+    path(
+        "stats/downloadhist/",
+        views.StatDownloadHist.as_view(),
+        name="api-stats-downloadhist",
+    ),
+    path(
+        "stats/biggestchannels/",
+        views.StatBiggestChannel.as_view(),
+        name="api-stats-biggestchannels",
+    ),
 ]

tubearchivist/api/views.py

@@ -1,7 +1,12 @@
 """all API views"""
 
+from api.src.aggs import BiggestChannel, DownloadHist, Primary, WatchProgress
 from api.src.search_processor import SearchProcess
 from home.src.download.queue import PendingInteract
+from home.src.download.subscriptions import (
+    ChannelSubscription,
+    PlaylistSubscription,
+)
 from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.connect import ElasticWrap
 from home.src.es.snapshot import ElasticSnapshot
@@ -9,10 +14,11 @@ from home.src.frontend.searching import SearchForm
 from home.src.frontend.watched import WatchState
 from home.src.index.channel import YoutubeChannel
 from home.src.index.generic import Pagination
+from home.src.index.playlist import YoutubePlaylist
 from home.src.index.reindex import ReindexProgress
 from home.src.index.video import SponsorBlock, YoutubeVideo
-from home.src.ta.config import AppConfig
-from home.src.ta.ta_redis import RedisArchivist, RedisQueue
+from home.src.ta.config import AppConfig, ReleaseVersion
+from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.task_manager import TaskCommand, TaskManager
 from home.src.ta.urlparser import Parser
 from home.tasks import (
@@ -38,8 +44,8 @@ class ApiBaseView(APIView):
 
     authentication_classes = [SessionAuthentication, TokenAuthentication]
     permission_classes = [IsAuthenticated]
-    search_base = False
-    data = False
+    search_base = ""
+    data = ""
 
     def __init__(self):
         super().__init__()
@@ -134,7 +140,7 @@ class VideoApiListView(ApiBaseView):
 
 
 class VideoProgressView(ApiBaseView):
-    """resolves to /api/video/<video_id>/
+    """resolves to /api/video/<video_id>/progress/
     handle progress status for video
     """
 
@@ -189,7 +195,7 @@ class VideoCommentView(ApiBaseView):
 
 class VideoSimilarView(ApiBaseView):
     """resolves to /api/video/<video-id>/similar/
-    GET: return max 3 videos similar to this
+    GET: return max 6 videos similar to this
     """
 
     search_base = "ta_video/_search/"
@@ -223,6 +229,10 @@ class VideoSponsorView(ApiBaseView):
         # pylint: disable=unused-argument
 
         self.get_document(video_id)
+        if not self.response.get("data"):
+            message = {"message": "video not found"}
+            return Response(message, status=404)
+
         sponsorblock = self.response["data"].get("sponsorblock")
 
         return Response(sponsorblock)
@@ -318,9 +328,8 @@ class ChannelApiListView(ApiBaseView):
 
         return Response(self.response)
 
-    @staticmethod
-    def post(request):
-        """subscribe to list of channels"""
+    def post(self, request):
+        """subscribe/unsubscribe to list of channels"""
         data = request.data
         try:
             to_add = data["data"]
@@ -329,12 +338,58 @@ class ChannelApiListView(ApiBaseView):
             print(message)
             return Response({"message": message}, status=400)
 
-        pending = [i["channel_id"] for i in to_add if i["channel_subscribed"]]
-        url_str = " ".join(pending)
-        subscribe_to.delay(url_str)
+        pending = []
+        for channel_item in to_add:
+            channel_id = channel_item["channel_id"]
+            if channel_item["channel_subscribed"]:
+                pending.append(channel_id)
+            else:
+                self._unsubscribe(channel_id)
+
+        if pending:
+            url_str = " ".join(pending)
+            subscribe_to.delay(url_str, expected_type="channel")
 
         return Response(data)
 
+    @staticmethod
+    def _unsubscribe(channel_id: str):
+        """unsubscribe"""
+        print(f"[{channel_id}] unsubscribe from channel")
+        ChannelSubscription().change_subscribe(
+            channel_id, channel_subscribed=False
+        )
+
+
+class ChannelApiSearchView(ApiBaseView):
+    """resolves to /api/channel/search/
+    search for channel
+    """
+
+    search_base = "ta_channel/_doc/"
+
+    def get(self, request):
+        """handle get request, search with s parameter"""
+
+        query = request.GET.get("q")
+        if not query:
+            message = "missing expected q parameter"
+            return Response({"message": message, "data": False}, status=400)
+
+        try:
+            parsed = Parser(query).parse()[0]
+        except (ValueError, IndexError, AttributeError):
+            message = f"channel not found: {query}"
+            return Response({"message": message, "data": False}, status=404)
+
+        if not parsed["type"] == "channel":
+            message = "expected type channel"
+            return Response({"message": message, "data": False}, status=400)
+
+        self.get_document(parsed["url"])
+
+        return Response(self.response, status=self.status_code)
+
 
 class ChannelApiVideoView(ApiBaseView):
     """resolves to /api/channel/<channel-id>/video
@@ -373,6 +428,38 @@ class PlaylistApiListView(ApiBaseView):
         self.get_document_list(request)
         return Response(self.response)
 
+    def post(self, request):
+        """subscribe/unsubscribe to list of playlists"""
+        data = request.data
+        try:
+            to_add = data["data"]
+        except KeyError:
+            message = "missing expected data key"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        pending = []
+        for playlist_item in to_add:
+            playlist_id = playlist_item["playlist_id"]
+            if playlist_item["playlist_subscribed"]:
+                pending.append(playlist_id)
+            else:
+                self._unsubscribe(playlist_id)
+
+        if pending:
+            url_str = " ".join(pending)
+            subscribe_to.delay(url_str, expected_type="playlist")
+
+        return Response(data)
+
+    @staticmethod
+    def _unsubscribe(playlist_id: str):
+        """unsubscribe"""
+        print(f"[{playlist_id}] unsubscribe from playlist")
+        PlaylistSubscription().change_subscribe(
+            playlist_id, subscribe_status=False
+        )
+
 
 class PlaylistApiView(ApiBaseView):
     """resolves to /api/playlist/<playlist_id>/
@@ -387,6 +474,17 @@ class PlaylistApiView(ApiBaseView):
         self.get_document(playlist_id)
         return Response(self.response, status=self.status_code)
 
+    def delete(self, request, playlist_id):
+        """delete playlist"""
+        print(f"{playlist_id}: delete playlist")
+        delete_videos = request.GET.get("delete-videos", False)
+        if delete_videos:
+            YoutubePlaylist(playlist_id).delete_videos_playlist()
+        else:
+            YoutubePlaylist(playlist_id).delete_metadata()
+
+        return Response({"success": True})
+
 
 class PlaylistApiVideoView(ApiBaseView):
     """resolves to /api/playlist/<playlist_id>/video
@@ -436,12 +534,9 @@ class DownloadApiView(ApiBaseView):
             return Response({"message": message}, status=404)
 
         print(f"{video_id}: change status to {item_status}")
+        PendingInteract(video_id, item_status).update_status()
         if item_status == "priority":
-            PendingInteract(youtube_id=video_id).prioritize()
-            download_pending.delay(from_queue=False)
-        else:
-            PendingInteract(video_id, item_status).update_status()
-            RedisQueue(queue_name="dl_queue").clear_item(video_id)
+            download_pending.delay(auto_only=True)
 
         return Response(request.data)
 
@@ -494,6 +589,7 @@ class DownloadApiListView(ApiBaseView):
     def post(request):
         """add list of videos to download queue"""
         data = request.data
+        auto_start = bool(request.GET.get("autostart"))
         try:
             to_add = data["data"]
         except KeyError:
@@ -510,7 +606,7 @@ class DownloadApiListView(ApiBaseView):
             print(message)
             return Response({"message": message}, status=400)
 
-        extrac_dl.delay(youtube_ids)
+        extrac_dl.delay(youtube_ids, auto_start=auto_start)
 
         return Response(data)
 
@@ -537,7 +633,11 @@ class PingView(ApiBaseView):
     @staticmethod
     def get(request):
         """get pong"""
-        data = {"response": "pong", "user": request.user.id}
+        data = {
+            "response": "pong",
+            "user": request.user.id,
+            "version": ReleaseVersion().get_local_version(),
+        }
         return Response(data)
 
 
@@ -563,7 +663,7 @@ class LoginApiView(ObtainAuthToken):
 
 class SnapshotApiListView(ApiBaseView):
     """resolves to /api/snapshot/
-    GET: returns snashot config plus list of existing snapshots
+    GET: returns snapshot config plus list of existing snapshots
     POST: take snapshot now
     """
 
@@ -876,3 +976,58 @@ class NotificationView(ApiBaseView):
             query = f"{query}:{filter_by}"
 
         return Response(RedisArchivist().list_items(query))
+
+
+class StatPrimaryView(ApiBaseView):
+    """resolves to /api/stats/primary/
+    GET: return document count
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Primary().process())
+
+
+class StatWatchProgress(ApiBaseView):
+    """resolves to /api/stats/watchprogress/
+    GET: return watch/unwatch progress stats
+    """
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+
+        return Response(WatchProgress().process())
+
+
+class StatDownloadHist(ApiBaseView):
+    """resolves to /api/stats/downloadhist/
+    GET: return download video count histogram for last days
+    """
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+
+        return Response(DownloadHist().process())
+
+
+class StatBiggestChannel(ApiBaseView):
+    """resolves to /api/stats/biggestchannels/
+    GET: return biggest channels
+    param: order
+    """
+
+    order_choices = ["doc_count", "duration", "media_size"]
+
+    def get(self, request):
+        """handle get request"""
+
+        order = request.GET.get("order", "doc_count")
+        if order and order not in self.order_choices:
+            message = {"message": f"invalid order parameter {order}"}
+            return Response(message, status=400)
+
+        return Response(BiggestChannel(order).process())

tubearchivist/config/management/commands/ta_connection.py

@@ -3,6 +3,7 @@ Functionality:
 - check that all connections are working
 """
 
+from os import environ
 from time import sleep
 
 import requests
@@ -81,11 +82,16 @@ class Command(BaseCommand):
                 _, status_code = ElasticWrap("/").get(
                     timeout=1, print_error=False
                 )
-            except requests.exceptions.ConnectionError:
+            except (
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout,
+            ):
                 sleep(5)
                 continue
 
             if status_code and status_code == 200:
+                path = "_cluster/health?wait_for_status=yellow&timeout=60s"
+                _, _ = ElasticWrap(path).get(timeout=60)
                 self.stdout.write(
                     self.style.SUCCESS("    ✓ ES connection established")
                 )
@@ -116,7 +122,7 @@ class Command(BaseCommand):
             return
 
         message = (
-            "    🗙 ES connection failed. "
+            "    🗙 ES version check failed. "
             + f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}"
         )
         self.stdout.write(self.style.ERROR(f"{message}"))
@@ -127,7 +133,19 @@ class Command(BaseCommand):
         """check that path.repo var is set"""
         self.stdout.write("[5] check ES path.repo env var")
         response, _ = ElasticWrap("_nodes/_all/settings").get()
+        snaphost_roles = [
+            "data",
+            "data_cold",
+            "data_content",
+            "data_frozen",
+            "data_hot",
+            "data_warm",
+            "master",
+        ]
         for node in response["nodes"].values():
+            if not (set(node["roles"]) & set(snaphost_roles)):
+                continue
+
             if node["settings"]["path"].get("repo"):
                 self.stdout.write(
                     self.style.SUCCESS("    ✓ path.repo env var is set")
@@ -137,7 +155,10 @@ class Command(BaseCommand):
             message = (
                 "    🗙 path.repo env var not found. "
                 + "set the following env var to the ES container:\n"
-                + "    path.repo=/usr/share/elasticsearch/data/snapshot"
+                + "    path.repo="
+                + environ.get(
+                    "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
+                ),
             )
             self.stdout.write(self.style.ERROR(f"{message}"))
             sleep(60)

tubearchivist/config/management/commands/ta_migpath.py

@@ -0,0 +1,182 @@
+"""filepath migration from v0.3.6 to v0.3.7"""
+
+import json
+import os
+import shutil
+
+from django.core.management.base import BaseCommand
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import ignore_filelist
+
+TOPIC = """
+
+########################
+# Filesystem Migration #
+########################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """run commands"""
+        self.stdout.write(TOPIC)
+
+        handler = FolderMigration()
+        to_migrate = handler.get_to_migrate()
+        if not to_migrate:
+            self.stdout.write(
+                self.style.SUCCESS("    no channel migration needed\n")
+            )
+            return
+
+        self.stdout.write(self.style.SUCCESS("    migrating channels"))
+        total_channels = handler.create_folders(to_migrate)
+        self.stdout.write(
+            self.style.SUCCESS(f"    created {total_channels} channels")
+        )
+
+        self.stdout.write(
+            self.style.SUCCESS(f"    migrating {len(to_migrate)} videos")
+        )
+        handler.migrate_videos(to_migrate)
+        self.stdout.write(self.style.SUCCESS("    update videos in index"))
+        handler.send_bulk()
+
+        self.stdout.write(self.style.SUCCESS("    cleanup old folders"))
+        handler.delete_old()
+
+        self.stdout.write(self.style.SUCCESS("    ✓ migration completed\n"))
+
+
+class FolderMigration:
+    """migrate video archive folder"""
+
+    def __init__(self):
+        self.config = AppConfig().config
+        self.videos = self.config["application"]["videos"]
+        self.bulk_list = []
+
+    def get_to_migrate(self):
+        """get videos to migrate"""
+        script = (
+            "doc['media_url'].value == "
+            + "doc['channel.channel_id'].value + '/'"
+            + " + doc['youtube_id'].value + '.mp4'"
+        )
+        data = {
+            "query": {"bool": {"must_not": [{"script": {"script": script}}]}},
+            "_source": [
+                "youtube_id",
+                "media_url",
+                "channel.channel_id",
+                "subtitles",
+            ],
+        }
+        response = IndexPaginate("ta_video", data).get_results()
+
+        return response
+
+    def create_folders(self, to_migrate):
+        """create required channel folders"""
+        host_uid = self.config["application"]["HOST_UID"]
+        host_gid = self.config["application"]["HOST_GID"]
+        all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate}
+
+        for channel_id in all_channel_ids:
+            new_folder = os.path.join(self.videos, channel_id)
+            os.makedirs(new_folder, exist_ok=True)
+            if host_uid and host_gid:
+                os.chown(new_folder, host_uid, host_gid)
+
+        return len(all_channel_ids)
+
+    def migrate_videos(self, to_migrate):
+        """migrate all videos of channel"""
+        total = len(to_migrate)
+        for idx, video in enumerate(to_migrate):
+            new_media_url = self._move_video_file(video)
+            if not new_media_url:
+                continue
+
+            all_subtitles = self._move_subtitles(video)
+            action = {
+                "update": {"_id": video["youtube_id"], "_index": "ta_video"}
+            }
+            source = {"doc": {"media_url": new_media_url}}
+            if all_subtitles:
+                source["doc"].update({"subtitles": all_subtitles})
+
+            self.bulk_list.append(json.dumps(action))
+            self.bulk_list.append(json.dumps(source))
+            if idx % 1000 == 0:
+                print(f"processing migration [{idx}/{total}]")
+                self.send_bulk()
+
+    def _move_video_file(self, video):
+        """move video file to new location"""
+        old_path = os.path.join(self.videos, video["media_url"])
+        if not os.path.exists(old_path):
+            print(f"did not find expected video at {old_path}")
+            return False
+
+        new_media_url = os.path.join(
+            video["channel"]["channel_id"], video["youtube_id"] + ".mp4"
+        )
+        new_path = os.path.join(self.videos, new_media_url)
+        os.rename(old_path, new_path)
+
+        return new_media_url
+
+    def _move_subtitles(self, video):
+        """move subtitle files to new location"""
+        all_subtitles = video.get("subtitles")
+        if not all_subtitles:
+            return False
+
+        for subtitle in all_subtitles:
+            old_path = os.path.join(self.videos, subtitle["media_url"])
+            if not os.path.exists(old_path):
+                print(f"did not find expected subtitle at {old_path}")
+                continue
+
+            new_media_url = os.path.join(
+                video["channel"]["channel_id"],
+                f"{video.get('youtube_id')}.{subtitle.get('lang')}.vtt",
+            )
+            new_path = os.path.join(self.videos, new_media_url)
+            os.rename(old_path, new_path)
+            subtitle["media_url"] = new_media_url
+
+        return all_subtitles
+
+    def send_bulk(self):
+        """send bulk request to update index with new urls"""
+        if not self.bulk_list:
+            print("nothing to update")
+            return
+
+        self.bulk_list.append("\n")
+        path = "_bulk?refresh=true"
+        data = "\n".join(self.bulk_list)
+        response, status = ElasticWrap(path).post(data=data, ndjson=True)
+        if not status == 200:
+            print(response)
+
+        self.bulk_list = []
+
+    def delete_old(self):
+        """delete old empty folders"""
+        all_folders = ignore_filelist(os.listdir(self.videos))
+        for folder in all_folders:
+            folder_path = os.path.join(self.videos, folder)
+            if not os.path.isdir(folder_path):
+                continue
+
+            if not ignore_filelist(os.listdir(folder_path)):
+                shutil.rmtree(folder_path)

tubearchivist/config/management/commands/ta_startup.py

@@ -8,13 +8,15 @@ import os
 from time import sleep
 
 from django.core.management.base import BaseCommand, CommandError
-from home.src.es.connect import ElasticWrap
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.es.index_setup import ElasitIndexWrap
 from home.src.es.snapshot import ElasticSnapshot
+from home.src.index.video_streams import MediaStreamExtractor
 from home.src.ta.config import AppConfig, ReleaseVersion
 from home.src.ta.helper import clear_dl_cache
 from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.task_manager import TaskManager
+from home.src.ta.users import UserConfig
 
 TOPIC = """
 
@@ -41,7 +43,9 @@ class Command(BaseCommand):
         self._version_check()
         self._mig_index_setup()
         self._mig_snapshot_check()
-        self._mig_set_vid_type()
+        self._mig_set_streams()
+        self._mig_set_autostart()
+        self._mig_move_users_to_es()
 
     def _sync_redis_state(self):
         """make sure redis gets new config.json values"""
@@ -145,51 +149,165 @@ class Command(BaseCommand):
         self.stdout.write("[MIGRATION] setup snapshots")
         ElasticSnapshot().setup()
 
-    def _mig_set_vid_type(self):
-        """migration: update 0.3.0 to 0.3.1 set vid_type default"""
-        self.stdout.write("[MIGRATION] set default vid_type")
-        index_list = ["ta_video", "ta_download"]
+    def _mig_set_streams(self):
+        """migration: update from 0.3.5 to 0.3.6, set streams and media_size"""
+        self.stdout.write("[MIGRATION] index streams and media size")
+        videos = AppConfig().config["application"]["videos"]
         data = {
             "query": {
-                "bool": {
-                    "should": [
-                        {
-                            "bool": {
-                                "must_not": [{"exists": {"field": "vid_type"}}]
-                            }
-                        },
-                        {"term": {"vid_type": {"value": "unknown"}}},
-                    ]
-                }
+                "bool": {"must_not": [{"exists": {"field": "streams"}}]}
             },
-            "script": {"source": "ctx._source['vid_type'] = 'videos'"},
+            "_source": ["media_url", "youtube_id"],
         }
+        all_missing = IndexPaginate("ta_video", data).get_results()
+        if not all_missing:
+            self.stdout.write("    no videos need updating")
+            return
 
-        for index_name in index_list:
-            path = f"{index_name}/_update_by_query"
-            response, status_code = ElasticWrap(path).post(data=data)
-            if status_code == 503:
-                message = f"    🗙 {index_name} retry failed migration."
-                self.stdout.write(self.style.ERROR(message))
-                sleep(10)
-                response, status_code = ElasticWrap(path).post(data=data)
+        total = len(all_missing)
+        for idx, missing in enumerate(all_missing):
+            media_url = missing["media_url"]
+            youtube_id = missing["youtube_id"]
+            media_path = os.path.join(videos, media_url)
+            if not os.path.exists(media_path):
+                self.stdout.write(f"    file not found: {media_path}")
+                self.stdout.write("    run file system rescan to fix")
+                continue
 
-            if status_code == 200:
-                updated = response.get("updated", 0)
-                if not updated:
-                    self.stdout.write(
-                        f"    no videos needed updating in {index_name}"
-                    )
-                    continue
+            media = MediaStreamExtractor(media_path)
+            vid_data = {
+                "doc": {
+                    "streams": media.extract_metadata(),
+                    "media_size": media.get_file_size(),
+                }
+            }
+            path = f"ta_video/_update/{youtube_id}"
+            response, status_code = ElasticWrap(path).post(data=vid_data)
+            if not status_code == 200:
+                self.stdout.errors(
+                    f"    update failed: {path}, {response}, {status_code}"
+                )
 
+            if idx % 100 == 0:
+                self.stdout.write(f"    progress {idx}/{total}")
+
+    def _mig_set_autostart(self):
+        """migration: update from 0.3.5 to 0.3.6 set auto_start to false"""
+        self.stdout.write("[MIGRATION] set default download auto_start")
+        data = {
+            "query": {
+                "bool": {"must_not": [{"exists": {"field": "auto_start"}}]}
+            },
+            "script": {"source": "ctx._source['auto_start'] = false"},
+        }
+        path = "ta_download/_update_by_query"
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code == 200:
+            updated = response.get("updated", 0)
+            if updated:
                 self.stdout.write(
                     self.style.SUCCESS(
-                        f"    ✓ {updated} videos updated in {index_name}"
+                        f"    ✓ {updated} videos updated in ta_download"
                     )
                 )
             else:
-                message = f"    🗙 {index_name} vid_type update failed"
-                self.stdout.write(self.style.ERROR(message))
-                self.stdout.write(response)
-                sleep(60)
-                raise CommandError(message)
+                self.stdout.write(
+                    "    no videos needed updating in ta_download"
+                )
+            return
+
+        message = "    🗙 ta_download auto_start update failed"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)
+
+    def _mig_move_users_to_es(self):  # noqa: C901
+        """migration: update from 0.4.1 to 0.4.2 move user config to ES"""
+        self.stdout.write("[MIGRATION] move user configuration to ES")
+        redis = RedisArchivist()
+
+        # 1: Find all users in Redis
+        users = {i.split(":")[0] for i in redis.list_keys("[0-9]*:")}
+        if not users:
+            self.stdout.write("    no users needed migrating to ES")
+            return
+
+        # 2: Write all Redis user settings to ES
+        # 3: Remove user settings from Redis
+        try:
+            for user in users:
+                new_conf = UserConfig(user)
+
+                colors_key = f"{user}:colors"
+                colors = redis.get_message(colors_key).get("status")
+                if colors:
+                    new_conf.set_value("colors", colors)
+                    redis.del_message(colors_key)
+
+                sort_by_key = f"{user}:sort_by"
+                sort_by = redis.get_message(sort_by_key).get("status")
+                if sort_by:
+                    new_conf.set_value("sort_by", sort_by)
+                    redis.del_message(sort_by_key)
+
+                page_size_key = f"{user}:page_size"
+                page_size = redis.get_message(page_size_key).get("status")
+                if page_size:
+                    new_conf.set_value("page_size", page_size)
+                    redis.del_message(page_size_key)
+
+                sort_order_key = f"{user}:sort_order"
+                sort_order = redis.get_message(sort_order_key).get("status")
+                if sort_order:
+                    new_conf.set_value("sort_order", sort_order)
+                    redis.del_message(sort_order_key)
+
+                grid_items_key = f"{user}:grid_items"
+                grid_items = redis.get_message(grid_items_key).get("status")
+                if grid_items:
+                    new_conf.set_value("grid_items", grid_items)
+                    redis.del_message(grid_items_key)
+
+                hide_watch_key = f"{user}:hide_watched"
+                hide_watch = redis.get_message(hide_watch_key).get("status")
+                if hide_watch:
+                    new_conf.set_value("hide_watched", hide_watch)
+                    redis.del_message(hide_watch_key)
+
+                ignore_only_key = f"{user}:show_ignored_only"
+                ignore_only = redis.get_message(ignore_only_key).get("status")
+                if ignore_only:
+                    new_conf.set_value("show_ignored_only", ignore_only)
+                    redis.del_message(ignore_only_key)
+
+                subed_only_key = f"{user}:show_subed_only"
+                subed_only = redis.get_message(subed_only_key).get("status")
+                if subed_only:
+                    new_conf.set_value("show_subed_only", subed_only)
+                    redis.del_message(subed_only_key)
+
+                for view in ["channel", "playlist", "home", "downloads"]:
+                    view_key = f"{user}:view:{view}"
+                    view_style = redis.get_message(view_key).get("status")
+                    if view_style:
+                        new_conf.set_value(f"view_style_{view}", view_style)
+                        redis.del_message(view_key)
+
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"    ✓ Settings for user '{user}' migrated to ES"
+                    )
+                )
+        except Exception as e:
+            message = "    🗙 user migration to ES failed"
+            self.stdout.write(self.style.ERROR(message))
+            self.stdout.write(self.style.ERROR(e))
+            sleep(60)
+            raise CommandError(message)
+        else:
+            self.stdout.write(
+                self.style.SUCCESS(
+                    "    ✓ Settings for all users migrated to ES"
+                )
+            )

tubearchivist/config/settings.py

@@ -64,6 +64,7 @@ MIDDLEWARE = [
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "home.src.ta.health.HealthCheckMiddleware",
 ]
 
 ROOT_URLCONF = "config.urls"
@@ -174,7 +175,6 @@ if bool(environ.get("TA_LDAP")):
             ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
         }
 
-    global AUTHENTICATION_BACKENDS
     AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
 
 # Database
@@ -210,6 +210,19 @@ AUTH_PASSWORD_VALIDATORS = [
 
 AUTH_USER_MODEL = "home.Account"
 
+# Forward-auth authentication
+if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
+    TA_AUTH_PROXY_USERNAME_HEADER = (
+        environ.get("TA_AUTH_PROXY_USERNAME_HEADER") or "HTTP_REMOTE_USER"
+    )
+    TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")
+
+    MIDDLEWARE.append("home.src.ta.auth.HttpRemoteUserMiddleware")
+
+    AUTHENTICATION_BACKENDS = (
+        "django.contrib.auth.backends.RemoteUserBackend",
+    )
+
 
 # Internationalization
 # https://docs.djangoproject.com/en/3.2/topics/i18n/
@@ -256,4 +269,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.3.5"
+TA_VERSION = "v0.4.2"

tubearchivist/home/config.json

@@ -1,29 +1,16 @@
 {
-    "archive": {
-        "sort_by": "published",
-        "sort_order": "desc",
-        "page_size": 12
-    },
-    "default_view": {
-        "home": "grid",
-        "channel": "list",
-        "downloads": "list",
-        "playlist": "grid",
-        "grid_items": 3
-    },
     "subscriptions": {
-        "auto_search": false,
-        "auto_download": false,
         "channel_size": 50,
         "live_channel_size": 50,
-        "shorts_channel_size": 50
+        "shorts_channel_size": 50,
+        "auto_start": false
     },
     "downloads": {
-        "limit_count": false,
         "limit_speed": false,
         "sleep_interval": 3,
         "autodelete_days": false,
         "format": false,
+        "format_sort": false,
         "add_metadata": false,
         "add_thumbnail": false,
         "subtitle": false,
@@ -33,6 +20,7 @@
         "comment_sort": "top",
         "cookie_import": false,
         "throttledratelimit": false,
+        "extractor_lang": false,
         "integrate_ryd": false,
         "integrate_sponsorblock": false
     },
@@ -40,14 +28,16 @@
         "app_root": "/app",
         "cache_dir": "/cache",
         "videos": "/youtube",
-        "colors": "dark",
         "enable_cast": false,
         "enable_snapshot": true
     },
     "scheduler": {
         "update_subscribed": false,
+        "update_subscribed_notify": false,
         "download_pending": false,
+        "download_pending_notify": false,
         "check_reindex":  {"minute": "0", "hour": "12", "day_of_week": "*"},
+        "check_reindex_notify": false,
         "check_reindex_days": 90,
         "thumbnail_check": {"minute": "0", "hour": "17", "day_of_week": "*"},
         "run_backup": false,

tubearchivist/home/src/download/queue.py

@@ -17,8 +17,7 @@ from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import DurationConverter, is_shorts
-from home.src.ta.ta_redis import RedisQueue
+from home.src.ta.helper import get_duration_str, is_shorts
 
 
 class PendingIndex:
@@ -112,20 +111,20 @@ class PendingInteract:
         _, _ = ElasticWrap(path).post(data=data)
 
     def update_status(self):
-        """update status field of pending item"""
-        data = {"doc": {"status": self.status}}
-        path = f"ta_download/_update/{self.youtube_id}"
-        _, _ = ElasticWrap(path).post(data=data)
+        """update status of pending item"""
+        if self.status == "priority":
+            data = {
+                "doc": {
+                    "status": "pending",
+                    "auto_start": True,
+                    "message": None,
+                }
+            }
+        else:
+            data = {"doc": {"status": self.status}}
 
-    def prioritize(self):
-        """prioritize pending item in redis queue"""
-        pending_video, _ = self.get_item()
-        vid_type = pending_video.get("vid_type", VideoTypeEnum.VIDEOS.value)
-        to_add = {
-            "youtube_id": pending_video["youtube_id"],
-            "vid_type": vid_type,
-        }
-        RedisQueue(queue_name="dl_queue").add_priority(to_add)
+        path = f"ta_download/_update/{self.youtube_id}/?refresh=true"
+        _, _ = ElasticWrap(path).post(data=data)
 
     def get_item(self):
         """return pending item dict"""
@@ -229,13 +228,18 @@ class PendingList(PendingIndex):
         """add all videos of playlist to list"""
         playlist = YoutubePlaylist(url)
         playlist.build_json()
+        if not playlist.json_data:
+            message = f"{playlist.youtube_id}: failed to extract metadata"
+            print(message)
+            raise ValueError(message)
+
         video_results = playlist.json_data.get("playlist_entries")
         youtube_ids = [i["youtube_id"] for i in video_results]
         for video_id in youtube_ids:
             # match vid_type later
             self._add_video(video_id, VideoTypeEnum.UNKNOWN)
 
-    def add_to_pending(self, status="pending"):
+    def add_to_pending(self, status="pending", auto_start=False):
         """add missing videos to pending list"""
         self.get_channels()
         bulk_list = []
@@ -251,7 +255,13 @@ class PendingList(PendingIndex):
             if not video_details:
                 continue
 
-            video_details["status"] = status
+            video_details.update(
+                {
+                    "status": status,
+                    "auto_start": auto_start,
+                }
+            )
+
             action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
             bulk_list.append(json.dumps(action))
             bulk_list.append(json.dumps(video_details))
@@ -273,7 +283,7 @@ class PendingList(PendingIndex):
         # add last newline
         bulk_list.append("\n")
         query_str = "\n".join(bulk_list)
-        _, _ = ElasticWrap("_bulk").post(query_str, ndjson=True)
+        _, _ = ElasticWrap("_bulk?refresh=true").post(query_str, ndjson=True)
 
     def _notify_add(self, idx, total):
         """send notification for adding videos to download queue"""
@@ -329,9 +339,6 @@ class PendingList(PendingIndex):
     def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS):
         """parse response"""
         vid_id = vid.get("id")
-        duration_str = DurationConverter.get_str(vid["duration"])
-        if duration_str == "NA":
-            print(f"skip extracting duration for: {vid_id}")
         published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
             "%Y-%m-%d"
         )
@@ -343,7 +350,7 @@ class PendingList(PendingIndex):
             "vid_thumb_url": vid["thumbnail"],
             "title": vid["title"],
             "channel_id": vid["channel_id"],
-            "duration": duration_str,
+            "duration": get_duration_str(vid["duration"]),
             "published": published,
             "timestamp": int(datetime.now().timestamp()),
             # Pulling enum value out so it is serializable

tubearchivist/home/src/download/subscriptions.py

@@ -12,7 +12,6 @@ from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
-from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.urlparser import Parser
 
 
@@ -175,10 +174,7 @@ class PlaylistSubscription:
 
     def process_url_str(self, new_playlists, subscribed=True):
         """process playlist subscribe form url_str"""
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"published": {"order": "desc"}}],
-        }
+        data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
         all_indexed = IndexPaginate("ta_video", data).get_results()
         all_youtube_ids = [i["youtube_id"] for i in all_indexed]
 
@@ -191,6 +187,11 @@ class PlaylistSubscription:
             playlist_h = YoutubePlaylist(playlist_id)
             playlist_h.all_youtube_ids = all_youtube_ids
             playlist_h.build_json()
+            if not playlist_h.json_data:
+                message = f"{playlist_h.youtube_id}: failed to extract data"
+                print(message)
+                raise ValueError(message)
+
             playlist_h.json_data["playlist_subscribed"] = subscribed
             playlist_h.upload_to_es()
             playlist_h.add_vids_to_playlist()
@@ -200,16 +201,13 @@ class PlaylistSubscription:
             thumb = ThumbManager(playlist_id, item_type="playlist")
             thumb.download_playlist_thumb(url)
 
-            # notify
-            message = {
-                "status": "message:subplaylist",
-                "level": "info",
-                "title": "Subscribing to Playlists",
-                "message": f"Processing {idx + 1} of {len(new_playlists)}",
-            }
-            RedisArchivist().set_message(
-                "message:subplaylist", message=message, expire=True
-            )
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[
+                        f"Processing {idx + 1} of {len(new_playlists)}"
+                    ],
+                    progress=(idx + 1) / len(new_playlists),
+                )
 
     @staticmethod
     def channel_validate(channel_id):
@@ -284,6 +282,7 @@ class SubscriptionScanner:
     def __init__(self, task=False):
         self.task = task
         self.missing_videos = False
+        self.auto_start = AppConfig().config["subscriptions"].get("auto_start")
 
     def scan(self):
         """scan channels and playlists"""
@@ -334,7 +333,7 @@ class SubscriptionHandler:
         self.task = task
         self.to_subscribe = False
 
-    def subscribe(self):
+    def subscribe(self, expected_type=False):
         """subscribe to url_str items"""
         if self.task:
             self.task.send_progress(["Processing form content."])
@@ -345,11 +344,16 @@ class SubscriptionHandler:
             if self.task:
                 self._notify(idx, item, total)
 
-            self.subscribe_type(item)
+            self.subscribe_type(item, expected_type=expected_type)
 
-    def subscribe_type(self, item):
+    def subscribe_type(self, item, expected_type):
         """process single item"""
         if item["type"] == "playlist":
+            if expected_type and expected_type != "playlist":
+                raise TypeError(
+                    f"expected {expected_type} url but got {item.get('type')}"
+                )
+
             PlaylistSubscription().process_url_str([item])
             return
 
@@ -362,6 +366,11 @@ class SubscriptionHandler:
         else:
             raise ValueError("failed to subscribe to: " + item["url"])
 
+        if expected_type and expected_type != "channel":
+            raise TypeError(
+                f"expected {expected_type} url but got {item.get('type')}"
+            )
+
         self._subscribe(channel_id)
 
     def _subscribe(self, channel_id):

tubearchivist/home/src/download/thumbnails.py

@@ -54,11 +54,14 @@ class ThumbManagerBase:
                 if response.status_code == 404:
                     return self.get_fallback()
 
-            except requests.exceptions.RequestException:
+            except (
+                requests.exceptions.RequestException,
+                requests.exceptions.ReadTimeout,
+            ):
                 print(f"{self.item_id}: retry thumbnail download {url}")
                 sleep((i + 1) ** i)
 
-        return False
+        return self.get_fallback()
 
     def get_fallback(self):
         """get fallback thumbnail if not available"""
@@ -243,9 +246,10 @@ class ThumbManager(ThumbManagerBase):
 class ValidatorCallback:
     """handle callback validate thumbnails page by page"""
 
-    def __init__(self, source, index_name):
+    def __init__(self, source, index_name, counter=0):
         self.source = source
         self.index_name = index_name
+        self.counter = counter
 
     def run(self):
         """run the task for page"""
@@ -270,7 +274,7 @@ class ValidatorCallback:
             urls = (
                 channel["_source"]["channel_thumb_url"],
                 channel["_source"]["channel_banner_url"],
-                channel["_source"]["channel_tvart_url"],
+                channel["_source"].get("channel_tvart_url", False),
             )
             handler = ThumbManager(channel["_source"]["channel_id"])
             handler.download_channel_art(urls, skip_existing=True)
@@ -381,9 +385,10 @@ class EmbedCallback:
     MEDIA_DIR = CONFIG["application"]["videos"]
     FORMAT = MP4Cover.FORMAT_JPEG
 
-    def __init__(self, source, index_name):
+    def __init__(self, source, index_name, counter=0):
         self.source = source
         self.index_name = index_name
+        self.counter = counter
 
     def run(self):
         """run embed"""

tubearchivist/home/src/download/yt_dlp_base.py

@@ -48,11 +48,14 @@ class YtWrap:
         with yt_dlp.YoutubeDL(self.obs) as ydl:
             try:
                 ydl.download([url])
-            except yt_dlp.utils.DownloadError:
-                print(f"{url}: failed to download.")
-                return False
+            except yt_dlp.utils.DownloadError as err:
+                print(f"{url}: failed to download with message {err}")
+                if "Temporary failure in name resolution" in str(err):
+                    raise ConnectionError("lost the internet, abort!") from err
 
-        return True
+                return False, str(err)
+
+        return True, True
 
     def extract(self, url):
         """make extract request"""
@@ -61,8 +64,17 @@ class YtWrap:
         except cookiejar.LoadError:
             print("cookie file is invalid")
             return False
-        except (yt_dlp.utils.ExtractorError, yt_dlp.utils.DownloadError):
-            print(f"{url}: failed to get info from youtube")
+        except yt_dlp.utils.ExtractorError as err:
+            print(f"{url}: failed to extract with message: {err}, continue...")
+            return False
+        except yt_dlp.utils.DownloadError as err:
+            if "This channel does not have a" in str(err):
+                return False
+
+            print(f"{url}: failed to get info from youtube with message {err}")
+            if "Temporary failure in name resolution" in str(err):
+                raise ConnectionError("lost the internet, abort!") from err
+
             return False
 
         return response
@@ -99,10 +111,10 @@ class CookieHandler:
         print("cookie: import successful")
 
     def set_cookie(self, cookie):
-        """set cookie str and activate in cofig"""
-        RedisArchivist().set_message("cookie", cookie)
+        """set cookie str and activate in config"""
+        RedisArchivist().set_message("cookie", cookie, save=True)
         path = ".downloads.cookie_import"
-        RedisArchivist().set_message("config", True, path=path)
+        RedisArchivist().set_message("config", True, path=path, save=True)
         self.config["downloads"]["cookie_import"] = True
         print("cookie: activated and stored in Redis")
 

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -6,14 +6,13 @@ functionality:
 - move to archive
 """
 
-import json
 import os
 import shutil
 from datetime import datetime
 
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import PlaylistSubscription
-from home.src.download.yt_dlp_base import CookieHandler, YtWrap
+from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
 from home.src.index.comments import CommentList
@@ -21,8 +20,7 @@ from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo, index_new_video
 from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import clean_string, ignore_filelist
-from home.src.ta.ta_redis import RedisQueue
+from home.src.ta.helper import ignore_filelist
 
 
 class DownloadPostProcess:
@@ -159,114 +157,81 @@ class VideoDownloader:
         self.channels = set()
         self.videos = set()
 
-    def run_queue(self):
+    def run_queue(self, auto_only=False):
         """setup download queue in redis loop until no more items"""
-        self._setup_queue()
-        queue = RedisQueue(queue_name="dl_queue")
-
-        limit_queue = self.config["downloads"]["limit_count"]
-        if limit_queue:
-            queue.trim(limit_queue - 1)
-
+        self._get_overwrites()
         while True:
-            youtube_data = queue.get_next()
-            if self.task.is_stopped() or not youtube_data:
-                queue.clear()
+            video_data = self._get_next(auto_only)
+            if self.task.is_stopped() or not video_data:
+                self._reset_auto()
                 break
 
-            youtube_data = json.loads(youtube_data)
-            youtube_id = youtube_data.get("youtube_id")
-
-            tmp_vid_type = youtube_data.get(
-                "vid_type", VideoTypeEnum.VIDEOS.value
-            )
-            video_type = VideoTypeEnum(tmp_vid_type)
-            print(f"{youtube_id}: Downloading type: {video_type}")
+            youtube_id = video_data.get("youtube_id")
+            print(f"{youtube_id}: Downloading video")
+            self._notify(video_data, "Validate download format")
 
             success = self._dl_single_vid(youtube_id)
             if not success:
                 continue
 
-            if self.task:
-                self.task.send_progress(
-                    [
-                        f"Processing video {youtube_id}",
-                        "Add video metadata to index.",
-                    ]
-                )
+            self._notify(video_data, "Add video metadata to index")
 
             vid_dict = index_new_video(
                 youtube_id,
                 video_overwrites=self.video_overwrites,
-                video_type=video_type,
+                video_type=VideoTypeEnum(video_data["vid_type"]),
             )
             self.channels.add(vid_dict["channel"]["channel_id"])
             self.videos.add(vid_dict["youtube_id"])
 
-            if self.task:
-                self.task.send_progress(
-                    [
-                        f"Processing video {youtube_id}",
-                        "Move downloaded file to archive.",
-                    ]
-                )
-
+            self._notify(video_data, "Move downloaded file to archive")
             self.move_to_archive(vid_dict)
-
-            if queue.has_item():
-                message = "Continue with next video."
-            else:
-                message = "Download queue is finished."
-
-            if self.task:
-                self.task.send_progress([message])
-
             self._delete_from_pending(youtube_id)
 
         # post processing
         self._add_subscribed_channels()
         DownloadPostProcess(self).run()
 
-    def _setup_queue(self):
-        """setup required and validate"""
-        if self.config["downloads"]["cookie_import"]:
-            valid = CookieHandler(self.config).validate()
-            if not valid:
-                return
+        return self.videos
 
+    def _notify(self, video_data, message):
+        """send progress notification to task"""
+        if not self.task:
+            return
+
+        typ = VideoTypeEnum(video_data["vid_type"]).value.rstrip("s").title()
+        title = video_data.get("title")
+        self.task.send_progress([f"Processing {typ}: {title}", message])
+
+    def _get_next(self, auto_only):
+        """get next item in queue"""
+        must_list = [{"term": {"status": {"value": "pending"}}}]
+        must_not_list = [{"exists": {"field": "message"}}]
+        if auto_only:
+            must_list.append({"term": {"auto_start": {"value": True}}})
+
+        data = {
+            "size": 1,
+            "query": {"bool": {"must": must_list, "must_not": must_not_list}},
+            "sort": [
+                {"auto_start": {"order": "desc"}},
+                {"timestamp": {"order": "asc"}},
+            ],
+        }
+        path = "ta_download/_search"
+        response, _ = ElasticWrap(path).get(data=data)
+        if not response["hits"]["hits"]:
+            return False
+
+        return response["hits"]["hits"][0]["_source"]
+
+    def _get_overwrites(self):
+        """get channel overwrites"""
         pending = PendingList()
         pending.get_download()
         pending.get_channels()
         self.video_overwrites = pending.video_overwrites
 
-    def add_pending(self):
-        """add pending videos to download queue"""
-        if self.task:
-            self.task.send_progress(["Scanning your download queue."])
-
-        pending = PendingList()
-        pending.get_download()
-        to_add = [
-            json.dumps(
-                {
-                    "youtube_id": i["youtube_id"],
-                    # Using .value in default val to match what would be
-                    # decoded when parsing json if not set
-                    "vid_type": i.get("vid_type", VideoTypeEnum.VIDEOS.value),
-                }
-            )
-            for i in pending.all_pending
-        ]
-        if not to_add:
-            # there is nothing pending
-            print("download queue is empty")
-            if self.task:
-                self.task.send_progress(["Download queue is empty."])
-
-            return
-
-        RedisQueue(queue_name="dl_queue").add_list(to_add)
-
     def _progress_hook(self, response):
         """process the progress_hooks from yt_dlp"""
         progress = False
@@ -312,6 +277,10 @@ class VideoDownloader:
         """build user customized options"""
         if self.config["downloads"]["format"]:
             self.obs["format"] = self.config["downloads"]["format"]
+        if self.config["downloads"]["format_sort"]:
+            format_sort = self.config["downloads"]["format_sort"]
+            format_sort_list = [i.strip() for i in format_sort.split(",")]
+            self.obs["format_sort"] = format_sort_list
         if self.config["downloads"]["limit_speed"]:
             self.obs["ratelimit"] = (
                 self.config["downloads"]["limit_speed"] * 1024
@@ -379,7 +348,9 @@ class VideoDownloader:
             if youtube_id in file_name:
                 obs["outtmpl"] = os.path.join(dl_cache, file_name)
 
-        success = YtWrap(obs, self.config).download(youtube_id)
+        success, message = YtWrap(obs, self.config).download(youtube_id)
+        if not success:
+            self._handle_error(youtube_id, message)
 
         if self.obs["writethumbnail"]:
             # webp files don't get cleaned up automatically
@@ -391,28 +362,27 @@ class VideoDownloader:
 
         return success
 
+    @staticmethod
+    def _handle_error(youtube_id, message):
+        """store error message"""
+        data = {"doc": {"message": message}}
+        _, _ = ElasticWrap(f"ta_download/_update/{youtube_id}").post(data=data)
+
     def move_to_archive(self, vid_dict):
         """move downloaded video from cache to archive"""
         videos = self.config["application"]["videos"]
         host_uid = self.config["application"]["HOST_UID"]
         host_gid = self.config["application"]["HOST_GID"]
-        channel_name = clean_string(vid_dict["channel"]["channel_name"])
-        if len(channel_name) <= 3:
-            # fall back to channel id
-            channel_name = vid_dict["channel"]["channel_id"]
-        # make archive folder with correct permissions
-        new_folder = os.path.join(videos, channel_name)
-        if not os.path.exists(new_folder):
-            os.makedirs(new_folder)
-        if host_uid and host_gid:
-            os.chown(new_folder, host_uid, host_gid)
-        # find real filename
+        # make folder
+        folder = os.path.join(videos, vid_dict["channel"]["channel_id"])
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+            if host_uid and host_gid:
+                os.chown(folder, host_uid, host_gid)
+        # move media file
+        media_file = vid_dict["youtube_id"] + ".mp4"
         cache_dir = self.config["application"]["cache_dir"]
-        all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))
-        for file_str in all_cached:
-            if vid_dict["youtube_id"] in file_str:
-                old_file = file_str
-        old_path = os.path.join(cache_dir, "download", old_file)
+        old_path = os.path.join(cache_dir, "download", media_file)
         new_path = os.path.join(videos, vid_dict["media_url"])
         # move media file and fix permission
         shutil.move(old_path, new_path, copy_function=shutil.copyfile)
@@ -422,7 +392,7 @@ class VideoDownloader:
     @staticmethod
     def _delete_from_pending(youtube_id):
         """delete downloaded video from pending index if its there"""
-        path = f"ta_download/_doc/{youtube_id}"
+        path = f"ta_download/_doc/{youtube_id}?refresh=true"
         _, _ = ElasticWrap(path).delete()
 
     def _add_subscribed_channels(self):
@@ -436,3 +406,18 @@ class VideoDownloader:
             self.channels.add(channel_id)
 
         return
+
+    def _reset_auto(self):
+        """reset autostart to defaults after queue stop"""
+        path = "ta_download/_update_by_query"
+        data = {
+            "query": {"term": {"auto_start": {"value": True}}},
+            "script": {
+                "source": "ctx._source.auto_start = false",
+                "lang": "painless",
+            },
+        }
+        response, _ = ElasticWrap(path).post(data=data)
+        updated = response.get("updated")
+        if updated:
+            print(f"[download] reset auto start on {updated} videos.")

tubearchivist/home/src/es/backup.py

@@ -18,6 +18,8 @@ from home.src.ta.helper import get_mapping, ignore_filelist
 class ElasticBackup:
     """dump index to nd-json files for later bulk import"""
 
+    INDEX_SPLIT = ["comment"]
+
     def __init__(self, reason=False, task=False):
         self.config = AppConfig().config
         self.cache_dir = self.config["application"]["cache_dir"]
@@ -32,7 +34,8 @@ class ElasticBackup:
         if not self.reason:
             raise ValueError("missing backup reason in ElasticBackup")
 
-        self.task.send_progress(["Scanning your index."])
+        if self.task:
+            self.task.send_progress(["Scanning your index."])
         for index in self.index_config:
             index_name = index["index_name"]
             print(f"backup: export in progress for {index_name}")
@@ -42,21 +45,26 @@ class ElasticBackup:
 
             self.backup_index(index_name)
 
-        self.task.send_progress(["Compress files to zip archive."])
+        if self.task:
+            self.task.send_progress(["Compress files to zip archive."])
         self.zip_it()
         if self.reason == "auto":
             self.rotate_backup()
 
     def backup_index(self, index_name):
         """export all documents of a single index"""
-        paginate = IndexPaginate(
-            f"ta_{index_name}",
-            data={"query": {"match_all": {}}},
-            keep_source=True,
-            callback=BackupCallback,
-            task=self.task,
-            total=self._get_total(index_name),
-        )
+        paginate_kwargs = {
+            "data": {"query": {"match_all": {}}},
+            "keep_source": True,
+            "callback": BackupCallback,
+            "task": self.task,
+            "total": self._get_total(index_name),
+        }
+
+        if index_name in self.INDEX_SPLIT:
+            paginate_kwargs.update({"size": 200})
+
+        paginate = IndexPaginate(f"ta_{index_name}", **paginate_kwargs)
         _ = paginate.get_results()
 
     @staticmethod
@@ -204,9 +212,10 @@ class ElasticBackup:
 class BackupCallback:
     """handle backup ndjson writer as callback for IndexPaginate"""
 
-    def __init__(self, source, index_name):
+    def __init__(self, source, index_name, counter=0):
         self.source = source
         self.index_name = index_name
+        self.counter = counter
         self.timestamp = datetime.now().strftime("%Y%m%d")
 
     def run(self):
@@ -235,7 +244,8 @@ class BackupCallback:
     def _write_es_json(self, file_content):
         """write nd-json file for es _bulk API to disk"""
         cache_dir = AppConfig().config["application"]["cache_dir"]
-        file_name = f"es_{self.index_name.lstrip('ta_')}-{self.timestamp}.json"
+        index = self.index_name.lstrip("ta_")
+        file_name = f"es_{index}-{self.timestamp}-{self.counter}.json"
         file_path = os.path.join(cache_dir, "backup", file_name)
         with open(file_path, "a+", encoding="utf-8") as f:
             f.write(file_content)

tubearchivist/home/src/es/connect.py

@@ -6,9 +6,11 @@ functionality:
 # pylint: disable=missing-timeout
 
 import json
+import os
+from typing import Any
 
 import requests
-from home.src.ta.config import AppConfig
+import urllib3
 
 
 class ElasticWrap:
@@ -16,61 +18,96 @@ class ElasticWrap:
     returns response json and status code tuple
     """
 
-    def __init__(self, path, config=False):
-        self.url = False
-        self.auth = False
-        self.path = path
-        self.config = config
-        self._get_config()
+    ES_URL: str = str(os.environ.get("ES_URL"))
+    ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD"))
+    ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic")
+    ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL"))
 
-    def _get_config(self):
-        """add config if not passed"""
-        if not self.config:
-            self.config = AppConfig().config
+    def __init__(self, path: str):
+        self.url: str = f"{self.ES_URL}/{path}"
+        self.auth: tuple[str, str] = (self.ES_USER, self.ES_PASS)
 
-        es_url = self.config["application"]["es_url"]
-        self.auth = self.config["application"]["es_auth"]
-        self.url = f"{es_url}/{self.path}"
+        if self.ES_DISABLE_VERIFY_SSL:
+            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
-    def get(self, data=False, timeout=10, print_error=True):
+    def get(
+        self,
+        data: bool | dict = False,
+        timeout: int = 10,
+        print_error: bool = True,
+    ) -> tuple[dict, int]:
         """get data from es"""
+
+        kwargs: dict[str, Any] = {
+            "auth": self.auth,
+            "timeout": timeout,
+        }
+
+        if self.ES_DISABLE_VERIFY_SSL:
+            kwargs["verify"] = False
+
         if data:
-            response = requests.get(
-                self.url, json=data, auth=self.auth, timeout=timeout
-            )
-        else:
-            response = requests.get(self.url, auth=self.auth, timeout=timeout)
+            kwargs["json"] = data
+
+        response = requests.get(self.url, **kwargs)
+
         if print_error and not response.ok:
             print(response.text)
 
         return response.json(), response.status_code
 
-    def post(self, data=False, ndjson=False):
+    def post(
+        self, data: bool | dict = False, ndjson: bool = False
+    ) -> tuple[dict, int]:
         """post data to es"""
-        if ndjson:
-            headers = {"Content-type": "application/x-ndjson"}
-            payload = data
-        else:
-            headers = {"Content-type": "application/json"}
-            payload = json.dumps(data)
 
-        if data:
-            response = requests.post(
-                self.url, data=payload, headers=headers, auth=self.auth
+        kwargs: dict[str, Any] = {"auth": self.auth}
+
+        if ndjson and data:
+            kwargs.update(
+                {
+                    "headers": {"Content-type": "application/x-ndjson"},
+                    "data": data,
+                }
             )
-        else:
-            response = requests.post(self.url, headers=headers, auth=self.auth)
+        elif data:
+            kwargs.update(
+                {
+                    "headers": {"Content-type": "application/json"},
+                    "data": json.dumps(data),
+                }
+            )
+
+        if self.ES_DISABLE_VERIFY_SSL:
+            kwargs["verify"] = False
+
+        response = requests.post(self.url, **kwargs)
 
         if not response.ok:
             print(response.text)
 
         return response.json(), response.status_code
 
-    def put(self, data, refresh=False):
+    def put(
+        self,
+        data: bool | dict = False,
+        refresh: bool = False,
+    ) -> tuple[dict, Any]:
         """put data to es"""
+
         if refresh:
             self.url = f"{self.url}/?refresh=true"
-        response = requests.put(f"{self.url}", json=data, auth=self.auth)
+
+        kwargs: dict[str, Any] = {
+            "json": data,
+            "auth": self.auth,
+        }
+
+        if self.ES_DISABLE_VERIFY_SSL:
+            kwargs["verify"] = False
+
+        response = requests.put(self.url, **kwargs)
+
         if not response.ok:
             print(response.text)
             print(data)
@@ -78,14 +115,25 @@ class ElasticWrap:
 
         return response.json(), response.status_code
 
-    def delete(self, data=False, refresh=False):
+    def delete(
+        self,
+        data: bool | dict = False,
+        refresh: bool = False,
+    ) -> tuple[dict, Any]:
         """delete document from es"""
+
         if refresh:
             self.url = f"{self.url}/?refresh=true"
+
+        kwargs: dict[str, Any] = {"auth": self.auth}
+
         if data:
-            response = requests.delete(self.url, json=data, auth=self.auth)
-        else:
-            response = requests.delete(self.url, auth=self.auth)
+            kwargs["json"] = data
+
+        if self.ES_DISABLE_VERIFY_SSL:
+            kwargs["verify"] = False
+
+        response = requests.delete(self.url, **kwargs)
 
         if not response.ok:
             print(response.text)
@@ -127,6 +175,12 @@ class IndexPaginate:
 
     def validate_data(self):
         """add pit and size to data"""
+        if not self.data:
+            self.data = {}
+
+        if "query" not in self.data.keys():
+            self.data.update({"query": {"match_all": {}}})
+
         if "sort" not in self.data.keys():
             self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
 
@@ -150,7 +204,9 @@ class IndexPaginate:
                     all_results.append(hit["_source"])
 
             if self.kwargs.get("callback"):
-                self.kwargs.get("callback")(all_hits, self.index_name).run()
+                self.kwargs.get("callback")(
+                    all_hits, self.index_name, counter=counter
+                ).run()
 
             if self.kwargs.get("task"):
                 print(f"{self.index_name}: processing page {counter}")

tubearchivist/home/src/es/index_mapping.json

@@ -1,5 +1,17 @@
 {
     "index_config": [{
+            "index_name": "config",
+            "expected_map": {
+                "config": {
+                    "type": "object",
+                    "enabled": false
+                }
+            },
+            "expected_set": {
+                "number_of_replicas": "0"
+            }
+        },
+        {
             "index_name": "channel",
             "expected_map": {
                 "channel_id": {
@@ -37,7 +49,18 @@
                     "type": "text"
                 },
                 "channel_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
+                },
+                "channel_tags": {
+                    "type": "text",
+                    "analyzer": "english",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
                 },
                 "channel_overwrites": {
                     "properties": {
@@ -80,7 +103,8 @@
                     "index": false
                 },
                 "date_downloaded": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "channel": {
                     "properties": {
@@ -119,7 +143,18 @@
                             "type": "text"
                         },
                         "channel_last_refresh": {
-                            "type": "date"
+                            "type": "date",
+                            "format": "epoch_second"
+                        },
+                        "channel_tags": {
+                            "type": "text",
+                            "analyzer": "english",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "ignore_above": 256
+                                }
+                            }
                         },
                         "channel_overwrites": {
                             "properties": {
@@ -146,6 +181,9 @@
                     "type": "keyword",
                     "index": false
                 },
+                "media_size": {
+                    "type": "long"
+                },
                 "tags": {
                     "type": "text",
                     "analyzer": "english",
@@ -173,7 +211,8 @@
                     }
                 },
                 "vid_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "youtube_id": {
                     "type": "keyword"
@@ -239,10 +278,35 @@
                         }
                     }
                 },
+                "streams": {
+                    "properties": {
+                        "type": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "index": {
+                            "type": "short",
+                            "index": false
+                        },
+                        "codec": {
+                            "type": "text"
+                        },
+                        "width": {
+                            "type": "short"
+                        },
+                        "height": {
+                            "type": "short"
+                        },
+                        "bitrate": {
+                            "type": "integer"
+                        }
+                    }
+                },
                 "sponsorblock": {
                     "properties": {
                         "last_refresh": {
-                            "type": "date"
+                            "type": "date",
+                            "format": "epoch_second"
                         },
                         "has_unlocked": {
                             "type": "boolean"
@@ -294,7 +358,8 @@
             "index_name": "download",
             "expected_map": {
                 "timestamp": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "channel_id": {
                     "type": "keyword"
@@ -330,6 +395,12 @@
                 },
                 "vid_type": {
                     "type": "keyword"
+                },
+                "auto_start": {
+                    "type": "boolean"
+                },
+                "message": {
+                    "type": "text"
                 }
             },
             "expected_set": {
@@ -386,7 +457,8 @@
                     "type": "keyword"
                 },
                 "playlist_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 }
             },
             "expected_set": {
@@ -440,7 +512,8 @@
                     "type": "text"
                 },
                 "subtitle_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "subtitle_index": {
                     "type" : "long"
@@ -475,7 +548,8 @@
                     "type": "keyword"
                 },
                 "comment_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "comment_channel_id": {
                     "type": "keyword"
@@ -489,7 +563,8 @@
                             "type" : "text"
                         },
                         "comment_timestamp": {
-                            "type": "date"
+                            "type": "date",
+                            "format": "epoch_second"
                         },
                         "comment_time_text": {
                             "type" : "text"
@@ -538,4 +613,4 @@
             }
         }
     ]
-}
+}

tubearchivist/home/src/es/snapshot.py

@@ -19,7 +19,9 @@ class ElasticSnapshot:
     REPO_SETTINGS = {
         "compress": "true",
         "chunk_size": "1g",
-        "location": "/usr/share/elasticsearch/data/snapshot",
+        "location": environ.get(
+            "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
+        ),
     }
     POLICY = "ta_daily"
 

tubearchivist/home/src/frontend/api_calls.py

@@ -4,14 +4,8 @@ Functionality:
 - called via user input
 """
 
-from home.src.download.subscriptions import (
-    ChannelSubscription,
-    PlaylistSubscription,
-)
-from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.ta_redis import RedisArchivist
-from home.src.ta.urlparser import Parser
-from home.tasks import run_restore_backup, subscribe_to
+from home.src.ta.users import UserConfig
+from home.tasks import run_restore_backup
 
 
 class PostData:
@@ -36,24 +30,19 @@ class PostData:
         exec_map = {
             "change_view": self._change_view,
             "change_grid": self._change_grid,
-            "unsubscribe": self._unsubscribe,
-            "subscribe": self._subscribe,
             "sort_order": self._sort_order,
             "hide_watched": self._hide_watched,
             "show_subed_only": self._show_subed_only,
             "show_ignored_only": self._show_ignored_only,
             "db-restore": self._db_restore,
-            "delete-playlist": self._delete_playlist,
         }
 
         return exec_map[self.to_exec]
 
     def _change_view(self):
         """process view changes in home, channel, and downloads"""
-        origin, new_view = self.exec_val.split(":")
-        key = f"{self.current_user}:view:{origin}"
-        print(f"change view: {key} to {new_view}")
-        RedisArchivist().set_message(key, {"status": new_view})
+        view, setting = self.exec_val.split(":")
+        UserConfig(self.current_user).set_value(f"view_style_{view}", setting)
         return {"success": True}
 
     def _change_grid(self):
@@ -61,76 +50,38 @@ class PostData:
         grid_items = int(self.exec_val)
         grid_items = max(grid_items, 3)
         grid_items = min(grid_items, 7)
-
-        key = f"{self.current_user}:grid_items"
-        print(f"change grid items: {grid_items}")
-        RedisArchivist().set_message(key, {"status": grid_items})
-        return {"success": True}
-
-    def _unsubscribe(self):
-        """unsubscribe from channels or playlists"""
-        id_unsub = self.exec_val
-        print(f"{id_unsub}: unsubscribe")
-        to_unsub_list = Parser(id_unsub).parse()
-        for to_unsub in to_unsub_list:
-            unsub_type = to_unsub["type"]
-            unsub_id = to_unsub["url"]
-            if unsub_type == "playlist":
-                PlaylistSubscription().change_subscribe(
-                    unsub_id, subscribe_status=False
-                )
-            elif unsub_type == "channel":
-                ChannelSubscription().change_subscribe(
-                    unsub_id, channel_subscribed=False
-                )
-            else:
-                raise ValueError("failed to process " + id_unsub)
-
-        return {"success": True}
-
-    def _subscribe(self):
-        """subscribe to channel or playlist, called from js buttons"""
-        id_sub = self.exec_val
-        print(f"{id_sub}: subscribe")
-        subscribe_to.delay(id_sub)
+        UserConfig(self.current_user).set_value("grid_items", grid_items)
         return {"success": True}
 
     def _sort_order(self):
         """change the sort between published to downloaded"""
-        sort_order = {"status": self.exec_val}
         if self.exec_val in ["asc", "desc"]:
-            RedisArchivist().set_message(
-                f"{self.current_user}:sort_order", sort_order
+            UserConfig(self.current_user).set_value(
+                "sort_order", self.exec_val
             )
         else:
-            RedisArchivist().set_message(
-                f"{self.current_user}:sort_by", sort_order
-            )
+            UserConfig(self.current_user).set_value("sort_by", self.exec_val)
         return {"success": True}
 
     def _hide_watched(self):
         """toggle if to show watched vids or not"""
-        key = f"{self.current_user}:hide_watched"
-        message = {"status": bool(int(self.exec_val))}
-        print(f"toggle {key}: {message}")
-        RedisArchivist().set_message(key, message)
+        UserConfig(self.current_user).set_value(
+            "hide_watched", bool(int(self.exec_val))
+        )
         return {"success": True}
 
     def _show_subed_only(self):
         """show or hide subscribed channels only on channels page"""
-        key = f"{self.current_user}:show_subed_only"
-        message = {"status": bool(int(self.exec_val))}
-        print(f"toggle {key}: {message}")
-        RedisArchivist().set_message(key, message)
+        UserConfig(self.current_user).set_value(
+            "show_subed_only", bool(int(self.exec_val))
+        )
         return {"success": True}
 
     def _show_ignored_only(self):
         """switch view on /downloads/ to show ignored only"""
-        show_value = self.exec_val
-        key = f"{self.current_user}:show_ignored_only"
-        value = {"status": show_value}
-        print(f"Filter download view ignored only: {show_value}")
-        RedisArchivist().set_message(key, value)
+        UserConfig(self.current_user).set_value(
+            "show_ignored_only", bool(int(self.exec_val))
+        )
         return {"success": True}
 
     def _db_restore(self):
@@ -139,16 +90,3 @@ class PostData:
         filename = self.exec_val
         run_restore_backup.delay(filename)
         return {"success": True}
-
-    def _delete_playlist(self):
-        """delete playlist, only metadata or incl all videos"""
-        playlist_dict = self.exec_val
-        playlist_id = playlist_dict["playlist-id"]
-        playlist_action = playlist_dict["playlist-action"]
-        print(f"{playlist_id}: delete playlist {playlist_action}")
-        if playlist_action == "metadata":
-            YoutubePlaylist(playlist_id).delete_metadata()
-        elif playlist_action == "all":
-            YoutubePlaylist(playlist_id).delete_videos_playlist()
-
-        return {"success": True}

tubearchivist/home/src/frontend/forms.py

@@ -44,6 +44,12 @@ class UserSettingsForm(forms.Form):
 class ApplicationSettingsForm(forms.Form):
     """handle all application settings"""
 
+    AUTOSTART_CHOICES = [
+        ("", "-- change subscription autostart --"),
+        ("0", "disable auto start"),
+        ("1", "enable auto start"),
+    ]
+
     METADATA_CHOICES = [
         ("", "-- change metadata embed --"),
         ("0", "don't embed metadata"),
@@ -107,12 +113,16 @@ class ApplicationSettingsForm(forms.Form):
     subscriptions_shorts_channel_size = forms.IntegerField(
         required=False, min_value=0
     )
-    downloads_limit_count = forms.IntegerField(required=False)
+    subscriptions_auto_start = forms.ChoiceField(
+        widget=forms.Select, choices=AUTOSTART_CHOICES, required=False
+    )
     downloads_limit_speed = forms.IntegerField(required=False)
     downloads_throttledratelimit = forms.IntegerField(required=False)
     downloads_sleep_interval = forms.IntegerField(required=False)
     downloads_autodelete_days = forms.IntegerField(required=False)
     downloads_format = forms.CharField(required=False)
+    downloads_format_sort = forms.CharField(required=False)
+    downloads_extractor_lang = forms.CharField(required=False)
     downloads_add_metadata = forms.ChoiceField(
         widget=forms.Select, choices=METADATA_CHOICES, required=False
     )
@@ -147,9 +157,41 @@ class ApplicationSettingsForm(forms.Form):
 class SchedulerSettingsForm(forms.Form):
     """handle scheduler settings"""
 
+    HELP_TEXT = "Add Apprise notification URLs, one per line"
+
     update_subscribed = forms.CharField(required=False)
+    update_subscribed_notify = forms.CharField(
+        label=False,
+        widget=forms.Textarea(
+            attrs={
+                "rows": 2,
+                "placeholder": HELP_TEXT,
+            }
+        ),
+        required=False,
+    )
     download_pending = forms.CharField(required=False)
+    download_pending_notify = forms.CharField(
+        label=False,
+        widget=forms.Textarea(
+            attrs={
+                "rows": 2,
+                "placeholder": HELP_TEXT,
+            }
+        ),
+        required=False,
+    )
     check_reindex = forms.CharField(required=False)
+    check_reindex_notify = forms.CharField(
+        label=False,
+        widget=forms.Textarea(
+            attrs={
+                "rows": 2,
+                "placeholder": HELP_TEXT,
+            }
+        ),
+        required=False,
+    )
     check_reindex_days = forms.IntegerField(required=False)
     thumbnail_check = forms.CharField(required=False)
     run_backup = forms.CharField(required=False)

tubearchivist/home/src/frontend/searching.py

@@ -6,154 +6,19 @@ Functionality:
 - calculate pagination values
 """
 
-import urllib.parse
-from datetime import datetime
 
-from home.src.download.thumbnails import ThumbManager
+from api.src.search_processor import SearchProcess
 from home.src.es.connect import ElasticWrap
-from home.src.ta.config import AppConfig
-
-
-class SearchHandler:
-    """search elastic search"""
-
-    def __init__(self, path, config, data=False):
-        self.max_hits = None
-        self.path = path
-        self.config = config
-        self.data = data
-
-    def get_data(self):
-        """get the data"""
-        response, _ = ElasticWrap(self.path, config=self.config).get(self.data)
-
-        if "hits" in response.keys():
-            self.max_hits = response["hits"]["total"]["value"]
-            return_value = response["hits"]["hits"]
-        else:
-            # simulate list for single result to reuse rest of class
-            return_value = [response]
-
-        # stop if empty
-        if not return_value:
-            return False
-
-        all_videos = []
-        all_channels = []
-        for idx, hit in enumerate(return_value):
-            return_value[idx] = self.hit_cleanup(hit)
-            if hit["_index"] == "ta_video":
-                video_dict, channel_dict = self.vid_cache_link(hit)
-                if video_dict not in all_videos:
-                    all_videos.append(video_dict)
-                if channel_dict not in all_channels:
-                    all_channels.append(channel_dict)
-            elif hit["_index"] == "ta_channel":
-                channel_dict = self.channel_cache_link(hit)
-                if channel_dict not in all_channels:
-                    all_channels.append(channel_dict)
-
-        return return_value
-
-    @staticmethod
-    def vid_cache_link(hit):
-        """download thumbnails into cache"""
-        vid_thumb = hit["source"]["vid_thumb_url"]
-        youtube_id = hit["source"]["youtube_id"]
-        channel_id_hit = hit["source"]["channel"]["channel_id"]
-        chan_thumb = hit["source"]["channel"]["channel_thumb_url"]
-        try:
-            chan_banner = hit["source"]["channel"]["channel_banner_url"]
-        except KeyError:
-            chan_banner = False
-        video_dict = {"youtube_id": youtube_id, "vid_thumb": vid_thumb}
-        channel_dict = {
-            "channel_id": channel_id_hit,
-            "chan_thumb": chan_thumb,
-            "chan_banner": chan_banner,
-        }
-        return video_dict, channel_dict
-
-    @staticmethod
-    def channel_cache_link(hit):
-        """build channel thumb links"""
-        channel_id_hit = hit["source"]["channel_id"]
-        chan_thumb = hit["source"]["channel_thumb_url"]
-        try:
-            chan_banner = hit["source"]["channel_banner_url"]
-        except KeyError:
-            chan_banner = False
-        channel_dict = {
-            "channel_id": channel_id_hit,
-            "chan_thumb": chan_thumb,
-            "chan_banner": chan_banner,
-        }
-        return channel_dict
-
-    @staticmethod
-    def hit_cleanup(hit):
-        """clean up and parse data from a single hit"""
-        hit["source"] = hit.pop("_source")
-        hit_keys = hit["source"].keys()
-        if "media_url" in hit_keys:
-            parsed_url = urllib.parse.quote(hit["source"]["media_url"])
-            hit["source"]["media_url"] = parsed_url
-
-        if "published" in hit_keys:
-            published = hit["source"]["published"]
-            date_pub = datetime.strptime(published, "%Y-%m-%d")
-            date_str = datetime.strftime(date_pub, "%d %b, %Y")
-            hit["source"]["published"] = date_str
-
-        if "vid_last_refresh" in hit_keys:
-            vid_last_refresh = hit["source"]["vid_last_refresh"]
-            date_refresh = datetime.fromtimestamp(vid_last_refresh)
-            date_str = datetime.strftime(date_refresh, "%d %b, %Y")
-            hit["source"]["vid_last_refresh"] = date_str
-
-        if "playlist_last_refresh" in hit_keys:
-            playlist_last_refresh = hit["source"]["playlist_last_refresh"]
-            date_refresh = datetime.fromtimestamp(playlist_last_refresh)
-            date_str = datetime.strftime(date_refresh, "%d %b, %Y")
-            hit["source"]["playlist_last_refresh"] = date_str
-
-        if "vid_thumb_url" in hit_keys:
-            youtube_id = hit["source"]["youtube_id"]
-            thumb_path = ThumbManager(youtube_id).vid_thumb_path()
-            hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
-
-        if "channel_last_refresh" in hit_keys:
-            refreshed = hit["source"]["channel_last_refresh"]
-            date_refresh = datetime.fromtimestamp(refreshed)
-            date_str = datetime.strftime(date_refresh, "%d %b, %Y")
-            hit["source"]["channel_last_refresh"] = date_str
-
-        if "channel" in hit_keys:
-            channel_keys = hit["source"]["channel"].keys()
-            if "channel_last_refresh" in channel_keys:
-                refreshed = hit["source"]["channel"]["channel_last_refresh"]
-                date_refresh = datetime.fromtimestamp(refreshed)
-                date_str = datetime.strftime(date_refresh, "%d %b, %Y")
-                hit["source"]["channel"]["channel_last_refresh"] = date_str
-
-        if "subtitle_fragment_id" in hit_keys:
-            youtube_id = hit["source"]["youtube_id"]
-            thumb_path = ThumbManager(youtube_id).vid_thumb_path()
-            hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
-
-        return hit
 
 
 class SearchForm:
     """build query from search form data"""
 
-    CONFIG = AppConfig().config
-
     def multi_search(self, search_query):
         """searching through index"""
         path, query, query_type = SearchParser(search_query).run()
-        look_up = SearchHandler(path, config=self.CONFIG, data=query)
-        search_results = look_up.get_data()
+        response, _ = ElasticWrap(path).get(data=query)
+        search_results = SearchProcess(response).process()
         all_results = self.build_results(search_results)
 
         return {"results": all_results, "queryType": query_type}
@@ -429,6 +294,7 @@ class QueryBuilder:
                             "channel_name._2gram^2",
                             "channel_name._3gram^2",
                             "channel_name.search_as_you_type^2",
+                            "channel_tags",
                         ],
                     }
                 }
@@ -502,7 +368,6 @@ class QueryBuilder:
 
         query = {
             "size": 30,
-            "_source": {"excludes": "subtitle_line"},
             "query": {"bool": {"must": must_list}},
             "highlight": {
                 "fields": {

tubearchivist/home/src/index/channel.py

@@ -6,158 +6,14 @@ functionality:
 
 import json
 import os
-import re
 from datetime import datetime
 
-import requests
-from bs4 import BeautifulSoup
 from home.src.download import queue  # partial import
 from home.src.download.thumbnails import ThumbManager
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.generic import YouTubeItem
 from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.helper import clean_string, requests_headers
-
-
-class ChannelScraper:
-    """custom scraper using bs4 to scrape channel about page
-    will be able to be integrated into yt-dlp
-    once #2237 and #2350 are merged upstream
-    """
-
-    def __init__(self, channel_id):
-        self.channel_id = channel_id
-        self.soup = False
-        self.yt_json = False
-        self.json_data = False
-
-    def get_json(self):
-        """main method to return channel dict"""
-        self.get_soup()
-        self._extract_yt_json()
-        if self._is_deactivated():
-            return False
-
-        self._parse_channel_main()
-        self._parse_channel_meta()
-        return self.json_data
-
-    def get_soup(self):
-        """return soup from youtube"""
-        print(f"{self.channel_id}: scrape channel data from youtube")
-        url = f"https://www.youtube.com/channel/{self.channel_id}/about?hl=en"
-        cookies = {"CONSENT": "YES+xxxxxxxxxxxxxxxxxxxxxxxxxxx"}
-        response = requests.get(
-            url, cookies=cookies, headers=requests_headers(), timeout=10
-        )
-        if response.ok:
-            channel_page = response.text
-        else:
-            print(f"{self.channel_id}: failed to extract channel info")
-            raise ConnectionError
-        self.soup = BeautifulSoup(channel_page, "html.parser")
-
-    def _extract_yt_json(self):
-        """parse soup and get ytInitialData json"""
-        all_scripts = self.soup.find("body").find_all("script")
-        for script in all_scripts:
-            if "var ytInitialData = " in str(script):
-                script_content = str(script)
-                break
-        # extract payload
-        script_content = script_content.split("var ytInitialData = ")[1]
-        json_raw = script_content.rstrip(";</script>")
-        self.yt_json = json.loads(json_raw)
-
-    def _is_deactivated(self):
-        """check if channel is deactivated"""
-        alerts = self.yt_json.get("alerts")
-        if not alerts:
-            return False
-
-        for alert in alerts:
-            alert_text = alert["alertRenderer"]["text"]["simpleText"]
-            print(f"{self.channel_id}: failed to extract, {alert_text}")
-            return True
-
-    def _parse_channel_main(self):
-        """extract maintab values from scraped channel json data"""
-        main_tab = self.yt_json["header"]["c4TabbedHeaderRenderer"]
-        # build and return dict
-        self.json_data = {
-            "channel_active": True,
-            "channel_last_refresh": int(datetime.now().timestamp()),
-            "channel_subs": self._get_channel_subs(main_tab),
-            "channel_name": main_tab["title"],
-            "channel_banner_url": self._get_thumbnails(main_tab, "banner"),
-            "channel_tvart_url": self._get_thumbnails(main_tab, "tvBanner"),
-            "channel_id": self.channel_id,
-            "channel_subscribed": False,
-        }
-
-    @staticmethod
-    def _get_thumbnails(main_tab, thumb_name):
-        """extract banner url from main_tab"""
-        try:
-            all_banners = main_tab[thumb_name]["thumbnails"]
-            banner = sorted(all_banners, key=lambda k: k["width"])[-1]["url"]
-        except KeyError:
-            banner = False
-
-        return banner
-
-    @staticmethod
-    def _get_channel_subs(main_tab):
-        """process main_tab to get channel subs as int"""
-        try:
-            sub_text_simple = main_tab["subscriberCountText"]["simpleText"]
-            sub_text = sub_text_simple.split(" ")[0]
-            if sub_text[-1] == "K":
-                channel_subs = int(float(sub_text.replace("K", "")) * 1000)
-            elif sub_text[-1] == "M":
-                channel_subs = int(float(sub_text.replace("M", "")) * 1000000)
-            elif int(sub_text) >= 0:
-                channel_subs = int(sub_text)
-            else:
-                message = f"{sub_text} not dealt with"
-                print(message)
-        except KeyError:
-            channel_subs = 0
-
-        return channel_subs
-
-    def _parse_channel_meta(self):
-        """extract meta tab values from channel payload"""
-        # meta tab
-        meta_tab = self.yt_json["metadata"]["channelMetadataRenderer"]
-        all_thumbs = meta_tab["avatar"]["thumbnails"]
-        thumb_url = sorted(all_thumbs, key=lambda k: k["width"])[-1]["url"]
-        # stats tab
-        renderer = "twoColumnBrowseResultsRenderer"
-        all_tabs = self.yt_json["contents"][renderer]["tabs"]
-        for tab in all_tabs:
-            if "tabRenderer" in tab.keys():
-                if tab["tabRenderer"]["title"] == "About":
-                    about_tab = tab["tabRenderer"]["content"][
-                        "sectionListRenderer"
-                    ]["contents"][0]["itemSectionRenderer"]["contents"][0][
-                        "channelAboutFullMetadataRenderer"
-                    ]
-                    break
-        try:
-            channel_views_text = about_tab["viewCountText"]["simpleText"]
-            channel_views = int(re.sub(r"\D", "", channel_views_text))
-        except KeyError:
-            channel_views = 0
-
-        self.json_data.update(
-            {
-                "channel_description": meta_tab["description"],
-                "channel_thumb_url": thumb_url,
-                "channel_views": channel_views,
-            }
-        )
 
 
 class YoutubeChannel(YouTubeItem):
@@ -166,36 +22,95 @@ class YoutubeChannel(YouTubeItem):
     es_path = False
     index_name = "ta_channel"
     yt_base = "https://www.youtube.com/channel/"
+    yt_obs = {
+        "extract_flat": True,
+        "allow_playlist_files": True,
+    }
 
     def __init__(self, youtube_id, task=False):
         super().__init__(youtube_id)
-        self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.all_playlists = False
         self.task = task
 
+    def build_yt_url(self):
+        """overwrite base to use channel about page"""
+        return f"{self.yt_base}{self.youtube_id}/about"
+
     def build_json(self, upload=False, fallback=False):
         """get from es or from youtube"""
         self.get_from_es()
         if self.json_data:
             return
 
-        self.get_from_youtube(fallback)
+        self.get_from_youtube()
+        if not self.youtube_meta and fallback:
+            self._video_fallback(fallback)
+        else:
+            self.process_youtube_meta()
+            self.get_channel_art()
 
         if upload:
             self.upload_to_es()
-        return
 
-    def get_from_youtube(self, fallback=False):
-        """use bs4 to scrape channel about page"""
-        self.json_data = ChannelScraper(self.youtube_id).get_json()
+    def process_youtube_meta(self):
+        """extract relevant fields"""
+        self.youtube_meta["thumbnails"].reverse()
+        channel_subs = self.youtube_meta.get("channel_follower_count") or 0
+        self.json_data = {
+            "channel_active": True,
+            "channel_description": self.youtube_meta.get("description", False),
+            "channel_id": self.youtube_id,
+            "channel_last_refresh": int(datetime.now().timestamp()),
+            "channel_name": self.youtube_meta["uploader"],
+            "channel_subs": channel_subs,
+            "channel_subscribed": False,
+            "channel_tags": self._parse_tags(self.youtube_meta.get("tags")),
+            "channel_banner_url": self._get_banner_art(),
+            "channel_thumb_url": self._get_thumb_art(),
+            "channel_tvart_url": self._get_tv_art(),
+            "channel_views": self.youtube_meta.get("view_count", 0),
+        }
 
-        if not self.json_data and fallback:
-            self._video_fallback(fallback)
+    def _parse_tags(self, tags):
+        """parse channel tags"""
+        if not tags:
+            return False
 
-        if not self.json_data:
-            return
+        joined = " ".join(tags)
+        return [i.strip() for i in joined.split('"') if i and not i == " "]
 
-        self.get_channel_art()
+    def _get_thumb_art(self):
+        """extract thumb art"""
+        for i in self.youtube_meta["thumbnails"]:
+            if not i.get("width"):
+                continue
+            if i.get("width") == i.get("height"):
+                return i["url"]
+
+        return False
+
+    def _get_tv_art(self):
+        """extract tv artwork"""
+        for i in self.youtube_meta["thumbnails"]:
+            if i.get("id") == "banner_uncropped":
+                return i["url"]
+        for i in self.youtube_meta["thumbnails"]:
+            if not i.get("width"):
+                continue
+            if i["width"] // i["height"] < 2 and not i["width"] == i["height"]:
+                return i["url"]
+
+        return False
+
+    def _get_banner_art(self):
+        """extract banner artwork"""
+        for i in self.youtube_meta["thumbnails"]:
+            if not i.get("width"):
+                continue
+            if i["width"] // i["height"] > 5:
+                return i["url"]
+
+        return False
 
     def _video_fallback(self, fallback):
         """use video metadata as fallback"""
@@ -209,6 +124,7 @@ class YoutubeChannel(YouTubeItem):
             "channel_tvart_url": False,
             "channel_id": self.youtube_id,
             "channel_subscribed": False,
+            "channel_tags": False,
             "channel_description": False,
             "channel_thumb_url": False,
             "channel_views": 0,
@@ -261,12 +177,10 @@ class YoutubeChannel(YouTubeItem):
 
     def get_folder_path(self):
         """get folder where media files get stored"""
-        channel_name = self.json_data["channel_name"]
-        folder_name = clean_string(channel_name)
-        if len(folder_name) <= 3:
-            # fall back to channel id
-            folder_name = self.json_data["channel_id"]
-        folder_path = os.path.join(self.app_conf["videos"], folder_name)
+        folder_path = os.path.join(
+            self.app_conf["videos"],
+            self.json_data["channel_id"],
+        )
         return folder_path
 
     def delete_es_videos(self):

tubearchivist/home/src/index/comments.py

@@ -63,6 +63,7 @@ class Comments:
             "check_formats": None,
             "skip_download": True,
             "getcomments": True,
+            "ignoreerrors": True,
             "extractor_args": {
                 "youtube": {
                     "max_comments": max_comments_list,
@@ -119,12 +120,14 @@ class Comments:
             "comment_text": comment["text"].replace("\xa0", ""),
             "comment_timestamp": comment["timestamp"],
             "comment_time_text": time_text,
-            "comment_likecount": comment["like_count"],
-            "comment_is_favorited": comment["is_favorited"],
+            "comment_likecount": comment.get("like_count", None),
+            "comment_is_favorited": comment.get("is_favorited", False),
             "comment_author": comment["author"],
             "comment_author_id": comment["author_id"],
             "comment_author_thumbnail": comment["author_thumbnail"],
-            "comment_author_is_uploader": comment["author_is_uploader"],
+            "comment_author_is_uploader": comment.get(
+                "comment_author_is_uploader", False
+            ),
             "comment_parent": comment["parent"],
         }
 

tubearchivist/home/src/index/filesystem.py

@@ -1,194 +1,85 @@
 """
 Functionality:
-- reindexing old documents
-- syncing updated values between indexes
 - scan the filesystem to delete or index
 """
 
-import json
 import os
 
-from home.src.download.queue import PendingList
-from home.src.es.connect import ElasticWrap
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.comments import CommentList
-from home.src.index.video import index_new_video
+from home.src.index.video import YoutubeVideo, index_new_video
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import clean_string, ignore_filelist
-from PIL import ImageFile
-
-ImageFile.LOAD_TRUNCATED_IMAGES = True
+from home.src.ta.helper import ignore_filelist
 
 
-class ScannerBase:
-    """scan the filesystem base class"""
+class Scanner:
+    """scan index and filesystem"""
 
-    CONFIG = AppConfig().config
-    VIDEOS = CONFIG["application"]["videos"]
+    VIDEOS: str = AppConfig().config["application"]["videos"]
 
-    def __init__(self):
-        self.to_index = False
-        self.to_delete = False
-        self.mismatch = False
-        self.to_rename = False
-
-    def scan(self):
-        """entry point, scan and compare"""
-        all_downloaded = self._get_all_downloaded()
-        all_indexed = self._get_all_indexed()
-        self.list_comarison(all_downloaded, all_indexed)
-
-    def _get_all_downloaded(self):
-        """get a list of all video files downloaded"""
-        channels = os.listdir(self.VIDEOS)
-        all_channels = ignore_filelist(channels)
-        all_channels.sort()
-        all_downloaded = []
-        for channel_name in all_channels:
-            channel_path = os.path.join(self.VIDEOS, channel_name)
-            channel_files = os.listdir(channel_path)
-            channel_files_clean = ignore_filelist(channel_files)
-            all_videos = [i for i in channel_files_clean if i.endswith(".mp4")]
-            for video in all_videos:
-                youtube_id = video[9:20]
-                all_downloaded.append((channel_name, video, youtube_id))
-
-        return all_downloaded
-
-    @staticmethod
-    def _get_all_indexed():
-        """get a list of all indexed videos"""
-        index_handler = PendingList()
-        index_handler.get_download()
-        index_handler.get_indexed()
-
-        all_indexed = []
-        for video in index_handler.all_videos:
-            youtube_id = video["youtube_id"]
-            media_url = video["media_url"]
-            published = video["published"]
-            title = video["title"]
-            all_indexed.append((youtube_id, media_url, published, title))
-        return all_indexed
-
-    def list_comarison(self, all_downloaded, all_indexed):
-        """compare the lists to figure out what to do"""
-        self._find_unindexed(all_downloaded, all_indexed)
-        self._find_missing(all_downloaded, all_indexed)
-        self._find_bad_media_url(all_downloaded, all_indexed)
-
-    def _find_unindexed(self, all_downloaded, all_indexed):
-        """find video files without a matching document indexed"""
-        all_indexed_ids = [i[0] for i in all_indexed]
-        self.to_index = []
-        for downloaded in all_downloaded:
-            if downloaded[2] not in all_indexed_ids:
-                self.to_index.append(downloaded)
-
-    def _find_missing(self, all_downloaded, all_indexed):
-        """find indexed videos without matching media file"""
-        all_downloaded_ids = [i[2] for i in all_downloaded]
-        self.to_delete = []
-        for video in all_indexed:
-            youtube_id = video[0]
-            if youtube_id not in all_downloaded_ids:
-                self.to_delete.append(video)
-
-    def _find_bad_media_url(self, all_downloaded, all_indexed):
-        """rename media files not matching the indexed title"""
-        self.mismatch = []
-        self.to_rename = []
-
-        for downloaded in all_downloaded:
-            channel, filename, downloaded_id = downloaded
-            # find in indexed
-            for indexed in all_indexed:
-                indexed_id, media_url, published, title = indexed
-                if indexed_id == downloaded_id:
-                    # found it
-                    pub = published.replace("-", "")
-                    expected = f"{pub}_{indexed_id}_{clean_string(title)}.mp4"
-                    new_url = os.path.join(channel, expected)
-                    if expected != filename:
-                        # file to rename
-                        self.to_rename.append((channel, filename, expected))
-                    if media_url != new_url:
-                        # media_url to update in es
-                        self.mismatch.append((indexed_id, new_url))
-
-                    break
-
-
-class Filesystem(ScannerBase):
-    """handle scanning and fixing from filesystem"""
-
-    def __init__(self, task=False):
-        super().__init__()
+    def __init__(self, task=False) -> None:
         self.task = task
+        self.to_delete: set[str] = set()
+        self.to_index: set[str] = set()
 
-    def process(self):
-        """entry point"""
-        self.task.send_progress(["Scanning your archive and index."])
-        self.scan()
-        self.rename_files()
-        self.send_mismatch_bulk()
-        self.delete_from_index()
-        self.add_missing()
+    def scan(self) -> None:
+        """scan the filesystem"""
+        downloaded: set[str] = self._get_downloaded()
+        indexed: set[str] = self._get_indexed()
+        self.to_index = downloaded - indexed
+        self.to_delete = indexed - downloaded
 
-    def rename_files(self):
-        """rename media files as identified by find_bad_media_url"""
-        if not self.to_rename:
-            return
+    def _get_downloaded(self) -> set[str]:
+        """get downloaded ids"""
+        if self.task:
+            self.task.send_progress(["Scan your filesystem for videos."])
 
-        total = len(self.to_rename)
-        self.task.send_progress([f"Rename {total} media files."])
-        for bad_filename in self.to_rename:
-            channel, filename, expected_filename = bad_filename
-            print(f"renaming [{filename}] to [{expected_filename}]")
-            old_path = os.path.join(self.VIDEOS, channel, filename)
-            new_path = os.path.join(self.VIDEOS, channel, expected_filename)
-            os.rename(old_path, new_path)
+        downloaded: set = set()
+        channels = ignore_filelist(os.listdir(self.VIDEOS))
+        for channel in channels:
+            folder = os.path.join(self.VIDEOS, channel)
+            files = ignore_filelist(os.listdir(folder))
+            downloaded.update({i.split(".")[0] for i in files})
 
-    def send_mismatch_bulk(self):
-        """build bulk update"""
-        if not self.mismatch:
-            return
+        return downloaded
 
-        total = len(self.mismatch)
-        self.task.send_progress([f"Fix media urls for {total} files"])
-        bulk_list = []
-        for video_mismatch in self.mismatch:
-            youtube_id, media_url = video_mismatch
-            print(f"{youtube_id}: fixing media url {media_url}")
-            action = {"update": {"_id": youtube_id, "_index": "ta_video"}}
-            source = {"doc": {"media_url": media_url}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(source))
-        # add last newline
-        bulk_list.append("\n")
-        data = "\n".join(bulk_list)
-        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
+    def _get_indexed(self) -> set:
+        """get all indexed ids"""
+        if self.task:
+            self.task.send_progress(["Get all videos indexed."])
 
-    def delete_from_index(self):
-        """find indexed but deleted mediafile"""
+        data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
+        response = IndexPaginate("ta_video", data).get_results()
+        return {i["youtube_id"] for i in response}
+
+    def apply(self) -> None:
+        """apply all changes"""
+        self.delete()
+        self.index()
+        self.url_fix()
+
+    def delete(self) -> None:
+        """delete videos from index"""
         if not self.to_delete:
+            print("nothing to delete")
             return
 
-        total = len(self.to_delete)
-        self.task.send_progress([f"Clean up {total} items from index."])
-        for indexed in self.to_delete:
-            youtube_id = indexed[0]
-            print(f"deleting {youtube_id} from index")
-            path = f"ta_video/_doc/{youtube_id}"
-            _, _ = ElasticWrap(path).delete()
+        if self.task:
+            self.task.send_progress(
+                [f"Remove {len(self.to_delete)} videos from index."]
+            )
 
-    def add_missing(self):
-        """add missing videos to index"""
-        video_ids = [i[2] for i in self.to_index]
-        if not video_ids:
+        for youtube_id in self.to_delete:
+            YoutubeVideo(youtube_id).delete_media_file()
+
+    def index(self) -> None:
+        """index new"""
+        if not self.to_index:
+            print("nothing to index")
             return
 
-        total = len(video_ids)
-        for idx, youtube_id in enumerate(video_ids):
+        total = len(self.to_index)
+        for idx, youtube_id in enumerate(self.to_index):
             if self.task:
                 self.task.send_progress(
                     message_lines=[
@@ -198,4 +89,36 @@ class Filesystem(ScannerBase):
                 )
             index_new_video(youtube_id)
 
-        CommentList(video_ids, task=self.task).index()
+        CommentList(self.to_index, task=self.task).index()
+
+    def url_fix(self) -> None:
+        """
+        update path v0.3.6 to v0.3.7
+        fix url not matching channel-videoid pattern
+        """
+        bool_must = (
+            "doc['media_url'].value == "
+            + "(doc['channel.channel_id'].value + '/' + "
+            + "doc['youtube_id'].value) + '.mp4'"
+        )
+        to_update = (
+            "ctx._source['media_url'] = "
+            + "ctx._source.channel['channel_id'] + '/' + "
+            + "ctx._source['youtube_id'] + '.mp4'"
+        )
+        data = {
+            "query": {
+                "bool": {
+                    "must_not": [{"script": {"script": {"source": bool_must}}}]
+                }
+            },
+            "script": {"source": to_update},
+        }
+        response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data)
+        updated = response.get("updates")
+        if updated:
+            print(f"updated {updated} bad media_url")
+            if self.task:
+                self.task.send_progress(
+                    [f"Updated {updated} wrong media urls."]
+                )

tubearchivist/home/src/index/generic.py

@@ -8,15 +8,15 @@ import math
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
-from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.users import UserConfig
 
 
 class YouTubeItem:
     """base class for youtube"""
 
     es_path = False
-    index_name = False
-    yt_base = False
+    index_name = ""
+    yt_base = ""
     yt_obs = {
         "skip_download": True,
         "noplaylist": True,
@@ -24,18 +24,27 @@ class YouTubeItem:
 
     def __init__(self, youtube_id):
         self.youtube_id = youtube_id
+        self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.config = AppConfig().config
         self.app_conf = self.config["application"]
         self.youtube_meta = False
         self.json_data = False
 
+    def build_yt_url(self):
+        """build youtube url"""
+        return self.yt_base + self.youtube_id
+
     def get_from_youtube(self):
         """use yt-dlp to get meta data from youtube"""
         print(f"{self.youtube_id}: get metadata from youtube")
-        url = self.yt_base + self.youtube_id
-        response = YtWrap(self.yt_obs, self.config).extract(url)
+        obs_request = self.yt_obs.copy()
+        if self.config["downloads"]["extractor_lang"]:
+            langs = self.config["downloads"]["extractor_lang"]
+            langs_list = [i.strip() for i in langs.split(",")]
+            obs_request["extractor_args"] = {"youtube": {"lang": langs_list}}
 
-        self.youtube_meta = response
+        url = self.build_yt_url()
+        self.youtube_meta = YtWrap(obs_request, self.config).extract(url)
 
     def get_from_es(self):
         """get indexed data from elastic search"""
@@ -91,13 +100,7 @@ class Pagination:
 
     def get_page_size(self):
         """get default or user modified page_size"""
-        key = f"{self.request.user.id}:page_size"
-        page_size = RedisArchivist().get_message(key)["status"]
-        if not page_size:
-            config = AppConfig().config
-            page_size = config["archive"]["page_size"]
-
-        return page_size
+        return UserConfig(self.request.user.id).get_value("page_size")
 
     def first_guess(self):
         """build first guess before api call"""

tubearchivist/home/src/index/manual.py

@@ -31,6 +31,9 @@ class ImportFolderScanner:
     CACHE_DIR = CONFIG["application"]["cache_dir"]
     IMPORT_DIR = os.path.join(CACHE_DIR, "import")
 
+    """All extensions should be in lowercase until better handling is in place.
+    Described in Issue #502.
+    """
     EXT_MAP = {
         "media": [".mp4", ".mkv", ".webm"],
         "metadata": [".json"],
@@ -118,7 +121,7 @@ class ImportFolderScanner:
         """detect metadata type for file"""
 
         for key, value in self.EXT_MAP.items():
-            if ext in value:
+            if ext.lower() in value:
                 return key, file_path
 
         return False, False

tubearchivist/home/src/index/playlist.py

@@ -26,7 +26,6 @@ class YoutubePlaylist(YouTubeItem):
 
     def __init__(self, youtube_id):
         super().__init__(youtube_id)
-        self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.all_members = False
         self.nav = False
         self.all_youtube_ids = []

tubearchivist/home/src/index/reindex.py

@@ -6,7 +6,6 @@ functionality:
 
 import json
 import os
-import shutil
 from datetime import datetime
 from time import sleep
 
@@ -14,7 +13,6 @@ from home.src.download.queue import PendingList
 from home.src.download.subscriptions import ChannelSubscription
 from home.src.download.thumbnails import ThumbManager
 from home.src.download.yt_dlp_base import CookieHandler
-from home.src.download.yt_dlp_handler import VideoDownloader
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
 from home.src.index.comments import Comments
@@ -54,6 +52,7 @@ class ReindexBase:
     def __init__(self):
         self.config = AppConfig().config
         self.now = int(datetime.now().timestamp())
+        self.total = None
 
     def populate(self, all_ids, reindex_config):
         """add all to reindex ids to redis queue"""
@@ -61,6 +60,7 @@ class ReindexBase:
             return
 
         RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
+        self.total = None
 
 
 class ReindexPopulate(ReindexBase):
@@ -227,6 +227,11 @@ class Reindex(ReindexBase):
         super().__init__()
         self.task = task
         self.all_indexed_ids = False
+        self.processed = {
+            "videos": 0,
+            "channels": 0,
+            "playlists": 0,
+        }
 
     def reindex_all(self):
         """reindex all in queue"""
@@ -238,18 +243,19 @@ class Reindex(ReindexBase):
             if not RedisQueue(index_config["queue_name"]).has_item():
                 continue
 
-            total = RedisQueue(index_config["queue_name"]).length()
+            self.total = RedisQueue(index_config["queue_name"]).length()
             while True:
-                has_next = self.reindex_index(name, index_config, total)
+                has_next = self.reindex_index(name, index_config)
                 if not has_next:
                     break
 
-    def reindex_index(self, name, index_config, total):
+    def reindex_index(self, name, index_config):
         """reindex all of a single index"""
         reindex = self.get_reindex_map(index_config["index_name"])
         youtube_id = RedisQueue(index_config["queue_name"]).get_next()
         if youtube_id:
-            self._notify(name, index_config, total)
+            if self.task:
+                self._notify(name, index_config)
             reindex(youtube_id)
             sleep_interval = self.config["downloads"].get("sleep_interval", 0)
             sleep(sleep_interval)
@@ -266,23 +272,18 @@ class Reindex(ReindexBase):
 
         return def_map.get(index_name)
 
-    def _notify(self, name, index_config, total):
+    def _notify(self, name, index_config):
         """send notification back to task"""
+        if self.total is None:
+            self.total = RedisQueue(index_config["queue_name"]).length()
+
         remaining = RedisQueue(index_config["queue_name"]).length()
-        idx = total - remaining
-        message = [f"Reindexing {name.title()}s {idx}/{total}"]
-        progress = idx / total
+        idx = self.total - remaining
+        message = [f"Reindexing {name.title()}s {idx}/{self.total}"]
+        progress = idx / self.total
         self.task.send_progress(message, progress=progress)
 
     def _reindex_single_video(self, youtube_id):
-        """wrapper to handle channel name changes"""
-        try:
-            self._reindex_single_video_call(youtube_id)
-        except FileNotFoundError:
-            ChannelUrlFixer(youtube_id, self.config).run()
-            self._reindex_single_video_call(youtube_id)
-
-    def _reindex_single_video_call(self, youtube_id):
         """refresh data for single video"""
         video = YoutubeVideo(youtube_id)
 
@@ -291,7 +292,10 @@ class Reindex(ReindexBase):
         es_meta = video.json_data.copy()
 
         # get new
-        video.build_json()
+        media_url = os.path.join(
+            self.config["application"]["videos"], es_meta["media_url"]
+        )
+        video.build_json(media_path=media_url)
         if not video.youtube_meta:
             video.deactivate()
             return
@@ -317,6 +321,7 @@ class Reindex(ReindexBase):
         thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
 
         Comments(youtube_id, config=self.config).reindex_comments()
+        self.processed["videos"] += 1
 
         return
 
@@ -328,27 +333,33 @@ class Reindex(ReindexBase):
         new_path = os.path.join(videos, media_url_should)
         os.rename(old_path, new_path)
 
-    @staticmethod
-    def _reindex_single_channel(channel_id):
+    def _reindex_single_channel(self, channel_id):
         """refresh channel data and sync to videos"""
+        # read current state
         channel = YoutubeChannel(channel_id)
         channel.get_from_es()
-        subscribed = channel.json_data["channel_subscribed"]
-        overwrites = channel.json_data.get("channel_overwrites", False)
+        es_meta = channel.json_data.copy()
+
+        # get new
         channel.get_from_youtube()
-        if not channel.json_data:
+        if not channel.youtube_meta:
             channel.deactivate()
             channel.get_from_es()
             channel.sync_to_videos()
             return
 
-        channel.json_data["channel_subscribed"] = subscribed
+        channel.process_youtube_meta()
+        channel.get_channel_art()
+
+        # add back
+        channel.json_data["channel_subscribed"] = es_meta["channel_subscribed"]
+        overwrites = es_meta.get("channel_overwrites")
         if overwrites:
             channel.json_data["channel_overwrites"] = overwrites
-        channel.upload_to_es()
-        channel.sync_to_videos()
 
+        channel.upload_to_es()
         ChannelFullScan(channel_id).scan()
+        self.processed["channels"] += 1
 
     def _reindex_single_playlist(self, playlist_id):
         """refresh playlist data"""
@@ -364,6 +375,7 @@ class Reindex(ReindexBase):
 
         playlist.json_data["playlist_subscribed"] = subscribed
         playlist.upload_to_es()
+        self.processed["playlists"] += 1
         return
 
     def _get_all_videos(self):
@@ -385,6 +397,18 @@ class Reindex(ReindexBase):
         valid = CookieHandler(self.config).validate()
         return valid
 
+    def build_message(self):
+        """build progress message"""
+        message = ""
+        for key, value in self.processed.items():
+            if value:
+                message = message + f"{value} {key}, "
+
+        if message:
+            message = f"reindexed {message.rstrip(', ')}"
+
+        return message
+
 
 class ReindexProgress(ReindexBase):
     """
@@ -460,65 +484,6 @@ class ReindexProgress(ReindexBase):
         return state_dict
 
 
-class ChannelUrlFixer:
-    """fix not matching channel names in reindex"""
-
-    def __init__(self, youtube_id, config):
-        self.youtube_id = youtube_id
-        self.config = config
-        self.video = False
-
-    def run(self):
-        """check and run if needed"""
-        print(f"{self.youtube_id}: failed to build channel path, try to fix.")
-        video_path_is, video_folder_is = self.get_as_is()
-        if not os.path.exists(video_path_is):
-            print(f"giving up reindex, video in video: {self.video.json_data}")
-            raise ValueError
-
-        _, video_folder_should = self.get_as_should()
-
-        if video_folder_is != video_folder_should:
-            self.process(video_path_is)
-        else:
-            print(f"{self.youtube_id}: skip channel url fixer")
-
-    def get_as_is(self):
-        """get video object as is"""
-        self.video = YoutubeVideo(self.youtube_id)
-        self.video.get_from_es()
-        video_path_is = os.path.join(
-            self.config["application"]["videos"],
-            self.video.json_data["media_url"],
-        )
-        video_folder_is = os.path.split(video_path_is)[0]
-
-        return video_path_is, video_folder_is
-
-    def get_as_should(self):
-        """add fresh metadata from remote"""
-        self.video.get_from_youtube()
-        self.video.add_file_path()
-
-        video_path_should = os.path.join(
-            self.config["application"]["videos"],
-            self.video.json_data["media_url"],
-        )
-        video_folder_should = os.path.split(video_path_should)[0]
-        return video_path_should, video_folder_should
-
-    def process(self, video_path_is):
-        """fix filepath"""
-        print(f"{self.youtube_id}: fixing channel rename.")
-        cache_dir = self.config["application"]["cache_dir"]
-        new_path = os.path.join(
-            cache_dir, "download", self.youtube_id + ".mp4"
-        )
-        shutil.move(video_path_is, new_path, copy_function=shutil.copyfile)
-        VideoDownloader().move_to_archive(self.video.json_data)
-        self.video.update_media_url()
-
-
 class ChannelFullScan:
     """
     update from v0.3.0 to v0.3.1

tubearchivist/home/src/index/subtitle.py

@@ -62,7 +62,12 @@ class YoutubeSubtitle:
         if not all_formats:
             return False
 
-        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
+        subtitle_json3 = [i for i in all_formats if i["ext"] == "json3"]
+        if not subtitle_json3:
+            print(f"{self.video.youtube_id}-{lang}: json3 not processed")
+            return False
+
+        subtitle = subtitle_json3[0]
         subtitle.update(
             {"lang": lang, "source": "auto", "media_url": media_url}
         )

tubearchivist/home/src/index/video.py

@@ -16,8 +16,9 @@ from home.src.index import playlist as ta_playlist
 from home.src.index.generic import YouTubeItem
 from home.src.index.subtitle import YoutubeSubtitle
 from home.src.index.video_constants import VideoTypeEnum
-from home.src.ta.helper import DurationConverter, clean_string, randomizor
-from home.src.ta.ta_redis import RedisArchivist
+from home.src.index.video_streams import MediaStreamExtractor
+from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor
+from home.src.ta.users import UserConfig
 from ryd_client import ryd_client
 
 
@@ -31,17 +32,16 @@ class SponsorBlock:
         self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
         self.last_refresh = int(datetime.now().timestamp())
 
-    def get_sb_id(self):
-        """get sponsorblock userid or generate if needed"""
+    def get_sb_id(self) -> str:
+        """get sponsorblock for the userid or generate if needed"""
         if not self.user_id:
-            print("missing request user id")
-            raise ValueError
+            raise ValueError("missing request user id")
 
-        key = f"{self.user_id}:id_sponsorblock"
-        sb_id = RedisArchivist().get_message(key)
-        if not sb_id["status"]:
-            sb_id = {"status": randomizor(32)}
-            RedisArchivist().set_message(key, sb_id)
+        user = UserConfig(self.user_id)
+        sb_id = user.get_value("sponsorblock_id")
+        if not sb_id:
+            sb_id = randomizor(32)
+            user.set_value("sponsorblock_id", sb_id)
 
         return sb_id
 
@@ -87,7 +87,7 @@ class SponsorBlock:
 
     def post_timestamps(self, youtube_id, start_time, end_time):
         """post timestamps to api"""
-        user_id = self.get_sb_id().get("status")
+        user_id = self.get_sb_id()
         data = {
             "videoID": youtube_id,
             "startTime": start_time,
@@ -104,7 +104,7 @@ class SponsorBlock:
 
     def vote_on_segment(self, uuid, vote):
         """send vote on existing segment"""
-        user_id = self.get_sb_id().get("status")
+        user_id = self.get_sb_id()
         data = {
             "UUID": uuid,
             "userID": user_id,
@@ -134,7 +134,6 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         self.channel_id = False
         self.video_overwrites = video_overwrites
         self.video_type = video_type
-        self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.offline_import = False
 
     def build_json(self, youtube_meta_overwrite=False, media_path=False):
@@ -152,6 +151,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         self._add_stats()
         self.add_file_path()
         self.add_player(media_path)
+        self.add_streams(media_path)
         if self.config["downloads"]["integrate_ryd"]:
             self._get_ryd_stats()
 
@@ -227,69 +227,53 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
     def build_dl_cache_path(self):
         """find video path in dl cache"""
         cache_dir = self.app_conf["cache_dir"]
-        cache_path = f"{cache_dir}/download/"
-        all_cached = os.listdir(cache_path)
-        for file_cached in all_cached:
-            if self.youtube_id in file_cached:
-                vid_path = os.path.join(cache_path, file_cached)
-                return vid_path
+        video_id = self.json_data["youtube_id"]
+        cache_path = f"{cache_dir}/download/{video_id}.mp4"
+        if os.path.exists(cache_path):
+            return cache_path
+
+        channel_path = os.path.join(
+            self.app_conf["videos"],
+            self.json_data["channel"]["channel_id"],
+            f"{video_id}.mp4",
+        )
+        if os.path.exists(channel_path):
+            return channel_path
 
         raise FileNotFoundError
 
     def add_player(self, media_path=False):
         """add player information for new videos"""
-        vid_path = self._get_vid_path(media_path)
+        vid_path = media_path or self.build_dl_cache_path()
+        duration = get_duration_sec(vid_path)
 
-        duration_handler = DurationConverter()
-        duration = duration_handler.get_sec(vid_path)
-        duration_str = duration_handler.get_str(duration)
         self.json_data.update(
             {
                 "player": {
                     "watched": False,
                     "duration": duration,
-                    "duration_str": duration_str,
+                    "duration_str": get_duration_str(duration),
                 }
             }
         )
 
-    def _get_vid_path(self, media_path=False):
-        """get path of media file"""
-        if media_path:
-            return media_path
-
-        try:
-            # when indexing from download task
-            vid_path = self.build_dl_cache_path()
-        except FileNotFoundError as err:
-            # when reindexing needs to handle title rename
-            channel = os.path.split(self.json_data["media_url"])[0]
-            channel_dir = os.path.join(self.app_conf["videos"], channel)
-            all_files = os.listdir(channel_dir)
-            for file in all_files:
-                if self.youtube_id in file and file.endswith(".mp4"):
-                    vid_path = os.path.join(channel_dir, file)
-                    break
-            else:
-                raise FileNotFoundError("could not find video file") from err
-
-        return vid_path
+    def add_streams(self, media_path=False):
+        """add stream metadata"""
+        vid_path = media_path or self.build_dl_cache_path()
+        media = MediaStreamExtractor(vid_path)
+        self.json_data.update(
+            {
+                "streams": media.extract_metadata(),
+                "media_size": media.get_file_size(),
+            }
+        )
 
     def add_file_path(self):
         """build media_url for where file will be located"""
-        channel_name = self.json_data["channel"]["channel_name"]
-        clean_channel_name = clean_string(channel_name)
-        if len(clean_channel_name) <= 3:
-            # fall back to channel id
-            clean_channel_name = self.json_data["channel"]["channel_id"]
-
-        timestamp = self.json_data["published"].replace("-", "")
-        youtube_id = self.json_data["youtube_id"]
-        title = self.json_data["title"]
-        clean_title = clean_string(title)
-        filename = f"{timestamp}_{youtube_id}_{clean_title}.mp4"
-        media_url = os.path.join(clean_channel_name, filename)
-        self.json_data["media_url"] = media_url
+        self.json_data["media_url"] = os.path.join(
+            self.json_data["channel"]["channel_id"],
+            self.json_data["youtube_id"] + ".mp4",
+        )
 
     def delete_media_file(self):
         """delete video file, meta data"""

tubearchivist/home/src/index/video_streams.py

@@ -0,0 +1,81 @@
+"""extract metadata from video streams"""
+
+import json
+import subprocess
+from os import stat
+
+
+class MediaStreamExtractor:
+    """extract stream metadata"""
+
+    def __init__(self, media_path):
+        self.media_path = media_path
+        self.metadata = []
+
+    def extract_metadata(self):
+        """entry point to extract metadata"""
+
+        cmd = [
+            "ffprobe",
+            "-v",
+            "quiet",
+            "-print_format",
+            "json",
+            "-show_streams",
+            "-show_format",
+            self.media_path,
+        ]
+        result = subprocess.run(
+            cmd, capture_output=True, text=True, check=False
+        )
+
+        if result.returncode != 0:
+            return self.metadata
+
+        streams = json.loads(result.stdout).get("streams")
+        for stream in streams:
+            self.process_stream(stream)
+
+        return self.metadata
+
+    def process_stream(self, stream):
+        """parse stream to metadata"""
+        codec_type = stream.get("codec_type")
+        if codec_type == "video":
+            self._extract_video_metadata(stream)
+        elif codec_type == "audio":
+            self._extract_audio_metadata(stream)
+        else:
+            return
+
+    def _extract_video_metadata(self, stream):
+        """parse video metadata"""
+        if "bit_rate" not in stream:
+            # is probably thumbnail
+            return
+
+        self.metadata.append(
+            {
+                "type": "video",
+                "index": stream["index"],
+                "codec": stream["codec_name"],
+                "width": stream["width"],
+                "height": stream["height"],
+                "bitrate": int(stream["bit_rate"]),
+            }
+        )
+
+    def _extract_audio_metadata(self, stream):
+        """extract audio metadata"""
+        self.metadata.append(
+            {
+                "type": "audio",
+                "index": stream["index"],
+                "codec": stream.get("codec_name", "undefined"),
+                "bitrate": int(stream.get("bit_rate", 0)),
+            }
+        )
+
+    def get_file_size(self):
+        """get filesize in bytes"""
+        return stat(self.media_path).st_size

tubearchivist/home/src/ta/auth.py

@@ -0,0 +1,10 @@
+from django.conf import settings
+from django.contrib.auth.middleware import PersistentRemoteUserMiddleware
+
+
+class HttpRemoteUserMiddleware(PersistentRemoteUserMiddleware):
+    """This class allows authentication via HTTP_REMOTE_USER which is set for
+    example by certain SSO applications.
+    """
+
+    header = settings.TA_AUTH_PROXY_USERNAME_HEADER

tubearchivist/home/src/ta/config.py

@@ -8,6 +8,7 @@ import json
 import os
 import re
 from random import randint
+from time import sleep
 
 import requests
 from celery.schedules import crontab
@@ -16,12 +17,10 @@ from home.src.ta.ta_redis import RedisArchivist
 
 
 class AppConfig:
-    """handle user settings and application variables"""
+    """handle application variables"""
 
-    def __init__(self, user_id=False):
-        self.user_id = user_id
+    def __init__(self):
         self.config = self.get_config()
-        self.colors = self.get_colors()
 
     def get_config(self):
         """get config from default file or redis if changed"""
@@ -29,12 +28,6 @@ class AppConfig:
         if not config:
             config = self.get_config_file()
 
-        if self.user_id:
-            key = f"{self.user_id}:page_size"
-            page_size = RedisArchivist().get_message(key)["status"]
-            if page_size:
-                config["archive"]["page_size"] = page_size
-
         config["application"].update(self.get_config_env())
         return config
 
@@ -49,14 +42,12 @@ class AppConfig:
 
     @staticmethod
     def get_config_env():
-        """read environment application variables"""
-        es_pass = os.environ.get("ELASTIC_PASSWORD")
-        es_user = os.environ.get("ELASTIC_USER", default="elastic")
+        """read environment application variables.
+
+        Connection to ES is managed in ElasticWrap and the
+        connection to Redis is managed in RedisArchivist."""
 
         application = {
-            "REDIS_HOST": os.environ.get("REDIS_HOST"),
-            "es_url": os.environ.get("ES_URL"),
-            "es_auth": (es_user, es_pass),
             "HOST_UID": int(os.environ.get("HOST_UID", False)),
             "HOST_GID": int(os.environ.get("HOST_GID", False)),
             "enable_cast": bool(os.environ.get("ENABLE_CAST")),
@@ -67,11 +58,19 @@ class AppConfig:
     @staticmethod
     def get_config_redis():
         """read config json set from redis to overwrite defaults"""
-        config = RedisArchivist().get_message("config")
-        if not list(config.values())[0]:
-            return False
+        for i in range(10):
+            try:
+                config = RedisArchivist().get_message("config")
+                if not list(config.values())[0]:
+                    return False
 
-        return config
+                return config
+
+            except Exception:  # pylint: disable=broad-except
+                print(f"... Redis connection failed, retry [{i}/10]")
+                sleep(3)
+
+        raise ConnectionError("failed to connect to redis")
 
     def update_config(self, form_post):
         """update config values from settings form"""
@@ -91,33 +90,9 @@ class AppConfig:
             self.config[config_dict][config_value] = to_write
             updated.append((config_value, to_write))
 
-        RedisArchivist().set_message("config", self.config)
+        RedisArchivist().set_message("config", self.config, save=True)
         return updated
 
-    @staticmethod
-    def set_user_config(form_post, user_id):
-        """set values in redis for user settings"""
-        for key, value in form_post.items():
-            if not value:
-                continue
-
-            message = {"status": value}
-            redis_key = f"{user_id}:{key}"
-            RedisArchivist().set_message(redis_key, message)
-
-    def get_colors(self):
-        """overwrite config if user has set custom values"""
-        colors = False
-        if self.user_id:
-            col_dict = RedisArchivist().get_message(f"{self.user_id}:colors")
-            colors = col_dict["status"]
-
-        if not colors:
-            colors = self.config["application"]["colors"]
-
-        self.config["application"]["colors"] = colors
-        return colors
-
     @staticmethod
     def _build_rand_daily():
         """build random daily schedule per installation"""
@@ -175,6 +150,11 @@ class ScheduleBuilder:
         "version_check": "0 11 *",
     }
     CONFIG = ["check_reindex_days", "run_backup_rotate"]
+    NOTIFY = [
+        "update_subscribed_notify",
+        "download_pending_notify",
+        "check_reindex_notify",
+    ]
     MSG = "message:setting"
 
     def __init__(self):
@@ -191,10 +171,11 @@ class ScheduleBuilder:
                 except ValueError:
                     print(f"failed: {key} {value}")
                     mess_dict = {
-                        "status": self.MSG,
+                        "group": "setting:schedule",
                         "level": "error",
                         "title": "Scheduler update failed.",
-                        "message": "Invalid schedule input",
+                        "messages": ["Invalid schedule input"],
+                        "id": "0000",
                     }
                     RedisArchivist().set_message(
                         self.MSG, mess_dict, expire=True
@@ -204,12 +185,20 @@ class ScheduleBuilder:
                 redis_config["scheduler"][key] = to_write
             if key in self.CONFIG and value:
                 redis_config["scheduler"][key] = int(value)
-        RedisArchivist().set_message("config", redis_config)
+            if key in self.NOTIFY and value:
+                if value == "0":
+                    to_write = False
+                else:
+                    to_write = value
+                redis_config["scheduler"][key] = to_write
+
+        RedisArchivist().set_message("config", redis_config, save=True)
         mess_dict = {
-            "status": self.MSG,
+            "group": "setting:schedule",
             "level": "info",
             "title": "Scheduler changed.",
-            "message": "Please restart container for changes to take effect",
+            "messages": ["Restart container for changes to take effect"],
+            "id": "0000",
         }
         RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
 
@@ -317,8 +306,13 @@ class ReleaseVersion:
             RedisArchivist().set_message(self.NEW_KEY, message)
             print(f"[{self.local_version}]: found new version {new_version}")
 
+    def get_local_version(self):
+        """read version from local"""
+        return self.local_version
+
     def get_remote_version(self):
         """read version from remote"""
+        sleep(randint(0, 60))
         self.response = requests.get(self.REMOTE_URL, timeout=20).json()
         remote_version_str = self.response["release_version"]
         self.remote_version = self._parse_version(remote_version_str)

tubearchivist/home/src/ta/health.py

@@ -0,0 +1,11 @@
+from django.http import HttpResponse
+
+
+class HealthCheckMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if request.path == "/health":
+            return HttpResponse("ok")
+        return self.get_response(request)

tubearchivist/home/src/ta/helper.py

@@ -6,30 +6,23 @@ Loose collection of helper functions
 import json
 import os
 import random
-import re
 import string
 import subprocess
-import unicodedata
 from datetime import datetime
 from urllib.parse import urlparse
 
 import requests
 
 
-def clean_string(file_name):
-    """clean string to only asci characters"""
-    whitelist = "-_.() " + string.ascii_letters + string.digits
-    normalized = unicodedata.normalize("NFKD", file_name)
-    ascii_only = normalized.encode("ASCII", "ignore").decode().strip()
-    white_listed = "".join(c for c in ascii_only if c in whitelist)
-    cleaned = re.sub(r"[ ]{2,}", " ", white_listed)
-    return cleaned
-
-
-def ignore_filelist(filelist):
+def ignore_filelist(filelist: list[str]) -> list[str]:
     """ignore temp files for os.listdir sanitizer"""
-    to_ignore = ["Icon\r\r", "Temporary Items", "Network Trash Folder"]
-    cleaned = []
+    to_ignore = [
+        "@eaDir",
+        "Icon\r\r",
+        "Network Trash Folder",
+        "Temporary Items",
+    ]
+    cleaned: list[str] = []
     for file_name in filelist:
         if file_name.startswith(".") or file_name in to_ignore:
             continue
@@ -39,13 +32,13 @@ def ignore_filelist(filelist):
     return cleaned
 
 
-def randomizor(length):
+def randomizor(length: int) -> str:
     """generate random alpha numeric string"""
-    pool = string.digits + string.ascii_letters
+    pool: str = string.digits + string.ascii_letters
     return "".join(random.choice(pool) for i in range(length))
 
 
-def requests_headers():
+def requests_headers() -> dict[str, str]:
     """build header with random user agent for requests outside of yt-dlp"""
 
     chrome_versions = (
@@ -97,17 +90,17 @@ def requests_headers():
     return {"User-Agent": template}
 
 
-def date_praser(timestamp):
+def date_praser(timestamp: int | str) -> str:
     """return formatted date string"""
     if isinstance(timestamp, int):
         date_obj = datetime.fromtimestamp(timestamp)
     elif isinstance(timestamp, str):
         date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
 
-    return datetime.strftime(date_obj, "%d %b, %Y")
+    return date_obj.date().isoformat()
 
 
-def time_parser(timestamp):
+def time_parser(timestamp: str) -> float:
     """return seconds from timestamp, false on empty"""
     if not timestamp:
         return False
@@ -119,11 +112,11 @@ def time_parser(timestamp):
     return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
 
 
-def clear_dl_cache(config):
+def clear_dl_cache(config: dict) -> int:
     """clear leftover files from dl cache"""
     print("clear download cache")
     cache_dir = os.path.join(config["application"]["cache_dir"], "download")
-    leftover_files = os.listdir(cache_dir)
+    leftover_files = ignore_filelist(os.listdir(cache_dir))
     for cached in leftover_files:
         to_delete = os.path.join(cache_dir, cached)
         os.remove(to_delete)
@@ -131,15 +124,15 @@ def clear_dl_cache(config):
     return len(leftover_files)
 
 
-def get_mapping():
+def get_mapping() -> dict:
     """read index_mapping.json and get expected mapping and settings"""
     with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
-        index_config = json.load(f).get("index_config")
+        index_config: dict = json.load(f).get("index_config")
 
     return index_config
 
 
-def is_shorts(youtube_id):
+def is_shorts(youtube_id: str) -> bool:
     """check if youtube_id is a shorts video, bot not it it's not a shorts"""
     shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
     response = requests.head(
@@ -149,10 +142,57 @@ def is_shorts(youtube_id):
     return response.status_code == 200
 
 
-def ta_host_parser(ta_host):
+def get_duration_sec(file_path: str) -> int:
+    """get duration of media file from file path"""
+
+    duration = subprocess.run(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            file_path,
+        ],
+        capture_output=True,
+        check=True,
+    )
+    duration_raw = duration.stdout.decode().strip()
+    if duration_raw == "N/A":
+        return 0
+
+    duration_sec = int(float(duration_raw))
+    return duration_sec
+
+
+def get_duration_str(seconds: int) -> str:
+    """Return a human-readable duration string from seconds."""
+    if not seconds:
+        return "NA"
+
+    units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
+    duration_parts = []
+
+    for unit_label, unit_seconds in units:
+        if seconds >= unit_seconds:
+            unit_count, seconds = divmod(seconds, unit_seconds)
+            duration_parts.append(f"{unit_count}{unit_label}")
+
+    return " ".join(duration_parts)
+
+
+def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
     """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
-    allowed_hosts = []
-    csrf_trusted_origins = []
+    allowed_hosts: list[str] = [
+        "localhost",
+        "tubearchivist",
+    ]
+    csrf_trusted_origins: list[str] = [
+        "http://localhost",
+        "http://tubearchivist",
+    ]
     for host in ta_host.split():
         host_clean = host.strip()
         if not host_clean.startswith("http"):
@@ -163,54 +203,3 @@ def ta_host_parser(ta_host):
         csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}")
 
     return allowed_hosts, csrf_trusted_origins
-
-
-class DurationConverter:
-    """
-    using ffmpeg to get and parse duration from filepath
-    """
-
-    @staticmethod
-    def get_sec(file_path):
-        """read duration from file"""
-        duration = subprocess.run(
-            [
-                "ffprobe",
-                "-v",
-                "error",
-                "-show_entries",
-                "format=duration",
-                "-of",
-                "default=noprint_wrappers=1:nokey=1",
-                file_path,
-            ],
-            capture_output=True,
-            check=True,
-        )
-        duration_raw = duration.stdout.decode().strip()
-        if duration_raw == "N/A":
-            return 0
-
-        duration_sec = int(float(duration_raw))
-        return duration_sec
-
-    @staticmethod
-    def get_str(duration_sec):
-        """takes duration in sec and returns clean string"""
-        if not duration_sec:
-            # failed to extract
-            return "NA"
-
-        hours = duration_sec // 3600
-        minutes = (duration_sec - (hours * 3600)) // 60
-        secs = duration_sec - (hours * 3600) - (minutes * 60)
-
-        duration_str = str()
-        if hours:
-            duration_str = str(hours).zfill(2) + ":"
-        if minutes:
-            duration_str = duration_str + str(minutes).zfill(2) + ":"
-        else:
-            duration_str = duration_str + "00:"
-        duration_str = duration_str + str(secs).zfill(2)
-        return duration_str

tubearchivist/home/src/ta/notify.py

@@ -0,0 +1,55 @@
+"""send notifications using apprise"""
+
+import apprise
+from home.src.ta.config import AppConfig
+from home.src.ta.task_manager import TaskManager
+
+
+class Notifications:
+    """notification handler"""
+
+    def __init__(self, name: str, task_id: str, task_title: str):
+        self.name: str = name
+        self.task_id: str = task_id
+        self.task_title: str = task_title
+
+    def send(self) -> None:
+        """send notifications"""
+        apobj = apprise.Apprise()
+        hooks: str | None = self.get_url()
+        if not hooks:
+            return
+
+        hook_list: list[str] = self.parse_hooks(hooks=hooks)
+        title, body = self.build_message()
+
+        if not body:
+            return
+
+        for hook in hook_list:
+            apobj.add(hook)
+
+        apobj.notify(body=body, title=title)
+
+    def get_url(self) -> str | None:
+        """get apprise urls for task"""
+        config = AppConfig().config
+        hooks: str = config["scheduler"].get(f"{self.name}_notify")
+
+        return hooks
+
+    def parse_hooks(self, hooks: str) -> list[str]:
+        """create list of hooks"""
+
+        hook_list: list[str] = [i.strip() for i in hooks.split()]
+
+        return hook_list
+
+    def build_message(self) -> tuple[str, str | None]:
+        """build message to send notification"""
+        task = TaskManager().get_task(self.task_id)
+        status = task.get("status")
+        title: str = f"[TA] {self.task_title} process ended with {status}"
+        body: str | None = task.get("result")
+
+        return title, body

tubearchivist/home/src/ta/ta_redis.py

@@ -41,6 +41,7 @@ class RedisArchivist(RedisBase):
         message: dict,
         path: str = ".",
         expire: bool | int = False,
+        save: bool = False,
     ) -> None:
         """write new message to redis"""
         self.conn.execute_command(
@@ -54,6 +55,16 @@ class RedisArchivist(RedisBase):
                 secs = expire
             self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
 
+        if save:
+            self.bg_save()
+
+    def bg_save(self) -> None:
+        """save to aof"""
+        try:
+            self.conn.bgsave()
+        except redis.exceptions.ResponseError:
+            pass
+
     def get_message(self, key: str) -> dict:
         """get message dict from redis"""
         reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)

tubearchivist/home/src/ta/urlparser.py

@@ -92,7 +92,7 @@ class Parser:
             item_type = "video"
         elif len_id_str == 24:
             item_type = "channel"
-        elif len_id_str in (34, 18):
+        elif len_id_str in (34, 26, 18):
             item_type = "playlist"
         else:
             raise ValueError(f"not a valid id_str: {id_str}")

tubearchivist/home/src/ta/users.py

@@ -0,0 +1,104 @@
+"""
+Functionality:
+- read and write user config backed by ES
+- encapsulate persistence of user properties
+"""
+
+from typing import TypedDict
+
+from home.src.es.connect import ElasticWrap
+
+
+class UserConfigType(TypedDict, total=False):
+    """describes the user configuration"""
+
+    colors: str
+    page_size: int
+    sort_by: str
+    sort_order: str
+    view_style_home: str
+    view_style_channel: str
+    view_style_downloads: str
+    view_style_playlist: str
+    grid_items: int
+    hide_watched: bool
+    show_ignored_only: bool
+    show_subed_only: bool
+    sponsorblock_id: str
+
+
+class UserConfig:
+    """Handle settings for an individual user
+
+    Create getters and setters for usage in the application.
+    Although tedious it helps prevents everything caring about how properties
+    are persisted. Plus it allows us to save anytime any value is set.
+    """
+
+    _DEFAULT_USER_SETTINGS = UserConfigType(
+        colors="dark",
+        page_size=12,
+        sort_by="published",
+        sort_order="desc",
+        view_style_home="grid",
+        view_style_channel="list",
+        view_style_downloads="list",
+        view_style_playlist="grid",
+        grid_items=3,
+        hide_watched=False,
+        show_ignored_only=False,
+        show_subed_only=False,
+        sponsorblock_id=None,
+    )
+
+    def __init__(self, user_id: str):
+        self._user_id: str = user_id
+        self._config: UserConfigType = self._get_config()
+
+    def get_value(self, key: str):
+        """Get the given key from the users configuration
+
+        Throws a KeyError if the requested Key is not a permitted value"""
+        if key not in self._DEFAULT_USER_SETTINGS:
+            raise KeyError(f"Unable to read config for unknown key '{key}'")
+
+        return self._config.get(key) or self._DEFAULT_USER_SETTINGS.get(key)
+
+    def set_value(self, key: str, value: str | bool | int):
+        """Set or replace a configuration value for the user
+
+        Throws a KeyError if the requested Key is not a permitted value"""
+        if not self._user_id:
+            raise ValueError("Unable to persist config for null user_id")
+
+        if key not in self._DEFAULT_USER_SETTINGS:
+            raise KeyError(f"Unable to persist config for unknown key '{key}'")
+
+        old = self.get_value(key)
+        self._config[key] = value
+
+        # Upsert this property (creating a record if not exists)
+        es_payload = {"doc": {"config": {key: value}}, "doc_as_upsert": True}
+        es_document_path = f"ta_config/_update/user_{self._user_id}"
+        response, status = ElasticWrap(es_document_path).post(es_payload)
+        if status < 200 or status > 299:
+            raise ValueError(f"Failed storing user value {status}: {response}")
+
+        print(f"User {self._user_id} value '{key}' change: {old} > {value}")
+
+    def _get_config(self) -> UserConfigType:
+        """get config from ES or load from the application defaults"""
+        if not self._user_id:
+            # this is for a non logged-in user so use all the defaults
+            return {}
+
+        # Does this user have configuration stored in ES
+        es_document_path = f"ta_config/_doc/user_{self._user_id}"
+        response, status = ElasticWrap(es_document_path).get(print_error=False)
+        if status == 200 and "_source" in response.keys():
+            source = response.get("_source")
+            if "config" in source.keys():
+                return source.get("config")
+
+        # There is no config in ES
+        return {}

tubearchivist/home/tasks.py

@@ -19,12 +19,14 @@ from home.src.download.yt_dlp_handler import VideoDownloader
 from home.src.es.backup import ElasticBackup
 from home.src.es.index_setup import ElasitIndexWrap
 from home.src.index.channel import YoutubeChannel
-from home.src.index.filesystem import Filesystem
+from home.src.index.filesystem import Scanner
 from home.src.index.manual import ImportFolderScanner
 from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
 from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
+from home.src.ta.notify import Notifications
 from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.task_manager import TaskManager
+from home.src.ta.urlparser import Parser
 
 CONFIG = AppConfig().config
 REDIS_HOST = os.environ.get("REDIS_HOST")
@@ -112,7 +114,7 @@ class BaseTask(Task):
         """callback for task failure"""
         print(f"{task_id} Failed callback")
         message, key = self._build_message(level="error")
-        message.update({"messages": ["Task failed"]})
+        message.update({"messages": [f"Task failed: {exc}"]})
         RedisArchivist().set_message(key, message, expire=20)
 
     def on_success(self, retval, task_id, args, kwargs):
@@ -129,6 +131,12 @@ class BaseTask(Task):
         message.update({"messages": ["New task received."]})
         RedisArchivist().set_message(key, message)
 
+    def after_return(self, status, retval, task_id, args, kwargs, einfo):
+        """callback after task returns"""
+        print(f"{task_id} return callback")
+        task_title = self.TASK_CONFIG.get(self.name).get("title")
+        Notifications(self.name, task_id, task_title).send()
+
     def send_progress(self, message_lines, progress=False, title=False):
         """send progress message"""
         message, key = self._build_message()
@@ -168,38 +176,55 @@ def update_subscribed(self):
     if manager.is_pending(self):
         print(f"[task][{self.name}] rescan already running")
         self.send_progress("Rescan already in progress.")
-        return
+        return None
 
     manager.init(self)
-    missing_videos = SubscriptionScanner(task=self).scan()
+    handler = SubscriptionScanner(task=self)
+    missing_videos = handler.scan()
+    auto_start = handler.auto_start
     if missing_videos:
         print(missing_videos)
-        extrac_dl.delay(missing_videos)
+        extrac_dl.delay(missing_videos, auto_start=auto_start)
+        message = f"Found {len(missing_videos)} videos to add to the queue."
+        return message
+
+    return None
 
 
 @shared_task(name="download_pending", bind=True, base=BaseTask)
-def download_pending(self, from_queue=True):
+def download_pending(self, auto_only=False):
     """download latest pending videos"""
     manager = TaskManager()
     if manager.is_pending(self):
         print(f"[task][{self.name}] download queue already running")
         self.send_progress("Download Queue is already running.")
-        return
+        return None
 
     manager.init(self)
     downloader = VideoDownloader(task=self)
-    if from_queue:
-        downloader.add_pending()
-    downloader.run_queue()
+    videos_downloaded = downloader.run_queue(auto_only=auto_only)
+
+    if videos_downloaded:
+        return f"downloaded {len(videos_downloaded)} videos."
+
+    return None
 
 
 @shared_task(name="extract_download", bind=True, base=BaseTask)
-def extrac_dl(self, youtube_ids):
+def extrac_dl(self, youtube_ids, auto_start=False):
     """parse list passed and add to pending"""
     TaskManager().init(self)
-    pending_handler = PendingList(youtube_ids=youtube_ids, task=self)
+    if isinstance(youtube_ids, str):
+        to_add = Parser(youtube_ids).parse()
+    else:
+        to_add = youtube_ids
+
+    pending_handler = PendingList(youtube_ids=to_add, task=self)
     pending_handler.parse_url_list()
-    pending_handler.add_to_pending()
+    pending_handler.add_to_pending(auto_start=auto_start)
+
+    if auto_start:
+        download_pending.delay(auto_only=True)
 
 
 @shared_task(bind=True, name="check_reindex", base=BaseTask)
@@ -227,7 +252,10 @@ def check_reindex(self, data=False, extract_videos=False):
         self.send_progress("Add outdated documents to the reindex Queue.")
         populate.add_outdated()
 
-    Reindex(task=self).reindex_all()
+    handler = Reindex(task=self)
+    handler.reindex_all()
+
+    return handler.build_message()
 
 
 @shared_task(bind=True, name="manual_import", base=BaseTask)
@@ -282,7 +310,9 @@ def rescan_filesystem(self):
         return
 
     manager.init(self)
-    Filesystem(task=self).process()
+    handler = Scanner(task=self)
+    handler.scan()
+    handler.apply()
     ThumbValidator(task=self).validate()
 
 
@@ -313,9 +343,12 @@ def re_sync_thumbs(self):
 
 
 @shared_task(bind=True, name="subscribe_to", base=BaseTask)
-def subscribe_to(self, url_str):
-    """take a list of urls to subscribe to"""
-    SubscriptionHandler(url_str, task=self).subscribe()
+def subscribe_to(self, url_str: str, expected_type: str | bool = False):
+    """
+    take a list of urls to subscribe to
+    optionally validate expected_type channel / playlist
+    """
+    SubscriptionHandler(url_str, task=self).subscribe(expected_type)
 
 
 @shared_task(bind=True, name="index_playlists", base=BaseTask)

tubearchivist/home/templates/home/base_settings.html

@@ -0,0 +1,17 @@
+{# Base file for all of the settings pages to ensure a common menu #}
+{% extends "home/base.html" %}
+{% load static %}
+{% block content %}
+<div class="boxed-content">
+    <div class="info-box-item child-page-nav">
+        <a href="{% url 'settings' %}"><h3>Dashboard</h3></a>
+        <a href="{% url 'settings_user' %}"><h3>User</h3></a>
+        <a href="{% url 'settings_application' %}"><h3>Application</h3></a>
+        <a href="{% url 'settings_scheduling' %}"><h3>Scheduling</h3></a>
+        <a href="{% url 'settings_actions' %}"><h3>Actions</h3></a>
+    </div>
+    <div id="notifications" data=""></div>
+    {% block settings_content %}{% endblock %}
+</div>
+<script type="text/javascript" src="{% static 'progress.js' %}"></script>
+{% endblock content %}

tubearchivist/home/templates/home/channel.html

@@ -42,33 +42,33 @@
             {% for channel in results %}
                 <div class="channel-item {{ view_style }}">
                     <div class="channel-banner {{ view_style }}">
-                        <a href="{% url 'channel_id' channel.source.channel_id %}">
-                            <img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
+                        <a href="{% url 'channel_id' channel.channel_id %}">
+                            <img src="/cache/channels/{{ channel.channel_id }}_banner.jpg" alt="{{ channel.channel_id }}-banner">
                         </a>
                     </div>
                     <div class="info-box info-box-2 {{ view_style }}">
                         <div class="info-box-item">
                             <div class="round-img">
-                                <a href="{% url 'channel_id' channel.source.channel_id %}">
-                                    <img src="/cache/channels/{{ channel.source.channel_id }}_thumb.jpg" alt="channel-thumb">
+                                <a href="{% url 'channel_id' channel.channel_id %}">
+                                    <img src="/cache/channels/{{ channel.channel_id }}_thumb.jpg" alt="channel-thumb">
                                 </a>
                             </div>
                             <div>
-                                <h3><a href="{% url 'channel_id' channel.source.channel_id %}">{{ channel.source.channel_name }}</a></h3>
-                                {% if channel.source.channel_subs >= 1000000 %}
-                                    <p>Subscribers: {{ channel.source.channel_subs|intword }}</p>
+                                <h3><a href="{% url 'channel_id' channel.channel_id %}">{{ channel.channel_name }}</a></h3>
+                                {% if channel.channel_subs >= 1000000 %}
+                                    <p>Subscribers: {{ channel.channel_subs|intword }}</p>
                                 {% else %}
-                                    <p>Subscribers: {{ channel.source.channel_subs|intcomma }}</p>
+                                    <p>Subscribers: {{ channel.channel_subs|intcomma }}</p>
                                 {% endif %}
                             </div>
                         </div>
                         <div class="info-box-item">
                             <div>
-                                <p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
-                                {% if channel.source.channel_subscribed %}
-                                    <button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
+                                <p>Last refreshed: {{ channel.channel_last_refresh }}</p>
+                                {% if channel.channel_subscribed %}
+                                    <button class="unsubscribe" type="button" data-type="channel" data-subscribe="" data-id="{{ channel.channel_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ channel.channel_name }}">Unsubscribe</button>
                                 {% else %}
-                                    <button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
+                                    <button type="button" data-type="channel" data-subscribe="true" data-id="{{ channel.channel_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ channel.channel_name }}">Subscribe</button>
                                 {% endif %}
                             </div>
                         </div>

tubearchivist/home/templates/home/channel_id.html

@@ -6,7 +6,7 @@
     <div class="channel-banner">
         <a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
     </div>
-    <div class="info-box-item channel-nav">
+    <div class="info-box-item child-page-nav">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
         {% if has_streams %}
             <a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
@@ -38,19 +38,20 @@
                     <p>Subscribers: {{ channel_info.channel_subs|intcomma }}</p>
                 {% endif %}
                 {% if channel_info.channel_subscribed %}
-                    <button class="unsubscribe" type="button" id="{{ channel_info.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel_info.channel_name }}">Unsubscribe</button>
+                    <button class="unsubscribe" type="button" data-type="channel" data-subscribe="" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ channel_info.channel_name }}">Unsubscribe</button>
                 {% else %}
-                    <button type="button" id="{{ channel_info.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel_info.channel_name }}">Subscribe</button>
+                    <button type="button" data-type="channel" data-subscribe="true" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ channel_info.channel_name }}">Subscribe</button>
                 {% endif %}
             </div>
         </div>
         <div class="info-box-item">
-            <div>
-                {% if max_hits %}
-                    <p>Total Videos: {{ max_hits }}</p>
+            {% if aggs %}
+                <p>{{ aggs.total_items.value }} videos <span class="space-carrot">|</span> {{ aggs.total_duration.value_str }} playback <span class="space-carrot">|</span> Total size {{ aggs.total_size.value|filesizeformat }}</p>
+                <div class="button-box">
                     <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button>
-                {% endif %}
-            </div>
+                    <button title="Mark all videos from {{ channel_info.channel_name }} as unwatched" type="button" id="unwatched-button" data-id="{{ channel_info.channel_id }}" onclick="isUnwatchedButton(this)">Mark as unwatched</button>
+                </div>
+            {% endif %}
         </div>
     </div>
 </div>
@@ -105,14 +106,14 @@
         {% if results %}
             {% for video in results %}
                 <div class="video-item {{ view_style }}">
-                    <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
+                    <a href="#player" data-id="{{ video.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
-                                {% if video.source.player.progress %}
-                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                <img src="{{ video.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: {{video.player.progress}}%;"></div>
                                 {% else %}
-                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -121,16 +122,16 @@
                         </div>
                     </a>
                     <div class="video-desc {{ view_style }}">
-                        <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
-                            {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
+                        <div class="video-desc-player" id="video-info-{{ video.youtube_id }}">
+                            {% if video.player.watched %}
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
-                            <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
+                            <span>{{ video.published }} | {{ video.player.duration_str }}</span>
                         </div>
                         <div>
-                            <a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
+                            <a class="video-more" href="{% url 'video' video.youtube_id %}"><h2>{{ video.title }}</h2></a>
                         </div>
                     </div>
                 </div>

tubearchivist/home/templates/home/channel_id_about.html

@@ -6,7 +6,7 @@
     <div class="channel-banner">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
     </div>
-    <div class="info-box-item channel-nav">
+    <div class="info-box-item child-page-nav">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
         {% if has_streams %}
             <a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
@@ -81,11 +81,18 @@
             <button onclick="textExpand()" id="text-expand-button">Show more</button>
         </div>
     {% endif %}
-    <div class="description-box">
-        <h2>Customize {{ channel_info.channel_name }}</h2>
-    </div>
+    {% if channel_info.channel_tags %}
+        <div class="description-box">
+            <div class="video-tag-box">
+                {% for tag in channel_info.channel_tags %}
+                    <span class="video-tag">{{ tag }}</span>
+                {% endfor %}
+            </div>
+        </div>
+    {% endif %}
     <div id="overwrite-form" class="info-box">
         <div class="info-box-item">
+            <h2>Customize {{ channel_info.channel_name }}</h2>
             <form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/about/" method="POST">
                 {% csrf_token %}
                 <div class="overwrite-form-item">

tubearchivist/home/templates/home/channel_id_playlist.html

@@ -6,7 +6,7 @@
     <div class="channel-banner">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
     </div>
-    <div class="info-box-item channel-nav">
+    <div class="info-box-item child-page-nav">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
         {% if has_streams %}
             <a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
@@ -45,18 +45,18 @@
             {% for playlist in results %}
             <div class="playlist-item {{ view_style }}">
                 <div class="playlist-thumbnail">
-                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}">
-                        <img src="/cache/playlists/{{ playlist.source.playlist_id }}.jpg" alt="{{ playlist.source.playlist_id }}-thumbnail">
+                    <a href="{% url 'playlist_id' playlist.playlist_id %}">
+                        <img src="/cache/playlists/{{ playlist.playlist_id }}.jpg" alt="{{ playlist.playlist_id }}-thumbnail">
                     </a>
                 </div>
                 <div class="playlist-desc {{ view_style }}">
-                    <a href="{% url 'channel_id' playlist.source.playlist_channel_id %}"><h3>{{ playlist.source.playlist_channel }}</h3></a>
-                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
-                    <p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
-                    {% if playlist.source.playlist_subscribed %}
-                        <button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
+                    <a href="{% url 'channel_id' playlist.playlist_channel_id %}"><h3>{{ playlist.playlist_channel }}</h3></a>
+                    <a href="{% url 'playlist_id' playlist.playlist_id %}"><h2>{{ playlist.playlist_name }}</h2></a>
+                    <p>Last refreshed: {{ playlist.playlist_last_refresh }}</p>
+                    {% if playlist.playlist_subscribed %}
+                        <button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist.playlist_name }}">Unsubscribe</button>
                     {% else %}
-                        <button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
+                        <button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist.playlist_name }}">Subscribe</button>
                     {% endif %}
                 </div>
             </div>

tubearchivist/home/templates/home/downloads.html

@@ -20,11 +20,12 @@
             <img id="animate-icon" onclick="showForm()" src="{% static 'img/icon-add.svg' %}" alt="add-icon">
             <p>Add to download queue</p>
             <div class="show-form">
-                <form id='hidden-form' action="/downloads/" method="post">
+                <div id='hidden-form' novalidate>
                     {% csrf_token %}
                     {{ add_form }}
-                    <button type="submit">Add to download queue</button>
-                </form>
+                    <button onclick="addToQueue()">Add to queue</button>
+                    <button onclick="addToQueue(true)">Download now</button>
+                </div>
             </div>
         </div>
     </div>
@@ -69,37 +70,46 @@
     <div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
         {% if results %}
             {% for video in results %}
-                <div class="video-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
+                <div class="video-item {{ view_style }}" id="dl-{{ video.youtube_id }}">
                     <div class="video-thumb-wrap {{ view_style }}">
                         <div class="video-thumb">
-                            <img src="{{ video.source.vid_thumb_url }}" alt="video_thumb">
+                            <img src="{{ video.vid_thumb_url }}" alt="video_thumb">
                             <div class="video-tags">
                                 {% if show_ignored_only %}
                                     <span>ignored</span>
                                 {% else %}
                                     <span>queued</span>
                                 {% endif %}
-                                <span>{{ video.source.vid_type }}</span>
+                                <span>{{ video.vid_type }}</span>
+                                {% if video.auto_start %}
+                                    <span>auto</span>
+                                {% endif %}
                             </div>
                         </div>
                     </div>
                     <div class="video-desc {{ view_style }}">
                         <div>
-                            {% if video.source.channel_indexed %}
-                            <a href="{% url 'channel_id' video.source.channel_id %}">{{ video.source.channel_name }}</a>
+                            {% if video.channel_indexed %}
+                            <a href="{% url 'channel_id' video.channel_id %}">{{ video.channel_name }}</a>
                             {% else %}
-                                <span>{{ video.source.channel_name }}</span>
+                                <span>{{ video.channel_name }}</span>
                             {% endif %}
-                            <a href="https://www.youtube.com/watch?v={{ video.source.youtube_id }}" target="_blank"><h3>{{ video.source.title }}</h3></a>
+                            <a href="https://www.youtube.com/watch?v={{ video.youtube_id }}" target="_blank"><h3>{{ video.title }}</h3></a>
                         </div>
-                        <p>Published: {{ video.source.published }} | Duration: {{ video.source.duration }} | {{ video.source.youtube_id }}</p>
+                        <p>Published: {{ video.published }} | Duration: {{ video.duration }} | {{ video.youtube_id }}</p>
+                        {% if video.message %}
+                            <p class="danger-zone">{{ video.message }}</p>
+                        {% endif %}
                         <div>
                             {% if show_ignored_only %}
-                                <button data-id="{{ video.source.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>
-                                <button data-id="{{ video.source.youtube_id }}" onclick="addSingle(this)">Add to queue</button>
+                                <button data-id="{{ video.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>
+                                <button data-id="{{ video.youtube_id }}" onclick="addSingle(this)">Add to queue</button>
                             {% else %}
-                                <button data-id="{{ video.source.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
-                                <button id="{{ video.source.youtube_id }}" data-id="{{ video.source.youtube_id }}" onclick="downloadNow(this)">Download now</button>
+                                <button data-id="{{ video.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
+                                <button id="{{ video.youtube_id }}" data-id="{{ video.youtube_id }}" onclick="downloadNow(this)">Download now</button>
+                            {% endif %}
+                            {% if video.message %}
+                                <button class="danger-button" data-id="{{ video.youtube_id }}" onclick="forgetIgnore(this)">Delete</button>
                             {% endif %}
                         </div>
                     </div>

tubearchivist/home/templates/home/home.html

@@ -9,14 +9,14 @@
         <div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
             {% for video in continue_vids %}
                 <div class="video-item {{ view_style }}">
-                    <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
+                    <a href="#player" data-id="{{ video.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
-                                {% if video.source.player.progress %}
-                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                <img src="{{ video.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: {{video.player.progress}}%;"></div>
                                 {% else %}
-                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -25,17 +25,17 @@
                         </div>
                     </a>
                     <div class="video-desc {{ view_style }}">
-                        <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
-                            {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
+                        <div class="video-desc-player" id="video-info-{{ video.youtube_id }}">
+                            {% if video.player.watched %}
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
-                            <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
+                            <span>{{ video.published }} | {{ video.player.duration_str }}</span>
                         </div>
                         <div>
-                            <a href="{% url 'channel_id' video.source.channel.channel_id %}"><h3>{{ video.source.channel.channel_name }}</h3></a>
-                            <a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
+                            <a href="{% url 'channel_id' video.channel.channel_id %}"><h3>{{ video.channel.channel_name }}</h3></a>
+                            <a class="video-more" href="{% url 'video' video.youtube_id %}"><h2>{{ video.title }}</h2></a>
                         </div>
                     </div>
                 </div>
@@ -95,14 +95,14 @@
         {% if results %}
             {% for video in results %}
                 <div class="video-item {{ view_style }}">
-                    <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
+                    <a href="#player" data-id="{{ video.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
-                                {% if video.source.player.progress %}
-                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                <img src="{{ video.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: {{video.player.progress}}%;"></div>
                                 {% else %}
-                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -111,17 +111,17 @@
                         </div>
                     </a>
                     <div class="video-desc {{ view_style }}">
-                        <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
-                            {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
+                        <div class="video-desc-player" id="video-info-{{ video.youtube_id }}">
+                            {% if video.player.watched %}
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
-                            <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
+                            <span>{{ video.published }} | {{ video.player.duration_str }}</span>
                         </div>
                         <div>
-                            <a href="{% url 'channel_id' video.source.channel.channel_id %}"><h3>{{ video.source.channel.channel_name }}</h3></a>
-                            <a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
+                            <a href="{% url 'channel_id' video.channel.channel_id %}"><h3>{{ video.channel.channel_name }}</h3></a>
+                            <a class="video-more" href="{% url 'video' video.youtube_id %}"><h2>{{ video.title }}</h2></a>
                         </div>
                     </div>
                 </div>

tubearchivist/home/templates/home/playlist.html

@@ -40,18 +40,18 @@
             {% for playlist in results %}
             <div class="playlist-item {{ view_style }}">
                 <div class="playlist-thumbnail">
-                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}">
-                        <img src="/cache/playlists/{{ playlist.source.playlist_id }}.jpg" alt="{{ playlist.source.playlist_id }}-thumbnail">
+                    <a href="{% url 'playlist_id' playlist.playlist_id %}">
+                        <img src="/cache/playlists/{{ playlist.playlist_id }}.jpg" alt="{{ playlist.playlist_id }}-thumbnail">
                     </a>
                 </div>
                 <div class="playlist-desc {{ view_style }}">
-                    <a href="{% url 'channel_id' playlist.source.playlist_channel_id %}"><h3>{{ playlist.source.playlist_channel }}</h3></a>
-                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
-                    <p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
-                    {% if playlist.source.playlist_subscribed %}
-                        <button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
+                    <a href="{% url 'channel_id' playlist.playlist_channel_id %}"><h3>{{ playlist.playlist_channel }}</h3></a>
+                    <a href="{% url 'playlist_id' playlist.playlist_id %}"><h2>{{ playlist.playlist_name }}</h2></a>
+                    <p>Last refreshed: {{ playlist.playlist_last_refresh }}</p>
+                    {% if playlist.playlist_subscribed %}
+                        <button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist.playlist_name }}">Unsubscribe</button>
                     {% else %}
-                        <button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
+                        <button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist.playlist_name }}">Subscribe</button>
                     {% endif %}
                 </div>
             </div>

tubearchivist/home/templates/home/playlist_id.html

@@ -27,9 +27,9 @@
                 <p>Last refreshed: {{ playlist_info.playlist_last_refresh }}</p>
                 <p>Playlist:
                     {% if playlist_info.playlist_subscribed %}
-                        <button class="unsubscribe" type="button" id="{{ playlist_info.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist_info.playlist_name }}">Unsubscribe</button>
+                        <button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist_info.playlist_name }}">Unsubscribe</button>
                     {% else %}
-                        <button type="button" id="{{ playlist_info.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist_info.playlist_name }}">Subscribe</button>
+                        <button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist_info.playlist_name }}">Subscribe</button>
                     {% endif %}
                 </p>
                 {% if playlist_info.playlist_active %}
@@ -40,8 +40,8 @@
                 <button onclick="deleteConfirm()" id="delete-item">Delete Playlist</button>
                 <div class="delete-confirm" id="delete-button">
                     <span>Delete {{ playlist_info.playlist_name }}?</span>
-                    <button onclick="deletePlaylist(this)" data-action="metadata" data-id="{{ playlist_info.playlist_id }}">Delete metadata</button>
-                    <button onclick="deletePlaylist(this)" data-action="all" class="danger-button" data-id="{{ playlist_info.playlist_id }}">Delete all</button><br>
+                    <button onclick="deletePlaylist(this)" data-action="" data-id="{{ playlist_info.playlist_id }}">Delete metadata</button>
+                    <button onclick="deletePlaylist(this)" data-action="delete-videos" class="danger-button" data-id="{{ playlist_info.playlist_id }}">Delete all</button><br>
                     <button onclick="cancelDelete()">Cancel</button>
                 </div>
             </div>
@@ -50,7 +50,10 @@
             <div>
                 {% if max_hits %}
                     <p>Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}</p>
-                    <p>Watched: <button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
+                    <div id="watched-button" class="button-box">
+                        <button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button>
+                        <button title="Mark all videos from {{ playlist_info.playlist_name }} as unwatched" type="button" id="unwatched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isUnwatchedButton(this)">Mark as unwatched</button>
+                    </div>
                 {% endif %}
                 {% if reindex %}
                     <p>Reindex scheduled</p>
@@ -107,14 +110,14 @@
         {% if results %}
             {% for video in results %}
                 <div class="video-item {{ view_style }}">
-                    <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
+                    <a href="#player" data-id="{{ video.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
-                                {% if video.source.player.progress %}
-                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                <img src="{{ video.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: {{video.player.progress}}%;"></div>
                                 {% else %}
-                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -123,16 +126,16 @@
                         </div>
                     </a>
                     <div class="video-desc {{ view_style }}">
-                        <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
-                            {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
+                        <div class="video-desc-player" id="video-info-{{ video.youtube_id }}">
+                            {% if video.player.watched %}
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
-                            <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
+                            <span>{{ video.published }} | {{ video.player.duration_str }}</span>
                         </div>
                         <div>
-                            <a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
+                            <a class="video-more" href="{% url 'video' video.youtube_id %}"><h2>{{ video.title }}</h2></a>
                         </div>
                     </div>
                 </div>

tubearchivist/home/templates/home/settings.html

@@ -1,387 +1,68 @@
-{% extends "home/base.html" %}
+{% extends "home/base_settings.html" %}
 {% load static %}
-{% block content %}
-<div class="boxed-content">
-    <div id="notifications" data="setting reindex"></div>
-    <div class="title-bar">
-        <h1>User Configurations</h1>
-    </div>
-    <form action="/settings/" method="POST" name="user-update">
-        {% csrf_token %}
-        <div class="settings-group">
-            <h2>Color scheme</h2>
-            <div class="settings-item">
-                <p>Current color scheme: <span class="settings-current">{{ config.application.colors }}</span></p>
-                <i>Select your preferred color scheme between dark and light mode.</i><br>
-                {{ user_form.colors }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>Archive View</h2>
-            <div class="settings-item">
-                <p>Current page size: <span class="settings-current">{{ config.archive.page_size }}</span></p>
-                <i>Result of videos showing in archive page</i><br>
-                {{ user_form.page_size }}
-            </div>
-        </div>
-        <button type="submit" name="user-settings">Update User Configurations</button>
-    </form>
-    <div class="title-bar">
-        <h1>Application Configurations</h1>
-    </div>
-    <form action="/settings/" method="POST" name="application-update">
-        {% csrf_token %}
-        <div class="settings-group">
-            <h2 id="subscriptions">Subscriptions</h2>
-            <p>Disable shorts or streams by setting their page size to 0 (zero).</p>
-            <div class="settings-item">
-                <p>YouTube page size: <span class="settings-current">{{ config.subscriptions.channel_size }}</span></p>
-                <i>Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
-                {{ app_form.subscriptions_channel_size }}
-            </div>
-            <div class="settings-item">
-                <p>YouTube Live page size: <span class="settings-current">{{ config.subscriptions.live_channel_size }}</span></p>
-                <i>Live Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
-                {{ app_form.subscriptions_live_channel_size }}
-            </div>
-            <div class="settings-item">
-                <p>YouTube Shorts page size: <span class="settings-current">{{ config.subscriptions.shorts_channel_size }}</span></p>
-                <i>Shorts Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
-                {{ app_form.subscriptions_shorts_channel_size }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="downloads">Downloads</h2>
-            <div class="settings-item">
-                <p>Current download limit: <span class="settings-current">{{ config.downloads.limit_count }}</span></p>
-                <i>Limit the number of videos getting downloaded on every run. 0 (zero) to deactivate.</i><br>
-                {{ app_form.downloads_limit_count }}
-            </div>
-            <div class="settings-item">
-                <p>Current download speed limit in KB/s: <span class="settings-current">{{ config.downloads.limit_speed }}</span></p>
-                <i>Limit download speed. 0 (zero) to deactivate, e.g. 1000 (1MB/s). Speeds are in KB/s. Setting takes effect on new download jobs or application restart.</i><br>
-                {{ app_form.downloads_limit_speed }}
-            </div>
-            <div class="settings-item">
-                <p>Current throttled rate limit in KB/s: <span class="settings-current">{{ config.downloads.throttledratelimit }}</span></p>
-                <i>Download will restart if speeds drop below specified amount. 0 (zero) to deactivate, e.g. 100. Speeds are in KB/s.</i><br>
-                {{ app_form.downloads_throttledratelimit }}
-            </div>
-            <div class="settings-item">
-                <p>Current scraping sleep interval: <span class="settings-current">{{ config.downloads.sleep_interval }}</p>
-                <i>Seconds to sleep between calls to YouTube. Might be necessary to avoid throttling. Recommended 3.</i><br>
-                {{ app_form.downloads_sleep_interval }}
-            </div>
-            <div class="settings-item">
-                <p><span class="danger-zone">Danger Zone</span>: Current auto delete watched videos: <span class="settings-current">{{ config.downloads.autodelete_days }}</span></p>
-                <i>Auto delete watched videos after x days, 0 (zero) to deactivate:</i><br>
-                {{ app_form.downloads_autodelete_days }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="format">Download Format</h2>
-            <div class="settings-item">
-                <p>Limit video and audio quality format for yt-dlp.<br>
-                    Currently: <span class="settings-current">{{ config.downloads.format }}</span>
-                </p>
-                <p>Example configurations:</p>
-                <ul>
-                    <li><span class="settings-current">bestvideo[height<=720]+bestaudio/best[height<=720]</span>: best audio and max video height of 720p.</li>
-                    <li><span class="settings-current">bestvideo[height<=1080]+bestaudio/best[height<=1080]</span>: best audio and max video height of 1080p.</li>
-                    <li><span class="settings-current">bestvideo[height<=1080][vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
-                    <li><span class="settings-current">0</span>: deactivate and download the best quality possible as decided by yt-dlp.</li>
-                </ul>
-                <i>Make sure your custom format gets merged into a single file. Check out the <a href="https://github.com/yt-dlp/yt-dlp#format-selection" target="_blank">documentation</a> for valid configurations.</i><br>
-                {{ app_form.downloads_format }}
-                <br>
-            </div>
-            <div class="settings-item">
-                <p>Current metadata embed setting: <span class="settings-current">{{ config.downloads.add_metadata }}</span></p>
-                <i>Metadata is not embedded into the downloaded files by default.</i><br>
-                {{ app_form.downloads_add_metadata }}
-            </div>
-            <div class="settings-item">
-                <p>Current thumbnail embed setting: <span class="settings-current">{{ config.downloads.add_thumbnail }}</span></p>
-                <i>Embed thumbnail into the mediafile.</i><br>
-                {{ app_form.downloads_add_thumbnail }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="format">Subtitles</h2>
-            <div class="settings-item">
-                <p>Subtitles download setting: <span class="settings-current">{{ config.downloads.subtitle }}</span><br>
-                <i>Choose which subtitles to download, add comma separated two letter language ISO code,<br>
-                    e.g. <span class="settings-current">en, de</span></i><br>
-                {{ app_form.downloads_subtitle }}</p>
-            </div>
-            <div class="settings-item">
-                <p>Subtitle source settings: <span class="settings-current">{{ config.downloads.subtitle_source }}</span></p>
-                <i>Download only user generated, or also less accurate auto generated subtitles.</i><br>
-                {{ app_form.downloads_subtitle_source }}
-            </div>
-            <div class="settings-item">
-                <p>Index and make subtitles searchable: <span class="settings-current">{{ config.downloads.subtitle_index }}</span></p>
-                <i>Store subtitle lines in Elasticsearch. Not recommended for low-end hardware.</i><br>
-                {{ app_form.downloads_subtitle_index }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="comments">Comments</h2>
-            <div class="settings-item">
-                <p>Download and index comments: <span class="settings-current">{{ config.downloads.comment_max }}</span><br>
-                <i>Follow the yt-dlp max_comments documentation, <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">max-comments,max-parents,max-replies,max-replies-per-thread</a>:</i><br>
-                <p>Example configurations:</p>
-                <ul>
-                    <li><span class="settings-current">all,100,all,30</span>: Get 100 max-parents and 30 max-replies-per-thread.</li>
-                    <li><span class="settings-current">1000,all,all,50</span>: Get a total of 1000 comments over all, 50 replies per thread.</li>
-                </ul>
-                {{ app_form.downloads_comment_max }}</p>
-            </div>
-            <div class="settings-item">
-                <p>Selected comment sort method: <span class="settings-current">{{ config.downloads.comment_sort }}</span><br>
-                <i>Select how many comments and threads to download:</i><br>
-                {{ app_form.downloads_comment_sort }}</p>
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="format">Cookie</h2>
-            <div class="settings-item">
-                <p>Import YouTube cookie: <span class="settings-current">{{ config.downloads.cookie_import }}</span><br></p>
-                <p>For automatic cookie import use <b>Tube Archivist Companion</b> <a href="https://github.com/tubearchivist/browser-extension" target="_blank">browser extension</a>.</p> 
-                <i>For manual cookie import, place your cookie file named <span class="settings-current">cookies.google.txt</span> in <span class="settings-current">cache/import</span> before enabling. Instructions in the <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki.</a></i><br>
-                {{ app_form.downloads_cookie_import }}<br>
-                {% if config.downloads.cookie_import %}
-                    <div id="cookieMessage">
-                        <button onclick="handleCookieValidate()" type="button" id="cookieButton">Validate Cookie File</button>
-                    </div>
-                {% endif %}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="integrations">Integrations</h2>
-            <div class="settings-item">
-                <p>API token: <button type="button" onclick="textReveal()" id="text-reveal-button">Show</button></p>
-                <div id="text-reveal" class="description-text">
-                    <p>{{ api_token }}</p>
-                    <button class="danger-button" type="button" onclick="resetToken()">Revoke</button>
-                </div>
-            </div>
-            <div class="settings-item">
-                <p>Integrate with <a href="https://returnyoutubedislike.com/" target="_blank">returnyoutubedislike.com</a> to get dislikes and average ratings back: <span class="settings-current">{{ config.downloads.integrate_ryd }}</span></p>
-                <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
-                {{ app_form.downloads_integrate_ryd }}
-            </div>
-            <div class="settings-item">
-                <p>Integrate with <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a> to get sponsored timestamps: <span class="settings-current">{{ config.downloads.integrate_sponsorblock }}</span></p>
-                <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
-                {{ app_form.downloads_integrate_sponsorblock }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="snapshots">Snapshots</h2>
-            <div class="settings-item">
-                <p>Current system snapshot: <span class="settings-current">{{ config.application.enable_snapshot }}</span></p>
-                <i>Automatically create daily deduplicated snapshots of the index, stored in Elasticsearch. Read first before activating: <a target="_blank" href="https://docs.tubearchivist.com/settings/#snapshots">Wiki</a>.</i><br>
-                {{ app_form.application_enable_snapshot }}
-            </div>
-            <div>
-                {% if snapshots %}
-                    <p>Create next snapshot: <span class="settings-current">{{ snapshots.next_exec_str }}</span>, snapshots expire after <span class="settings-current">{{ snapshots.expire_after }}</span>. <button onclick="createSnapshot()" id="createButton">Create snapshot now</button></p>
-                    <br>
-                    {% for snapshot in snapshots.snapshots %}
-                        <p><button id="{{ snapshot.id }}" onclick="restoreSnapshot(id)">Restore</button> Snapshot created on: <span class="settings-current">{{ snapshot.start_date }}</span>, took <span class="settings-current">{{ snapshot.duration_s }}s</span> to create. State: <i>{{ snapshot.state }}</i></p>
-                    {% endfor %}
-                {% endif %}
-            </div>
-        </div>
-        <button type="submit" name="application-settings">Update Application Configurations</button>
-    </form>
-    <div class="title-bar">
-        <h1>Scheduler Setup</h1>
-        <div class="settings-group">
-            <p>Schedule settings expect a cron like format, where the first value is minute, second is hour and third is day of the week.</p>
-            <p>Examples:</p>
-            <ul>
-                <li><span class="settings-current">0 15 *</span>: Run task every day at 15:00 in the afternoon.</li>
-                <li><span class="settings-current">30 8 */2</span>: Run task every second day of the week (Sun, Tue, Thu, Sat) at 08:30 in the morning.</li>
-                <li><span class="settings-current">auto</span>: Sensible default.</li>
-                <li><span class="settings-current">0</span>: (zero), deactivate that task.</li>
-            </ul>
-            <p>Note:</p>
-            <ul>
-                <li>Changes in the scheduler settings require a container restart to take effect.</li>
-                <li>Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason, the scheduler doesn't support schedules that trigger more than once per hour.</li>
-            </ul>
-        </div>
-    </div>
-    <form action="/settings/" method="POST" name="scheduler-update">
-        {% csrf_token %}
-        <div class="settings-group">
-            <h2>Rescan Subscriptions</h2>
-            <div class="settings-item">
-                <p>Current rescan schedule: <span class="settings-current">
-                    {% if config.scheduler.update_subscribed %}
-                        {% for key, value in config.scheduler.update_subscribed.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Become a sponsor and join <a href="https://members.tubearchivist.com/" target="_blank">members.tubearchivist.com</a> to get access to <span class="settings-current">real time</span> notifications for new videos uploaded by your favorite channels.</p>
-                <p>Periodically rescan your subscriptions:</p>
-                {{ scheduler_form.update_subscribed }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>Start download</h2>
-            <div class="settings-item">
-                <p>Current Download schedule: <span class="settings-current">
-                    {% if config.scheduler.download_pending %}
-                        {% for key, value in config.scheduler.download_pending.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Automatic video download schedule:</p>
-                {{ scheduler_form.download_pending }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>Refresh Metadata</h2>
-            <div class="settings-item">
-                <p>Current Metadata refresh schedule: <span class="settings-current">
-                    {% if config.scheduler.check_reindex %}
-                        {% for key, value in config.scheduler.check_reindex.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Daily schedule to refresh metadata from YouTube:</p>
-                {{ scheduler_form.check_reindex }}
-            </div>
-            <div class="settings-item">
-                <p>Current refresh for metadata older than x days: <span class="settings-current">{{ config.scheduler.check_reindex_days }}</span></p>
-                <p>Refresh older than x days, recommended 90:</p>
-                {{ scheduler_form.check_reindex_days }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>Thumbnail check</h2>
-            <div class="settings-item">
-                <p>Current thumbnail check schedule: <span class="settings-current">
-                    {% if config.scheduler.thumbnail_check %}
-                        {% for key, value in config.scheduler.thumbnail_check.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Periodically check and cleanup thumbnails:</p>
-                {{ scheduler_form.thumbnail_check }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>ZIP file index backup</h2>
-            <div class="settings-item">
-                <p><i>Zip file backups are very slow for large archives and consistency is not guaranteed, use snapshots instead. Make sure no other tasks are running when creating a Zip file backup.</i></p>
-                <p>Current index backup schedule: <span class="settings-current">
-                    {% if config.scheduler.run_backup %}
-                        {% for key, value in config.scheduler.run_backup.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Automatically backup metadata to a zip file:</p>
-                {{ scheduler_form.run_backup }}
-            </div>
-            <div class="settings-item">
-                <p>Current backup files to keep: <span class="settings-current">{{ config.scheduler.run_backup_rotate }}</span></p>
-                <p>Max auto backups to keep:</p>
-                {{ scheduler_form.run_backup_rotate }}
-            </div>
-        </div>
-        <button type="submit" name="scheduler-settings">Update Scheduler Settings</button>
-    </form>
-    <div class="title-bar">
-        <h1>Actions</h1>
-    </div>
-    <div class="settings-group">
-        <h2>Delete download queue</h2>
-        <p>Delete your pending or previously ignored videos from your download queue.<p>
-        <button onclick="deleteQueue(this)" id="ignore-button" data-id="ignore" title="Delete all previously ignored videos from the queue">Delete all ignored</button>
-        <button onclick="deleteQueue(this)" id="pending-button" data-id="pending" title="Delete all pending videos from the queue">Delete all queued</button>
-    </div>
-    <div class="settings-group">
-        <h2>Manual media files import.</h2>
-        <p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki</a>.</p>
-        <div id="manual-import">
-            <button onclick="manualImport()">Start import</button>
-        </div>
-    </div>
-    <div class="settings-group">
-        <h2>Embed thumbnails into media file.</h2>
-        <p>Set extracted youtube thumbnail as cover art of the media file.</p>
-        <div id="re-embed">
-            <button onclick="reEmbed()">Start process</button>
-        </div>
-    </div>
-    <div class="settings-group">
-        <h2>ZIP file index backup</h2>
-        <p>Export your database to a zip file stored at <span class="settings-current">cache/backup</span>.</p>
-        <p><i>Zip file backups are very slow for large archives and consistency is not guaranteed, use snapshots instead. Make sure no other tasks are running when creating a Zip file backup.</i></p>
-        <div id="db-backup">
-            <button onclick="dbBackup()">Start backup</button>
-        </div>
-    </div>
-    <div class="settings-group">
-        <h2>Restore from backup</h2>
-        <p><span class="danger-zone">Danger Zone</span>: This will replace your existing index with the backup.</p>
-        <p>Restore from available backup files from <span class="settings-current">cache/backup</span>.</p>
-        {% if available_backups %}
-            <div class="backup-grid-row">
-                <span></span>
-                <span>Timestamp</span>
-                <span>Source</span>
-                <span>Filename</span>
-            </div>
-            {% for backup in available_backups %}
-                <div class="backup-grid-row" id="{{ backup.filename }}">
-                    <button onclick="dbRestore(this)" data-id="{{ backup.filename }}">Restore</button>
-                    <span>{{ backup.timestamp }}</span>
-                    <span>{{ backup.reason }}</span>
-                    <span>{{ backup.filename }}</span>
-                </div>
-            {% endfor %}
-        {% else %}
-            <p>No backups found.</p>
-        {% endif %}
-    </div>
-    <div class="settings-group">
-        <h2>Rescan filesystem</h2>
-        <p><span class="danger-zone">Danger Zone</span>: This will delete the metadata of deleted videos from the filesystem.</p>
-        <p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki</a>.</p>
-        <div id="fs-rescan">
-            <button onclick="fsRescan()">Rescan filesystem</button>
-        </div>
-    </div>
-    {% if request.user.is_superuser %}
-        <div class="title-bar">
-            <h1>Users</h1>
-        </div>
-        <div class="settings-group">
-            <h2>User Management</h2>
-            <p>Access the admin interface for basic user management functionality like adding and deleting users, changing passwords and more.</p>
-            <a href="/admin/"><button>Admin Interface</button></a>
-        </div>
-    {% endif %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>Your Archive</h1>
 </div>
-<script type="text/javascript" src="{% static 'progress.js' %}"></script>
-{% endblock content %}
+<div class="settings-item">
+    <h2>Main overview</h2>
+    <div id="primaryBox" class="info-box info-box-4">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Watch Progress</h2>
+    <div id="watchBox" class="info-box info-box-3">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Download History</h2>
+    <div id="downHistBox" class="info-box info-box-4">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+
+<div class="settings-item">
+    <h2>Biggest Channels</h2>
+    <div class="info-box info-box-3">
+        <div class="info-box-item">
+            <table class="agg-channel-table">
+                <thead>
+                    <tr>
+                        <th>Name</th>
+                        <th class="agg-channel-right-align">Videos</th>
+                    </tr>
+                </thead>
+                <tbody id="biggestChannelTableVideos"></tbody>
+            </table>
+        </div>
+
+        <div class="info-box-item">
+            <table class="agg-channel-table">
+                <thead>
+                    <tr>
+                        <th>Name</th>
+                        <th class="agg-channel-right-align">Duration</th>
+                    </tr>
+                </thead>
+                <tbody id="biggestChannelTableDuration"></tbody>
+            </table>
+        </div>
+        
+        <div class="info-box-item">
+            <table class="agg-channel-table">
+                <thead>
+                    <tr>
+                        <th>Name</th>
+                        <th class="agg-channel-right-align">Media Size</th>
+                    </tr>
+                </thead>
+                <tbody id="biggestChannelTableMediaSize"></tbody>
+            </table>
+        </div>
+    </div>
+</div>
+
+<script type="text/javascript" src="{% static 'stats.js' %}"></script>
+{% endblock settings_content %}

tubearchivist/home/templates/home/settings_actions.html

@@ -0,0 +1,66 @@
+{% extends "home/base_settings.html" %}
+{% load static %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>Actions</h1>
+</div>
+<div class="settings-group">
+    <h2>Delete download queue</h2>
+    <p>Delete your pending or previously ignored videos from your download queue.<p>
+    <button onclick="deleteQueue(this)" id="ignore-button" data-id="ignore" title="Delete all previously ignored videos from the queue">Delete all ignored</button>
+    <button onclick="deleteQueue(this)" id="pending-button" data-id="pending" title="Delete all pending videos from the queue">Delete all queued</button>
+</div>
+<div class="settings-group">
+    <h2>Manual media files import.</h2>
+    <p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://docs.tubearchivist.com/settings/actions/#manual-media-files-import" target="_blank">Wiki</a>.</p>
+    <div id="manual-import">
+        <button onclick="manualImport()">Start import</button>
+    </div>
+</div>
+<div class="settings-group">
+    <h2>Embed thumbnails into media file.</h2>
+    <p>Set extracted youtube thumbnail as cover art of the media file.</p>
+    <div id="re-embed">
+        <button onclick="reEmbed()">Start process</button>
+    </div>
+</div>
+<div class="settings-group">
+    <h2>ZIP file index backup</h2>
+    <p>Export your database to a zip file stored at <span class="settings-current">cache/backup</span>.</p>
+    <p><i>Zip file backups are very slow for large archives and consistency is not guaranteed, use snapshots instead. Make sure no other tasks are running when creating a Zip file backup.</i></p>
+    <div id="db-backup">
+        <button onclick="dbBackup()">Start backup</button>
+    </div>
+</div>
+<div class="settings-group">
+    <h2>Restore from backup</h2>
+    <p><span class="danger-zone">Danger Zone</span>: This will replace your existing index with the backup.</p>
+    <p>Restore from available backup files from <span class="settings-current">cache/backup</span>.</p>
+    {% if available_backups %}
+        <div class="backup-grid-row">
+            <span></span>
+            <span>Timestamp</span>
+            <span>Source</span>
+            <span>Filename</span>
+        </div>
+        {% for backup in available_backups %}
+            <div class="backup-grid-row" id="{{ backup.filename }}">
+                <button onclick="dbRestore(this)" data-id="{{ backup.filename }}">Restore</button>
+                <span>{{ backup.timestamp }}</span>
+                <span>{{ backup.reason }}</span>
+                <span>{{ backup.filename }}</span>
+            </div>
+        {% endfor %}
+    {% else %}
+        <p>No backups found.</p>
+    {% endif %}
+</div>
+<div class="settings-group">
+    <h2>Rescan filesystem</h2>
+    <p><span class="danger-zone">Danger Zone</span>: This will delete the metadata of deleted videos from the filesystem.</p>
+    <p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://docs.tubearchivist.com/settings/actions/#rescan-filesystem" target="_blank">Wiki</a>.</p>
+    <div id="fs-rescan">
+        <button onclick="fsRescan()">Rescan filesystem</button>
+    </div>
+</div>
+{% endblock settings_content %}

tubearchivist/home/templates/home/settings_application.html

@@ -0,0 +1,192 @@
+{% extends "home/base_settings.html" %}
+{% load static %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>Application Configurations</h1>
+</div>
+<form action="{% url 'settings_application' %}" method="POST" name="application-update">
+    {% csrf_token %}
+    <div class="settings-group">
+        <h2 id="subscriptions">Subscriptions</h2>
+        <p>Disable shorts or streams by setting their page size to 0 (zero).</p>
+        <div class="settings-item">
+            <p>YouTube page size: <span class="settings-current">{{ config.subscriptions.channel_size }}</span></p>
+            <i>Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
+            {{ app_form.subscriptions_channel_size }}
+        </div>
+        <div class="settings-item">
+            <p>YouTube Live page size: <span class="settings-current">{{ config.subscriptions.live_channel_size }}</span></p>
+            <i>Live Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
+            {{ app_form.subscriptions_live_channel_size }}
+        </div>
+        <div class="settings-item">
+            <p>YouTube Shorts page size: <span class="settings-current">{{ config.subscriptions.shorts_channel_size }}</span></p>
+            <i>Shorts Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
+            {{ app_form.subscriptions_shorts_channel_size }}
+        </div>
+        <div class="settings-item">
+            <p>Auto start download from your subscriptions: <span class="settings-current">{{ config.subscriptions.auto_start}}</span></p>
+            <i>Enable this will automatically start and prioritize videos from your subscriptions.</i><br>
+            {{ app_form.subscriptions_auto_start }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="downloads">Downloads</h2>
+        <div class="settings-item">
+            <p>Current download speed limit in KB/s: <span class="settings-current">{{ config.downloads.limit_speed }}</span></p>
+            <i>Limit download speed. 0 (zero) to deactivate, e.g. 1000 (1MB/s). Speeds are in KB/s. Setting takes effect on new download jobs or application restart.</i><br>
+            {{ app_form.downloads_limit_speed }}
+        </div>
+        <div class="settings-item">
+            <p>Current throttled rate limit in KB/s: <span class="settings-current">{{ config.downloads.throttledratelimit }}</span></p>
+            <i>Download will restart if speeds drop below specified amount. 0 (zero) to deactivate, e.g. 100. Speeds are in KB/s.</i><br>
+            {{ app_form.downloads_throttledratelimit }}
+        </div>
+        <div class="settings-item">
+            <p>Current scraping sleep interval: <span class="settings-current">{{ config.downloads.sleep_interval }}</p>
+            <i>Seconds to sleep between calls to YouTube. Might be necessary to avoid throttling. Recommended 3.</i><br>
+            {{ app_form.downloads_sleep_interval }}
+        </div>
+        <div class="settings-item">
+            <p><span class="danger-zone">Danger Zone</span>: Current auto delete watched videos: <span class="settings-current">{{ config.downloads.autodelete_days }}</span></p>
+            <i>Auto delete watched videos after x days, 0 (zero) to deactivate:</i><br>
+            {{ app_form.downloads_autodelete_days }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="format">Download Format</h2>
+        <div class="settings-item">
+            <p>Limit video and audio quality format for yt-dlp.<br>
+                Currently: <span class="settings-current">{{ config.downloads.format }}</span>
+            </p>
+            <p>Example configurations:</p>
+            <ul>
+                <li><span class="settings-current">bestvideo[height<=720]+bestaudio/best[height<=720]</span>: best audio and max video height of 720p.</li>
+                <li><span class="settings-current">bestvideo[height<=1080]+bestaudio/best[height<=1080]</span>: best audio and max video height of 1080p.</li>
+                <li><span class="settings-current">bestvideo[height<=1080][vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
+                <li><span class="settings-current">0</span>: deactivate and download the best quality possible as decided by yt-dlp.</li>
+            </ul>
+            <i>Make sure your custom format gets merged into a single file. Check out the <a href="https://github.com/yt-dlp/yt-dlp#format-selection" target="_blank">documentation</a> for valid configurations.</i><br>
+            {{ app_form.downloads_format }}
+            <br>
+        </div>
+        <div class="settings-item">
+            <p>Force sort order to have precedence over all yt-dlp fields.<br>
+                Currently: <span class="settings-current">{{ config.downloads.format_sort }}</span>
+            </p>
+            <p>Example configurations:</p>
+            <ul>
+                <li><span class="settings-current">res,codec:av1</span>: prefer AV1 over all other video codecs.</li>
+                <li><span class="settings-current">0</span>: deactivate and keep the default as decided by yt-dlp.</li>
+            </ul>
+            <i>Not all codecs are supported by all browsers. The default value ensures best compatibility. Check out the <a href="https://github.com/yt-dlp/yt-dlp#sorting-formats" target="_blank">documentation</a> for valid configurations.</i><br>
+            {{ app_form.downloads_format_sort }}
+            <br>
+        </div>
+        <div class="settings-item">
+            <p>Prefer translated metadata language: <span class="settings-current">{{ config.downloads.extractor_lang }}</span></p>
+            <i>This will change the language this video gets indexed as. That will only be available if the uploader provides translations. Add as two letter ISO language code, check the <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">documentation</a> which languages are available.</i><br>
+            {{ app_form.downloads_extractor_lang}}
+        </div>
+        <div class="settings-item">
+            <p>Current metadata embed setting: <span class="settings-current">{{ config.downloads.add_metadata }}</span></p>
+            <i>Metadata is not embedded into the downloaded files by default.</i><br>
+            {{ app_form.downloads_add_metadata }}
+        </div>
+        <div class="settings-item">
+            <p>Current thumbnail embed setting: <span class="settings-current">{{ config.downloads.add_thumbnail }}</span></p>
+            <i>Embed thumbnail into the mediafile.</i><br>
+            {{ app_form.downloads_add_thumbnail }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="format">Subtitles</h2>
+        <div class="settings-item">
+            <p>Subtitles download setting: <span class="settings-current">{{ config.downloads.subtitle }}</span><br>
+            <i>Choose which subtitles to download, add comma separated language codes,<br>
+                e.g. <span class="settings-current">en, de, zh-Hans</span></i><br>
+            {{ app_form.downloads_subtitle }}</p>
+        </div>
+        <div class="settings-item">
+            <p>Subtitle source settings: <span class="settings-current">{{ config.downloads.subtitle_source }}</span></p>
+            <i>Download only user generated, or also less accurate auto generated subtitles.</i><br>
+            {{ app_form.downloads_subtitle_source }}
+        </div>
+        <div class="settings-item">
+            <p>Index and make subtitles searchable: <span class="settings-current">{{ config.downloads.subtitle_index }}</span></p>
+            <i>Store subtitle lines in Elasticsearch. Not recommended for low-end hardware.</i><br>
+            {{ app_form.downloads_subtitle_index }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="comments">Comments</h2>
+        <div class="settings-item">
+            <p>Download and index comments: <span class="settings-current">{{ config.downloads.comment_max }}</span><br>
+            <i>Follow the yt-dlp max_comments documentation, <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">max-comments,max-parents,max-replies,max-replies-per-thread</a>:</i><br>
+            <p>Example configurations:</p>
+            <ul>
+                <li><span class="settings-current">all,100,all,30</span>: Get 100 max-parents and 30 max-replies-per-thread.</li>
+                <li><span class="settings-current">1000,all,all,50</span>: Get a total of 1000 comments over all, 50 replies per thread.</li>
+            </ul>
+            {{ app_form.downloads_comment_max }}</p>
+        </div>
+        <div class="settings-item">
+            <p>Selected comment sort method: <span class="settings-current">{{ config.downloads.comment_sort }}</span><br>
+            <i>Select how many comments and threads to download:</i><br>
+            {{ app_form.downloads_comment_sort }}</p>
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="format">Cookie</h2>
+        <div class="settings-item">
+            <p>Import YouTube cookie: <span class="settings-current">{{ config.downloads.cookie_import }}</span><br></p>
+            <p>For automatic cookie import use <b>Tube Archivist Companion</b> <a href="https://github.com/tubearchivist/browser-extension" target="_blank">browser extension</a>.</p> 
+            <i>For manual cookie import, place your cookie file named <span class="settings-current">cookies.google.txt</span> in <span class="settings-current">cache/import</span> before enabling. Instructions in the <a href="https://docs.tubearchivist.com/settings/application/#cookie" target="_blank">Wiki.</a></i><br>
+            {{ app_form.downloads_cookie_import }}<br>
+            {% if config.downloads.cookie_import %}
+                <div id="cookieMessage">
+                    <button onclick="handleCookieValidate()" type="button" id="cookieButton">Validate Cookie File</button>
+                </div>
+            {% endif %}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="integrations">Integrations</h2>
+        <div class="settings-item">
+            <p>API token: <button type="button" onclick="textReveal(this)" id="text-reveal-button">Show</button></p>
+            <div id="text-reveal" class="description-text">
+                <p>{{ api_token }}</p>
+                <button class="danger-button" type="button" onclick="resetToken()">Revoke</button>
+            </div>
+        </div>
+        <div class="settings-item">
+            <p>Integrate with <a href="https://returnyoutubedislike.com/" target="_blank">returnyoutubedislike.com</a> to get dislikes and average ratings back: <span class="settings-current">{{ config.downloads.integrate_ryd }}</span></p>
+            <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
+            {{ app_form.downloads_integrate_ryd }}
+        </div>
+        <div class="settings-item">
+            <p>Integrate with <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a> to get sponsored timestamps: <span class="settings-current">{{ config.downloads.integrate_sponsorblock }}</span></p>
+            <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
+            {{ app_form.downloads_integrate_sponsorblock }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="snapshots">Snapshots</h2>
+        <div class="settings-item">
+            <p>Current system snapshot: <span class="settings-current">{{ config.application.enable_snapshot }}</span></p>
+            <i>Automatically create daily deduplicated snapshots of the index, stored in Elasticsearch. Read first before activating: <a target="_blank" href="https://docs.tubearchivist.com/settings/application/#snapshots">Wiki</a>.</i><br>
+            {{ app_form.application_enable_snapshot }}
+        </div>
+        <div>
+            {% if snapshots %}
+                <p>Create next snapshot: <span class="settings-current">{{ snapshots.next_exec_str }}</span>, snapshots expire after <span class="settings-current">{{ snapshots.expire_after }}</span>. <button onclick="createSnapshot()" id="createButton">Create snapshot now</button></p>
+                <br>
+                {% for snapshot in snapshots.snapshots %}
+                    <p><button id="{{ snapshot.id }}" onclick="restoreSnapshot(id)">Restore</button> Snapshot created on: <span class="settings-current">{{ snapshot.start_date }}</span>, took <span class="settings-current">{{ snapshot.duration_s }}s</span> to create. State: <i>{{ snapshot.state }}</i></p>
+                {% endfor %}
+            {% endif %}
+        </div>
+    </div>
+    <button type="submit" name="application-settings">Update Application Configurations</button>
+</form>
+{% endblock settings_content %}

tubearchivist/home/templates/home/settings_scheduling.html

@@ -0,0 +1,154 @@
+{% extends "home/base_settings.html" %}
+{% load static %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>Scheduler Setup</h1>
+    <div class="settings-group">
+        <p>Schedule settings expect a cron like format, where the first value is minute, second is hour and third is day of the week.</p>
+        <p>Examples:</p>
+        <ul>
+            <li><span class="settings-current">0 15 *</span>: Run task every day at 15:00 in the afternoon.</li>
+            <li><span class="settings-current">30 8 */2</span>: Run task every second day of the week (Sun, Tue, Thu, Sat) at 08:30 in the morning.</li>
+            <li><span class="settings-current">auto</span>: Sensible default.</li>
+            <li><span class="settings-current">0</span>: (zero), deactivate that task.</li>
+        </ul>
+        <p>Note:</p>
+        <ul>
+            <li>Changes in the scheduler settings require a container restart to take effect.</li>
+            <li>Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason, the scheduler doesn't support schedules that trigger more than once per hour.</li>
+        </ul>
+    </div>
+</div>
+<form action="{% url 'settings_scheduling' %}" method="POST" name="scheduler-update">
+    {% csrf_token %}
+    <div class="settings-group">
+        <h2>Rescan Subscriptions</h2>
+        <div class="settings-item">
+            <p>Current rescan schedule: <span class="settings-current">
+                {% if config.scheduler.update_subscribed %}
+                    {% for key, value in config.scheduler.update_subscribed.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Become a sponsor and join <a href="https://members.tubearchivist.com/" target="_blank">members.tubearchivist.com</a> to get access to <span class="settings-current">real time</span> notifications for new videos uploaded by your favorite channels.</p>
+            <p>Periodically rescan your subscriptions:</p>
+            {{ scheduler_form.update_subscribed }}
+        </div>
+        <div class="settings-item">
+            <p>Send notification on task completed:</p>
+            {% if config.scheduler.update_subscribed_notify %}
+                <p><button type="button" onclick="textReveal(this)" id="text-reveal-button">Show</button> stored notification links</p>
+                <div id="text-reveal" class="description-text">
+                    <p>{{ config.scheduler.update_subscribed_notify|linebreaks }}</p>
+                </div>
+            {% else %}
+                <p>Current notification urls: <span class="settings-current">{{ config.scheduler.update_subscribed_notify }}</span></p>
+            {% endif %}
+            {{ scheduler_form.update_subscribed_notify }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>Start download</h2>
+        <div class="settings-item">
+            <p>Current Download schedule: <span class="settings-current">
+                {% if config.scheduler.download_pending %}
+                    {% for key, value in config.scheduler.download_pending.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Automatic video download schedule:</p>
+            {{ scheduler_form.download_pending }}
+        </div>
+        <div class="settings-item">
+            <p>Send notification on task completed:</p>
+            {% if config.scheduler.download_pending_notify %}
+                <p><button type="button" onclick="textReveal(this)" id="text-reveal-button">Show</button> stored notification links</p>
+                <div id="text-reveal" class="description-text">
+                    <p>{{ config.scheduler.download_pending_notify|linebreaks }}</p>
+                </div>
+            {% else %}
+                <p>Current notification urls: <span class="settings-current">{{ config.scheduler.download_pending_notify }}</span></p>
+            {% endif %}
+            {{ scheduler_form.download_pending_notify }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>Refresh Metadata</h2>
+        <div class="settings-item">
+            <p>Current Metadata refresh schedule: <span class="settings-current">
+                {% if config.scheduler.check_reindex %}
+                    {% for key, value in config.scheduler.check_reindex.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Daily schedule to refresh metadata from YouTube:</p>
+            {{ scheduler_form.check_reindex }}
+        </div>
+        <div class="settings-item">
+            <p>Current refresh for metadata older than x days: <span class="settings-current">{{ config.scheduler.check_reindex_days }}</span></p>
+            <p>Refresh older than x days, recommended 90:</p>
+            {{ scheduler_form.check_reindex_days }}
+        </div>
+        <div class="settings-item">
+            <p>Send notification on task completed:</p>
+            {% if config.scheduler.check_reindex_notify %}
+                <p><button type="button" onclick="textReveal(this)" id="text-reveal-button">Show</button> stored notification links</p>
+                <div id="text-reveal" class="description-text">
+                    <p>{{ config.scheduler.check_reindex_notify|linebreaks }}</p>
+                </div>
+            {% else %}
+                <p>Current notification urls: <span class="settings-current">{{ config.scheduler.check_reindex_notify }}</span></p>
+            {% endif %}
+            {{ scheduler_form.check_reindex_notify }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>Thumbnail check</h2>
+        <div class="settings-item">
+            <p>Current thumbnail check schedule: <span class="settings-current">
+                {% if config.scheduler.thumbnail_check %}
+                    {% for key, value in config.scheduler.thumbnail_check.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Periodically check and cleanup thumbnails:</p>
+            {{ scheduler_form.thumbnail_check }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>ZIP file index backup</h2>
+        <div class="settings-item">
+            <p><i>Zip file backups are very slow for large archives and consistency is not guaranteed, use snapshots instead. Make sure no other tasks are running when creating a Zip file backup.</i></p>
+            <p>Current index backup schedule: <span class="settings-current">
+                {% if config.scheduler.run_backup %}
+                    {% for key, value in config.scheduler.run_backup.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Automatically backup metadata to a zip file:</p>
+            {{ scheduler_form.run_backup }}
+        </div>
+        <div class="settings-item">
+            <p>Current backup files to keep: <span class="settings-current">{{ config.scheduler.run_backup_rotate }}</span></p>
+            <p>Max auto backups to keep:</p>
+            {{ scheduler_form.run_backup_rotate }}
+        </div>
+    </div>
+    <button type="submit" name="scheduler-settings">Update Scheduler Settings</button>
+</form>
+{% endblock settings_content %}

tubearchivist/home/templates/home/settings_user.html

@@ -0,0 +1,37 @@
+{% extends "home/base_settings.html" %}
+{% load static %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>User Configurations</h1>
+</div>
+<form action="{% url 'settings_user' %}" method="POST" name="user-update">
+    {% csrf_token %}
+    <div class="settings-group">
+        <h2>Color scheme</h2>
+        <div class="settings-item">
+            <p>Current color scheme: <span class="settings-current">{{ colors }}</span></p>
+            <i>Select your preferred color scheme between dark and light mode.</i><br>
+            {{ user_form.colors }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>Archive View</h2>
+        <div class="settings-item">
+            <p>Current page size: <span class="settings-current">{{ page_size }}</span></p>
+            <i>Result of videos showing in archive page</i><br>
+            {{ user_form.page_size }}
+        </div>
+    </div>
+    <button type="submit" name="user-settings">Update User Configurations</button>
+</form>
+{% if request.user.is_superuser %}
+    <div class="title-bar">
+        <h1>Users</h1>
+    </div>
+    <div class="settings-group">
+        <h2>User Management</h2>
+        <p>Access the admin interface for basic user management functionality like adding and deleting users, changing passwords and more.</p>
+        <a href="/admin/"><button>Admin Interface</button></a>
+    </div>
+{% endif %}
+{% endblock settings_content %}

tubearchivist/home/templates/home/video.html

@@ -56,20 +56,6 @@
                 {% else %}
                     <p>Youtube: Deactivated</p>
                 {% endif %}
-                {% if reindex %}
-                    <p>Reindex scheduled</p>
-                {% else %}
-                    <div id="reindex-button" class="button-box">
-                        <button data-id="{{ video.youtube_id }}" data-type="video" onclick="reindex(this)" title="Reindex {{ video.title }}">Reindex</button>
-                    </div>
-                {% endif %}
-                <div class="button-box">
-                    <a download="" href="/media/{{ video.media_url }}"><button id="download-item">Download File</button></a>
-                    <button onclick="deleteConfirm()" id="delete-item">Delete Video</button>
-                    <div class="delete-confirm" id="delete-button">
-                        <span>Are you sure? </span><button class="danger-button" onclick="deleteVideo(this)" data-id="{{ video.youtube_id }}" data-redirect = "{{ video.channel.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
-                    </div>
-                </div>
             </div>
         </div>
         <div class="info-box-item">
@@ -89,6 +75,34 @@
             </div>
         </div>
     </div>
+    <div class="info-box info-box-2">
+        <div class="info-box-item">
+            <div class="button-box">
+                {% if reindex %}
+                    <p>Reindex scheduled</p>
+                {% else %}
+                    <div id="reindex-button" class="button-box">
+                        <button data-id="{{ video.youtube_id }}" data-type="video" onclick="reindex(this)" title="Reindex {{ video.title }}">Reindex</button>
+                    </div>
+                {% endif %}
+                <a download="" href="/media/{{ video.media_url }}"><button id="download-item">Download File</button></a>
+                <button onclick="deleteConfirm()" id="delete-item">Delete Video</button>
+                <div class="delete-confirm" id="delete-button">
+                    <span>Are you sure? </span><button class="danger-button" onclick="deleteVideo(this)" data-id="{{ video.youtube_id }}" data-redirect = "{{ video.channel.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
+                </div>
+            </div>
+        </div>
+        <div class="info-box-item">
+            {% if video.media_size %}
+                <p>File size: {{ video.media_size|filesizeformat }}</p>
+            {% endif %}
+            {% if video.streams %}
+                {% for stream in video.streams %}
+                    <p>{{ stream.type|title }}: {{ stream.codec }} {{ stream.bitrate|filesizeformat }}/s{% if stream.width %} <span class="space-carrot">|</span> {{ stream.width }}x{{ stream.height}}{% endif %}</p>
+                {% endfor %}
+            {% endif %}
+        </div>
+    </div>
     {% if video.tags %}
         <div class="description-box">
             <div class="video-tag-box">

tubearchivist/home/urls.py

@@ -3,18 +3,30 @@
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.views import LogoutView
+from django.shortcuts import redirect
 from django.urls import path
 from home import views
 
-urlpatterns = [
-    path("", login_required(views.HomeView.as_view()), name="home"),
-    path("login/", views.LoginView.as_view(), name="login"),
-    path(
+if hasattr(settings, "TA_AUTH_PROXY_LOGOUT_URL"):
+    logout_path = path(
+        "logout/",
+        lambda request: redirect(
+            settings.TA_AUTH_PROXY_LOGOUT_URL, permanent=False
+        ),
+        name="logout",
+    )
+else:
+    logout_path = path(
         "logout/",
         LogoutView.as_view(),
         {"next_page": settings.LOGOUT_REDIRECT_URL},
         name="logout",
-    ),
+    )
+
+urlpatterns = [
+    path("", login_required(views.HomeView.as_view()), name="home"),
+    path("login/", views.LoginView.as_view(), name="login"),
+    logout_path,
     path("about/", views.AboutView.as_view(), name="about"),
     path(
         "downloads/",
@@ -26,6 +38,26 @@ urlpatterns = [
         login_required(views.SettingsView.as_view()),
         name="settings",
     ),
+    path(
+        "settings/user/",
+        login_required(views.SettingsUserView.as_view()),
+        name="settings_user",
+    ),
+    path(
+        "settings/application/",
+        login_required(views.SettingsApplicationView.as_view()),
+        name="settings_application",
+    ),
+    path(
+        "settings/scheduling/",
+        login_required(views.SettingsSchedulingView.as_view()),
+        name="settings_scheduling",
+    ),
+    path(
+        "settings/actions/",
+        login_required(views.SettingsActionsView.as_view()),
+        name="settings_actions",
+    ),
     path("process/", login_required(views.process), name="process"),
     path(
         "channel/",

tubearchivist/home/views.py

@@ -8,11 +8,11 @@ import json
 import urllib.parse
 from time import sleep
 
-from api.src.search_processor import SearchProcess
+from api.src.search_processor import SearchProcess, process_aggs
 from django.conf import settings
 from django.contrib.auth import login
 from django.contrib.auth.forms import AuthenticationForm
-from django.http import JsonResponse
+from django.http import Http404, JsonResponse
 from django.shortcuts import redirect, render
 from django.views import View
 from home.src.download.queue import PendingInteract
@@ -32,7 +32,6 @@ from home.src.frontend.forms import (
     SubscribeToPlaylistForm,
     UserSettingsForm,
 )
-from home.src.frontend.searching import SearchHandler
 from home.src.index.channel import channel_overwrites
 from home.src.index.generic import Pagination
 from home.src.index.playlist import YoutubePlaylist
@@ -41,8 +40,8 @@ from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
 from home.src.ta.helper import time_parser
 from home.src.ta.ta_redis import RedisArchivist
-from home.src.ta.urlparser import Parser
-from home.tasks import extrac_dl, index_channel_playlists, subscribe_to
+from home.src.ta.users import UserConfig
+from home.tasks import index_channel_playlists, subscribe_to
 from rest_framework.authtoken.models import Token
 
 
@@ -53,93 +52,38 @@ class ArchivistViewConfig(View):
         super().__init__()
         self.view_origin = view_origin
         self.user_id = False
-        self.user_conf = False
+        self.user_conf: UserConfig = False
         self.default_conf = False
         self.context = False
 
-    def _get_sort_by(self):
-        """return sort_by config var"""
-        messag_key = f"{self.user_id}:sort_by"
-        sort_by = self.user_conf.get_message(messag_key)["status"]
-        if not sort_by:
-            sort_by = self.default_conf["archive"]["sort_by"]
-
-        return sort_by
-
-    def _get_sort_order(self):
-        """return sort_order config var"""
-        sort_order_key = f"{self.user_id}:sort_order"
-        sort_order = self.user_conf.get_message(sort_order_key)["status"]
-        if not sort_order:
-            sort_order = self.default_conf["archive"]["sort_order"]
-
-        return sort_order
-
-    def _get_view_style(self):
-        """return view_style config var"""
-        view_key = f"{self.user_id}:view:{self.view_origin}"
-        view_style = self.user_conf.get_message(view_key)["status"]
-        if not view_style:
-            view_style = self.default_conf["default_view"][self.view_origin]
-
-        return view_style
-
-    def _get_grid_items(self):
-        """return items per row to show in grid view"""
-        grid_key = f"{self.user_id}:grid_items"
-        grid_items = self.user_conf.get_message(grid_key)["status"]
-        if not grid_items:
-            grid_items = self.default_conf["default_view"]["grid_items"]
-
-        return grid_items
-
     def get_all_view_styles(self):
-        """get dict of all view stiles for search form"""
-        all_keys = ["channel", "playlist", "home"]
+        """get dict of all view styles for search form"""
         all_styles = {}
-        for view_origin in all_keys:
-            view_key = f"{self.user_id}:view:{view_origin}"
-            view_style = self.user_conf.get_message(view_key)["status"]
-            if not view_style:
-                view_style = self.default_conf["default_view"][view_origin]
-            all_styles[view_origin] = view_style
+        for view_origin in ["channel", "playlist", "home", "downloads"]:
+            all_styles[view_origin] = self.user_conf.get_value(
+                f"view_style_{view_origin}"
+            )
 
         return all_styles
 
-    def _get_hide_watched(self):
-        hide_watched_key = f"{self.user_id}:hide_watched"
-        hide_watched = self.user_conf.get_message(hide_watched_key)["status"]
-
-        return hide_watched
-
-    def _get_show_ignore_only(self):
-        ignored_key = f"{self.user_id}:show_ignored_only"
-        show_ignored_only = self.user_conf.get_message(ignored_key)["status"]
-
-        return show_ignored_only
-
-    def _get_show_subed_only(self):
-        sub_only_key = f"{self.user_id}:show_subed_only"
-        show_subed_only = self.user_conf.get_message(sub_only_key)["status"]
-
-        return show_subed_only
-
     def config_builder(self, user_id):
         """build default context for every view"""
         self.user_id = user_id
-        self.user_conf = RedisArchivist()
-        self.default_conf = AppConfig(self.user_id).config
+        self.user_conf = UserConfig(self.user_id)
+        self.default_conf = AppConfig().config
 
         self.context = {
-            "colors": self.default_conf["application"]["colors"],
+            "colors": self.user_conf.get_value("colors"),
             "cast": self.default_conf["application"]["enable_cast"],
-            "sort_by": self._get_sort_by(),
-            "sort_order": self._get_sort_order(),
-            "view_style": self._get_view_style(),
-            "grid_items": self._get_grid_items(),
-            "hide_watched": self._get_hide_watched(),
-            "show_ignored_only": self._get_show_ignore_only(),
-            "show_subed_only": self._get_show_subed_only(),
+            "sort_by": self.user_conf.get_value("sort_by"),
+            "sort_order": self.user_conf.get_value("sort_order"),
+            "view_style": self.user_conf.get_value(
+                f"view_style_{self.view_origin}"
+            ),
+            "grid_items": self.user_conf.get_value("grid_items"),
+            "hide_watched": self.user_conf.get_value("hide_watched"),
+            "show_ignored_only": self.user_conf.get_value("show_ignored_only"),
+            "show_subed_only": self.user_conf.get_value("show_subed_only"),
             "version": settings.TA_VERSION,
             "ta_update": ReleaseVersion().get_update(),
         }
@@ -148,8 +92,8 @@ class ArchivistViewConfig(View):
 class ArchivistResultsView(ArchivistViewConfig):
     """View class to inherit from when searching data in es"""
 
-    view_origin = False
-    es_search = False
+    view_origin = ""
+    es_search = ""
 
     def __init__(self):
         super().__init__(self.view_origin)
@@ -200,8 +144,7 @@ class ArchivistResultsView(ArchivistViewConfig):
         self.context["continue_vids"] = self.get_in_progress(results)
 
         in_progress = {i["youtube_id"]: i["position"] for i in results}
-        for hit in self.context["results"]:
-            video = hit["source"]
+        for video in self.context["results"]:
             if video["youtube_id"] in in_progress:
                 played_sec = in_progress.get(video["youtube_id"])
                 total = video["player"]["duration"]
@@ -213,32 +156,35 @@ class ArchivistResultsView(ArchivistViewConfig):
         """get all videos in progress"""
         ids = [{"match": {"youtube_id": i.get("youtube_id")}} for i in results]
         data = {
-            "size": self.default_conf["archive"]["page_size"],
+            "size": UserConfig(self.user_id).get_value("page_size"),
             "query": {"bool": {"should": ids}},
             "sort": [{"published": {"order": "desc"}}],
         }
-        search = SearchHandler(
-            "ta_video/_search", self.default_conf, data=data
-        )
-        videos = search.get_data()
+        response, _ = ElasticWrap("ta_video/_search").get(data)
+        videos = SearchProcess(response).process()
+
         if not videos:
             return False
 
         for video in videos:
-            youtube_id = video["source"]["youtube_id"]
+            youtube_id = video["youtube_id"]
             matched = [i for i in results if i["youtube_id"] == youtube_id]
             played_sec = matched[0]["position"]
-            total = video["source"]["player"]["duration"]
+            total = video["player"]["duration"]
             if not total:
                 total = matched[0].get("position") * 2
-            video["source"]["player"]["progress"] = 100 * (played_sec / total)
+            video["player"]["progress"] = 100 * (played_sec / total)
 
         return videos
 
     def single_lookup(self, es_path):
         """retrieve a single item from url"""
-        search = SearchHandler(es_path, config=self.default_conf)
-        result = search.get_data()[0]["source"]
+        response, status_code = ElasticWrap(es_path).get()
+        if not status_code == 200:
+            raise Http404
+
+        result = SearchProcess(response).process()
+
         return result
 
     def initiate_vars(self, request):
@@ -252,13 +198,19 @@ class ArchivistResultsView(ArchivistViewConfig):
 
     def find_results(self):
         """add results and pagination to context"""
-        search = SearchHandler(
-            self.es_search, config=self.default_conf, data=self.data
+        response, _ = ElasticWrap(self.es_search).get(self.data)
+        process_aggs(response)
+        results = SearchProcess(response).process()
+        max_hits = response["hits"]["total"]["value"]
+        self.pagination_handler.validate(max_hits)
+        self.context.update(
+            {
+                "results": results,
+                "max_hits": max_hits,
+                "pagination": self.pagination_handler.pagination,
+                "aggs": response.get("aggregations"),
+            }
         )
-        self.context["results"] = search.get_data()
-        self.pagination_handler.validate(search.max_hits)
-        self.context["max_hits"] = search.max_hits
-        self.context["pagination"] = self.pagination_handler.pagination
 
 
 class MinView(View):
@@ -268,7 +220,7 @@ class MinView(View):
     def get_min_context(request):
         """build minimal vars for context"""
         return {
-            "colors": AppConfig(request.user.id).colors,
+            "colors": UserConfig(request.user.id).get_value("colors"),
             "version": settings.TA_VERSION,
             "ta_update": ReleaseVersion().get_update(),
         }
@@ -367,7 +319,7 @@ class AboutView(MinView):
 
 class DownloadView(ArchivistResultsView):
     """resolves to /download/
-    takes POST for downloading youtube links
+    handle the download queue
     """
 
     view_origin = "downloads"
@@ -413,7 +365,10 @@ class DownloadView(ArchivistResultsView):
         self.data.update(
             {
                 "query": {"bool": {"must": must_list}},
-                "sort": [{"timestamp": {"order": "asc"}}],
+                "sort": [
+                    {"auto_start": {"order": "desc"}},
+                    {"timestamp": {"order": "asc"}},
+                ],
             }
         )
 
@@ -451,34 +406,6 @@ class DownloadView(ArchivistResultsView):
 
         return buckets_sorted
 
-    @staticmethod
-    def post(request):
-        """handle post requests"""
-        to_queue = AddToQueueForm(data=request.POST)
-        if to_queue.is_valid():
-            url_str = request.POST.get("vid_url")
-            print(url_str)
-            try:
-                youtube_ids = Parser(url_str).parse()
-            except ValueError:
-                # failed to process
-                key = "message:add"
-                print(f"failed to parse: {url_str}")
-                mess_dict = {
-                    "status": key,
-                    "level": "error",
-                    "title": "Failed to extract links.",
-                    "message": "Not a video, channel or playlist ID or URL",
-                }
-                RedisArchivist().set_message(key, mess_dict, expire=True)
-                return redirect("downloads")
-
-            print(youtube_ids)
-            extrac_dl.delay(youtube_ids)
-
-        sleep(2)
-        return redirect("downloads", permanent=True)
-
 
 class ChannelIdBaseView(ArchivistResultsView):
     """base class for all channel-id views"""
@@ -577,7 +504,7 @@ class ChannelIdView(ChannelIdBaseView):
         self.channel_pages(channel_id)
 
         if self.context["results"]:
-            channel_info = self.context["results"][0]["source"]["channel"]
+            channel_info = self.context["results"][0]["channel"]
             channel_name = channel_info["channel_name"]
         else:
             # fall back channel lookup if no videos found
@@ -613,6 +540,11 @@ class ChannelIdView(ChannelIdBaseView):
                 ]
             }
         }
+        self.data["aggs"] = {
+            "total_items": {"value_count": {"field": "youtube_id"}},
+            "total_size": {"sum": {"field": "media_size"}},
+            "total_duration": {"sum": {"field": "player.duration"}},
+        }
         self.data["sort"].append({"title.keyword": {"order": "asc"}})
 
         if self.context["hide_watched"]:
@@ -756,7 +688,7 @@ class ChannelView(ArchivistResultsView):
         if subscribe_form.is_valid():
             url_str = request.POST.get("subscribe")
             print(url_str)
-            subscribe_to.delay(url_str)
+            subscribe_to.delay(url_str, expected_type="channel")
 
         sleep(1)
         return redirect("channel", permanent=True)
@@ -899,7 +831,7 @@ class PlaylistView(ArchivistResultsView):
         if subscribe_form.is_valid():
             url_str = request.POST.get("subscribe")
             print(url_str)
-            subscribe_to.delay(url_str)
+            subscribe_to.delay(url_str, expected_type="playlist")
 
         sleep(1)
         return redirect("playlist")
@@ -912,9 +844,10 @@ class VideoView(MinView):
 
     def get(self, request, video_id):
         """get single video"""
-        config_handler = AppConfig(request.user.id)
-        look_up = SearchHandler(f"ta_video/_doc/{video_id}", config=False)
-        video_data = look_up.get_data()[0]["source"]
+        config_handler = AppConfig()
+        response, _ = ElasticWrap(f"ta_video/_doc/{video_id}").get()
+        video_data = SearchProcess(response).process()
+
         try:
             rating = video_data["stats"]["average_rating"]
             video_data["stats"]["average_rating"] = self.star_creator(rating)
@@ -982,7 +915,7 @@ class SearchView(ArchivistResultsView):
     """
 
     view_origin = "home"
-    es_search = False
+    es_search = ""
 
     def get(self, request):
         """handle get request"""
@@ -1001,7 +934,21 @@ class SearchView(ArchivistResultsView):
 
 class SettingsView(MinView):
     """resolves to /settings/
-    handle the settings page, display current settings,
+    handle the settings dashboard
+    """
+
+    def get(self, request):
+        """read and display the dashboard"""
+        context = self.get_min_context(request)
+        context.update({"title": "Settings Dashboard"})
+
+        return render(request, "home/settings.html", context)
+
+
+class SettingsUserView(MinView):
+    """resolves to /settings/user/
+    handle the settings sub-page for user settings,
+    display current settings,
     take post request from the form to update settings
     """
 
@@ -1010,18 +957,56 @@ class SettingsView(MinView):
         context = self.get_min_context(request)
         context.update(
             {
-                "title": "Settings",
-                "config": AppConfig(request.user.id).config,
-                "api_token": self.get_token(request),
-                "available_backups": ElasticBackup().get_all_backup_files(),
+                "title": "User Settings",
+                "page_size": UserConfig(request.user.id).get_value(
+                    "page_size"
+                ),
                 "user_form": UserSettingsForm(),
+            }
+        )
+
+        return render(request, "home/settings_user.html", context)
+
+    def post(self, request):
+        """handle form post to update settings"""
+        user_form = UserSettingsForm(request.POST)
+        config_handler = UserConfig(request.user.id)
+        if user_form.is_valid():
+            user_form_post = user_form.cleaned_data
+            if user_form_post.get("colors"):
+                config_handler.set_value(
+                    "colors", user_form_post.get("colors")
+                )
+            if user_form_post.get("page_size"):
+                config_handler.set_value(
+                    "page_size", user_form_post.get("page_size")
+                )
+
+        sleep(1)
+        return redirect("settings_user", permanent=True)
+
+
+class SettingsApplicationView(MinView):
+    """resolves to /settings/application/
+    handle the settings sub-page for application configuration,
+    display current settings,
+    take post request from the form to update settings
+    """
+
+    def get(self, request):
+        """read and display current application settings"""
+        context = self.get_min_context(request)
+        context.update(
+            {
+                "title": "Application Settings",
+                "config": AppConfig().config,
+                "api_token": self.get_token(request),
                 "app_form": ApplicationSettingsForm(),
-                "scheduler_form": SchedulerSettingsForm(),
                 "snapshots": ElasticSnapshot().get_snapshot_stats(),
             }
         )
 
-        return render(request, "home/settings.html", context)
+        return render(request, "home/settings_application.html", context)
 
     @staticmethod
     def get_token(request):
@@ -1032,12 +1017,7 @@ class SettingsView(MinView):
 
     def post(self, request):
         """handle form post to update settings"""
-        user_form = UserSettingsForm(request.POST)
         config_handler = AppConfig()
-        if user_form.is_valid():
-            user_form_post = user_form.cleaned_data
-            if any(user_form_post.values()):
-                config_handler.set_user_config(user_form_post, request.user.id)
 
         app_form = ApplicationSettingsForm(request.POST)
         if app_form.is_valid():
@@ -1047,15 +1027,8 @@ class SettingsView(MinView):
                 updated = config_handler.update_config(app_form_post)
                 self.post_process_updated(updated, config_handler.config)
 
-        scheduler_form = SchedulerSettingsForm(request.POST)
-        if scheduler_form.is_valid():
-            scheduler_form_post = scheduler_form.cleaned_data
-            if any(scheduler_form_post.values()):
-                print(scheduler_form_post)
-                ScheduleBuilder().update_schedule_conf(scheduler_form_post)
-
         sleep(1)
-        return redirect("settings", permanent=True)
+        return redirect("settings_application", permanent=True)
 
     def post_process_updated(self, updated, config):
         """apply changes for config"""
@@ -1093,13 +1066,66 @@ class SettingsView(MinView):
         key = "message:setting"
         message = {
             "status": key,
+            "group": "setting:application",
             "level": "error",
             "title": "Cookie import failed",
-            "message": message_line,
+            "messages": [message_line],
+            "id": "0000",
         }
         RedisArchivist().set_message(key, message=message, expire=True)
 
 
+class SettingsSchedulingView(MinView):
+    """resolves to /settings/scheduling/
+    handle the settings sub-page for scheduling settings,
+    display current settings,
+    take post request from the form to update settings
+    """
+
+    def get(self, request):
+        """read and display current settings"""
+        context = self.get_min_context(request)
+        context.update(
+            {
+                "title": "Scheduling Settings",
+                "config": AppConfig().config,
+                "scheduler_form": SchedulerSettingsForm(),
+            }
+        )
+
+        return render(request, "home/settings_scheduling.html", context)
+
+    def post(self, request):
+        """handle form post to update settings"""
+        scheduler_form = SchedulerSettingsForm(request.POST)
+        if scheduler_form.is_valid():
+            scheduler_form_post = scheduler_form.cleaned_data
+            if any(scheduler_form_post.values()):
+                print(scheduler_form_post)
+                ScheduleBuilder().update_schedule_conf(scheduler_form_post)
+
+        sleep(1)
+        return redirect("settings_scheduling", permanent=True)
+
+
+class SettingsActionsView(MinView):
+    """resolves to /settings/actions/
+    handle the settings actions sub-page
+    """
+
+    def get(self, request):
+        """read and display current settings"""
+        context = self.get_min_context(request)
+        context.update(
+            {
+                "title": "Actions",
+                "available_backups": ElasticBackup().get_all_backup_files(),
+            }
+        )
+
+        return render(request, "home/settings_actions.html", context)
+
+
 def process(request):
     """handle all the buttons calls via POST ajax"""
     if request.method == "POST":

tubearchivist/requirements.txt

@@ -1,13 +1,13 @@
-beautifulsoup4==4.12.2
-celery==5.2.7
-Django==4.2
-django-auth-ldap==4.2.0
-django-cors-headers==3.14.0
+apprise==1.5.0
+celery==5.3.4
+Django==4.2.6
+django-auth-ldap==4.6.0
+django-cors-headers==4.2.0
 djangorestframework==3.14.0
-Pillow==9.5.0
-redis==4.5.4
-requests==2.28.2
+Pillow==10.0.1
+redis==5.0.1
+requests==2.31.0
 ryd-client==0.0.6
-uWSGI==2.0.21
-whitenoise==6.4.0
-yt_dlp==2023.3.4
+uWSGI==2.0.22
+whitenoise==6.5.0
+yt-dlp==2023.10.7

tubearchivist/static/css/style.css

@@ -82,6 +82,7 @@ ul {
 td, th, span, label {
     font-family: Sen-Regular, sans-serif;
     color: var(--main-font);
+    text-align: left;
 }
 
 select, input {
@@ -369,6 +370,10 @@ button:hover {
     display: none;
 }
 
+#hidden-form button {
+    margin-right: 1rem;
+}
+
 #text-reveal {
     height: 0;
     overflow: hidden;
@@ -639,6 +644,10 @@ video:-webkit-full-screen {
     background-color: var(--highlight-bg);
 }
 
+.info-box-4 {
+    grid-template-columns: 1fr 1fr 1fr 1fr;
+}
+
 .info-box-3 {
     grid-template-columns: 1fr 1fr 1fr;
 }
@@ -647,6 +656,10 @@ video:-webkit-full-screen {
     grid-template-columns: 1fr 1fr;
 }
 
+.info-box-1 {
+    grid-template-columns: 1fr;
+}
+
 .info-box img {
     width: 80px;
     margin: 0 15px;
@@ -660,6 +673,10 @@ video:-webkit-full-screen {
     background-color: var(--highlight-bg);
 }
 
+.info-box-item p {
+    width: 100%;
+}
+
 .description-text {
     width: 100%;
 }
@@ -952,15 +969,15 @@ video:-webkit-full-screen {
     transform: translateX(-30%);
 }
 
-.info-box-item.channel-nav {
+.info-box-item.child-page-nav {
     justify-content: center;
 }
 
-.info-box-item.channel-nav a {
+.info-box-item.child-page-nav a {
     padding: 0 1rem;
 }
 
-.info-box-item.channel-nav a:hover {
+.info-box-item.child-page-nav a:hover {
     text-decoration: underline;
 }
 
@@ -1082,6 +1099,15 @@ video:-webkit-full-screen {
     min-width: 300px;
 }
 
+.settings-item .agg-channel-table {
+    width: 100%;
+}
+
+.settings-item .agg-channel-right-align {
+    white-space: nowrap;
+    text-align: right;
+}
+
 .danger-zone {
     background-color: var(--highlight-error);
     color: #fff;
@@ -1232,6 +1258,7 @@ video:-webkit-full-screen {
     .playlist-list.grid,
     .info-box-2,
     .info-box-3,
+    .info-box-4,
     .overwrite-form {
         grid-template-columns: 1fr;
     }
@@ -1298,4 +1325,7 @@ video:-webkit-full-screen {
     .playlist-nav-item img {
         width: 100%;
     }
+    .td, th, span, label {
+        text-align: unset;
+    }
 }

tubearchivist/static/progress.js

@@ -30,9 +30,15 @@ function getMessages(dataOrigin) {
 
 function buildMessage(responseData, dataOrigin) {
   // filter relevant messages
-  let messages = responseData.filter(function (value) {
-    return dataOrigin.split(' ').includes(value.group.split(':')[0]);
-  }, dataOrigin);
+  let messages;
+  if (dataOrigin) {
+    messages = responseData.filter(function (value) {
+      return dataOrigin.split(' ').includes(value.group.split(':')[0]);
+    }, dataOrigin);
+  } else {
+    messages = responseData;
+  }
+
   let notifications = document.getElementById('notifications');
   let currentNotifications = notifications.childElementCount;
 

tubearchivist/static/script.js

@@ -64,27 +64,42 @@ function isWatchedButton(button) {
   let youtube_id = button.getAttribute('data-id');
   let apiEndpoint = '/api/watched/';
   let data = { id: youtube_id, is_watched: true };
-  button.remove();
+  apiRequest(apiEndpoint, 'POST', data);
+  setTimeout(function () {
+    location.reload();
+  }, 1000);
+}
+function isUnwatchedButton(button) {
+  let youtube_id = button.getAttribute('data-id');
+  let apiEndpoint = '/api/watched/';
+  let data = { id: youtube_id, is_watched: false };
   apiRequest(apiEndpoint, 'POST', data);
   setTimeout(function () {
     location.reload();
   }, 1000);
 }
 
-function unsubscribe(id_unsub) {
-  let payload = JSON.stringify({ unsubscribe: id_unsub });
-  sendPost(payload);
+function subscribeStatus(subscribeButton) {
+  let id = subscribeButton.getAttribute('data-id');
+  let type = subscribeButton.getAttribute('data-type');
+  let subscribe = Boolean(subscribeButton.getAttribute('data-subscribe'));
+  let apiEndpoint;
+  let data;
+  if (type === 'channel') {
+    apiEndpoint = '/api/channel/';
+    data = { data: [{ channel_id: id, channel_subscribed: subscribe }] };
+  } else if (type === 'playlist') {
+    apiEndpoint = '/api/playlist/';
+    data = { data: [{ playlist_id: id, playlist_subscribed: subscribe }] };
+  }
+  apiRequest(apiEndpoint, 'POST', data);
   let message = document.createElement('span');
-  message.innerText = 'You are unsubscribed.';
-  document.getElementById(id_unsub).replaceWith(message);
-}
-
-function subscribe(id_sub) {
-  let payload = JSON.stringify({ subscribe: id_sub });
-  sendPost(payload);
-  let message = document.createElement('span');
-  message.innerText = 'You are subscribed.';
-  document.getElementById(id_sub).replaceWith(message);
+  if (subscribe) {
+    message.innerText = 'You are subscribed.';
+  } else {
+    message.innerText = 'You are unsubscribed.';
+  }
+  subscribeButton.replaceWith(message);
 }
 
 function changeView(image) {
@@ -160,6 +175,24 @@ function dlPending() {
   }, 500);
 }
 
+function addToQueue(autostart = false) {
+  let textArea = document.getElementById('id_vid_url');
+  if (textArea.value === '') {
+    return;
+  }
+  let toPost = { data: [{ youtube_id: textArea.value, status: 'pending' }] };
+  let apiEndpoint = '/api/download/';
+  if (autostart) {
+    apiEndpoint = `${apiEndpoint}?autostart=true`;
+  }
+  apiRequest(apiEndpoint, 'POST', toPost);
+  textArea.value = '';
+  setTimeout(function () {
+    checkMessages();
+  }, 500);
+  showForm();
+}
+
 function toIgnore(button) {
   let youtube_id = button.getAttribute('data-id');
   let apiEndpoint = '/api/download/' + youtube_id + '/';
@@ -356,13 +389,11 @@ function deleteChannel(button) {
 function deletePlaylist(button) {
   let playlist_id = button.getAttribute('data-id');
   let playlist_action = button.getAttribute('data-action');
-  let payload = JSON.stringify({
-    'delete-playlist': {
-      'playlist-id': playlist_id,
-      'playlist-action': playlist_action,
-    },
-  });
-  sendPost(payload);
+  let apiEndpoint = `/api/playlist/${playlist_id}/`;
+  if (playlist_action === 'delete-videos') {
+    apiEndpoint += '?delete-videos=true';
+  }
+  apiRequest(apiEndpoint, 'DELETE');
   setTimeout(function () {
     window.location.replace('/playlist/');
   }, 1000);
@@ -915,7 +946,7 @@ function populateMultiSearchResults(allResults, queryType) {
   videoBox.parentElement.style.display = 'block';
   if (allVideos.length > 0) {
     for (let index = 0; index < allVideos.length; index++) {
-      const video = allVideos[index].source;
+      const video = allVideos[index];
       const videoDiv = createVideo(video, defaultVideo);
       videoBox.appendChild(videoDiv);
     }
@@ -934,7 +965,7 @@ function populateMultiSearchResults(allResults, queryType) {
   channelBox.parentElement.style.display = 'block';
   if (allChannels.length > 0) {
     for (let index = 0; index < allChannels.length; index++) {
-      const channel = allChannels[index].source;
+      const channel = allChannels[index];
       const channelDiv = createChannel(channel, defaultChannel);
       channelBox.appendChild(channelDiv);
     }
@@ -953,7 +984,7 @@ function populateMultiSearchResults(allResults, queryType) {
   playlistBox.parentElement.style.display = 'block';
   if (allPlaylists.length > 0) {
     for (let index = 0; index < allPlaylists.length; index++) {
-      const playlist = allPlaylists[index].source;
+      const playlist = allPlaylists[index];
       const playlistDiv = createPlaylist(playlist, defaultPlaylist);
       playlistBox.appendChild(playlistDiv);
     }
@@ -972,7 +1003,7 @@ function populateMultiSearchResults(allResults, queryType) {
   if (allFullText.length > 0) {
     for (let i = 0; i < allFullText.length; i++) {
       const fullText = allFullText[i];
-      if ('highlight' in fullText) {
+      if ('subtitle_line' in fullText) {
         const fullTextDiv = createFulltext(fullText);
         fullTextBox.appendChild(fullTextDiv);
       }
@@ -1039,9 +1070,9 @@ function createChannel(channel, viewStyle) {
   const channelLastRefresh = channel.channel_last_refresh;
   let button;
   if (channel.channel_subscribed) {
-    button = `<button class="unsubscribe" type="button" id="${channelId}" onclick="unsubscribe(this.id)" title="Unsubscribe from ${channelName}">Unsubscribe</button>`;
+    button = `<button class="unsubscribe" type="button" data-id="${channelId}" data-subscribe="" data-type="channel" onclick="subscribeStatus(this)" title="Unsubscribe from ${channelName}">Unsubscribe</button>`;
   } else {
-    button = `<button type="button" id="${channelId}" onclick="subscribe(this.id)" title="Subscribe to ${channelName}">Subscribe</button>`;
+    button = `<button type="button" data-id="${channelId}" data-subscribe="true" data-type="channel" onclick="subscribeStatus(this)" title="Subscribe to ${channelName}">Subscribe</button>`;
   }
   // build markup
   const markup = `
@@ -1085,9 +1116,9 @@ function createPlaylist(playlist, viewStyle) {
   const playlistLastRefresh = playlist.playlist_last_refresh;
   let button;
   if (playlist.playlist_subscribed) {
-    button = `<button class="unsubscribe" type="button" id="${playlistId}" onclick="unsubscribe(this.id)" title="Unsubscribe from ${playlistName}">Unsubscribe</button>`;
+    button = `<button class="unsubscribe" type="button" data-id="${playlistId}" data-subscribe="" data-type="playlist" onclick="subscribeStatus(this)" title="Unsubscribe from ${playlistName}">Unsubscribe</button>`;
   } else {
-    button = `<button type="button" id="${playlistId}" onclick="subscribe(this.id)" title="Subscribe to ${playlistName}">Subscribe</button>`;
+    button = `<button type="button" data-id="${playlistId}" data-subscribe="true" data-type="playlist" onclick="subscribeStatus(this)" title="Subscribe to ${playlistName}">Subscribe</button>`;
   }
   const markup = `
     <div class="playlist-thumbnail">
@@ -1109,19 +1140,14 @@ function createPlaylist(playlist, viewStyle) {
 }
 
 function createFulltext(fullText) {
-  const videoId = fullText.source.youtube_id;
-  const videoTitle = fullText.source.title;
-  const thumbUrl = fullText.source.vid_thumb_url;
-  const channelId = fullText.source.subtitle_channel_id;
-  const channelName = fullText.source.subtitle_channel;
-  const subtitleLine = fullText.highlight.subtitle_line[0];
-  const subtitle_start = fullText.source.subtitle_start.split('.')[0];
-  const subtitle_end = fullText.source.subtitle_end.split('.')[0];
+  const videoId = fullText.youtube_id;
+  const subtitle_start = fullText.subtitle_start.split('.')[0];
+  const subtitle_end = fullText.subtitle_end.split('.')[0];
   const markup = `
     <a href="#player" data-id="${videoId}" data-position="${subtitle_start}" onclick="createPlayer(this)">
         <div class="video-thumb-wrap list">
             <div class="video-thumb">
-                <img src="${thumbUrl}" alt="video-thumb">
+                <img src="${fullText.vid_thumb_url}" alt="video-thumb">
             </div>
             <div class="video-play">
                 <img src="/static/img/icon-play.svg" alt="play-icon">
@@ -1129,12 +1155,13 @@ function createFulltext(fullText) {
         </div>
     </a>
     <div class="video-desc list">
-        <p>${subtitle_start} - ${subtitle_end}</p>
-        <p>${subtitleLine}</p>
         <div>
-            <a href="/channel/${channelId}/"><h3>${channelName}</h3></a>
-            <a class="video-more" href="/video/${videoId}/?t=${subtitle_start}"><h2>${videoTitle}</h2></a>
+          <a href="/channel/${fullText.subtitle_channel_id}/"><h3>${fullText.subtitle_channel}</h3></a>
+          <a class="video-more" href="/video/${videoId}/?t=${subtitle_start}"><h2>${fullText.title}</h2></a>
         </div>
+        <p>${subtitle_start} - ${subtitle_end}</p>
+        <p>${fullText.subtitle_line}</p>
+        <span class="settings-current">Score: ${fullText._score}</span>
     </div>
     `;
   const fullTextDiv = document.createElement('div');
@@ -1183,7 +1210,7 @@ function writeComments(allComments) {
 
 function createReplyButton(replyId, totalReplies) {
   let replyButton = document.createElement('button');
-  replyButton.innerHTML = `<span id="toggle-icon">+</span> ${totalReplies} replies`;
+  replyButton.innerHTML = `<span id="toggle-icon">▼</span> ${totalReplies} replies`;
   replyButton.setAttribute('data-id', replyId);
   replyButton.setAttribute('onclick', 'toggleCommentReplies(this)');
   return replyButton;
@@ -1195,10 +1222,10 @@ function toggleCommentReplies(button) {
 
   if (state === 'none' || state === '') {
     document.getElementById(commentReplyId).style.display = 'block';
-    button.querySelector('#toggle-icon').innerHTML = '-';
+    button.querySelector('#toggle-icon').innerHTML = '▲';
   } else {
     document.getElementById(commentReplyId).style.display = 'none';
-    button.querySelector('#toggle-icon').innerHTML = '+';
+    button.querySelector('#toggle-icon').innerHTML = '▼';
   }
 }
 
@@ -1300,9 +1327,9 @@ function getCookie(c_name) {
 
 // animations
 
-function textReveal() {
-  let textBox = document.getElementById('text-reveal');
-  let button = document.getElementById('text-reveal-button');
+function textReveal(button) {
+  let revealBox = button.parentElement.parentElement;
+  let textBox = revealBox.querySelector('#text-reveal');
   let textBoxHeight = textBox.style.height;
   if (textBoxHeight === 'unset') {
     textBox.style.height = '0px';
@@ -1460,6 +1487,21 @@ function doShortcut(e) {
       player.muted = !player.muted;
       break;
     }
+    case 'f': {
+      e.preventDefault();
+      if (document.fullscreenElement === null) {
+        player.requestFullscreen().catch(e => {
+          console.error(e);
+          showModal('Unable to enter fullscreen', 3000);
+        });
+      } else {
+        document.exitFullscreen().catch(e => {
+          console.error(e);
+          showModal('Unable to exit fullscreen', 3000);
+        });
+      }
+      break;
+    }
     case 'ArrowLeft': {
       e.preventDefault();
       showModal('- 5 seconds', 500);
@@ -1504,6 +1546,7 @@ function doShortcut(e) {
                 <table style="margin: auto; background: rgba(0,0,0,.5)"><tbody>
                 <tr><td>Show help</td><td>?</td>
                 <tr><td>Toggle mute</td><td>m</td>
+                <tr><td>Toggle fullscreen</td><td>f</td>
                 <tr><td>Toggle subtitles (if available)</td><td>c</td>
                 <tr><td>Increase speed</td><td>&gt;</td>
                 <tr><td>Decrease speed</td><td>&lt;</td>

tubearchivist/static/stats.js

@@ -0,0 +1,209 @@
+// build stats for settings page
+
+'use strict';
+
+/* globals apiRequest */
+
+function primaryStats() {
+  let apiEndpoint = '/api/stats/primary/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let primaryBox = document.getElementById('primaryBox');
+  clearLoading(primaryBox);
+  let videoTile = buildVideoTile(responseData);
+  primaryBox.appendChild(videoTile);
+  let channelTile = buildChannelTile(responseData);
+  primaryBox.appendChild(channelTile);
+  let playlistTile = buildPlaylistTile(responseData);
+  primaryBox.appendChild(playlistTile);
+  let downloadTile = buildDownloadTile(responseData);
+  primaryBox.appendChild(downloadTile);
+}
+
+function clearLoading(dashBox) {
+  dashBox.querySelector('#loading').remove();
+}
+
+function buildTile(titleText) {
+  let tile = document.createElement('div');
+  tile.classList.add('info-box-item');
+  let title = document.createElement('h3');
+  title.innerText = titleText;
+  tile.appendChild(title);
+  return tile;
+}
+
+function buildVideoTile(responseData) {
+  let tile = buildTile(`Total Videos: ${responseData.videos.total || 0}`);
+  let message = document.createElement('p');
+  message.innerHTML = `
+        videos: ${responseData.videos.videos || 0}<br>
+        shorts: ${responseData.videos.shorts || 0}<br>
+        streams: ${responseData.videos.streams || 0}<br>
+    `;
+  tile.appendChild(message);
+
+  return tile;
+}
+
+function buildChannelTile(responseData) {
+  let tile = buildTile(`Total Channels: ${responseData.channels.total || 0}`);
+  let message = document.createElement('p');
+  message.innerHTML = `subscribed: ${responseData.channels.sub_true || 0}`;
+  tile.appendChild(message);
+
+  return tile;
+}
+
+function buildPlaylistTile(responseData) {
+  let tile = buildTile(`Total Playlists: ${responseData.playlists.total || 0}`);
+  let message = document.createElement('p');
+  message.innerHTML = `subscribed: ${responseData.playlists.sub_true || 0}`;
+  tile.appendChild(message);
+
+  return tile;
+}
+
+function buildDownloadTile(responseData) {
+  let tile = buildTile('Downloads');
+  let message = document.createElement('p');
+  message.innerHTML = `
+        pending: ${responseData.downloads.pending || 0}<br>
+        ignored: ${responseData.downloads.ignore || 0}<br>
+    `;
+  tile.appendChild(message);
+
+  return tile;
+}
+
+function watchStats() {
+  let apiEndpoint = '/api/stats/watch/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let watchBox = document.getElementById('watchBox');
+  clearLoading(watchBox);
+
+  for (const property in responseData) {
+    let tile = buildWatchTile(property, responseData[property]);
+    watchBox.appendChild(tile);
+  }
+}
+
+function buildWatchTile(title, watchDetail) {
+  let tile = buildTile(`Total ${title}`);
+  let message = document.createElement('p');
+  message.innerHTML = `
+    ${watchDetail.items} Videos<br>
+    ${watchDetail.duration} Seconds<br>
+    ${watchDetail.duration_str} Playback
+  `;
+  if (watchDetail.progress) {
+    message.innerHTML += `<br>${Number(watchDetail.progress * 100).toFixed(2)}%`;
+  }
+  tile.appendChild(message);
+  return tile;
+}
+
+function downloadHist() {
+  let apiEndpoint = '/api/stats/downloadhist/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let histBox = document.getElementById('downHistBox');
+  clearLoading(histBox);
+  if (responseData.length === 0) {
+    let tile = buildTile('No recent downloads');
+    histBox.appendChild(tile);
+    return;
+  }
+
+  for (let i = 0; i < responseData.length; i++) {
+    const dailyStat = responseData[i];
+    let tile = buildDailyStat(dailyStat);
+    histBox.appendChild(tile);
+  }
+}
+
+function buildDailyStat(dailyStat) {
+  let tile = buildTile(dailyStat.date);
+  let message = document.createElement('p');
+  message.innerText = `new videos: ${dailyStat.count}`;
+  tile.appendChild(message);
+  return tile;
+}
+
+function humanFileSize(size) {
+  let i = size === 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024));
+  return (size / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
+}
+
+function buildChannelRow(id, name, value) {
+  let tableRow = document.createElement('tr');
+
+  tableRow.innerHTML = `
+    <td class="agg-channel-name"><a href="/channel/${id}/">${name}</a></td>
+    <td class="agg-channel-right-align">${value}</td>
+  `;
+
+  return tableRow;
+}
+
+function addBiggestChannelByDocCount() {
+  let tBody = document.getElementById('biggestChannelTableVideos');
+
+  let apiEndpoint = '/api/stats/biggestchannels/?order=doc_count';
+  const responseData = apiRequest(apiEndpoint, 'GET');
+
+  for (let i = 0; i < responseData.length; i++) {
+    const { id, name, doc_count } = responseData[i];
+
+    let tableRow = buildChannelRow(id, name, doc_count);
+
+    tBody.appendChild(tableRow);
+  }
+}
+
+function addBiggestChannelByDuration() {
+  const tBody = document.getElementById('biggestChannelTableDuration');
+
+  let apiEndpoint = '/api/stats/biggestchannels/?order=duration';
+  const responseData = apiRequest(apiEndpoint, 'GET');
+
+  for (let i = 0; i < responseData.length; i++) {
+    const { id, name, duration_str } = responseData[i];
+
+    let tableRow = buildChannelRow(id, name, duration_str);
+
+    tBody.appendChild(tableRow);
+  }
+}
+
+function addBiggestChannelByMediaSize() {
+  let tBody = document.getElementById('biggestChannelTableMediaSize');
+
+  let apiEndpoint = '/api/stats/biggestchannels/?order=media_size';
+  const responseData = apiRequest(apiEndpoint, 'GET');
+
+  for (let i = 0; i < responseData.length; i++) {
+    const { id, name, media_size } = responseData[i];
+
+    let tableRow = buildChannelRow(id, name, humanFileSize(media_size));
+
+    tBody.appendChild(tableRow);
+  }
+}
+
+function biggestChannel() {
+  addBiggestChannelByDocCount();
+  addBiggestChannelByDuration();
+  addBiggestChannelByMediaSize();
+}
+
+async function buildStats() {
+  primaryStats();
+  watchStats();
+  downloadHist();
+  biggestChannel();
+}
+
+document.addEventListener('DOMContentLoaded', () => {
+  window.requestAnimationFrame(() => {
+    buildStats();
+  });
+});