bump archivist-es

Changed:
- Added bulk clear error button
- bump yt-dlp

.gitattributes

@@ -1 +1 @@
-docker_assets\run.sh eol=lf
+* text=auto eol=lf

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -15,6 +15,7 @@ body:
       options:
         - label: I'm running the latest version of Tube Archivist and have read the [release notes](https://github.com/tubearchivist/tubearchivist/releases/latest).
           required: true
+        - label: I'm [beta testing](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#beta-testing) and am running the latest unstable build. 
         - label: I have read the [how to open an issue](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#how-to-open-an-issue) guide, particularly the [bug report](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#bug-report) section.
           required: true
 

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -10,3 +10,5 @@ body:
       options:
         - label: I understand that this issue will be closed without comment.
           required: true
+        - label: I will resist the temptation and I will not submit this issue. If I submit this, I understand I might get blocked from this repo.
+          required: true

.github/ISSUE_TEMPLATE/FRONTEND-MIGRATION.yml

@@ -1,23 +0,0 @@
-name: Frontend Migration
-description: Tracking our new React based frontend
-title: "[Frontend Migration]: "
-labels: ["react migration"]
-
-body:
-  - type: dropdown
-    id: domain
-    attributes:
-      label: Domain
-      options:
-        - Frontend
-        - Backend
-        - Combined
-    validations:
-      required: true
-  - type: textarea
-    id: description
-    attributes:
-      label: Description
-      placeholder: Organizing our React frontend migration
-    validations:
-      required: true

.gitignore

@@ -12,3 +12,5 @@ backend/.env
 
 # JavaScript stuff
 node_modules
+
+.editorconfig

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
         files: ^backend/
         args: ["--profile", "black", "-l 79"]
   - repo: https://github.com/pycqa/flake8
-    rev: 7.1.2
+    rev: 7.3.0
     hooks:
       - id: flake8
         alias: python
@@ -31,7 +31,7 @@ repos:
       - id: codespell
         exclude: ^frontend/package-lock.json
   - repo: https://github.com/pre-commit/mirrors-eslint
-    rev: v9.22.0
+    rev: v9.30.1
     hooks:
       - id: eslint
         name: eslint

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributing to Tube Archivist
 
-Welcome, and thanks for showing interest in improving Tube Archivist!  
+Welcome, and thanks for showing interest in improving Tube Archivist!
 
 ## Table of Content
 - [Beta Testing](#beta-testing)
@@ -16,20 +16,20 @@ Welcome, and thanks for showing interest in improving Tube Archivist!
 ---
 
 ## Beta Testing
-Be the first to help test new features and improvements and provide feedback! There are regular `:unstable` builds for easy access. That's for the tinkerers and the breave. Ideally use a testing environment first, before a release be the first to install it on your main system.
+Be the first to help test new features/improvements and provide feedback! Regular `:unstable` builds are available for early access. These are for the tinkerers and the brave. Ideally, use a testing environment first, before upgrading your main installation.
 
-There is always something that can get missed during development. Look at the commit messages tagged with `#build`, these are the unstable builds and give a quick overview what has changed.
+There is always something that can get missed during development. Look at the commit messages tagged with `#build` - these are the unstable builds and give a quick overview of what has changed.
 
 - Test the features mentioned, play around, try to break it.
-- Test the update path by installing the `:latest` release first, the upgrade to `:unstable` to check for any errors.
+- Test the update path by installing the `:latest` release first, then upgrade to `:unstable` to check for any errors.
 - Test the unstable build on a fresh install.
 
-Then provide feedback, if there is a problem but also if there is no problem. Reach out on [Discord](https://tubearchivist.com/discord) in the `#beta-testing` channel with your findings.
+Then provide feedback - even if you don't encounter any issues! You can do this in the `#beta-testing` channel on the [Discord](https://tubearchivist.com/discord) Discord server.
 
-This will help with a smooth update for the regular release. Plus you get to test things out early! 
+This helps ensure a smooth update for the stable release. Plus you get to test things out early!
 
 ## How to open an issue
-Please read this carefully before opening any [issue](https://github.com/tubearchivist/tubearchivist/issues) on GitHub. Make sure you read [Next Steps](#next-steps) above.
+Please read this carefully before opening any [issue](https://github.com/tubearchivist/tubearchivist/issues) on GitHub.
 
 **Do**:
 - Do provide details and context, this matters a lot and makes it easier for people to help.
@@ -37,7 +37,7 @@ Please read this carefully before opening any [issue](https://github.com/tubearc
 - Do respond to questions within a day or two so issues can progress. If the issue doesn't move forward due to a lack of response, we'll assume it's solved and we'll close it after some time to keep the list fresh.
 
 **Don't**:
-- Don't open *duplicates*, that includes open and closed issues.
+- Don't open *duplicates*, that includes open and closed issues. Also don't post the same issue on multiple platforms, that makes it unnecessarily hard for maintainers to keep up.
 - Don't open an issue for something that's already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), this needs your help to implement it, not another issue.
 - Don't open an issue for something that's a [known limitation](https://github.com/tubearchivist/tubearchivist#known-limitations). These are *known* by definition and don't need another reminder. Some limitations may be solved in the future, maybe by you?
 - Don't overwrite the *issue template*, they are there for a reason. Overwriting that shows that you don't really care about this project. It shows that you have a misunderstanding how open source collaboration works and just want to push your ideas through. Overwriting the template may result in a ban.
@@ -65,16 +65,22 @@ IMPORTANT: When receiving help, contribute back to the community by improving th
 
 ## How to make a Pull Request
 
-Make sure you read [Next Steps](#next-steps) above.
-
-Thank you for contributing and helping improve this project. Focus for the foreseeable future is on improving and building on existing functionality, *not* on adding and expanding the application.
+Focus for the foreseeable future is on improving and building on existing functionality, *not* on adding and expanding the application.
 
 This is a quick checklist to help streamline the process:
 
 - For **code changes**, make your PR against the [testing branch](https://github.com/tubearchivist/tubearchivist/tree/testing). That's where all active development happens. This simplifies the later merging into *master*, minimizes any conflicts and usually allows for easy and convenient *fast-forward* merging.
-- For **documentation changes**, make your PR directly against the *master* branch.
 - Show off your progress, even if not yet complete, by creating a [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) PR first and switch it as *ready* when you are ready.
-- Make sure all your code is linted and formatted correctly, see below. The automatic GH action unfortunately needs to be triggered manually by a maintainer for first time contributors, but will trigger automatically for existing contributors.
+- Make sure all your code is linted and formatted correctly, see below.
+
+### Documentation Changes
+
+All documentation is intended to represent the state of the [latest](https://github.com/tubearchivist/tubearchivist/releases/latest) release. 
+
+- If your PR with code changes also requires changes to documentation *.md files here in this repo, create a separate PR for that, so it can be merged separately at release.
+- You can make the PR directly against the *master* branch.
+- If your PR requires changes on the [tubearchivist/docs](https://github.com/tubearchivist/docs), make the PR over there.
+- Prepare your documentation updates at the same time as the code changes, so people testing your PR can consult the prepared docs if needed.
 
 ### Code formatting and linting
 
@@ -83,7 +89,7 @@ This project uses the excellent [pre-commit](https://github.com/pre-commit/pre-c
 **Quick Start**
 - Run `pre-commit install` from the root of the repo.
 - Next time you commit to your local git repo, the defined hooks will run.
-- On first run, this will download and install the needed environments to your local machine, that can take some time. But that will be reused on sunsequent commits. 
+- On first run, this will download and install the needed environments to your local machine, that can take some time. But that will be reused on sunsequent commits.
 
 That is also running as a Git Hub action.
 
@@ -93,7 +99,7 @@ That is also running as a Git Hub action.
 
 As you have read the [FAQ](https://docs.tubearchivist.com/faq/) and the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and have gotten an idea what this project tries to do, there will be some obvious shortcomings that stand out, that have been explicitly excluded from the scope of this project, at least for the time being.
 
-Extending the scope of this project will only be feasible with more [regular contributors](https://github.com/tubearchivist/tubearchivist/graphs/contributors) that are willing to help improve this project in the long run. Contributors that have an overall improvement of the project in mind and not just about implementing this *one* thing.  
+Extending the scope of this project will only be feasible with more [regular contributors](https://github.com/tubearchivist/tubearchivist/graphs/contributors) that are willing to help improve this project in the long run. Contributors that have an overall improvement of the project in mind and not just about implementing this *one* thing.
 
 Small minor additions, or making a PR for a documented feature request or bug, even if that was and will be your only contribution to this project, are always welcome and is *not* what this is about.
 
@@ -101,7 +107,7 @@ Beyond that, general rules to consider:
 
 - Maintainability is key: It's not just about implementing something and being done with it, it's about maintaining it, fixing bugs as they occur, improving on it and supporting it in the long run.
 - Others can do it better: Some problems have been solved by very talented developers. These things don't need to be reinvented again here in this project.
-- Develop for the 80%: New features and additions *should* be beneficial for 80% of the users. If you are trying to solve your own problem that only applies to you, maybe that would be better to do in your own fork or if possible by a standalone implementation using the API.
+- Develop for the 80%: New features and additions *should* be beneficial for 80% of the users. If you are trying to solve your own problem that only apply to you, maybe that would be better to do in your own fork or if possible by a standalone implementation using the API.
 - If all of that sounds too strict for you, as stated above, start becoming a regular contributor to this project.
 
 ---
@@ -127,7 +133,7 @@ The documentation available at [docs.tubearchivist.com](https://docs.tubearchivi
 This codebase is set up to be developed natively outside of docker as well as in a docker container. Developing outside of a docker container can be convenient, as IDE and hot reload usually works out of the box. But testing inside of a container is still essential, as there are subtle differences, especially when working with the filesystem and networking between containers.
 
 Note:
-- Subtitles currently fail to load with `DJANGO_DEBUG=True`, that is due to incorrect `Content-Type` error set by Django's static file implementation. That's only if you run the Django dev server, Nginx sets the correct headers.
+- Subtitles currently fail to load with `DJANGO_DEBUG=True`, that is due to incorrect `Content-Type` error set by Django's static file implementation. That's only if you run the Django dev server, Nginx sets the correct headers in the container.
 
 ### Native Instruction
 
@@ -177,12 +183,6 @@ And the frontend should be available at [localhost:3000](localhost:3000).
 
 ### Docker Instructions
 
-Set up docker on your development machine.
-
-Clone this repository.
-
-Functional changes should be made against the unstable `testing` branch, so check that branch out, then make a new branch for your work.
-
 Edit the `docker-compose.yml` file and replace the [`image: bbilly1/tubearchivist` line](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L7) with `build: .`. Also make any other changes to the environment variables and so on necessary to run the application, just like you're launching the application as normal.
 
 Run `docker compose up --build`. This will bring up the application. Kill it with `ctrl-c` or by running `docker compose down` from a new terminal window in the same directory.
@@ -231,7 +231,7 @@ If you want to run queries on the Elasticsearch container directly from your hos
 The token will get stored in ES in the `config` folder, and not in the `data` folder. To persist the token between ES container rebuilds, you'll need to persist the config folder as an additional volume:
 
 1. Create the token as described above
-2. While the container is running, copy the current config folder out of the container, e.g.: 
+2. While the container is running, copy the current config folder out of the container, e.g.:
 ```
 docker cp archivist-es:/usr/share/elasticsearch/config/ volume/es_config
 ```

Dockerfile

@@ -1,20 +1,24 @@
 # multi stage to build tube archivist
 # build python wheel, download and extract ffmpeg, copy into final image
 
+FROM node:lts-alpine AS npm-builder
+COPY frontend/package.json frontend/package-lock.json /
+RUN npm i
+
 FROM node:lts-alpine AS node-builder
 
 # RUN npm config set registry https://registry.npmjs.org/
 
+COPY --from=npm-builder ./node_modules /frontend/node_modules
 COPY ./frontend /frontend
-
 WORKDIR /frontend
-RUN npm i
+
 RUN npm run build:deploy
 
 WORKDIR /
 
 # First stage to build python wheel
-FROM python:3.11.8-slim-bookworm AS builder
+FROM python:3.11.13-slim-bookworm AS builder
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential gcc libldap2-dev libsasl2-dev libssl-dev git
@@ -24,7 +28,7 @@ COPY ./backend/requirements.txt /requirements.txt
 RUN pip install --user -r requirements.txt
 
 # build ffmpeg
-FROM python:3.11.8-slim-bookworm AS ffmpeg-builder
+FROM python:3.11.13-slim-bookworm AS ffmpeg-builder
 
 ARG TARGETPLATFORM
 
@@ -32,7 +36,7 @@ COPY docker_assets/ffmpeg_download.py ffmpeg_download.py
 RUN python ffmpeg_download.py $TARGETPLATFORM
 
 # build final image
-FROM python:3.11.8-slim-bookworm AS tubearchivist
+FROM python:3.11.13-slim-bookworm AS tubearchivist
 
 ARG INSTALL_DEBUG
 
@@ -54,9 +58,9 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm
 
 # install debug tools for testing environment
 RUN if [ "$INSTALL_DEBUG" ] ; then \
-        apt-get -y update && apt-get -y install --no-install-recommends \
-        vim htop bmon net-tools iputils-ping procps lsof \
-        && pip install --user ipython pytest pytest-django \
+    apt-get -y update && apt-get -y install --no-install-recommends \
+    vim htop bmon net-tools iputils-ping procps lsof \
+    && pip install --user ipython pytest pytest-django \
     ; fi
 
 # make folders
@@ -70,6 +74,7 @@ RUN sed -i 's/^user www\-data\;$/user root\;/' /etc/nginx/nginx.conf
 COPY ./backend /app
 COPY ./docker_assets/run.sh /app
 COPY ./docker_assets/backend_start.py /app
+COPY ./docker_assets/beat_auto_spawn.sh /app
 
 COPY --from=node-builder ./frontend/dist /app/static
 

README.md

@@ -1,4 +1,4 @@
-![Tube Archivist](assets/tube-archivist-front.jpg?raw=true "Tube Archivist Banner")  
+![Tube Archivist](assets/tube-archivist-front.jpg?raw=true "Tube Archivist Banner")
 [*more screenshots and video*](SHOWCASE.MD)
 
 <div align="center">
@@ -23,13 +23,13 @@
 ------------------------
 
 ## Core functionality
-Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. This includes:  
+Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. This includes:
 * Subscribe to your favorite YouTube channels
 * Download Videos using **[yt-dlp](https://github.com/yt-dlp/yt-dlp)**
 * Index and make videos searchable
 * Play videos
 * Keep track of viewed and unviewed videos
-  
+
 ## Resources
 - [Discord](https://www.tubearchivist.com/discord): Connect with us on our Discord server.
 - [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit.
@@ -38,8 +38,8 @@ Once your YouTube video collection grows, it becomes hard to search and find a s
 - [Plex Plugin](https://github.com/tubearchivist/tubearchivist-plex): Add your videos to Plex
 
 ## Installing
-For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. Minimal with dual core with 4 threads, better quad core plus. 
-This project requires docker. Ensure it is installed and running on your system.  
+For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. Minimal with dual core with 4 threads, better quad core plus.
+This project requires docker. Ensure it is installed and running on your system.
 
 The documentation has additional user provided instructions for [Unraid](https://docs.tubearchivist.com/installation/unraid/), [Synology](https://docs.tubearchivist.com/installation/synology/) and [Podman](https://docs.tubearchivist.com/installation/podman/).
 
@@ -49,7 +49,7 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
 
 All environment variables are explained in detail in the docs [here](https://docs.tubearchivist.com/installation/env-vars/).
 
-**TubeArchivist**:  
+**TubeArchivist**:
 | Environment Var | Value |  |
 | ----------- | ----------- | ----------- |
 | TA_HOST | Server IP or hostname `http://tubearchivist.local:8000` | Required |
@@ -71,9 +71,19 @@ All environment variables are explained in detail in the docs [here](https://doc
 | ELASTIC_USER | Change the default ElasticSearch user | Optional |
 | TA_LDAP | Configure TA to use LDAP Authentication | [Read more](https://docs.tubearchivist.com/configuration/ldap/) |
 | DISABLE_STATIC_AUTH | Remove authentication from media files, (Google Cast...) | [Read more](https://docs.tubearchivist.com/installation/env-vars/#disable_static_auth) |
+| TA_AUTO_UPDATE_YTDLP | Configure TA to automatically install the latest yt-dlp on container start | Optional |
 | DJANGO_DEBUG | Return additional error messages, for debug only | Optional |
+| TA_LOGIN_AUTH_MODE | Configure the order of login authentication backends (Default: single) | Optional |
 
-**ElasticSearch**  
+| TA_LOGIN_AUTH_MODE value | Description |
+| ------------------------ | ----------- |
+| single                   | Only use a single backend (default, or LDAP, or Forward auth, selected by TA_LDAP or TA_ENABLE_AUTH_PROXY) |
+| local                    | Use local password database only |
+| ldap                     | Use LDAP backend only |
+| forwardauth              | Use reverse proxy headers only |
+| ldap_local               | Use LDAP backend in addition to the local password database |
+
+**ElasticSearch**
 | Environment Var | Value | State |
 | ----------- | ----------- | ----------- |
 | ELASTIC_PASSWORD | Matching password `ELASTIC_PASSWORD` from TubeArchivist | Required |
@@ -81,58 +91,58 @@ All environment variables are explained in detail in the docs [here](https://doc
 
 
 ## Update
-Always use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tags are only for your testing environment, there might not be an update path for these testing builds. 
+Always use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tags see [CONTRIBUTING.md#beta-testing](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#beta-testing).
 
-You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. To update, you need to update the docker images, the method for which will depend on your platform. For example, if you're using `docker-compose`, run `docker-compose pull` and then restart with `docker-compose up -d`. After updating, check the footer to verify you are running the expected version.  
+You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. After updating, check the footer to verify you are running the expected version.
 
-  - This project is tested for updates between one or two releases maximum. Further updates back may or may not be supported and you might have to reset your index and configurations to update. Ideally apply new updates at least once per month.  
-  - There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.  
-  - All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version.   
+  - This project is tested for updates between one or two releases maximum. Further updates back may or may not be supported. Ideally apply new updates at least once per month.
+  - There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.
+  - All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated from time to time. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version.
 
 ## Getting Started
 1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS and some other browsers a compatible format must be specified. For example:
 ```
 bestvideo[vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4
 ```
-2. Subscribe to some of your favorite YouTube channels on the **channels** page. 
+2. Subscribe to some of your favorite YouTube channels on the **channels** page.
 3. On the **downloads** page, click on *Rescan subscriptions* to add videos from the subscribed channels to your Download queue or click on *Add to download queue* to manually add Video IDs, links, channels or playlists.
-4. Click on *Start download* and let **Tube Archivist** to it's thing. 
+4. Click on *Start download* and let **Tube Archivist** to it's thing.
 5. Enjoy your archived collection!
 
 
-### Port Collisions  
-If you have a collision on port `8000`, best solution is to use dockers *HOST_PORT* and *CONTAINER_PORT* distinction: To for example change the interface to port 9000 use `9000:8000` in your docker-compose file.  
+### Port Collisions
+If you have a collision on port `8000`, best solution is to use dockers *HOST_PORT* and *CONTAINER_PORT* distinction: To for example change the interface to port 9000 use `9000:8000` in your docker-compose file.
 
 For more information on port collisions, check the docs.
 
-## Common Errors  
-Here is a list of common errors and their solutions.  
+## Common Errors
+Here is a list of common errors and their solutions.
 
 ### `vm.max_map_count`
 **Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.
 
-To temporary set the value run:  
+To temporary set the value run:
 ```
 sudo sysctl -w vm.max_map_count=262144
-```  
-To apply the change permanently depends on your host operating system:  
+```
+To apply the change permanently depends on your host operating system:
 
  - For example on Ubuntu Server add `vm.max_map_count = 262144` to the file `/etc/sysctl.conf`.
- - On Arch based systems create a file `/etc/sysctl.d/max_map_count.conf` with the content `vm.max_map_count = 262144`. 
- - On any other platform look up in the documentation on how to pass kernel parameters.  
+ - On Arch based systems create a file `/etc/sysctl.d/max_map_count.conf` with the content `vm.max_map_count = 262144`.
+ - On any other platform look up in the documentation on how to pass kernel parameters.
 
 
 ### Permissions for elasticsearch
-If you see a message similar to `Unable to access 'path.repo' (/usr/share/elasticsearch/data/snapshot)` or `failed to obtain node locks, tried [/usr/share/elasticsearch/data]` and `maybe these locations are not writable` when initially starting elasticsearch, that probably means the container is not allowed to write files to the volume.  
+If you see a message similar to `Unable to access 'path.repo' (/usr/share/elasticsearch/data/snapshot)` or `failed to obtain node locks, tried [/usr/share/elasticsearch/data]` and `maybe these locations are not writable` when initially starting elasticsearch, that probably means the container is not allowed to write files to the volume.
 To fix that issue, shutdown the container and on your host machine run:
 ```
 chown 1000:0 -R /path/to/mount/point
 ```
-This will match the permissions with the **UID** and **GID** of elasticsearch process within the container and should fix the issue.  
+This will match the permissions with the **UID** and **GID** of elasticsearch process within the container and should fix the issue.
 
 
 ### Disk usage
-The Elasticsearch index will turn to ***read only*** if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`.  
+The Elasticsearch index will turn to ***read only*** if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`.
 
 Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
 
@@ -142,7 +152,7 @@ If you are seeing errors like `failed to create shim: OCI runtime create failed`
 This can happen if you have nested virtualizations, e.g. LXC running Docker in Proxmox.
 
 ## Known limitations
-- Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
+- Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection.
 - Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
 - There is no flexibility in naming of the media files.
 
@@ -191,7 +201,7 @@ Implemented:
 - [X] Scan your file system to index already downloaded videos [2021-09-14]
 
 ## User Scripts
-This is a list of useful user scripts, generously created from folks like you to extend this project and its functionality. Make sure to check the respective repository links for detailed license information.  
+This is a list of useful user scripts, generously created from folks like you to extend this project and its functionality. Make sure to check the respective repository links for detailed license information.
 
 This is your time to shine, [read this](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#user-scripts) then open a PR to add your script here.
 
@@ -200,13 +210,13 @@ This is your time to shine, [read this](https://github.com/tubearchivist/tubearc
 - [DarkFighterLuke/ta_base_url_nginx](https://gist.github.com/DarkFighterLuke/4561b6bfbf83720493dc59171c58ac36): Set base URL with Nginx when you can't use subdomains.
 - [lamusmaser/ta_migration_helper](https://github.com/lamusmaser/ta_migration_helper): Advanced helper script for migration issues to TubeArchivist v0.4.4 or later.
 - [lamusmaser/create_info_json](https://gist.github.com/lamusmaser/837fb58f73ea0cad784a33497932e0dd): Script to generate `.info.json` files using `ffmpeg` collecting information from downloaded videos.
-- [lamusmaser/ta_fix_for_video_redirection](https://github.com/lamusmaser/ta_fix_for_video_redirection): Script to fix videos that were incorrectly indexed by YouTube's "Video is Unavailable" response. 
+- [lamusmaser/ta_fix_for_video_redirection](https://github.com/lamusmaser/ta_fix_for_video_redirection): Script to fix videos that were incorrectly indexed by YouTube's "Video is Unavailable" response.
 - [RoninTech/ta-helper](https://github.com/RoninTech/ta-helper): Helper script to provide a symlink association to reference TubeArchivist videos with their original titles.
 - [tangyjoust/Tautulli-Notify-TubeArchivist-of-Plex-Watched-State](https://github.com/tangyjoust/Tautulli-Notify-TubeArchivist-of-Plex-Watched-State) Mark videos watched in Plex (through streaming not manually) through Tautulli back to TubeArchivist
 - [Dhs92/delete_shorts](https://github.com/Dhs92/delete_shorts): A script to delete ALL YouTube Shorts from TubeArchivist
 
 ## Donate
-The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.  
+The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.
 Second best way to support the development is to provide for caffeinated beverages:
 * [GitHub Sponsor](https://github.com/sponsors/bbilly1) become a sponsor here on GitHub
 * [Paypal.me](https://paypal.me/bbilly1) for a one time coffee

assets/tube-archivist-logo-dark.svg

@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg id="Layer_1" xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 1000 1000">
+  <!-- Generator: Adobe Illustrator 29.5.0, SVG Export Plug-In . SVG Version: 2.1.0 Build 137)  -->
+  <defs>
+    <style>
+      .st0 {
+        fill: #fff;
+      }
+
+      .st1 {
+        fill: #039a86;
+      }
+
+      .st2 {
+        fill: none;
+      }
+
+      .st3 {
+        clip-path: url(#clippath-1);
+      }
+
+      .st4 {
+        fill: #06131a;
+      }
+
+      .st5 {
+        clip-path: url(#clippath-3);
+      }
+
+      .st6 {
+        display: none;
+      }
+
+      .st7 {
+        clip-path: url(#clippath-2);
+      }
+
+      .st8 {
+        clip-path: url(#clippath);
+      }
+    </style>
+    <clipPath id="clippath">
+      <rect class="st2" x="25.6" y="22.9" width="948.9" height="954.2"/>
+    </clipPath>
+    <clipPath id="clippath-1">
+      <rect class="st2" x="25.6" y="22.9" width="948.9" height="954.2"/>
+    </clipPath>
+    <clipPath id="clippath-2">
+      <rect class="st2" x="25.6" y="22.9" width="948.9" height="954.2"/>
+    </clipPath>
+    <clipPath id="clippath-3">
+      <rect class="st2" x="25.6" y="22.9" width="948.9" height="954.2"/>
+    </clipPath>
+  </defs>
+  <g id="Artwork_1" class="st6">
+    <g class="st8">
+      <g class="st3">
+        <path class="st1" d="M447.2,22.9v15.2C269.3,59.3,118.8,179.4,58.6,348.1l76,21.8c49.9-135.2,169.9-232.2,312.6-252.7v15.4h35.3s0-109.7,0-109.7h-35.3ZM523,34.5v79.1c142.3,7.7,269.2,91.9,331.7,219.9l-14.8,4.2,9.7,33.7,106.6-30.3-9.7-33.9-14.9,4.3c-73.1-161.9-231-269-408.5-277M957.6,382.9l-75.8,21.7c8.9,32.9,13.6,66.8,13.8,100.8-.2,103.8-41.6,203.3-114.9,276.8l-9.4-12.6-28.6,20.8,11.9,16,46.5,64,6.6,9.1,28.6-20.8-8.8-12.1c93.6-88.8,146.7-212.1,147-341.1-.2-41.4-5.9-82.6-16.8-122.6M35.3,383.5l-9.7,33.9,14,4c-5.3,27.7-8.1,55.8-8.4,84,0,145.5,67.3,282.8,182.1,372.1l46.5-64c-94.4-74.4-149.6-187.9-149.8-308.1.3-20.8,2.2-41.6,5.8-62.1l15.1,4.1,9.7-33.9-17.9-4.9-75.7-21.7-11.6-3.3ZM303.8,820.6l-64.8,88.8,28.6,20.8,8.5-11.7c69.4,38.3,147.4,58.5,226.7,58.7,94.9,0,187.7-28.7,266.1-82.2l-46.6-64.1c-64.8,43.9-141.2,67.3-219.5,67.5-62.6-.3-124.2-15.5-179.8-44.4l9.4-12.6-28.6-20.8Z"/>
+        <polygon class="st4" points="114.9 238.4 115.1 324.3 261.3 324.3 261.1 458.5 351.9 458.5 352.1 324.3 495.9 324.3 495.6 238 114.9 238.4"/>
+        <rect class="st4" x="261.1" y="554.4" width="90.8" height="200.1"/>
+        <polygon class="st4" points="622.7 244.2 429.6 754.5 526.4 754.4 666.6 361.6 806 754.4 902.9 754.4 710.4 244.2 622.7 244.2"/>
+        <path class="st1" d="M255.5,476.4c-16.5,0-29.9,13.6-29.9,30.1.2,17.6,16.1,30.1,30,30.1,34.5,0,69.9,0,103.3,0,16.1,0,28.9-14,28.9-30.1,0-16.1-12.2-30.1-28.8-30.1-35.8,0-72.8,0-103.4,0"/>
+        <path class="st1" d="M665.5,483.6c-16.1,0-29.8,12.2-29.8,28.8v172l-37.8-38.9-25,24.5,92.2,93.8,94.3-93.8-25-24.5-38.9,38.9c0-23.6,0-40.8,0-68.6-.3-34.5,0-69,0-103.6,0-16.1-13.7-28.6-29.8-28.6h0Z"/>
+      </g>
+    </g>
+  </g>
+  <g id="Artwork_2">
+    <g class="st7">
+      <g class="st5">
+        <path class="st1" d="M447.2,22.9v15.2C269.3,59.3,118.8,179.4,58.6,348.1l76,21.8c49.9-135.2,169.9-232.2,312.6-252.7v15.4h35.3s0-109.7,0-109.7h-35.3ZM523,34.5v79.1c142.3,7.7,269.2,91.9,331.7,219.9l-14.8,4.2,9.7,33.7,106.6-30.3-9.7-33.9-14.9,4.3c-73.1-161.9-231-269-408.5-277M957.6,382.9l-75.8,21.7c8.9,32.9,13.6,66.8,13.8,100.8-.2,103.8-41.6,203.3-114.9,276.8l-9.4-12.6-28.6,20.8,11.9,16,46.5,64,6.6,9.1,28.6-20.8-8.8-12.1c93.6-88.8,146.7-212.1,147-341.1-.2-41.4-5.9-82.6-16.8-122.6M35.3,383.5l-9.7,33.9,14,4c-5.3,27.7-8.1,55.8-8.4,84,0,145.5,67.3,282.8,182.1,372.1l46.5-64c-94.4-74.4-149.6-187.9-149.8-308.1.3-20.8,2.2-41.6,5.8-62.1l15.1,4.1,9.7-33.9-17.9-4.9-75.7-21.7-11.6-3.3ZM303.8,820.6l-64.8,88.8,28.6,20.8,8.5-11.7c69.4,38.3,147.4,58.5,226.7,58.7,94.9,0,187.7-28.7,266.1-82.2l-46.6-64.1c-64.8,43.9-141.2,67.3-219.5,67.5-62.6-.3-124.2-15.5-179.8-44.4l9.4-12.6-28.6-20.8Z"/>
+        <polygon class="st0" points="114.9 238.4 115.1 324.3 261.3 324.3 261.1 458.5 351.9 458.5 352.1 324.3 495.9 324.3 495.6 238 114.9 238.4"/>
+        <rect class="st0" x="261.1" y="554.4" width="90.8" height="200.1"/>
+        <polygon class="st0" points="622.7 244.2 429.6 754.5 526.4 754.4 666.6 361.6 806 754.4 902.9 754.4 710.4 244.2 622.7 244.2"/>
+        <path class="st1" d="M255.5,476.4c-16.5,0-29.9,13.6-29.9,30.1.2,17.6,16.1,30.1,30,30.1,34.5,0,69.9,0,103.3,0,16.1,0,28.9-14,28.9-30.1,0-16.1-12.2-30.1-28.8-30.1-35.8,0-72.8,0-103.4,0"/>
+        <path class="st1" d="M665.5,483.6c-16.1,0-29.8,12.2-29.8,28.8v172l-37.8-38.9-25,24.5,92.2,93.8,94.3-93.8-25-24.5-38.9,38.9c0-23.6,0-40.8,0-68.6-.3-34.5,0-69,0-103.6,0-16.1-13.7-28.6-29.8-28.6h0Z"/>
+      </g>
+    </g>
+  </g>
+</svg>

assets/tube-archivist-logo-light.svg

@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg id="Layer_1" xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 1000 1000">
+  <!-- Generator: Adobe Illustrator 29.5.0, SVG Export Plug-In . SVG Version: 2.1.0 Build 137)  -->
+  <defs>
+    <style>
+      .st0 {
+        fill: #fff;
+      }
+
+      .st1 {
+        fill: #039a86;
+      }
+
+      .st2 {
+        fill: none;
+      }
+
+      .st3 {
+        clip-path: url(#clippath-1);
+      }
+
+      .st4 {
+        fill: #06131a;
+      }
+
+      .st5 {
+        clip-path: url(#clippath-3);
+      }
+
+      .st6 {
+        display: none;
+      }
+
+      .st7 {
+        clip-path: url(#clippath-2);
+      }
+
+      .st8 {
+        clip-path: url(#clippath);
+      }
+    </style>
+    <clipPath id="clippath">
+      <rect class="st2" x="25.6" y="22.9" width="948.9" height="954.2"/>
+    </clipPath>
+    <clipPath id="clippath-1">
+      <rect class="st2" x="25.6" y="22.9" width="948.9" height="954.2"/>
+    </clipPath>
+    <clipPath id="clippath-2">
+      <rect class="st2" x="25.6" y="22.9" width="948.9" height="954.2"/>
+    </clipPath>
+    <clipPath id="clippath-3">
+      <rect class="st2" x="25.6" y="22.9" width="948.9" height="954.2"/>
+    </clipPath>
+  </defs>
+  <g id="Artwork_1">
+    <g class="st8">
+      <g class="st3">
+        <path class="st1" d="M447.2,22.9v15.2C269.3,59.3,118.8,179.4,58.6,348.1l76,21.8c49.9-135.2,169.9-232.2,312.6-252.7v15.4h35.3s0-109.7,0-109.7h-35.3ZM523,34.5v79.1c142.3,7.7,269.2,91.9,331.7,219.9l-14.8,4.2,9.7,33.7,106.6-30.3-9.7-33.9-14.9,4.3c-73.1-161.9-231-269-408.5-277M957.6,382.9l-75.8,21.7c8.9,32.9,13.6,66.8,13.8,100.8-.2,103.8-41.6,203.3-114.9,276.8l-9.4-12.6-28.6,20.8,11.9,16,46.5,64,6.6,9.1,28.6-20.8-8.8-12.1c93.6-88.8,146.7-212.1,147-341.1-.2-41.4-5.9-82.6-16.8-122.6M35.3,383.5l-9.7,33.9,14,4c-5.3,27.7-8.1,55.8-8.4,84,0,145.5,67.3,282.8,182.1,372.1l46.5-64c-94.4-74.4-149.6-187.9-149.8-308.1.3-20.8,2.2-41.6,5.8-62.1l15.1,4.1,9.7-33.9-17.9-4.9-75.7-21.7-11.6-3.3ZM303.8,820.6l-64.8,88.8,28.6,20.8,8.5-11.7c69.4,38.3,147.4,58.5,226.7,58.7,94.9,0,187.7-28.7,266.1-82.2l-46.6-64.1c-64.8,43.9-141.2,67.3-219.5,67.5-62.6-.3-124.2-15.5-179.8-44.4l9.4-12.6-28.6-20.8Z"/>
+        <polygon class="st4" points="114.9 238.4 115.1 324.3 261.3 324.3 261.1 458.5 351.9 458.5 352.1 324.3 495.9 324.3 495.6 238 114.9 238.4"/>
+        <rect class="st4" x="261.1" y="554.4" width="90.8" height="200.1"/>
+        <polygon class="st4" points="622.7 244.2 429.6 754.5 526.4 754.4 666.6 361.6 806 754.4 902.9 754.4 710.4 244.2 622.7 244.2"/>
+        <path class="st1" d="M255.5,476.4c-16.5,0-29.9,13.6-29.9,30.1.2,17.6,16.1,30.1,30,30.1,34.5,0,69.9,0,103.3,0,16.1,0,28.9-14,28.9-30.1,0-16.1-12.2-30.1-28.8-30.1-35.8,0-72.8,0-103.4,0"/>
+        <path class="st1" d="M665.5,483.6c-16.1,0-29.8,12.2-29.8,28.8v172l-37.8-38.9-25,24.5,92.2,93.8,94.3-93.8-25-24.5-38.9,38.9c0-23.6,0-40.8,0-68.6-.3-34.5,0-69,0-103.6,0-16.1-13.7-28.6-29.8-28.6h0Z"/>
+      </g>
+    </g>
+  </g>
+  <g id="Artwork_2" class="st6">
+    <g class="st7">
+      <g class="st5">
+        <path class="st1" d="M447.2,22.9v15.2C269.3,59.3,118.8,179.4,58.6,348.1l76,21.8c49.9-135.2,169.9-232.2,312.6-252.7v15.4h35.3s0-109.7,0-109.7h-35.3ZM523,34.5v79.1c142.3,7.7,269.2,91.9,331.7,219.9l-14.8,4.2,9.7,33.7,106.6-30.3-9.7-33.9-14.9,4.3c-73.1-161.9-231-269-408.5-277M957.6,382.9l-75.8,21.7c8.9,32.9,13.6,66.8,13.8,100.8-.2,103.8-41.6,203.3-114.9,276.8l-9.4-12.6-28.6,20.8,11.9,16,46.5,64,6.6,9.1,28.6-20.8-8.8-12.1c93.6-88.8,146.7-212.1,147-341.1-.2-41.4-5.9-82.6-16.8-122.6M35.3,383.5l-9.7,33.9,14,4c-5.3,27.7-8.1,55.8-8.4,84,0,145.5,67.3,282.8,182.1,372.1l46.5-64c-94.4-74.4-149.6-187.9-149.8-308.1.3-20.8,2.2-41.6,5.8-62.1l15.1,4.1,9.7-33.9-17.9-4.9-75.7-21.7-11.6-3.3ZM303.8,820.6l-64.8,88.8,28.6,20.8,8.5-11.7c69.4,38.3,147.4,58.5,226.7,58.7,94.9,0,187.7-28.7,266.1-82.2l-46.6-64.1c-64.8,43.9-141.2,67.3-219.5,67.5-62.6-.3-124.2-15.5-179.8-44.4l9.4-12.6-28.6-20.8Z"/>
+        <polygon class="st0" points="114.9 238.4 115.1 324.3 261.3 324.3 261.1 458.5 351.9 458.5 352.1 324.3 495.9 324.3 495.6 238 114.9 238.4"/>
+        <rect class="st0" x="261.1" y="554.4" width="90.8" height="200.1"/>
+        <polygon class="st0" points="622.7 244.2 429.6 754.5 526.4 754.4 666.6 361.6 806 754.4 902.9 754.4 710.4 244.2 622.7 244.2"/>
+        <path class="st1" d="M255.5,476.4c-16.5,0-29.9,13.6-29.9,30.1.2,17.6,16.1,30.1,30,30.1,34.5,0,69.9,0,103.3,0,16.1,0,28.9-14,28.9-30.1,0-16.1-12.2-30.1-28.8-30.1-35.8,0-72.8,0-103.4,0"/>
+        <path class="st1" d="M665.5,483.6c-16.1,0-29.8,12.2-29.8,28.8v172l-37.8-38.9-25,24.5,92.2,93.8,94.3-93.8-25-24.5-38.9,38.9c0-23.6,0-40.8,0-68.6-.3-34.5,0-69,0-103.6,0-16.1-13.7-28.6-29.8-28.6h0Z"/>
+      </g>
+    </g>
+  </g>
+</svg>

backend/appsettings/index_mapping.json

@@ -62,6 +62,9 @@
                         }
                     }
                 },
+                "channel_tabs": {
+                    "type": "keyword"
+                },
                 "channel_overwrites": {
                     "properties": {
                         "download_format": {
@@ -165,6 +168,9 @@
                                 }
                             }
                         },
+                        "channel_tabs": {
+                            "type": "keyword"
+                        },
                         "channel_overwrites": {
                             "properties": {
                                 "download_format": {
@@ -239,7 +245,8 @@
                     "type": "keyword"
                 },
                 "published": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second||strict_date_optional_time"
                 },
                 "playlist": {
                     "type": "text",
@@ -460,6 +467,15 @@
                 "playlist_description": {
                     "type": "text"
                 },
+                "playlist_subscribed": {
+                    "type": "boolean"
+                },
+                "playlist_type": {
+                    "type": "keyword"
+                },
+                "playlist_active": {
+                    "type": "boolean"
+                },
                 "playlist_name": {
                     "type": "text",
                     "analyzer": "english",
@@ -496,6 +512,9 @@
                     "type": "date",
                     "format": "epoch_second"
                 },
+                "playlist_sort_order": {
+                    "type": "keyword"
+                },
                 "playlist_entries": {
                     "properties": {
                         "downloaded": {

backend/appsettings/serializers.py

@@ -21,10 +21,16 @@ class AppConfigSubSerializer(
 ):
     """serialize app config subscriptions"""
 
-    channel_size = serializers.IntegerField(required=False)
-    live_channel_size = serializers.IntegerField(required=False)
-    shorts_channel_size = serializers.IntegerField(required=False)
+    channel_size = serializers.IntegerField(required=False, allow_null=True)
+    live_channel_size = serializers.IntegerField(
+        required=False, allow_null=True
+    )
+    shorts_channel_size = serializers.IntegerField(
+        required=False, allow_null=True
+    )
+    playlist_size = serializers.IntegerField(required=False, allow_null=True)
     auto_start = serializers.BooleanField(required=False)
+    extract_flat = serializers.BooleanField(required=False)
 
 
 class AppConfigDownloadsSerializer(

backend/appsettings/src/config.py

@@ -21,7 +21,9 @@ class SubscriptionsConfigType(TypedDict):
     channel_size: int
     live_channel_size: int
     shorts_channel_size: int
+    playlist_size: int
     auto_start: bool
+    extract_flat: bool
 
 
 class DownloadsConfigType(TypedDict):
@@ -72,7 +74,9 @@ class AppConfig:
             "channel_size": 50,
             "live_channel_size": 50,
             "shorts_channel_size": 50,
+            "playlist_size": 50,
             "auto_start": False,
+            "extract_flat": False,
         },
         "downloads": {
             "limit_speed": None,

backend/appsettings/src/index_setup.py

@@ -201,7 +201,15 @@ class ElasitIndexWrap:
         if self.backup_run:
             return
 
-        config = AppConfig().config
+        try:
+            config = AppConfig().config
+        except ValueError:
+            # create defaults in ES if config not found
+            print("AppConfig not found, creating defaults...")
+            handler = AppConfig.__new__(AppConfig)
+            handler.sync_defaults()
+            config = AppConfig.CONFIG_DEFAULTS
+
         if config["application"]["enable_snapshot"]:
             # take snapshot if enabled
             ElasticSnapshot().take_snapshot_now(wait=True)

backend/appsettings/src/reindex.py

@@ -11,11 +11,11 @@ from typing import Callable, TypedDict
 
 from appsettings.src.config import AppConfig
 from channel.src.index import YoutubeChannel
+from channel.src.remote_query import get_last_channel_videos
 from common.src.env_settings import EnvironmentSettings
 from common.src.es_connect import ElasticWrap, IndexPaginate
 from common.src.helper import rand_sleep
 from common.src.ta_redis import RedisQueue
-from download.src.subscriptions import ChannelSubscription
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import CookieHandler
 from playlist.src.index import YoutubePlaylist
@@ -376,7 +376,7 @@ class Reindex(ReindexBase):
 
         channel.upload_to_es()
         channel.sync_to_videos()
-        ChannelFullScan(channel_id).scan()
+        ChannelFullScan(channel_id, self.config).scan()
         self.processed["channels"] += 1
 
     def _reindex_single_playlist(self, playlist_id: str) -> None:
@@ -493,13 +493,11 @@ class ReindexProgress(ReindexBase):
 
 
 class ChannelFullScan:
-    """
-    update from v0.3.0 to v0.3.1
-    full scan of channel to fix vid_type mismatch
-    """
+    """full scan of channel to fix vid_type mismatch"""
 
-    def __init__(self, channel_id):
+    def __init__(self, channel_id, config):
         self.channel_id = channel_id
+        self.config = config
         self.to_update = False
 
     def scan(self):
@@ -510,12 +508,14 @@ class ChannelFullScan:
         self.to_update = []
         for video in all_local_videos:
             video_id = video["youtube_id"]
-            remote_match = [i for i in all_remote_videos if i[0] == video_id]
+            remote_match = [
+                i for i in all_remote_videos if i["id"] == video_id
+            ]
             if not remote_match:
                 print(f"{video_id}: no remote match found")
                 continue
 
-            expected_type = remote_match[0][-1]
+            expected_type = remote_match[0]["vid_type"]
             if video["vid_type"] != expected_type:
                 self.to_update.append(
                     {
@@ -528,9 +528,8 @@ class ChannelFullScan:
 
     def _get_all_remote(self):
         """get all channel videos"""
-        sub = ChannelSubscription()
-        all_remote_videos = sub.get_last_youtube_videos(
-            self.channel_id, limit=False
+        all_remote_videos = get_last_channel_videos(
+            self.channel_id, self.config, limit=False
         )
 
         return all_remote_videos

backend/channel/serializers.py

@@ -4,6 +4,7 @@
 
 from common.serializers import PaginationSerializer, ValidateUnknownFieldsMixin
 from rest_framework import serializers
+from video.src.constants import VideoTypeEnum
 
 
 class ChannelOverwriteSerializer(
@@ -45,6 +46,9 @@ class ChannelSerializer(serializers.Serializer):
     channel_tags = serializers.ListField(
         child=serializers.CharField(), required=False
     )
+    channel_tabs = serializers.ListField(
+        child=serializers.ChoiceField(VideoTypeEnum.values_known())
+    )
     channel_views = serializers.IntegerField()
     _index = serializers.CharField(required=False)
     _score = serializers.IntegerField(required=False)
@@ -90,6 +94,7 @@ class ChannelNavSerializer(serializers.Serializer):
     """serialize channel navigation"""
 
     has_pending = serializers.BooleanField()
+    has_ignored = serializers.BooleanField()
     has_playlists = serializers.BooleanField()
     has_videos = serializers.BooleanField()
     has_streams = serializers.BooleanField()

backend/channel/src/index.py

@@ -4,17 +4,17 @@ functionality:
 - index and update in es
 """
 
-import json
 import os
 from datetime import datetime
 
+from channel.src.remote_query import get_last_channel_videos
 from common.src.env_settings import EnvironmentSettings
 from common.src.es_connect import ElasticWrap, IndexPaginate
 from common.src.helper import rand_sleep
 from common.src.index_generic import YouTubeItem
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import YtWrap
-from playlist.src.index import YoutubePlaylist
+from video.src.constants import VideoTypeEnum
 
 
 class YoutubeChannel(YouTubeItem):
@@ -70,6 +70,7 @@ class YoutubeChannel(YouTubeItem):
             "channel_thumb_url": self._get_thumb_art(),
             "channel_tvart_url": self._get_tv_art(),
             "channel_views": self.youtube_meta.get("view_count") or 0,
+            "channel_tabs": self.get_channel_tabs(),
         }
 
     def _get_thumb_art(self):
@@ -105,6 +106,31 @@ class YoutubeChannel(YouTubeItem):
 
         return False
 
+    def get_channel_tabs(self) -> list[str]:
+        """get channel tabs"""
+        tabs = VideoTypeEnum.values_known()
+        config_cp = self.config.copy()
+        config_cp["subscriptions"] = {
+            "channel_size": 1,
+            "live_channel_size": 1,
+            "shorts_channel_size": 1,
+        }
+        tabs = []
+        for query_filter in VideoTypeEnum:
+            if query_filter == VideoTypeEnum.UNKNOWN:
+                continue
+
+            videos = get_last_channel_videos(
+                channel_id=self.youtube_id,
+                config=config_cp,
+                limit=True,
+                query_filter=query_filter,
+            )
+            if videos:
+                tabs.append(query_filter.value)
+
+        return tabs
+
     def _video_fallback(self, fallback):
         """use video metadata as fallback"""
         print(f"{self.youtube_id}: fallback to video metadata")
@@ -122,27 +148,6 @@ class YoutubeChannel(YouTubeItem):
             "channel_thumb_url": False,
             "channel_views": 0,
         }
-        self._info_json_fallback()
-
-    def _info_json_fallback(self):
-        """read channel info.json for additional metadata"""
-        info_json = os.path.join(
-            EnvironmentSettings.CACHE_DIR,
-            "import",
-            f"{self.youtube_id}.info.json",
-        )
-        if os.path.exists(info_json):
-            print(f"{self.youtube_id}: read info.json file")
-            with open(info_json, "r", encoding="utf-8") as f:
-                content = json.loads(f.read())
-
-            self.json_data.update(
-                {
-                    "channel_subs": content.get("channel_follower_count", 0),
-                    "channel_description": content.get("description", False),
-                }
-            )
-            os.remove(info_json)
 
     def get_channel_art(self):
         """download channel art for new channels"""
@@ -178,46 +183,15 @@ class YoutubeChannel(YouTubeItem):
         update_path = f"ta_video/_update_by_query?pipeline={self.youtube_id}"
         _, _ = ElasticWrap(update_path).post(data)
 
-    def get_folder_path(self):
-        """get folder where media files get stored"""
-        folder_path = os.path.join(
-            EnvironmentSettings.MEDIA_DIR,
-            self.json_data["channel_id"],
-        )
-        return folder_path
+    def change_subscribe(self, new_subscribe_state: bool):
+        """change subscribe status"""
+        if not self.json_data:
+            self.build_json()
 
-    def delete_es_videos(self):
-        """delete all channel documents from elasticsearch"""
-        data = {
-            "query": {
-                "term": {"channel.channel_id": {"value": self.youtube_id}}
-            }
-        }
-        _, _ = ElasticWrap("ta_video/_delete_by_query").post(data)
-
-    def delete_es_comments(self):
-        """delete all comments from this channel"""
-        data = {
-            "query": {
-                "term": {"comment_channel_id": {"value": self.youtube_id}}
-            }
-        }
-        _, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
-
-    def delete_es_subtitles(self):
-        """delete all subtitles from this channel"""
-        data = {
-            "query": {
-                "term": {"subtitle_channel_id": {"value": self.youtube_id}}
-            }
-        }
-        _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data)
-
-    def delete_playlists(self):
-        """delete all indexed playlist from es"""
-        all_playlists = self.get_indexed_playlists()
-        for playlist in all_playlists:
-            YoutubePlaylist(playlist["playlist_id"]).delete_metadata()
+        self.json_data["channel_subscribed"] = new_subscribe_state
+        self.upload_to_es()
+        self.sync_to_videos()
+        return self.json_data
 
     def delete_channel(self):
         """delete channel and all videos"""
@@ -226,24 +200,7 @@ class YoutubeChannel(YouTubeItem):
         if not self.json_data:
             raise FileNotFoundError
 
-        folder_path = self.get_folder_path()
-        print(f"{self.youtube_id}: delete all media files")
-        try:
-            all_videos = os.listdir(folder_path)
-            for video in all_videos:
-                video_path = os.path.join(folder_path, video)
-                os.remove(video_path)
-            os.rmdir(folder_path)
-        except FileNotFoundError:
-            print(f"no videos found for {folder_path}")
-
-        print(f"{self.youtube_id}: delete indexed playlists")
-        self.delete_playlists()
-        print(f"{self.youtube_id}: delete indexed videos")
-        self.delete_es_videos()
-        self.delete_es_comments()
-        self.delete_es_subtitles()
-        self.del_in_es()
+        ChannelDelete(json_data=self.json_data).delete()
 
     def index_channel_playlists(self):
         """add all playlists of channel to index"""
@@ -265,6 +222,21 @@ class YoutubeChannel(YouTubeItem):
             print("add playlist: " + playlist[1])
             rand_sleep(self.config)
 
+    def get_all_playlists(self):
+        """get all playlists owned by this channel"""
+        url = (
+            f"https://www.youtube.com/channel/{self.youtube_id}"
+            + "/playlists?view=1&sort=dd&shelf_id=0"
+        )
+        obs = {"skip_download": True, "extract_flat": True}
+        playlists, _ = YtWrap(obs, self.config).extract(url)
+        if not playlists:
+            self.all_playlists = []
+            return
+
+        all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
+        self.all_playlists = all_entries
+
     def _notify_single_playlist(self, idx, total):
         """send notification"""
         channel_name = self.json_data["channel_name"]
@@ -277,6 +249,8 @@ class YoutubeChannel(YouTubeItem):
     @staticmethod
     def _index_single_playlist(playlist):
         """add single playlist if needed"""
+        from playlist.src.index import YoutubePlaylist
+
         playlist = YoutubePlaylist(playlist[0])
         playlist.update_playlist(skip_on_empty=True)
 
@@ -291,34 +265,6 @@ class YoutubeChannel(YouTubeItem):
         all_videos = IndexPaginate("ta_video", data).get_results()
         return all_videos
 
-    def get_all_playlists(self):
-        """get all playlists owned by this channel"""
-        url = (
-            f"https://www.youtube.com/channel/{self.youtube_id}"
-            + "/playlists?view=1&sort=dd&shelf_id=0"
-        )
-        obs = {"skip_download": True, "extract_flat": True}
-        playlists = YtWrap(obs, self.config).extract(url)
-        if not playlists:
-            self.all_playlists = []
-            return
-
-        all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
-        self.all_playlists = all_entries
-
-    def get_indexed_playlists(self, active_only=False):
-        """get all indexed playlists from channel"""
-        must_list = [
-            {"term": {"playlist_channel_id": {"value": self.youtube_id}}}
-        ]
-        if active_only:
-            must_list.append({"term": {"playlist_active": {"value": True}}})
-
-        data = {"query": {"bool": {"must": must_list}}}
-
-        all_playlists = IndexPaginate("ta_playlist", data).get_results()
-        return all_playlists
-
     def get_overwrites(self) -> dict:
         """get all per channel overwrites"""
         return self.json_data.get("channel_overwrites", {})
@@ -349,6 +295,93 @@ class YoutubeChannel(YouTubeItem):
         self.json_data["channel_overwrites"] = to_write
 
 
+class ChannelDelete(YouTubeItem):
+    """delete and cleanup"""
+
+    index_name = "ta_channel"
+
+    def __init__(self, json_data):
+        super().__init__(youtube_id=json_data["channel_id"])
+        self.json_data = json_data
+
+    def delete(self):
+        """delete channel and all videos"""
+        folder_path = self._get_folder_path()
+        print(f"{self.youtube_id}: delete all media files")
+        try:
+            all_videos = os.listdir(folder_path)
+            for video in all_videos:
+                video_path = os.path.join(folder_path, video)
+                os.remove(video_path)
+            os.rmdir(folder_path)
+        except FileNotFoundError:
+            print(f"no videos found for {folder_path}")
+
+        print(f"{self.youtube_id}: delete indexed playlists")
+        self._delete_playlists()
+        print(f"{self.youtube_id}: delete indexed videos")
+        self._delete_es_videos()
+        self._delete_es_comments()
+        self._delete_es_subtitles()
+        self.del_in_es()
+
+    def _get_folder_path(self):
+        """get folder where media files get stored"""
+        folder_path = os.path.join(
+            EnvironmentSettings.MEDIA_DIR,
+            self.json_data["channel_id"],
+        )
+        return folder_path
+
+    def _delete_es_videos(self):
+        """delete all channel documents from elasticsearch"""
+        data = {
+            "query": {
+                "term": {"channel.channel_id": {"value": self.youtube_id}}
+            }
+        }
+        _, _ = ElasticWrap("ta_video/_delete_by_query").post(data)
+
+    def _delete_es_comments(self):
+        """delete all comments from this channel"""
+        data = {
+            "query": {
+                "term": {"comment_channel_id": {"value": self.youtube_id}}
+            }
+        }
+        _, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
+
+    def _delete_es_subtitles(self):
+        """delete all subtitles from this channel"""
+        data = {
+            "query": {
+                "term": {"subtitle_channel_id": {"value": self.youtube_id}}
+            }
+        }
+        _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data)
+
+    def _delete_playlists(self):
+        """delete all indexed playlist from es"""
+        from playlist.src.index import YoutubePlaylist
+
+        all_playlists = self._get_indexed_playlists()
+        for playlist in all_playlists:
+            YoutubePlaylist(playlist["playlist_id"]).delete_metadata()
+
+    def _get_indexed_playlists(self, active_only=False):
+        """get all indexed playlists from channel"""
+        must_list = [
+            {"term": {"playlist_channel_id": {"value": self.youtube_id}}}
+        ]
+        if active_only:
+            must_list.append({"term": {"playlist_active": {"value": True}}})
+
+        data = {"query": {"bool": {"must": must_list}}}
+
+        all_playlists = IndexPaginate("ta_playlist", data).get_results()
+        return all_playlists
+
+
 def channel_overwrites(channel_id, overwrites):
     """collection to overwrite settings per channel"""
     channel = YoutubeChannel(channel_id)

backend/channel/src/nav.py

@@ -13,6 +13,7 @@ class ChannelNav:
         """build nav items"""
         nav = {
             "has_pending": self._get_has_pending(),
+            "has_ignored": self._get_has_ignored(),
             "has_playlists": self._get_has_playlists(),
         }
         nav.update(self._get_vid_types())
@@ -63,6 +64,24 @@ class ChannelNav:
 
         return bool(response["hits"]["hits"])
 
+    def _get_has_ignored(self):
+        """Check if there are ignored videos in the download queue"""
+        data = {
+            "size": 1,
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {"status": {"value": "ignore"}}},
+                        {"term": {"channel_id": {"value": self.channel_id}}},
+                    ]
+                }
+            },
+            "_source": False,
+        }
+        response, _ = ElasticWrap("ta_download/_search").get(data=data)
+
+        return bool(response["hits"]["hits"])
+
     def _get_has_playlists(self):
         """check if channel has playlists"""
         path = "ta_playlist/_search"

backend/channel/src/remote_query.py

@@ -0,0 +1,136 @@
+"""build queries for video extraction from channel subscriptions"""
+
+from download.src.yt_dlp_base import YtWrap
+from video.src.constants import VideoTypeEnum
+
+
+class VideoQueryBuilder:
+    """Build queries for yt-dlp."""
+
+    def __init__(self, config: dict, channel_overwrites: dict | None = None):
+        self.config = config
+        self.channel_overwrites = channel_overwrites or {}
+
+    def build_queries(
+        self,
+        video_type: VideoTypeEnum | list[VideoTypeEnum] | None,
+        limit: bool = True,
+    ) -> list[tuple[VideoTypeEnum, int | None]]:
+        """Build queries for all or specific video type."""
+        query_methods = {
+            VideoTypeEnum.VIDEOS: self.videos_query,
+            VideoTypeEnum.STREAMS: self.streams_query,
+            VideoTypeEnum.SHORTS: self.shorts_query,
+        }
+
+        if video_type and video_type != VideoTypeEnum.UNKNOWN:
+            # build query for specific type/s
+            if not isinstance(video_type, list):
+                video_type = [video_type]
+
+            queries = []
+            for video_type_item in video_type:
+                query_method = query_methods.get(video_type_item)
+                if not query_method:
+                    continue
+
+                query = query_method(limit)
+                if query[1] != 0:
+                    queries.append(query)
+
+            return queries
+
+        # Build and return queries for all video types
+        queries = []
+        for build_query in query_methods.values():
+            query = build_query(limit)
+            if query[1] != 0:
+                queries.append(query)
+
+        return queries
+
+    def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
+        """Build query for videos."""
+        return self._build_generic_query(
+            video_type=VideoTypeEnum.VIDEOS,
+            overwrite_key="subscriptions_channel_size",
+            config_key="channel_size",
+            limit=limit,
+        )
+
+    def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
+        """Build query for streams."""
+        return self._build_generic_query(
+            video_type=VideoTypeEnum.STREAMS,
+            overwrite_key="subscriptions_live_channel_size",
+            config_key="live_channel_size",
+            limit=limit,
+        )
+
+    def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
+        """Build query for shorts."""
+        return self._build_generic_query(
+            video_type=VideoTypeEnum.SHORTS,
+            overwrite_key="subscriptions_shorts_channel_size",
+            config_key="shorts_channel_size",
+            limit=limit,
+        )
+
+    def _build_generic_query(
+        self,
+        video_type: VideoTypeEnum,
+        overwrite_key: str,
+        config_key: str,
+        limit: bool,
+    ) -> tuple[VideoTypeEnum, int | None]:
+        """Generic query for video page scraping."""
+        app_config_size = self.config["subscriptions"].get(config_key)
+        if not limit or app_config_size is None:
+            # treat None as unlimited
+            return (video_type, None)
+
+        if (
+            overwrite_key in self.channel_overwrites
+            and self.channel_overwrites[overwrite_key] is not None
+        ):
+            overwrite = self.channel_overwrites[overwrite_key]
+            return (video_type, overwrite)
+
+        if app_config_size:
+            return (video_type, app_config_size)
+
+        return (video_type, 0)
+
+
+def get_last_channel_videos(
+    channel_id,
+    config,
+    limit=None,
+    query_filter=None,
+    channel_overwrites=None,
+):
+    """get a list of last videos from channel"""
+    query_handler = VideoQueryBuilder(config, channel_overwrites)
+    queries = query_handler.build_queries(query_filter)
+    last_videos = []
+
+    for vid_type_enum, limit_amount in queries:
+        obs = {
+            "skip_download": True,
+            "extract_flat": True,
+        }
+        vid_type = vid_type_enum.value
+
+        if limit is not None:
+            obs.update({"playlist_items": f":{limit_amount}:1"})
+
+        url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
+        channel_query, _ = YtWrap(obs, config).extract(url)
+        if not channel_query:
+            continue
+
+        for entry in channel_query["entries"]:
+            entry["vid_type"] = vid_type
+            last_videos.append(entry)
+
+    return last_videos

backend/channel/tests/test_src/test_remote_query.py

@@ -0,0 +1,128 @@
+"""test video query building"""
+
+# pylint: disable=redefined-outer-name
+
+from enum import Enum
+
+import pytest
+from channel.src.remote_query import VideoQueryBuilder
+from video.src.constants import VideoTypeEnum
+
+
+@pytest.fixture
+def default_config():
+    """from appsettings"""
+    return {
+        "subscriptions": {
+            "channel_size": 5,
+            "live_channel_size": 3,
+            "shorts_channel_size": 2,
+        }
+    }
+
+
+@pytest.fixture
+def empty_overwrites():
+    """from channel overwrites"""
+    return {}
+
+
+@pytest.fixture
+def overwrites():
+    """from channel overwrites"""
+    return {
+        "subscriptions_channel_size": 10,
+        "subscriptions_live_channel_size": 0,
+        "subscriptions_shorts_channel_size": None,
+    }
+
+
+def test_build_all_queries_with_limit(default_config, empty_overwrites):
+    """default, empty overwrite"""
+    builder = VideoQueryBuilder(default_config, empty_overwrites)
+    result = builder.build_queries(None, limit=True)
+    expected = [
+        (VideoTypeEnum.VIDEOS, 5),
+        (VideoTypeEnum.STREAMS, 3),
+        (VideoTypeEnum.SHORTS, 2),
+    ]
+    assert result == expected
+
+
+def test_build_all_queries_without_limit(default_config, empty_overwrites):
+    """limit disabled"""
+    builder = VideoQueryBuilder(default_config, empty_overwrites)
+    result = builder.build_queries(None, limit=False)
+    expected = [
+        (VideoTypeEnum.VIDEOS, None),
+        (VideoTypeEnum.STREAMS, None),
+        (VideoTypeEnum.SHORTS, None),
+    ]
+    assert result == expected
+
+
+def test_build_specific_query(default_config, empty_overwrites):
+    """single vid_type"""
+    builder = VideoQueryBuilder(default_config, empty_overwrites)
+    result = builder.build_queries(VideoTypeEnum.VIDEOS)
+    assert result == [(VideoTypeEnum.VIDEOS, 5)]
+
+
+def test_build_multiple_queries(default_config, empty_overwrites):
+    """vid_type list"""
+    builder = VideoQueryBuilder(default_config, empty_overwrites)
+    result = builder.build_queries(
+        [VideoTypeEnum.VIDEOS, VideoTypeEnum.SHORTS]
+    )
+    assert result == [(VideoTypeEnum.VIDEOS, 5), (VideoTypeEnum.SHORTS, 2)]
+
+
+def test_build_unknown_queries(default_config, empty_overwrites):
+    """vid_type unknown"""
+    builder = VideoQueryBuilder(default_config, empty_overwrites)
+    result = builder.build_queries(VideoTypeEnum.UNKNOWN)
+    assert result == [
+        (VideoTypeEnum.VIDEOS, 5),
+        (VideoTypeEnum.STREAMS, 3),
+        (VideoTypeEnum.SHORTS, 2),
+    ]
+
+
+def test_overwrite_applied(default_config, overwrites):
+    """with overwrite from channel config"""
+    builder = VideoQueryBuilder(default_config, overwrites)
+    result = builder.build_queries(None, limit=True)
+    expected = [
+        (VideoTypeEnum.VIDEOS, 10),  # Overwritten
+        # STREAMS is overwritten to 0, should be excluded
+        (VideoTypeEnum.SHORTS, 2),  # None in overwrite, fallback to config
+    ]
+    assert result == expected
+
+
+def test_no_limit_ignores_config_and_overwrites(default_config, overwrites):
+    """no limit single vid_type"""
+    builder = VideoQueryBuilder(default_config, overwrites)
+    result = builder.build_queries([VideoTypeEnum.STREAMS], limit=False)
+    assert result == [(VideoTypeEnum.STREAMS, None)]
+
+
+def test_zero_query_not_included(default_config):
+    """overwrite to zero to disable"""
+    overwrites = {"subscriptions_live_channel_size": 0}
+    builder = VideoQueryBuilder(default_config, overwrites)
+    result = builder.build_queries([VideoTypeEnum.STREAMS], limit=True)
+    assert not result  # Should be skipped due to 0
+
+
+def test_invalid_video_type_is_ignored(default_config):
+    """invalid enum"""
+    builder = VideoQueryBuilder(default_config)
+
+    class FakeEnum(Enum):
+        """invalid"""
+
+        INVALID = "invalid"
+
+    result = builder.build_queries([FakeEnum.INVALID], limit=True)
+    assert not result

backend/channel/views.py

@@ -14,7 +14,6 @@ from channel.src.nav import ChannelNav
 from common.serializers import ErrorResponseSerializer
 from common.src.urlparser import Parser
 from common.views_base import AdminWriteOnly, ApiBaseView
-from download.src.subscriptions import ChannelSubscription
 from drf_spectacular.utils import (
     OpenApiParameter,
     OpenApiResponse,
@@ -89,9 +88,7 @@ class ChannelApiListView(ApiBaseView):
     def _unsubscribe(channel_id: str):
         """unsubscribe"""
         print(f"[{channel_id}] unsubscribe from channel")
-        ChannelSubscription().change_subscribe(
-            channel_id, channel_subscribed=False
-        )
+        YoutubeChannel(channel_id).change_subscribe(new_subscribe_state=False)
 
 
 class ChannelApiView(ApiBaseView):
@@ -146,7 +143,9 @@ class ChannelApiView(ApiBaseView):
 
         subscribed = validated_data.get("channel_subscribed")
         if subscribed is not None:
-            ChannelSubscription().change_subscribe(channel_id, subscribed)
+            YoutubeChannel(channel_id).change_subscribe(
+                new_subscribe_state=subscribed
+            )
 
         overwrites = validated_data.get("channel_overwrites")
         if overwrites:

backend/common/serializers.py

@@ -69,6 +69,7 @@ class NotificationSerializer(serializers.Serializer):
     level = serializers.ChoiceField(choices=["info", "error"])
     messages = serializers.ListField(child=serializers.CharField())
     progress = serializers.FloatField(required=False)
+    command = serializers.ChoiceField(choices=["STOP", "KILL"], required=False)
 
 
 class NotificationQueryFilterSerializer(serializers.Serializer):

backend/common/src/es_connect.py

@@ -56,7 +56,7 @@ class ElasticWrap:
         return response.json(), response.status_code
 
     def post(
-        self, data: bool | dict = False, ndjson: bool = False
+        self, data: bool | dict | str = False, ndjson: bool = False
     ) -> tuple[dict, int]:
         """post data to es"""
 

backend/common/src/helper.py

@@ -1,299 +1,352 @@
-"""
-Loose collection of helper functions
-- don't import AppConfig class here to avoid circular imports
-"""
-
-import json
-import os
-import random
-import string
-import subprocess
-from datetime import datetime
-from time import sleep
-from typing import Any
-from urllib.parse import urlparse
-
-import requests
-from common.src.es_connect import IndexPaginate
-
-
-def ignore_filelist(filelist: list[str]) -> list[str]:
-    """ignore temp files for os.listdir sanitizer"""
-    to_ignore = [
-        "@eaDir",
-        "Icon\r\r",
-        "Network Trash Folder",
-        "Temporary Items",
-    ]
-    cleaned: list[str] = []
-    for file_name in filelist:
-        if file_name.startswith(".") or file_name in to_ignore:
-            continue
-
-        cleaned.append(file_name)
-
-    return cleaned
-
-
-def randomizor(length: int) -> str:
-    """generate random alpha numeric string"""
-    pool: str = string.digits + string.ascii_letters
-    return "".join(random.choice(pool) for i in range(length))
-
-
-def rand_sleep(config) -> None:
-    """randomized sleep based on config"""
-    sleep_config = config["downloads"].get("sleep_interval")
-    if not sleep_config:
-        return
-
-    secs = random.randrange(int(sleep_config * 0.5), int(sleep_config * 1.5))
-    sleep(secs)
-
-
-def requests_headers() -> dict[str, str]:
-    """build header with random user agent for requests outside of yt-dlp"""
-
-    chrome_versions = (
-        "90.0.4430.212",
-        "90.0.4430.24",
-        "90.0.4430.70",
-        "90.0.4430.72",
-        "90.0.4430.85",
-        "90.0.4430.93",
-        "91.0.4472.101",
-        "91.0.4472.106",
-        "91.0.4472.114",
-        "91.0.4472.124",
-        "91.0.4472.164",
-        "91.0.4472.19",
-        "91.0.4472.77",
-        "92.0.4515.107",
-        "92.0.4515.115",
-        "92.0.4515.131",
-        "92.0.4515.159",
-        "92.0.4515.43",
-        "93.0.4556.0",
-        "93.0.4577.15",
-        "93.0.4577.63",
-        "93.0.4577.82",
-        "94.0.4606.41",
-        "94.0.4606.54",
-        "94.0.4606.61",
-        "94.0.4606.71",
-        "94.0.4606.81",
-        "94.0.4606.85",
-        "95.0.4638.17",
-        "95.0.4638.50",
-        "95.0.4638.54",
-        "95.0.4638.69",
-        "95.0.4638.74",
-        "96.0.4664.18",
-        "96.0.4664.45",
-        "96.0.4664.55",
-        "96.0.4664.93",
-        "97.0.4692.20",
-    )
-    template = (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        + "AppleWebKit/537.36 (KHTML, like Gecko) "
-        + f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
-    )
-
-    return {"User-Agent": template}
-
-
-def date_parser(timestamp: int | str) -> str:
-    """return formatted date string"""
-    if isinstance(timestamp, int):
-        date_obj = datetime.fromtimestamp(timestamp)
-    elif isinstance(timestamp, str):
-        date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
-    else:
-        raise TypeError(f"invalid timestamp: {timestamp}")
-
-    return date_obj.date().isoformat()
-
-
-def time_parser(timestamp: str) -> float:
-    """return seconds from timestamp, false on empty"""
-    if not timestamp:
-        return False
-
-    if timestamp.isnumeric():
-        return int(timestamp)
-
-    hours, minutes, seconds = timestamp.split(":", maxsplit=3)
-    return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
-
-
-def clear_dl_cache(cache_dir: str) -> int:
-    """clear leftover files from dl cache"""
-    print("clear download cache")
-    download_cache_dir = os.path.join(cache_dir, "download")
-    leftover_files = ignore_filelist(os.listdir(download_cache_dir))
-    for cached in leftover_files:
-        to_delete = os.path.join(download_cache_dir, cached)
-        os.remove(to_delete)
-
-    return len(leftover_files)
-
-
-def get_mapping() -> dict:
-    """read index_mapping.json and get expected mapping and settings"""
-    with open("appsettings/index_mapping.json", "r", encoding="utf-8") as f:
-        index_config: dict = json.load(f).get("index_config")
-
-    return index_config
-
-
-def is_shorts(youtube_id: str) -> bool:
-    """check if youtube_id is a shorts video, bot not it it's not a shorts"""
-    shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
-    cookies = {"SOCS": "CAI"}
-    response = requests.head(
-        shorts_url, cookies=cookies, headers=requests_headers(), timeout=10
-    )
-
-    return response.status_code == 200
-
-
-def get_duration_sec(file_path: str) -> int:
-    """get duration of media file from file path"""
-
-    duration = subprocess.run(
-        [
-            "ffprobe",
-            "-v",
-            "error",
-            "-show_entries",
-            "format=duration",
-            "-of",
-            "default=noprint_wrappers=1:nokey=1",
-            file_path,
-        ],
-        capture_output=True,
-        check=True,
-    )
-    duration_raw = duration.stdout.decode().strip()
-    if duration_raw == "N/A":
-        return 0
-
-    duration_sec = int(float(duration_raw))
-    return duration_sec
-
-
-def get_duration_str(seconds: int) -> str:
-    """Return a human-readable duration string from seconds."""
-    if not seconds:
-        return "NA"
-
-    units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
-    duration_parts = []
-
-    for unit_label, unit_seconds in units:
-        if seconds >= unit_seconds:
-            unit_count, seconds = divmod(seconds, unit_seconds)
-            duration_parts.append(f"{unit_count:02}{unit_label}")
-
-    duration_parts[0] = duration_parts[0].lstrip("0")
-
-    return " ".join(duration_parts)
-
-
-def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
-    """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
-    allowed_hosts: list[str] = [
-        "localhost",
-        "tubearchivist",
-    ]
-    csrf_trusted_origins: list[str] = [
-        "http://localhost",
-        "http://tubearchivist",
-    ]
-    for host in ta_host.split():
-        host_clean = host.strip()
-        if not host_clean.startswith("http"):
-            host_clean = f"http://{host_clean}"
-
-        parsed = urlparse(host_clean)
-        allowed_hosts.append(f"{parsed.hostname}")
-        cors_url = f"{parsed.scheme}://{parsed.hostname}"
-
-        if parsed.port:
-            cors_url = f"{cors_url}:{parsed.port}"
-
-        csrf_trusted_origins.append(cors_url)
-
-    return allowed_hosts, csrf_trusted_origins
-
-
-def get_stylesheets() -> list:
-    """Get all valid stylesheets from /static/css"""
-
-    stylesheets = [
-        "dark.css",
-        "light.css",
-        "matrix.css",
-        "midnight.css",
-        "custom.css",
-    ]
-    return stylesheets
-
-
-def check_stylesheet(stylesheet: str):
-    """Check if a stylesheet exists. Return dark.css as a fallback"""
-    if stylesheet in get_stylesheets():
-        return stylesheet
-
-    return "dark.css"
-
-
-def is_missing(
-    to_check: str | list[str],
-    index_name: str = "ta_video,ta_download",
-    on_key: str = "youtube_id",
-) -> list[str]:
-    """id or list of ids that are missing from index_name"""
-    if isinstance(to_check, str):
-        to_check = [to_check]
-
-    data = {
-        "query": {"terms": {on_key: to_check}},
-        "_source": [on_key],
-    }
-    result = IndexPaginate(index_name, data=data).get_results()
-    existing_ids = [i[on_key] for i in result]
-    dl = [i for i in to_check if i not in existing_ids]
-
-    return dl
-
-
-def get_channel_overwrites() -> dict[str, dict[str, Any]]:
-    """get overwrites indexed my channel_id"""
-    data = {
-        "query": {
-            "bool": {"must": [{"exists": {"field": "channel_overwrites"}}]}
-        },
-        "_source": ["channel_id", "channel_overwrites"],
-    }
-    result = IndexPaginate("ta_channel", data).get_results()
-    overwrites = {i["channel_id"]: i["channel_overwrites"] for i in result}
-
-    return overwrites
-
-
-def calc_is_watched(duration: float, position: float) -> bool:
-    """considered watched based on duration position"""
-
-    if not duration or duration <= 0:
-        return False
-
-    if duration < 60:
-        threshold = 0.5
-    elif duration > 900:
-        threshold = 1 - (180 / duration)
-    else:
-        threshold = 0.9
-
-    return position >= duration * threshold
+"""
+Loose collection of helper functions
+- don't import AppConfig class here to avoid circular imports
+"""
+
+import json
+import os
+import random
+import string
+import subprocess
+from datetime import datetime, timezone
+from time import sleep
+from typing import Any
+from urllib.parse import urlparse
+
+import requests
+from common.src.es_connect import IndexPaginate
+
+
+def ignore_filelist(filelist: list[str]) -> list[str]:
+    """ignore temp files for os.listdir sanitizer"""
+    to_ignore = [
+        "@eaDir",
+        "Icon\r\r",
+        "Network Trash Folder",
+        "Temporary Items",
+    ]
+    cleaned: list[str] = []
+    for file_name in filelist:
+        if file_name.startswith(".") or file_name in to_ignore:
+            continue
+
+        cleaned.append(file_name)
+
+    return cleaned
+
+
+def randomizor(length: int) -> str:
+    """generate random alpha numeric string"""
+    pool: str = string.digits + string.ascii_letters
+    return "".join(random.choice(pool) for i in range(length))
+
+
+def rand_sleep(config) -> None:
+    """randomized sleep based on config"""
+    sleep_config = config["downloads"].get("sleep_interval")
+    if not sleep_config:
+        return
+
+    secs = random.randrange(int(sleep_config * 0.5), int(sleep_config * 1.5))
+    sleep(secs)
+
+
+def requests_headers() -> dict[str, str]:
+    """build header with random user agent for requests outside of yt-dlp"""
+
+    chrome_versions = (
+        "90.0.4430.212",
+        "90.0.4430.24",
+        "90.0.4430.70",
+        "90.0.4430.72",
+        "90.0.4430.85",
+        "90.0.4430.93",
+        "91.0.4472.101",
+        "91.0.4472.106",
+        "91.0.4472.114",
+        "91.0.4472.124",
+        "91.0.4472.164",
+        "91.0.4472.19",
+        "91.0.4472.77",
+        "92.0.4515.107",
+        "92.0.4515.115",
+        "92.0.4515.131",
+        "92.0.4515.159",
+        "92.0.4515.43",
+        "93.0.4556.0",
+        "93.0.4577.15",
+        "93.0.4577.63",
+        "93.0.4577.82",
+        "94.0.4606.41",
+        "94.0.4606.54",
+        "94.0.4606.61",
+        "94.0.4606.71",
+        "94.0.4606.81",
+        "94.0.4606.85",
+        "95.0.4638.17",
+        "95.0.4638.50",
+        "95.0.4638.54",
+        "95.0.4638.69",
+        "95.0.4638.74",
+        "96.0.4664.18",
+        "96.0.4664.45",
+        "96.0.4664.55",
+        "96.0.4664.93",
+        "97.0.4692.20",
+    )
+    template = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        + "AppleWebKit/537.36 (KHTML, like Gecko) "
+        + f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
+    )
+
+    return {"User-Agent": template}
+
+
+def date_parser(timestamp: int | str | None) -> str | None:
+    """return formatted date string"""
+    if timestamp is None:
+        return None
+
+    if isinstance(timestamp, int):
+        date_obj = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+    elif isinstance(timestamp, str):
+        date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
+        date_obj = date_obj.replace(tzinfo=timezone.utc)
+    else:
+        raise TypeError(f"invalid timestamp: {timestamp}")
+
+    return date_obj.isoformat()
+
+
+def time_parser(timestamp: str) -> float:
+    """return seconds from timestamp, false on empty"""
+    if not timestamp:
+        return False
+
+    if timestamp.isnumeric():
+        return int(timestamp)
+
+    hours, minutes, seconds = timestamp.split(":", maxsplit=3)
+    return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
+
+
+def clear_dl_cache(cache_dir: str) -> int:
+    """clear leftover files from dl cache"""
+    print("clear download cache")
+    download_cache_dir = os.path.join(cache_dir, "download")
+    leftover_files = ignore_filelist(os.listdir(download_cache_dir))
+    for cached in leftover_files:
+        to_delete = os.path.join(download_cache_dir, cached)
+        os.remove(to_delete)
+
+    return len(leftover_files)
+
+
+def get_mapping() -> dict:
+    """read index_mapping.json and get expected mapping and settings"""
+    with open("appsettings/index_mapping.json", "r", encoding="utf-8") as f:
+        index_config: dict = json.load(f).get("index_config")
+
+    return index_config
+
+
+def is_shorts(youtube_id: str) -> bool:
+    """check if youtube_id is a shorts video, bot not it it's not a shorts"""
+    shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
+    cookies = {"SOCS": "CAI"}
+    try:
+        response = requests.head(
+            shorts_url, cookies=cookies, headers=requests_headers(), timeout=10
+        )
+    except requests.exceptions.RequestException:
+        # assume video on error
+        return False
+
+    return response.status_code == 200
+
+
+def get_duration_sec(file_path: str) -> int:
+    """get duration of media file from file path"""
+
+    duration = subprocess.run(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            file_path,
+        ],
+        capture_output=True,
+        check=True,
+    )
+    duration_raw = duration.stdout.decode().strip()
+    if duration_raw == "N/A":
+        return 0
+
+    duration_sec = int(float(duration_raw))
+    return duration_sec
+
+
+def get_duration_str(seconds: int | float) -> str:
+    """Return a human-readable duration string from seconds."""
+    if not seconds:
+        return "NA"
+
+    seconds = int(seconds)
+    units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
+    duration_parts = []
+
+    for unit_label, unit_seconds in units:
+        if seconds >= unit_seconds:
+            unit_count, seconds = divmod(seconds, unit_seconds)
+            duration_parts.append(f"{unit_count:02}{unit_label}")
+
+    duration_parts[0] = duration_parts[0].lstrip("0")
+
+    return " ".join(duration_parts)
+
+
+def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
+    """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
+    allowed_hosts: list[str] = [
+        "localhost",
+        "tubearchivist",
+    ]
+    csrf_trusted_origins: list[str] = [
+        "http://localhost",
+        "http://tubearchivist",
+    ]
+    for host in ta_host.split():
+        host_clean = host.strip()
+        if not host_clean.startswith("http"):
+            host_clean = f"http://{host_clean}"
+
+        parsed = urlparse(host_clean)
+        allowed_hosts.append(f"{parsed.hostname}")
+        cors_url = f"{parsed.scheme}://{parsed.hostname}"
+
+        if parsed.port:
+            cors_url = f"{cors_url}:{parsed.port}"
+
+        csrf_trusted_origins.append(cors_url)
+
+    return allowed_hosts, csrf_trusted_origins
+
+
+def get_stylesheets() -> list:
+    """Get all valid stylesheets from /static/css"""
+
+    stylesheets = [
+        "dark.css",
+        "light.css",
+        "matrix.css",
+        "midnight.css",
+        "custom.css",
+    ]
+    return stylesheets
+
+
+def check_stylesheet(stylesheet: str):
+    """Check if a stylesheet exists. Return dark.css as a fallback"""
+    if stylesheet in get_stylesheets():
+        return stylesheet
+
+    return "dark.css"
+
+
+def is_missing(
+    to_check: str | list[str],
+    index_name: str = "ta_video,ta_download",
+    on_key: str = "youtube_id",
+) -> list[str]:
+    """id or list of ids that are missing from index_name"""
+    if isinstance(to_check, str):
+        to_check = [to_check]
+
+    data = {
+        "query": {"terms": {on_key: to_check}},
+        "_source": [on_key],
+    }
+    result = IndexPaginate(index_name, data=data).get_results()
+    existing_ids = [i[on_key] for i in result]
+    dl = [i for i in to_check if i not in existing_ids]
+
+    return dl
+
+
+def get_channel_overwrites() -> dict[str, dict[str, Any]]:
+    """get overwrites indexed my channel_id"""
+    data = {
+        "query": {
+            "bool": {"must": [{"exists": {"field": "channel_overwrites"}}]}
+        },
+        "_source": ["channel_id", "channel_overwrites"],
+    }
+    result = IndexPaginate("ta_channel", data).get_results()
+    overwrites = {i["channel_id"]: i["channel_overwrites"] for i in result}
+
+    return overwrites
+
+
+def get_channels(
+    subscribed_only: bool, source: list[str] | None = None
+) -> list[dict]:
+    """get a list of all channels"""
+    data = {
+        "sort": [{"channel_name.keyword": {"order": "asc"}}],
+    }
+    if subscribed_only:
+        query = {"term": {"channel_subscribed": {"value": True}}}
+    else:
+        query = {"match_all": {}}
+
+    data["query"] = query  # type: ignore
+
+    if source:
+        data["_source"] = source  # type: ignore
+
+    all_channels = IndexPaginate("ta_channel", data).get_results()
+
+    return all_channels
+
+
+def get_playlists(
+    subscribed_only: bool, source: list[str] | None = None
+) -> list[dict]:
+    """get list of playlists"""
+
+    data = {
+        "sort": [{"playlist_channel.keyword": {"order": "desc"}}],
+    }
+
+    must_list = [{"term": {"playlist_active": {"value": True}}}]
+    if subscribed_only:
+        must_list.append({"term": {"playlist_subscribed": {"value": True}}})
+
+    data = {"query": {"bool": {"must": must_list}}}  # type: ignore
+    if source:
+        data["_source"] = source  # type: ignore
+
+    all_playlists = IndexPaginate("ta_playlist", data).get_results()
+
+    return all_playlists
+
+
+def calc_is_watched(duration: float, position: float) -> bool:
+    """considered watched based on duration position"""
+
+    if not duration or duration <= 0:
+        return False
+
+    if duration < 60:
+        threshold = 0.5
+    elif duration > 900:
+        threshold = 1 - (180 / duration)
+    else:
+        threshold = 0.9
+
+    return position >= duration * threshold

backend/common/src/index_generic.py

@@ -26,6 +26,7 @@ class YouTubeItem:
         self.youtube_id = youtube_id
         self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.config = AppConfig().config
+        self.error = None
         self.youtube_meta = False
         self.json_data = False
 
@@ -33,17 +34,24 @@ class YouTubeItem:
         """build youtube url"""
         return self.yt_base + self.youtube_id
 
-    def get_from_youtube(self):
+    def get_from_youtube(self, obs_overwrite: dict | None = None):
         """use yt-dlp to get meta data from youtube"""
         print(f"{self.youtube_id}: get metadata from youtube")
         obs_request = self.yt_obs.copy()
         if self.config["downloads"]["extractor_lang"]:
             langs = self.config["downloads"]["extractor_lang"]
             langs_list = [i.strip() for i in langs.split(",")]
-            obs_request["extractor_args"] = {"youtube": {"lang": langs_list}}
+            obs_request["extractor_args"] = {
+                "youtube": {"lang": langs_list}
+            }  # type: ignore
+
+        if obs_overwrite:
+            obs_request.update(obs_overwrite)
 
         url = self.build_yt_url()
-        self.youtube_meta = YtWrap(obs_request, self.config).extract(url)
+        self.youtube_meta, self.error = YtWrap(
+            obs_request, self.config
+        ).extract(url)
 
     def get_from_es(self):
         """get indexed data from elastic search"""

backend/common/src/search_processor.py

@@ -165,15 +165,17 @@ class SearchProcess:
 
     def _process_download(self, download_dict):
         """run on single download item"""
-        video_id = download_dict["youtube_id"]
-        cache_root = EnvironmentSettings().get_cache_root()
-        vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
-        published = date_parser(download_dict["published"])
+        vid_thumb_url = None
+        if download_dict.get("vid_thumb_url"):
+            video_id = download_dict["youtube_id"]
+            cache_root = EnvironmentSettings().get_cache_root()
+            relative_path = ThumbManager(video_id).vid_thumb_path()
+            vid_thumb_url = f"{cache_root}/{relative_path}"
 
         download_dict.update(
             {
-                "vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
-                "published": published,
+                "vid_thumb_url": vid_thumb_url,
+                "published": date_parser(download_dict["published"]),
             }
         )
         return dict(sorted(download_dict.items()))

backend/common/src/urlparser.py

@@ -4,6 +4,7 @@ Functionality:
 - identify vid_type if possible
 """
 
+from typing import Literal, NotRequired, TypedDict
 from urllib.parse import parse_qs, urlparse
 
 from common.src.ta_redis import RedisArchivist
@@ -11,19 +12,28 @@ from download.src.yt_dlp_base import YtWrap
 from video.src.constants import VideoTypeEnum
 
 
+class ParsedURLType(TypedDict):
+    """represents single parsed url"""
+
+    type: Literal["video", "channel", "playlist"]
+    url: str
+    vid_type: VideoTypeEnum
+    limit: NotRequired[int | None]
+
+
 class Parser:
     """
     take a multi line string and detect valid youtube ids
     channel handle lookup is cached, can be disabled for unittests
     """
 
-    def __init__(self, url_str, use_cache=True):
+    def __init__(self, url_str: str, use_cache: bool = True):
         self.url_list = [i.strip() for i in url_str.split()]
         self.use_cache = use_cache
 
-    def parse(self):
+    def parse(self) -> list[ParsedURLType]:
         """parse the list"""
-        ids = []
+        ids: list[ParsedURLType] = []
         for url in self.url_list:
             parsed = urlparse(url)
             if parsed.netloc:
@@ -124,9 +134,9 @@ class Parser:
             "extract_flat": True,
             "playlistend": 0,
         }
-        url_info = YtWrap(obs_request).extract(url)
+        url_info, error = YtWrap(obs_request).extract(url)
         if not url_info:
-            raise ValueError(f"failed to retrieve content from URL: {url}")
+            raise ValueError(f"failed to retrieve URL: {error}")
 
         channel_id = url_info.get("channel_id", False)
         if channel_id:

backend/common/src/watched.py

@@ -28,6 +28,11 @@ class WatchState:
             self.change_vid_state()
             return
 
+        if url_type == "channel":
+            self.reset_channel_progress()
+        if url_type == "playlist":
+            self.reset_playlist_progress()
+
         self._add_pipeline()
         path = f"ta_video/_update_by_query?pipeline=watch_{self.youtube_id}"
         data = self._build_update_data(url_type)
@@ -53,6 +58,31 @@ class WatchState:
             print(response)
             raise ValueError("failed to mark video as watched")
 
+    def reset_channel_progress(self):
+        """reset channel progress positions"""
+        from channel.src.index import YoutubeChannel
+
+        videos = YoutubeChannel(self.youtube_id).get_channel_videos()
+        video_ids = [i["youtube_id"] for i in videos]
+        self._reset_list(video_ids)
+
+    def reset_playlist_progress(self):
+        """reset playlist progress positions"""
+        from playlist.src.index import YoutubePlaylist
+
+        videos = YoutubePlaylist(self.youtube_id).get_playlist_videos()
+        video_ids = [i["youtube_id"] for i in videos]
+        self._reset_list(video_ids)
+
+    def _reset_list(self, video_ids: list[str]):
+        """reset list of video ids"""
+        redis_con = RedisArchivist()
+        all_ids = redis_con.list_keys(f"{self.user_id}:progress")
+        for progress_id in all_ids:
+            video_id = progress_id.split(":")[-1]
+            if video_id in video_ids:
+                redis_con.del_message(progress_id)
+
     def _build_update_data(self, url_type):
         """build update by query data based on url_type"""
         term_key_map = {

backend/common/tests/test_src/test_helper.py

@@ -22,14 +22,14 @@ def test_randomizor_with_positive_length():
 def test_date_parser_with_int():
     """unix timestamp"""
     timestamp = 1621539600
-    expected_date = "2021-05-20"
+    expected_date = "2021-05-20T19:40:00+00:00"
     assert date_parser(timestamp) == expected_date
 
 
 def test_date_parser_with_str():
     """iso timestamp"""
     date_str = "2021-05-21"
-    expected_date = "2021-05-21"
+    expected_date = "2021-05-21T00:00:00+00:00"
     assert date_parser(date_str) == expected_date
 
 

backend/config/fwd_auth_settings.py

@@ -0,0 +1,6 @@
+from os import environ
+
+TA_AUTH_PROXY_USERNAME_HEADER = (
+    environ.get("TA_AUTH_PROXY_USERNAME_HEADER") or "HTTP_REMOTE_USER"
+)
+TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")

backend/config/ldap_settings.py

@@ -0,0 +1,111 @@
+from os import environ
+
+import ldap
+from django_auth_ldap.config import LDAPSearch
+
+AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
+
+AUTH_LDAP_BIND_DN = environ.get("TA_LDAP_BIND_DN")
+
+AUTH_LDAP_BIND_PASSWORD = environ.get("TA_LDAP_BIND_PASSWORD")
+
+"""
+Given Names are *_technically_* different from Personal names, as people
+who change their names have different given names and personal names,
+and they go by personal names. Additionally, "LastName" is actually
+incorrect for many cultures, such as Korea, where the
+family name comes first, and the personal name comes last.
+
+But we all know people are going to try to guess at these, so still want
+to include names that people will guess, hence using first/last as well.
+"""
+
+AUTH_LDAP_USER_ATTR_MAP_USERNAME = (
+    environ.get("TA_LDAP_USER_ATTR_MAP_USERNAME")
+    or environ.get("TA_LDAP_USER_ATTR_MAP_UID")
+    or "uid"
+)
+
+AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME = (
+    environ.get("TA_LDAP_USER_ATTR_MAP_PERSONALNAME")
+    or environ.get("TA_LDAP_USER_ATTR_MAP_FIRSTNAME")
+    or environ.get("TA_LDAP_USER_ATTR_MAP_GIVENNAME")
+    or "givenName"
+)
+
+AUTH_LDAP_USER_ATTR_MAP_SURNAME = (
+    environ.get("TA_LDAP_USER_ATTR_MAP_SURNAME")
+    or environ.get("TA_LDAP_USER_ATTR_MAP_LASTNAME")
+    or environ.get("TA_LDAP_USER_ATTR_MAP_FAMILYNAME")
+    or "sn"
+)
+
+AUTH_LDAP_USER_ATTR_MAP_EMAIL = (
+    environ.get("TA_LDAP_USER_ATTR_MAP_EMAIL")
+    or environ.get("TA_LDAP_USER_ATTR_MAP_MAIL")
+    or "mail"
+)
+
+AUTH_LDAP_USER_BASE = environ.get("TA_LDAP_USER_BASE")
+
+AUTH_LDAP_USER_FILTER = environ.get("TA_LDAP_USER_FILTER")
+
+# pylint: disable=no-member
+AUTH_LDAP_USER_SEARCH = LDAPSearch(
+    AUTH_LDAP_USER_BASE,
+    ldap.SCOPE_SUBTREE,
+    "(&("
+    + AUTH_LDAP_USER_ATTR_MAP_USERNAME
+    + "=%(user)s)"
+    + AUTH_LDAP_USER_FILTER
+    + ")",
+)
+
+AUTH_LDAP_USER_ATTR_MAP = {
+    "username": AUTH_LDAP_USER_ATTR_MAP_USERNAME,
+    "first_name": AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME,
+    "last_name": AUTH_LDAP_USER_ATTR_MAP_SURNAME,
+    "email": AUTH_LDAP_USER_ATTR_MAP_EMAIL,
+}
+
+if bool(environ.get("TA_LDAP_DISABLE_CERT_CHECK")):
+    # pylint: disable=global-at-module-level
+    global AUTH_LDAP_GLOBAL_OPTIONS
+    AUTH_LDAP_GLOBAL_OPTIONS = {
+        ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
+    }
+
+# Promote specific usernames to staff or superuser permission levels
+_ldap_superuser_username_config = (
+    environ.get("TA_LDAP_PROMOTE_USERNAMES_TO_SUPERUSER") or ""
+)
+_ldap_superuser_usernames = []
+if _ldap_superuser_username_config:
+    _ldap_superuser_usernames = [
+        u.strip() for u in _ldap_superuser_username_config.split(",")
+    ]
+
+_ldap_staff_username_config = (
+    environ.get("TA_LDAP_PROMOTE_USERNAMES_TO_STAFF") or ""
+)
+_ldap_staff_usernames = []
+if _ldap_staff_username_config:
+    _ldap_staff_usernames = [
+        u.strip() for u in _ldap_staff_username_config.split(",")
+    ]
+
+if _ldap_staff_usernames or _ldap_superuser_usernames:
+    import django_auth_ldap.backend
+
+    def create_user(sender, user=None, ldap_user=None, **kwargs):
+        if user.ldap_username in _ldap_superuser_usernames and not (
+            user.is_superuser and user.is_staff
+        ):
+            user.is_staff = True
+            user.is_superuser = True
+            user.save()
+        elif user.ldap_username in _ldap_staff_usernames and not user.is_staff:
+            user.is_staff = True
+            user.save()
+
+    django_auth_ldap.backend.populate_user.connect(create_user)

backend/config/management/commands/ta_config_backup.py

@@ -1,76 +0,0 @@
-"""backup config for sqlite reset and restore"""
-
-import json
-from pathlib import Path
-
-from django.contrib.auth import get_user_model
-from django.core.management.base import BaseCommand
-from home.models import CustomPeriodicTask
-from home.src.ta.settings import EnvironmentSettings
-from rest_framework.authtoken.models import Token
-
-User = get_user_model()
-
-
-class Command(BaseCommand):
-    """export"""
-
-    help = "Exports all users and their auth tokens to a JSON file"
-    FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
-
-    def handle(self, *args, **kwargs):
-        """entry point"""
-
-        data = {
-            "user_data": self.get_users(),
-            "schedule_data": self.get_schedules(),
-        }
-
-        with open(self.FILE, "w", encoding="utf-8") as json_file:
-            json_file.write(json.dumps(data))
-
-    def get_users(self):
-        """get users"""
-
-        users = User.objects.all()
-
-        user_data = []
-
-        for user in users:
-            user_info = {
-                "username": user.name,
-                "is_staff": user.is_staff,
-                "is_superuser": user.is_superuser,
-                "password": user.password,
-                "tokens": [],
-            }
-
-            try:
-                token = Token.objects.get(user=user)
-                user_info["tokens"] = [token.key]
-            except Token.DoesNotExist:
-                user_info["tokens"] = []
-
-            user_data.append(user_info)
-
-        return user_data
-
-    def get_schedules(self):
-        """get schedules"""
-
-        all_schedules = CustomPeriodicTask.objects.all()
-        schedule_data = []
-
-        for schedule in all_schedules:
-            schedule_info = {
-                "name": schedule.name,
-                "crontab": {
-                    "minute": schedule.crontab.minute,
-                    "hour": schedule.crontab.hour,
-                    "day_of_week": schedule.crontab.day_of_week,
-                },
-            }
-
-            schedule_data.append(schedule_info)
-
-        return schedule_data

backend/config/management/commands/ta_config_restore.py

@@ -1,89 +0,0 @@
-"""restore config from backup"""
-
-import json
-from pathlib import Path
-
-from common.src.env_settings import EnvironmentSettings
-from django.core.management.base import BaseCommand
-from django_celery_beat.models import CrontabSchedule
-from rest_framework.authtoken.models import Token
-from task.models import CustomPeriodicTask
-from task.src.task_config import TASK_CONFIG
-from user.models import Account
-
-
-class Command(BaseCommand):
-    """export"""
-
-    help = "Exports all users and their auth tokens to a JSON file"
-    FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
-
-    def handle(self, *args, **options):
-        """handle"""
-        self.stdout.write("restore users and schedules")
-        data = self.get_config()
-        self.restore_users(data["user_data"])
-        self.restore_schedules(data["schedule_data"])
-        self.stdout.write(
-            self.style.SUCCESS(
-                "    ✓ restore completed. Please restart the container."
-            )
-        )
-
-    def get_config(self) -> dict:
-        """get config from backup"""
-        with open(self.FILE, "r", encoding="utf-8") as json_file:
-            data = json.loads(json_file.read())
-
-        self.stdout.write(
-            self.style.SUCCESS(f"    ✓ json file found: {self.FILE}")
-        )
-
-        return data
-
-    def restore_users(self, user_data: list[dict]) -> None:
-        """restore users from config"""
-        self.stdout.write("delete existing users")
-        Account.objects.all().delete()
-
-        self.stdout.write("recreate users")
-        for user_info in user_data:
-            user = Account.objects.create(
-                name=user_info["username"],
-                is_staff=user_info["is_staff"],
-                is_superuser=user_info["is_superuser"],
-                password=user_info["password"],
-            )
-            for token in user_info["tokens"]:
-                Token.objects.create(user=user, key=token)
-
-            self.stdout.write(
-                self.style.SUCCESS(
-                    f"    ✓ recreated user with name: {user_info['username']}"
-                )
-            )
-
-    def restore_schedules(self, schedule_data: list[dict]) -> None:
-        """restore schedules"""
-        self.stdout.write("delete existing schedules")
-        CustomPeriodicTask.objects.all().delete()
-
-        self.stdout.write("recreate schedules")
-        for schedule in schedule_data:
-            task_name = schedule["name"]
-            description = TASK_CONFIG[task_name].get("title")
-            crontab, _ = CrontabSchedule.objects.get_or_create(
-                minute=schedule["crontab"]["minute"],
-                hour=schedule["crontab"]["hour"],
-                day_of_week=schedule["crontab"]["day_of_week"],
-                timezone=EnvironmentSettings.TZ,
-            )
-            task = CustomPeriodicTask.objects.create(
-                name=task_name,
-                task=task_name,
-                description=description,
-                crontab=crontab,
-            )
-            self.stdout.write(
-                self.style.SUCCESS(f"    ✓ recreated schedule: {task}")
-            )

backend/config/management/commands/ta_connection.py

@@ -24,7 +24,7 @@ class Command(BaseCommand):
     """command framework"""
 
     TIMEOUT = 120
-    MIN_MAJOR, MAX_MAJOR = 8, 8
+    MIN_MAJOR, MAX_MAJOR = 8, 9
     MIN_MINOR = 0
 
     # pylint: disable=no-member
@@ -99,7 +99,12 @@ class Command(BaseCommand):
                 continue
 
             if status_code and status_code == 200:
-                path = "_cluster/health?wait_for_status=yellow&timeout=60s"
+                path = (
+                    "_cluster/health?"
+                    "wait_for_status=yellow&"
+                    "timeout=60s&"
+                    "wait_for_active_shards=1"
+                )
                 _, _ = ElasticWrap(path).get(timeout=60)
                 self.stdout.write(
                     self.style.SUCCESS("    ✓ ES connection established")

backend/config/management/commands/ta_envcheck.py

@@ -81,6 +81,7 @@ class Command(BaseCommand):
         """run all commands"""
         self.stdout.write(LOGO)
         self.stdout.write(TOPIC)
+        self._additional_auth_vars_expectations()
         self._expected_vars()
         self._unexpected_vars()
         self._elastic_user_overwrite()
@@ -89,6 +90,50 @@ class Command(BaseCommand):
         self._disable_static_auth()
         self._create_superuser()
 
+    def _additional_auth_vars_expectations(self):
+        """conditionally add additional expectations for auth modes"""
+        ldap_required_env = [
+            "TA_LDAP_SERVER_URI",
+            "TA_LDAP_BIND_DN",
+            "TA_LDAP_BIND_PASSWORD",
+            "TA_LDAP_USER_BASE",
+            "TA_LDAP_USER_FILTER",
+        ]
+        _login_auth_mode = (
+            os.environ.get("TA_LOGIN_AUTH_MODE") or "single"
+        ).casefold()
+        if _login_auth_mode == "local":
+            UNEXPECTED_ENV_VARS["TA_LDAP"] = (
+                "TA_LDAP is not valid with current auth mode"
+            )
+            UNEXPECTED_ENV_VARS["TA_ENABLE_AUTH_PROXY"] = (
+                "TA_ENABLE_AUTH_PROXY is not valid with current auth mode"
+            )
+        elif _login_auth_mode == "ldap":
+            EXPECTED_ENV_VARS.extend(ldap_required_env)
+            UNEXPECTED_ENV_VARS["TA_ENABLE_AUTH_PROXY"] = (
+                "TA_ENABLE_AUTH_PROXY is not valid with current auth mode"
+            )
+        elif _login_auth_mode == "forwardauth":
+            UNEXPECTED_ENV_VARS["TA_LDAP"] = (
+                "TA_LDAP is not valid with current auth mode"
+            )
+        elif _login_auth_mode == "ldap_local":
+            EXPECTED_ENV_VARS.extend(ldap_required_env)
+            UNEXPECTED_ENV_VARS["TA_ENABLE_AUTH_PROXY"] = (
+                "TA_ENABLE_AUTH_PROXY is not valid with current auth mode"
+            )
+        else:
+            if bool(os.environ.get("TA_LDAP")):
+                EXPECTED_ENV_VARS.extend(ldap_required_env)
+                UNEXPECTED_ENV_VARS["TA_ENABLE_AUTH_PROXY"] = (
+                    "TA_ENABLE_AUTH_PROXY is not valid with current auth mode"
+                )
+            if bool(os.environ.get("TA_ENABLE_AUTH_PROXY")):
+                UNEXPECTED_ENV_VARS["TA_LDAP"] = (
+                    "TA_LDAP is not valid with current auth mode"
+                )
+
     def _expected_vars(self):
         """check if expected env vars are set"""
         self.stdout.write("[1] checking expected env vars")

backend/config/management/commands/ta_index_channel_tabs.py

@@ -0,0 +1,36 @@
+"""
+migration for 0.5.4 to 0.5.5
+index channel_tabs for subscribed channels
+"""
+
+import time
+
+from channel.src.index import YoutubeChannel
+from common.src.helper import get_channels
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    """command"""
+
+    def handle(self, *args, **kwargs):
+        """handle task"""
+
+        self.stdout.write("channel tags initial index")
+
+        channels = get_channels(subscribed_only=True, source=["channel_id"])
+
+        for es_channel in channels:
+            channel = YoutubeChannel(es_channel["channel_id"])
+            channel.get_from_es()
+            channel_name = channel.json_data["channel_name"]
+            channel_tabs = channel.get_channel_tabs()
+            channel.json_data["channel_tabs"] = channel_tabs
+            channel.upload_to_es()
+            channel.sync_to_videos()
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"    ✓ updated '{channel_name}' tabs: {channel_tabs}"
+                )
+            )
+            time.sleep(5)

backend/config/management/commands/ta_startup.py

@@ -19,11 +19,11 @@ from common.src.ta_redis import RedisArchivist
 from django.core.management.base import BaseCommand, CommandError
 from django.utils import dateformat
 from django_celery_beat.models import CrontabSchedule, PeriodicTasks
-from redis.exceptions import ResponseError
 from task.models import CustomPeriodicTask
 from task.src.config_schedule import ScheduleBuilder
 from task.src.task_manager import TaskManager
 from task.tasks import version_check
+from video.src.constants import VideoTypeEnum
 
 TOPIC = """
 
@@ -49,12 +49,13 @@ class Command(BaseCommand):
         self._version_check()
         self._index_setup()
         self._snapshot_check()
-        self._mig_app_settings()
         self._create_default_schedules()
         self._update_schedule_tz()
         self._init_app_config()
-        self._mig_channel_tags()
-        self._mig_video_channel_tags()
+        self._mig_fix_download_channel_indexed()
+        self._mig_add_default_playlist_sort()
+        self._mig_set_channel_tabs()
+        self._mig_set_video_channel_tabs()
 
     def _make_folders(self):
         """make expected cache folders"""
@@ -157,39 +158,6 @@ class Command(BaseCommand):
         self.stdout.write("[7] setup snapshots")
         ElasticSnapshot().setup()
 
-    def _mig_app_settings(self) -> None:
-        """update from v0.4.13 to v0.5.0, migrate application settings"""
-        self.stdout.write("[MIGRATION] move appconfig to ES")
-        try:
-            config = RedisArchivist().get_message("config")
-        except ResponseError:
-            self.stdout.write(
-                self.style.SUCCESS("    Redis does not support JSON decoding")
-            )
-            return
-
-        if not config or config == {"status": False}:
-            self.stdout.write(
-                self.style.SUCCESS("    no config values to migrate")
-            )
-            return
-
-        path = "ta_config/_doc/appsettings"
-        response, status_code = ElasticWrap(path).post(config)
-
-        if status_code in [200, 201]:
-            self.stdout.write(
-                self.style.SUCCESS("    ✓ migrated appconfig to ES")
-            )
-            RedisArchivist().del_message("config", save=True)
-            return
-
-        message = "    🗙 failed to migrate app config"
-        self.stdout.write(self.style.ERROR(message))
-        self.stdout.write(response)
-        sleep(60)
-        raise CommandError(message)
-
     def _create_default_schedules(self) -> None:
         """create default schedules for new installations"""
         self.stdout.write("[8] create initial schedules")
@@ -262,7 +230,7 @@ class Command(BaseCommand):
     def _init_app_config(self) -> None:
         """init default app config to ES"""
         self.stdout.write("[10] Check AppConfig")
-        _, status_code = ElasticWrap("ta_config/_doc/appsettings").get()
+        response, status_code = ElasticWrap("ta_config/_doc/appsettings").get()
         if status_code in [200, 201]:
             self.stdout.write(
                 self.style.SUCCESS("    skip completed appsettings init")
@@ -275,6 +243,13 @@ class Command(BaseCommand):
 
             return
 
+        if status_code != 404:
+            message = "    🗙 ta_config index lookup failed"
+            self.stdout.write(self.style.ERROR(message))
+            self.stdout.write(response)
+            sleep(60)
+            raise CommandError(message)
+
         handler = AppConfig.__new__(AppConfig)
         _, status_code = handler.sync_defaults()
         self.stdout.write(
@@ -284,44 +259,18 @@ class Command(BaseCommand):
             self.style.SUCCESS(f"      Status code: {status_code}")
         )
 
-    def _mig_channel_tags(self) -> None:
-        """update from v0.4.13 to v0.5.0, migrate incorrect data types"""
-        self.stdout.write("[MIGRATION] fix incorrect channel tags types")
-        path = "ta_channel/_update_by_query"
-        data = {
-            "query": {"match": {"channel_tags": False}},
-            "script": {
-                "source": "ctx._source.channel_tags = []",
-                "lang": "painless",
-            },
-        }
-        response, status_code = ElasticWrap(path).post(data)
-        if status_code in [200, 201]:
-            updated = response.get("updated")
-            if updated:
-                self.stdout.write(
-                    self.style.SUCCESS(f"    ✓ fixed {updated} channel tags")
-                )
-            else:
-                self.stdout.write(
-                    self.style.SUCCESS("    no channel tags needed fixing")
-                )
-            return
-
-        message = "    🗙 failed to fix channel tags"
-        self.stdout.write(self.style.ERROR(message))
-        self.stdout.write(response)
-        sleep(60)
-        raise CommandError(message)
-
-    def _mig_video_channel_tags(self) -> None:
-        """update from v0.4.13 to v0.5.0, migrate incorrect data types"""
+    def _mig_fix_download_channel_indexed(self) -> None:
+        """migrate from v0.5.2 to 0.5.3, fix missing channel_indexed"""
         self.stdout.write("[MIGRATION] fix incorrect video channel tags types")
-        path = "ta_video/_update_by_query"
+        path = "ta_download/_update_by_query"
         data = {
-            "query": {"match": {"channel.channel_tags": False}},
+            "query": {
+                "bool": {
+                    "must_not": [{"exists": {"field": "channel_indexed"}}]
+                }
+            },
             "script": {
-                "source": "ctx._source.channel.channel_tags = []",
+                "source": "ctx._source.channel_indexed = false",
                 "lang": "painless",
             },
         }
@@ -330,15 +279,11 @@ class Command(BaseCommand):
             updated = response.get("updated")
             if updated:
                 self.stdout.write(
-                    self.style.SUCCESS(
-                        f"    ✓ fixed {updated} video channel tags"
-                    )
+                    self.style.SUCCESS(f"    ✓ fixed {updated} queued videos")
                 )
             else:
                 self.stdout.write(
-                    self.style.SUCCESS(
-                        "    no video channel tags needed fixing"
-                    )
+                    self.style.SUCCESS("    no queued videos to fix")
                 )
             return
 
@@ -347,3 +292,107 @@ class Command(BaseCommand):
         self.stdout.write(response)
         sleep(60)
         raise CommandError(message)
+
+    def _mig_add_default_playlist_sort(self) -> None:
+        """migrate from 0.5.4 to 0.5.5 set default playlist sortorder"""
+        self.stdout.write("[MIGRATION] set default playlist sort order")
+        path = "ta_playlist/_update_by_query"
+        data = {
+            "query": {
+                "bool": {
+                    "must_not": [{"exists": {"field": "playlist_sort_order"}}]
+                }
+            },
+            "script": {
+                "source": "ctx._source.playlist_sort_order = 'top'",
+                "lang": "painless",
+            },
+        }
+        response, status_code = ElasticWrap(path).post(data)
+        if status_code in [200, 201]:
+            updated = response.get("updated")
+            if updated:
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ updated {updated} playlists")
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS("    no playlists need updating")
+                )
+            return
+
+        message = "    🗙 failed to set default playlist sort order"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)
+
+    def _mig_set_channel_tabs(self) -> None:
+        """migrate from 0.5.4 to 0.5.5 set initial channel tabs"""
+        self.stdout.write("[MIGRATION] set default channel_tabs")
+
+        path = "ta_channel/_update_by_query"
+        tabs = VideoTypeEnum.values_known()
+        data = {
+            "query": {
+                "bool": {"must_not": [{"exists": {"field": "channel_tabs"}}]}
+            },
+            "script": {
+                "source": f"ctx._source.channel_tabs = {tabs}",
+                "lang": "painless",
+            },
+        }
+        response, status_code = ElasticWrap(path).post(data)
+        if status_code in [200, 201]:
+            updated = response.get("updated")
+            if updated:
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ updated {updated} channels")
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS("    no channels need updating")
+                )
+            return
+
+        message = "    🗙 failed to set default channel_tabs"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)
+
+    def _mig_set_video_channel_tabs(self) -> None:
+        """migrate from 0.5.4 to 0.5.5 set initial video channel tabs"""
+        self.stdout.write("[MIGRATION] set default channel_tabs for videos")
+
+        path = "ta_video/_update_by_query"
+        tabs = VideoTypeEnum.values_known()
+        data = {
+            "query": {
+                "bool": {
+                    "must_not": [{"exists": {"field": "channel.channel_tabs"}}]
+                }
+            },
+            "script": {
+                "source": f"ctx._source.channel.channel_tabs = {tabs}",
+                "lang": "painless",
+            },
+        }
+        response, status_code = ElasticWrap(path).post(data)
+        if status_code in [200, 201]:
+            updated = response.get("updated")
+            if updated:
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ updated {updated} videos")
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS("    no videos need updating")
+                )
+            return
+
+        message = "    🗙 failed to set default channel_tabs"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)

backend/config/settings.py

@@ -104,99 +104,6 @@ TEMPLATES = [
 
 WSGI_APPLICATION = "config.wsgi.application"
 
-if bool(environ.get("TA_LDAP")):
-    # pylint: disable=global-at-module-level
-    import ldap
-    from django_auth_ldap.config import LDAPSearch
-
-    global AUTH_LDAP_SERVER_URI
-    AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
-
-    global AUTH_LDAP_BIND_DN
-    AUTH_LDAP_BIND_DN = environ.get("TA_LDAP_BIND_DN")
-
-    global AUTH_LDAP_BIND_PASSWORD
-    AUTH_LDAP_BIND_PASSWORD = environ.get("TA_LDAP_BIND_PASSWORD")
-
-    """
-    Since these are new environment variables, taking the opporunity to use
-    more accurate env names.
-    Given Names are *_technically_* different from Personal names, as people
-    who change their names have different given names and personal names,
-    and they go by personal names. Additionally, "LastName" is actually
-    incorrect for many cultures, such as Korea, where the
-    family name comes first, and the personal name comes last.
-
-    But we all know people are going to try to guess at these, so still want
-    to include names that people will guess, hence using first/last as well.
-    """
-
-    # Attribute mapping options
-
-    global AUTH_LDAP_USER_ATTR_MAP_USERNAME
-    AUTH_LDAP_USER_ATTR_MAP_USERNAME = (
-        environ.get("TA_LDAP_USER_ATTR_MAP_USERNAME")
-        or environ.get("TA_LDAP_USER_ATTR_MAP_UID")
-        or "uid"
-    )
-
-    global AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME
-    AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME = (
-        environ.get("TA_LDAP_USER_ATTR_MAP_PERSONALNAME")
-        or environ.get("TA_LDAP_USER_ATTR_MAP_FIRSTNAME")
-        or environ.get("TA_LDAP_USER_ATTR_MAP_GIVENNAME")
-        or "givenName"
-    )
-
-    global AUTH_LDAP_USER_ATTR_MAP_SURNAME
-    AUTH_LDAP_USER_ATTR_MAP_SURNAME = (
-        environ.get("TA_LDAP_USER_ATTR_MAP_SURNAME")
-        or environ.get("TA_LDAP_USER_ATTR_MAP_LASTNAME")
-        or environ.get("TA_LDAP_USER_ATTR_MAP_FAMILYNAME")
-        or "sn"
-    )
-
-    global AUTH_LDAP_USER_ATTR_MAP_EMAIL
-    AUTH_LDAP_USER_ATTR_MAP_EMAIL = (
-        environ.get("TA_LDAP_USER_ATTR_MAP_EMAIL")
-        or environ.get("TA_LDAP_USER_ATTR_MAP_MAIL")
-        or "mail"
-    )
-
-    global AUTH_LDAP_USER_BASE
-    AUTH_LDAP_USER_BASE = environ.get("TA_LDAP_USER_BASE")
-
-    global AUTH_LDAP_USER_FILTER
-    AUTH_LDAP_USER_FILTER = environ.get("TA_LDAP_USER_FILTER")
-
-    global AUTH_LDAP_USER_SEARCH
-    # pylint: disable=no-member
-    AUTH_LDAP_USER_SEARCH = LDAPSearch(
-        AUTH_LDAP_USER_BASE,
-        ldap.SCOPE_SUBTREE,
-        "(&("
-        + AUTH_LDAP_USER_ATTR_MAP_USERNAME
-        + "=%(user)s)"
-        + AUTH_LDAP_USER_FILTER
-        + ")",
-    )
-
-    global AUTH_LDAP_USER_ATTR_MAP
-    AUTH_LDAP_USER_ATTR_MAP = {
-        "username": AUTH_LDAP_USER_ATTR_MAP_USERNAME,
-        "first_name": AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME,
-        "last_name": AUTH_LDAP_USER_ATTR_MAP_SURNAME,
-        "email": AUTH_LDAP_USER_ATTR_MAP_EMAIL,
-    }
-
-    if bool(environ.get("TA_LDAP_DISABLE_CERT_CHECK")):
-        global AUTH_LDAP_GLOBAL_OPTIONS
-        AUTH_LDAP_GLOBAL_OPTIONS = {
-            ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
-        }
-
-    AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
-
 # Database
 # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
 
@@ -230,19 +137,41 @@ AUTH_PASSWORD_VALIDATORS = [
 
 AUTH_USER_MODEL = "user.Account"
 
-# Forward-auth authentication
-if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
-    TA_AUTH_PROXY_USERNAME_HEADER = (
-        environ.get("TA_AUTH_PROXY_USERNAME_HEADER") or "HTTP_REMOTE_USER"
+# Configure Authentication Backend Combinations
+_login_auth_mode = (environ.get("TA_LOGIN_AUTH_MODE") or "single").casefold()
+if _login_auth_mode == "local":
+    AUTHENTICATION_BACKENDS: tuple = (
+        "django.contrib.auth.backends.ModelBackend",
     )
-    TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")
-
-    MIDDLEWARE.append("user.src.remote_user_auth.HttpRemoteUserMiddleware")
+elif _login_auth_mode == "ldap":
+    AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
+    from .ldap_settings import *  # noqa: F403 F401
+elif _login_auth_mode == "forwardauth":
+    from .fwd_auth_settings import *  # noqa: F403 F401
 
     AUTHENTICATION_BACKENDS = (
         "django.contrib.auth.backends.RemoteUserBackend",
     )
+    MIDDLEWARE.append("user.src.remote_user_auth.HttpRemoteUserMiddleware")
+elif _login_auth_mode == "ldap_local":
+    AUTHENTICATION_BACKENDS = (
+        "django_auth_ldap.backend.LDAPBackend",
+        "django.contrib.auth.backends.ModelBackend",
+    )
+    from .ldap_settings import *  # noqa: F403 F401
+else:
+    # If none of these cases match, AUTHENTICATION_BACKENDS is unset, which
+    # means the ModelBackend should be used by default
+    if bool(environ.get("TA_LDAP")):
+        AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
+        from .ldap_settings import *  # noqa: F403 F401
+    if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
+        from .fwd_auth_settings import *  # noqa: F403 F401
 
+        AUTHENTICATION_BACKENDS = (
+            "django.contrib.auth.backends.RemoteUserBackend",
+        )
+        MIDDLEWARE.append("user.src.remote_user_auth.HttpRemoteUserMiddleware")
 
 # Internationalization
 # https://docs.djangoproject.com/en/3.2/topics/i18n/
@@ -294,7 +223,7 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.5.2"
+TA_VERSION = "v0.5.5"
 
 # API
 REST_FRAMEWORK = {
@@ -307,3 +236,21 @@ SPECTACULAR_SETTINGS = {
     "VERSION": TA_VERSION,
     "SERVE_INCLUDE_SCHEMA": False,
 }
+
+# Logging configuration
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+        },
+    },
+    "loggers": {
+        "apprise": {
+            "handlers": ["console"],
+            "level": "DEBUG",
+            "propagate": True,
+        },
+    },
+}

backend/download/serializers.py

@@ -15,11 +15,11 @@ class DownloadItemSerializer(serializers.Serializer):
     channel_indexed = serializers.BooleanField()
     channel_name = serializers.CharField()
     duration = serializers.CharField()
-    published = serializers.CharField()
+    published = serializers.CharField(allow_null=True)
     status = serializers.ChoiceField(choices=["pending", "ignore"])
-    timestamp = serializers.IntegerField()
+    timestamp = serializers.IntegerField(allow_null=True)
     title = serializers.CharField()
-    vid_thumb_url = serializers.CharField()
+    vid_thumb_url = serializers.CharField(allow_null=True)
     vid_type = serializers.ChoiceField(choices=VideoTypeEnum.values())
     youtube_id = serializers.CharField()
     message = serializers.CharField(required=False)
@@ -42,14 +42,23 @@ class DownloadListQuerySerializer(
     filter = serializers.ChoiceField(
         choices=["pending", "ignore"], required=False
     )
+    vid_type = serializers.ChoiceField(
+        choices=VideoTypeEnum.values_known(), required=False
+    )
     channel = serializers.CharField(required=False, help_text="channel ID")
     page = serializers.IntegerField(required=False)
+    q = serializers.CharField(required=False, help_text="Search Query")
+    error = serializers.BooleanField(required=False, allow_null=True)
 
 
 class DownloadListQueueDeleteQuerySerializer(serializers.Serializer):
     """serialize bulk delete download queue query string"""
 
     filter = serializers.ChoiceField(choices=["pending", "ignore"])
+    channel = serializers.CharField(required=False, help_text="channel ID")
+    vid_type = serializers.ChoiceField(
+        choices=VideoTypeEnum.values_known(), required=False
+    )
 
 
 class AddDownloadItemSerializer(serializers.Serializer):
@@ -69,6 +78,26 @@ class AddToDownloadQuerySerializer(serializers.Serializer):
     """add to queue query serializer"""
 
     autostart = serializers.BooleanField(required=False)
+    flat = serializers.BooleanField(required=False)
+
+
+class BulkUpdateDowloadQuerySerializer(serializers.Serializer):
+    """serialize bulk update query"""
+
+    filter = serializers.ChoiceField(choices=["pending", "ignore", "priority"])
+    channel = serializers.CharField(required=False)
+    vid_type = serializers.ChoiceField(
+        choices=VideoTypeEnum.values_known(), required=False
+    )
+    error = serializers.BooleanField(required=False, allow_null=True)
+
+
+class BulkUpdateDowloadDataSerializer(serializers.Serializer):
+    """serialize data"""
+
+    status = serializers.ChoiceField(
+        choices=["pending", "ignore", "priority", "clear_error"]
+    )
 
 
 class DownloadQueueItemUpdateSerializer(serializers.Serializer):

backend/download/src/queue.py

@@ -4,16 +4,25 @@ Functionality:
 - linked with ta_dowload index
 """
 
+import json
 from datetime import datetime
 
 from appsettings.src.config import AppConfig
+from channel.src.index import YoutubeChannel
+from channel.src.remote_query import get_last_channel_videos
 from common.src.es_connect import ElasticWrap, IndexPaginate
-from common.src.helper import get_duration_str, is_shorts, rand_sleep
-from download.src.subscriptions import ChannelSubscription
+from common.src.helper import (
+    get_channels,
+    get_duration_str,
+    is_shorts,
+    rand_sleep,
+)
+from common.src.urlparser import ParsedURLType
+from download.src.queue_interact import PendingInteract
 from download.src.thumbnails import ThumbManager
-from download.src.yt_dlp_base import YtWrap
 from playlist.src.index import YoutubePlaylist
 from video.src.constants import VideoTypeEnum
+from video.src.index import YoutubeVideo
 
 
 class PendingIndex:
@@ -61,11 +70,7 @@ class PendingIndex:
         """get a list of all channels indexed"""
         self.all_channels = []
         self.channel_overwrites = {}
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"channel_id": {"order": "asc"}}],
-        }
-        channels = IndexPaginate("ta_channel", data).get_results()
+        channels = get_channels(subscribed_only=False)
 
         for channel in channels:
             channel_id = channel["channel_id"]
@@ -88,67 +93,6 @@ class PendingIndex:
                 self.video_overwrites.update({video_id: overwrites})
 
 
-class PendingInteract:
-    """interact with items in download queue"""
-
-    def __init__(self, youtube_id=False, status=False):
-        self.youtube_id = youtube_id
-        self.status = status
-
-    def delete_item(self):
-        """delete single item from pending"""
-        path = f"ta_download/_doc/{self.youtube_id}"
-        _, _ = ElasticWrap(path).delete(refresh=True)
-
-    def delete_by_status(self):
-        """delete all matching item by status"""
-        data = {"query": {"term": {"status": {"value": self.status}}}}
-        path = "ta_download/_delete_by_query"
-        _, _ = ElasticWrap(path).post(data=data)
-
-    def update_status(self):
-        """update status of pending item"""
-        if self.status == "priority":
-            data = {
-                "doc": {
-                    "status": "pending",
-                    "auto_start": True,
-                    "message": None,
-                }
-            }
-        else:
-            data = {"doc": {"status": self.status}}
-
-        path = f"ta_download/_update/{self.youtube_id}/?refresh=true"
-        _, _ = ElasticWrap(path).post(data=data)
-
-    def get_item(self):
-        """return pending item dict"""
-        path = f"ta_download/_doc/{self.youtube_id}"
-        response, status_code = ElasticWrap(path).get()
-        return response["_source"], status_code
-
-    def get_channel(self):
-        """
-        get channel metadata from queue to not depend on channel to be indexed
-        """
-        data = {
-            "size": 1,
-            "query": {"term": {"channel_id": {"value": self.youtube_id}}},
-        }
-        response, _ = ElasticWrap("ta_download/_search").get(data=data)
-        hits = response["hits"]["hits"]
-        if not hits:
-            channel_name = "NA"
-        else:
-            channel_name = hits[0]["_source"].get("channel_name", "NA")
-
-        return {
-            "channel_id": self.youtube_id,
-            "channel_name": channel_name,
-        }
-
-
 class PendingList(PendingIndex):
     """manage the pending videos list"""
 
@@ -159,120 +103,380 @@ class PendingList(PendingIndex):
         "check_formats": None,
     }
 
-    def __init__(self, youtube_ids=False, task=False):
+    def __init__(
+        self,
+        youtube_ids: list[ParsedURLType],
+        task=None,
+        auto_start=False,
+        flat=False,
+    ):
         super().__init__()
         self.config = AppConfig().config
         self.youtube_ids = youtube_ids
         self.task = task
+        self.auto_start = auto_start
+        self.flat = flat
         self.to_skip = False
-        self.missing_videos = False
+        self.missing_videos: list[dict] = []
+        self.added = 0
 
-    def parse_url_list(self):
+    def parse_url_list(self, status="pending") -> int:
         """extract youtube ids from list"""
-        self.missing_videos = []
         self.get_download()
         self.get_indexed()
-        total = len(self.youtube_ids)
-        for idx, entry in enumerate(self.youtube_ids):
-            self._process_entry(entry)
-            if not self.task:
-                continue
-
-            self.task.send_progress(
-                message_lines=[f"Extracting items {idx + 1}/{total}"],
-                progress=(idx + 1) / total,
-            )
-
-    def _process_entry(self, entry):
-        """process single entry from url list"""
-        vid_type = self._get_vid_type(entry)
-        if entry["type"] == "video":
-            self._add_video(entry["url"], vid_type)
-        elif entry["type"] == "channel":
-            self._parse_channel(entry["url"], vid_type)
-        elif entry["type"] == "playlist":
-            self._parse_playlist(entry["url"])
-        else:
-            raise ValueError(f"invalid url_type: {entry}")
-
-    @staticmethod
-    def _get_vid_type(entry):
-        """add vid type enum if available"""
-        vid_type_str = entry.get("vid_type")
-        if not vid_type_str:
-            return VideoTypeEnum.UNKNOWN
-
-        return VideoTypeEnum(vid_type_str)
-
-    def _add_video(self, url, vid_type):
-        """add video to list"""
-        if url not in self.missing_videos and url not in self.to_skip:
-            self.missing_videos.append((url, vid_type))
-        else:
-            print(f"{url}: skipped adding already indexed video to download.")
-
-    def _parse_channel(self, url, vid_type):
-        """add all videos of channel to list"""
-        video_results = ChannelSubscription().get_last_youtube_videos(
-            url, limit=False, query_filter=vid_type
-        )
-        for video_id, _, vid_type in video_results:
-            self._add_video(video_id, vid_type)
-
-    def _parse_playlist(self, url):
-        """add all videos of playlist to list"""
-        playlist = YoutubePlaylist(url)
-        is_active = playlist.update_playlist()
-        if not is_active:
-            message = f"{playlist.youtube_id}: failed to extract metadata"
-            print(message)
-            raise ValueError(message)
-
-        entries = playlist.json_data["playlist_entries"]
-        to_add = [i["youtube_id"] for i in entries if not i["downloaded"]]
-        if not to_add:
-            return
-
-        for video_id in to_add:
-            # match vid_type later
-            self._add_video(video_id, VideoTypeEnum.UNKNOWN)
-
-    def add_to_pending(self, status="pending", auto_start=False):
-        """add missing videos to pending list"""
         self.get_channels()
+        total = len(self.youtube_ids)
+        for idx, entry in enumerate(self.youtube_ids, start=1):
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[f"Extracting URL {idx}/{total}"],
+                    progress=idx / total,
+                )
+
+            self._process_entry(entry, idx, total)
+
+            if self.missing_videos:
+                self.added += self.add_to_pending(status)
+                self.missing_videos = []
 
-        total = len(self.missing_videos)
-        videos_added = []
-        for idx, (youtube_id, vid_type) in enumerate(self.missing_videos):
             if self.task and self.task.is_stopped():
                 break
 
-            print(f"{youtube_id}: [{idx + 1}/{total}]: add to queue")
-            self._notify_add(idx, total)
-            video_details = self.get_youtube_details(youtube_id, vid_type)
-            if not video_details:
-                rand_sleep(self.config)
+            rand_sleep(self.config)
+
+        return self.added
+
+    def _process_entry(self, entry: ParsedURLType, idx: int, total: int):
+        """process single entry from url list"""
+        if entry["type"] == "video":
+            to_add = self._add_video(entry["url"], entry["vid_type"])
+            if to_add:
+                self.__notify_add(
+                    item_type="video",
+                    name=to_add["title"],
+                    idx=idx,
+                    total=total,
+                )
+
+        elif entry["type"] == "channel":
+            self._parse_channel(entry)
+        elif entry["type"] == "playlist":
+            self._parse_playlist(entry["url"], entry.get("limit"))
+        else:
+            raise ValueError(f"invalid url_type: {entry}")
+
+    def _add_video(self, url, vid_type) -> dict | None:
+        """add video to list"""
+        if self.auto_start and url in set(
+            i["youtube_id"] for i in self.all_pending
+        ):
+            PendingInteract(youtube_id=url, status="priority").update_status()
+            return None
+
+        if url in self.missing_videos or url in self.to_skip:
+            print(f"{url}: skipped adding already indexed video to download.")
+            return None
+
+        to_add = self._parse_video(url, vid_type)
+        if to_add:
+            self.missing_videos.append(to_add)
+
+        return to_add
+
+    def _parse_channel(self, entry):
+        """parse channel"""
+        url = entry["url"]
+        vid_type = entry["vid_type"]
+        if isinstance(vid_type, str):
+            # lookup enum
+            vid_type = getattr(VideoTypeEnum, vid_type.upper())
+
+        limit = entry.get("limit")
+        video_results = get_last_channel_videos(
+            channel_id=url,
+            config=self.config,
+            limit=limit,
+            query_filter=vid_type,
+        )
+        if not video_results:
+            print(f"{url}: no videos to add from channel, skipping")
+            return
+
+        channel_handler = YoutubeChannel(url)
+        channel_handler.build_json(upload=False)
+        if not channel_handler.json_data:
+            print(f"{url}: channel metadata extraction failed, skipping")
+            return
+
+        total = len(video_results)
+        for idx, video_data in enumerate(video_results, start=1):
+            to_add = self.__parse_channel_video(
+                video_data, vid_type, channel_handler.json_data
+            )
+            if self.task and self.task.is_stopped():
+                break
+
+            if not to_add:
                 continue
 
-            video_details.update(
-                {
-                    "status": status,
-                    "auto_start": auto_start,
-                }
+            self.missing_videos.append(to_add)
+            self.__notify_add(
+                item_type="channel",
+                name=channel_handler.json_data["channel_name"],
+                idx=idx,
+                total=total,
             )
 
-            url = video_details["vid_thumb_url"]
-            ThumbManager(youtube_id).download_video_thumb(url)
-            es_url = f"ta_download/_doc/{youtube_id}"
-            _, _ = ElasticWrap(es_url).put(video_details)
-            videos_added.append(youtube_id)
+    def __parse_channel_video(
+        self, video_data, vid_type, channel_json
+    ) -> dict | None:
+        """parse video of channel"""
+        video_id = video_data["id"]
+        if video_id in self.to_skip:
+            return None
 
-            if idx != total:
-                rand_sleep(self.config)
+        # fallback
+        channel_name = channel_json["channel_name"]
+        channel_id = channel_json["channel_id"]
 
-        return videos_added
+        if self.flat:
+            if not video_data.get("channel"):
+                video_data["channel"] = channel_name
 
-    def _notify_add(self, idx, total):
+            if not video_data.get("channel_id"):
+                video_data["channel_id"] = channel_id
+
+            to_add = self._parse_entry(
+                youtube_id=video_id,
+                video_data=video_data,
+            )
+        else:
+            to_add = self._parse_video(video_id, vid_type)
+
+        return to_add
+
+    def _parse_playlist(self, url: str, limit: int | None):
+        """fast parse playlist"""
+        playlist = YoutubePlaylist(url)
+        playlist.update_playlist(limit=limit)
+        if not playlist.youtube_meta:
+            print(f"{url}: playlist metadata extraction failed, skipping")
+            return
+
+        video_results = playlist.youtube_meta["entries"]
+
+        total = len(video_results)
+        for idx, video_data in enumerate(video_results, start=1):
+            video_id = video_data["id"]
+            if video_id in self.to_skip:
+                continue
+
+            if self.task and self.task.is_stopped():
+                break
+
+            if self.flat:
+                if not video_data.get("channel"):
+                    video_data["channel"] = playlist.youtube_meta["channel"]
+
+                if not video_data.get("channel_id"):
+                    channel_id = playlist.youtube_meta["channel_id"]
+                    video_data["channel_id"] = channel_id
+
+                to_add = self._parse_entry(video_id, video_data)
+            else:
+                to_add = self._parse_video(video_id, vid_type=None)
+
+            if not to_add:
+                continue
+
+            self.missing_videos.append(to_add)
+            self.__notify_add(
+                item_type="playlist",
+                name=playlist.json_data["playlist_name"],
+                idx=idx,
+                total=total,
+            )
+
+    def _parse_video(self, url: str, vid_type) -> dict | None:
+        """parse video when not flat, fetch from YT"""
+        video = YoutubeVideo(youtube_id=url)
+        video.get_from_youtube()
+
+        if not video.youtube_meta:
+            print(f"{url}: video metadata extraction failed, skipping")
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[
+                        "Video extraction failed.",
+                        f"{video.error}",
+                    ],
+                    level="error",
+                )
+            return None
+
+        video.youtube_meta["vid_type"] = vid_type
+        to_add = self._parse_entry(
+            youtube_id=url,
+            video_data=video.youtube_meta,
+        )
+        if not to_add:
+            return None
+
+        ThumbManager(item_id=url).download_video_thumb(to_add["vid_thumb_url"])
+        rand_sleep(self.config)
+
+        return to_add
+
+    def _parse_entry(
+        self,
+        youtube_id: str,
+        video_data: dict,
+    ) -> dict | None:
+        """parse entry"""
+        if video_data.get("id") != youtube_id:
+            # skip premium videos with different id or redirects
+            print(f"{youtube_id}: skipping redirect, id not matching")
+            return None
+
+        if video_data.get("live_status") in ["is_upcoming", "is_live"]:
+            print(f"{youtube_id}: skip is_upcoming or is_live")
+            return None
+
+        to_add = {
+            "youtube_id": video_data["id"],
+            "title": video_data["title"],
+            "vid_thumb_url": self.__extract_thumb(video_data),
+            "duration": get_duration_str(video_data.get("duration", 0)),
+            "published": self.__extract_published(video_data),
+            "timestamp": int(datetime.now().timestamp()),
+            "vid_type": self.__extract_vid_type(video_data),
+            "channel_name": video_data["channel"],
+            "channel_id": video_data["channel_id"],
+            "channel_indexed": video_data["channel_id"] in self.all_channels,
+        }
+
+        return to_add
+
+    def __extract_thumb(self, video_data) -> str | None:
+        """extract thumb"""
+        if "thumbnail" in video_data:
+            return video_data["thumbnail"]
+
+        if video_data.get("thumbnails"):
+            return video_data["thumbnails"][-1]["url"]
+
+        return None
+
+    def __extract_published(self, video_data) -> str | int | None:
+        """build published date or timestamp"""
+        timestamp = video_data.get("timestamp")
+        if timestamp:
+            return timestamp
+
+        upload_date = video_data.get("upload_date")
+        if not upload_date:
+            return None
+
+        upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
+        published = upload_date_time.strftime("%Y-%m-%d")
+
+        return published
+
+    def __extract_vid_type(self, video_data) -> str:
+        """build vid type"""
+        if (
+            "vid_type" in video_data
+            and video_data["vid_type"]
+            and str(video_data["vid_type"]) in VideoTypeEnum.values_known()
+        ):
+            return VideoTypeEnum(video_data["vid_type"]).value
+
+        if video_data.get("live_status") == "was_live":
+            return VideoTypeEnum.STREAMS.value
+
+        if video_data.get("width", 0) > video_data.get("height", 0):
+            return VideoTypeEnum.VIDEOS.value
+
+        duration = video_data.get("duration")
+        if duration and isinstance(duration, int):
+            if duration > 3 * 60:
+                return VideoTypeEnum.VIDEOS.value
+
+        if is_shorts(video_data["id"]):
+            return VideoTypeEnum.SHORTS.value
+
+        return VideoTypeEnum.VIDEOS.value
+
+    def add_to_pending(self, status="pending") -> int:
+        """add missing videos to pending list"""
+
+        total = len(self.missing_videos)
+
+        if not self.missing_videos:
+            self._notify_empty()
+            return 0
+
+        self._notify_start(total)
+        bulk_list = []
+        for video_entry in self.missing_videos:
+            video_entry.update(
+                {
+                    "status": status,
+                    "auto_start": self.auto_start,
+                }
+            )
+            video_id = video_entry["youtube_id"]
+            action = {"index": {"_index": "ta_download", "_id": video_id}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(video_entry))
+
+        # add last newline
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+        _, status_code = ElasticWrap("_bulk").post(query_str, ndjson=True)
+        if status_code != 200:
+            self._notify_fail(status_code)
+        else:
+            self._notify_done(total)
+
+        return len(self.missing_videos)
+
+    def __notify_add(
+        self, item_type: str, name: str, idx: int, total: int
+    ) -> None:
+        """notify"""
+        if not self.task:
+            return
+
+        if self.flat:
+            lines = [
+                f"Bulk extracting {item_type.title()}: '{name}'.",
+                f"Fast adding item {idx}/{total}.",
+            ]
+        else:
+            lines = [
+                f"Full extracting {item_type.title()}: '{name}'",
+                f"Parsing item {idx}/{total}.",
+            ]
+
+        self.task.send_progress(
+            message_lines=lines,
+            progress=idx / total,
+        )
+
+    def _notify_empty(self):
+        """notify nothing to add"""
+        if not self.task:
+            return
+
+        self.task.send_progress(
+            message_lines=[
+                "Extracting videos completed.",
+                "No new videos found to add.",
+            ]
+        )
+
+    def _notify_start(self, total):
         """send notification for adding videos to download queue"""
         if not self.task:
             return
@@ -280,75 +484,31 @@ class PendingList(PendingIndex):
         self.task.send_progress(
             message_lines=[
                 "Adding new videos to download queue.",
-                f"Extracting items {idx + 1}/{total}",
+                f"Bulk adding {total} videos",
+            ]
+        )
+
+    def _notify_done(self, total):
+        """send done notification"""
+        if not self.task:
+            return
+
+        self.task.send_progress(
+            message_lines=[
+                "Adding new videos to the queue completed.",
+                f"Added {total} videos.",
+            ]
+        )
+
+    def _notify_fail(self, status_code):
+        """failed to add"""
+        if not self.task:
+            return
+
+        self.task.send_progress(
+            message_lines=[
+                "Adding extracted videos failed.",
+                f"Status code: {status_code}",
             ],
-            progress=(idx + 1) / total,
+            level="error",
         )
-
-    def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEOS):
-        """get details from youtubedl for single pending video"""
-        vid = YtWrap(self.yt_obs, self.config).extract(youtube_id)
-        if not vid:
-            return False
-
-        if vid.get("id") != youtube_id:
-            # skip premium videos with different id
-            print(f"{youtube_id}: skipping premium video, id not matching")
-            return False
-        # stop if video is streaming live now
-        if vid["live_status"] in ["is_upcoming", "is_live"]:
-            print(f"{youtube_id}: skip is_upcoming or is_live")
-            return False
-
-        if vid["live_status"] == "was_live":
-            vid_type = VideoTypeEnum.STREAMS
-        else:
-            if self._check_shorts(vid):
-                vid_type = VideoTypeEnum.SHORTS
-            else:
-                vid_type = VideoTypeEnum.VIDEOS
-
-        if not vid.get("channel"):
-            print(f"{youtube_id}: skip video not part of channel")
-            return False
-
-        return self._parse_youtube_details(vid, vid_type)
-
-    @staticmethod
-    def _check_shorts(vid):
-        """check if vid is shorts video"""
-        if vid["width"] > vid["height"]:
-            return False
-
-        duration = vid.get("duration")
-        if duration and isinstance(duration, int):
-            if duration > 3 * 60:
-                return False
-
-        return is_shorts(vid["id"])
-
-    def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS):
-        """parse response"""
-        vid_id = vid.get("id")
-        published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
-            "%Y-%m-%d"
-        )
-
-        # build dict
-        youtube_details = {
-            "youtube_id": vid_id,
-            "channel_name": vid["channel"],
-            "vid_thumb_url": vid["thumbnail"],
-            "title": vid["title"],
-            "channel_id": vid["channel_id"],
-            "duration": get_duration_str(vid["duration"]),
-            "published": published,
-            "timestamp": int(datetime.now().timestamp()),
-            # Pulling enum value out so it is serializable
-            "vid_type": vid_type.value,
-        }
-        if self.all_channels:
-            youtube_details.update(
-                {"channel_indexed": vid["channel_id"] in self.all_channels}
-            )
-        return youtube_details

backend/download/src/queue_interact.py

@@ -0,0 +1,115 @@
+"""interact with queue items"""
+
+from common.src.es_connect import ElasticWrap
+
+
+class PendingInteract:
+    """interact with items in download queue"""
+
+    def __init__(self, youtube_id=False, status=False):
+        self.youtube_id = youtube_id
+        self.status = status
+
+    def delete_item(self):
+        """delete single item from pending"""
+        path = f"ta_download/_doc/{self.youtube_id}"
+        _, _ = ElasticWrap(path).delete(refresh=True)
+
+    def delete_bulk(self, channel_id: str | None, vid_type: str | None):
+        """delete all matching item by status"""
+        must_list = [{"term": {"status": {"value": self.status}}}]
+        if channel_id:
+            must_list.append({"term": {"channel_id": {"value": channel_id}}})
+
+        if vid_type:
+            must_list.append({"term": {"vid_type": {"value": vid_type}}})
+
+        data = {"query": {"bool": {"must": must_list}}}
+
+        path = "ta_download/_delete_by_query?refresh=true"
+        _, _ = ElasticWrap(path).post(data=data)
+
+    def update_bulk(
+        self,
+        channel_id: str | None,
+        vid_type: str | None,
+        new_status: str,
+        error: bool | None = None,
+    ):
+        """update status in bulk"""
+        must_list = [{"term": {"status": {"value": self.status}}}]
+        must_not_list = []
+
+        if channel_id:
+            must_list.append({"term": {"channel_id": {"value": channel_id}}})
+
+        if vid_type:
+            must_list.append({"term": {"vid_type": {"value": vid_type}}})
+
+        if error is not None:
+            exists = {"exists": {"field": "message"}}
+            if error:
+                must_list.append(exists)  # type: ignore
+            else:
+                must_not_list.append(exists)
+
+        if new_status == "priority":
+            source = """
+            ctx._source.status = 'pending';
+            ctx._source.auto_start = true;
+            ctx._source.message = null;
+            """
+        elif new_status == "clear_error":
+            source = "ctx._source.message = null"
+        else:
+            source = f"ctx._source.status = '{new_status}'"
+
+        data = {
+            "query": {"bool": {"must": must_list, "must_not": must_not_list}},
+            "script": {"source": source, "lang": "painless"},
+        }
+
+        path = "ta_download/_update_by_query?refresh=true"
+        _, _ = ElasticWrap(path).post(data)
+
+    def update_status(self):
+        """update status of pending item"""
+        if self.status == "priority":
+            data = {
+                "doc": {
+                    "status": "pending",
+                    "auto_start": True,
+                    "message": None,
+                }
+            }
+        else:
+            data = {"doc": {"status": self.status}}
+
+        path = f"ta_download/_update/{self.youtube_id}/?refresh=true"
+        _, _ = ElasticWrap(path).post(data=data)
+
+    def get_item(self):
+        """return pending item dict"""
+        path = f"ta_download/_doc/{self.youtube_id}"
+        response, status_code = ElasticWrap(path).get()
+        return response["_source"], status_code
+
+    def get_channel(self):
+        """
+        get channel metadata from queue to not depend on channel to be indexed
+        """
+        data = {
+            "size": 1,
+            "query": {"term": {"channel_id": {"value": self.youtube_id}}},
+        }
+        response, _ = ElasticWrap("ta_download/_search").get(data=data)
+        hits = response["hits"]["hits"]
+        if not hits:
+            channel_name = "NA"
+        else:
+            channel_name = hits[0]["_source"].get("channel_name", "NA")
+
+        return {
+            "channel_id": self.youtube_id,
+            "channel_name": channel_name,
+        }

backend/download/src/subscriptions.py

@@ -6,324 +6,114 @@ Functionality:
 
 from appsettings.src.config import AppConfig
 from channel.src.index import YoutubeChannel
-from common.src.es_connect import IndexPaginate
-from common.src.helper import is_missing, rand_sleep
-from common.src.urlparser import Parser
-from download.src.thumbnails import ThumbManager
-from download.src.yt_dlp_base import YtWrap
+from channel.src.remote_query import VideoQueryBuilder
+from common.src.helper import get_channels, get_playlists
+from common.src.urlparser import ParsedURLType, Parser
+from download.src.queue import PendingList
 from playlist.src.index import YoutubePlaylist
 from video.src.constants import VideoTypeEnum
 from video.src.index import YoutubeVideo
 
 
 class ChannelSubscription:
-    """manage the list of channels subscribed"""
+    """scan subscribed channels to find missing videos to add to pending"""
 
-    def __init__(self, task=False):
+    def __init__(self, task=None):
         self.config = AppConfig().config
         self.task = task
 
-    @staticmethod
-    def get_channels(subscribed_only=True):
-        """get a list of all channels subscribed to"""
-        data = {
-            "sort": [{"channel_name.keyword": {"order": "asc"}}],
-        }
-        if subscribed_only:
-            data["query"] = {"term": {"channel_subscribed": {"value": True}}}
-        else:
-            data["query"] = {"match_all": {}}
+    def find_missing(self) -> int:
+        """find missing videos from channel subscriptions"""
+        if self.task:
+            self.task.send_progress(["Looking up channels."])
 
-        all_channels = IndexPaginate("ta_channel", data).get_results()
-
-        return all_channels
-
-    def get_last_youtube_videos(
-        self,
-        channel_id,
-        limit=True,
-        query_filter=None,
-        channel_overwrites=None,
-    ):
-        """get a list of last videos from channel"""
-        query_handler = VideoQueryBuilder(self.config, channel_overwrites)
-        queries = query_handler.build_queries(query_filter)
-        last_videos = []
-
-        for vid_type_enum, limit_amount in queries:
-            obs = {
-                "skip_download": True,
-                "extract_flat": True,
-            }
-            vid_type = vid_type_enum.value
-
-            if limit:
-                obs["playlistend"] = limit_amount
-
-            url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
-            channel_query = YtWrap(obs, self.config).extract(url)
-            if not channel_query:
-                continue
-
-            last_videos.extend(
-                [
-                    (i["id"], i["title"], vid_type)
-                    for i in channel_query["entries"]
-                ]
-            )
-
-        return last_videos
-
-    def find_missing(self):
-        """add missing videos from subscribed channels to pending"""
-        all_channels = self.get_channels()
+        all_channels = get_channels(
+            subscribed_only=True,
+            source=["channel_id", "channel_overwrites", "channel_tabs"],
+        )
         if not all_channels:
-            return False
+            return 0
 
-        missing_videos = []
+        all_channel_urls = self._process_channel_urls(all_channels)
 
-        total = len(all_channels)
-        for idx, channel in enumerate(all_channels):
-            channel_id = channel["channel_id"]
-            print(f"{channel_id}: find missing videos.")
-            last_videos = self.get_last_youtube_videos(
-                channel_id,
-                channel_overwrites=channel.get("channel_overwrites"),
-            )
+        if self.task:
+            self.task.send_progress([f"Scanning {len(all_channels)} channels"])
 
-            if last_videos:
-                ids_to_add = is_missing([i[0] for i in last_videos])
-                for video_id, _, vid_type in last_videos:
-                    if video_id in ids_to_add:
-                        missing_videos.append((video_id, vid_type))
+        pending_handler = PendingList(
+            youtube_ids=all_channel_urls,
+            task=self.task,
+            auto_start=self.config["subscriptions"].get("auto_start", False),
+            flat=self.config["subscriptions"].get("extract_flat", False),
+        )
+        added = pending_handler.parse_url_list()
 
-            if not self.task:
+        return added
+
+    def _process_channel_urls(self, all_channels: list[dict]):
+        """process channels, build queries"""
+
+        all_channel_urls: list[ParsedURLType] = []
+
+        for channel in all_channels:
+            channel_tabs = channel["channel_tabs"]
+            if not channel_tabs:
                 continue
 
-            if self.task.is_stopped():
-                self.task.send_progress(["Received Stop signal."])
-                break
+            enums = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs]
+            queries = VideoQueryBuilder(
+                config=self.config,
+                channel_overwrites=channel.get("channel_overwrites", {}),
+            ).build_queries(video_type=enums)
 
-            self.task.send_progress(
-                message_lines=[f"Scanning Channel {idx + 1}/{total}"],
-                progress=(idx + 1) / total,
-            )
-            rand_sleep(self.config)
+            for query in queries:
+                all_channel_urls.append(
+                    ParsedURLType(
+                        type="channel",
+                        url=channel["channel_id"],
+                        vid_type=query[0],
+                        limit=query[1],
+                    )
+                )
 
-        return missing_videos
-
-    @staticmethod
-    def change_subscribe(channel_id, channel_subscribed):
-        """subscribe or unsubscribe from channel and update"""
-        channel = YoutubeChannel(channel_id)
-        channel.build_json()
-        channel.json_data["channel_subscribed"] = channel_subscribed
-        channel.upload_to_es()
-        channel.sync_to_videos()
-
-        return channel.json_data
-
-
-class VideoQueryBuilder:
-    """Build queries for yt-dlp."""
-
-    def __init__(self, config: dict, channel_overwrites: dict | None = None):
-        self.config = config
-        self.channel_overwrites = channel_overwrites or {}
-
-    def build_queries(
-        self, video_type: VideoTypeEnum | None, limit: bool = True
-    ) -> list[tuple[VideoTypeEnum, int | None]]:
-        """Build queries for all or specific video type."""
-        query_methods = {
-            VideoTypeEnum.VIDEOS: self.videos_query,
-            VideoTypeEnum.STREAMS: self.streams_query,
-            VideoTypeEnum.SHORTS: self.shorts_query,
-        }
-
-        if video_type:
-            # build query for specific type
-            query_method = query_methods.get(video_type)
-            if query_method:
-                query = query_method(limit)
-                if query[1] != 0:
-                    return [query]
-                return []
-
-        # Build and return queries for all video types
-        queries = []
-        for build_query in query_methods.values():
-            query = build_query(limit)
-            if query[1] != 0:
-                queries.append(query)
-
-        return queries
-
-    def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
-        """Build query for videos."""
-        return self._build_generic_query(
-            video_type=VideoTypeEnum.VIDEOS,
-            overwrite_key="subscriptions_channel_size",
-            config_key="channel_size",
-            limit=limit,
-        )
-
-    def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
-        """Build query for streams."""
-        return self._build_generic_query(
-            video_type=VideoTypeEnum.STREAMS,
-            overwrite_key="subscriptions_live_channel_size",
-            config_key="live_channel_size",
-            limit=limit,
-        )
-
-    def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
-        """Build query for shorts."""
-        return self._build_generic_query(
-            video_type=VideoTypeEnum.SHORTS,
-            overwrite_key="subscriptions_shorts_channel_size",
-            config_key="shorts_channel_size",
-            limit=limit,
-        )
-
-    def _build_generic_query(
-        self,
-        video_type: VideoTypeEnum,
-        overwrite_key: str,
-        config_key: str,
-        limit: bool,
-    ) -> tuple[VideoTypeEnum, int | None]:
-        """Generic query for video page scraping."""
-        if not limit:
-            return (video_type, None)
-
-        if (
-            overwrite_key in self.channel_overwrites
-            and self.channel_overwrites[overwrite_key] is not None
-        ):
-            overwrite = self.channel_overwrites[overwrite_key]
-            return (video_type, overwrite)
-
-        if overwrite := self.config["subscriptions"].get(config_key):
-            return (video_type, overwrite)
-
-        return (video_type, 0)
+        return all_channel_urls
 
 
 class PlaylistSubscription:
-    """manage the playlist download functionality"""
+    """scan subscribed playlists for videos to add to pending"""
 
-    def __init__(self, task=False):
+    def __init__(self, task=None):
         self.config = AppConfig().config
         self.task = task
 
-    @staticmethod
-    def get_playlists(subscribed_only=True):
-        """get a list of all active playlists"""
-        data = {
-            "sort": [{"playlist_channel.keyword": {"order": "desc"}}],
-        }
-        data["query"] = {
-            "bool": {"must": [{"term": {"playlist_active": {"value": True}}}]}
-        }
-        if subscribed_only:
-            data["query"]["bool"]["must"].append(
-                {"term": {"playlist_subscribed": {"value": True}}}
-            )
-
-        all_playlists = IndexPaginate("ta_playlist", data).get_results()
-
-        return all_playlists
-
-    def process_url_str(self, new_playlists, subscribed=True):
-        """process playlist subscribe form url_str"""
-        for idx, playlist in enumerate(new_playlists):
-            playlist_id = playlist["url"]
-            if not playlist["type"] == "playlist":
-                print(f"{playlist_id} not a playlist, skipping...")
-                continue
-
-            playlist_h = YoutubePlaylist(playlist_id)
-            playlist_h.build_json()
-            if not playlist_h.json_data:
-                message = f"{playlist_h.youtube_id}: failed to extract data"
-                print(message)
-                raise ValueError(message)
-
-            playlist_h.json_data["playlist_subscribed"] = subscribed
-            playlist_h.upload_to_es()
-            playlist_h.add_vids_to_playlist()
-            self.channel_validate(playlist_h.json_data["playlist_channel_id"])
-
-            url = playlist_h.json_data["playlist_thumbnail"]
-            thumb = ThumbManager(playlist_id, item_type="playlist")
-            thumb.download_playlist_thumb(url)
-
-            if self.task:
-                self.task.send_progress(
-                    message_lines=[
-                        f"Processing {idx + 1} of {len(new_playlists)}"
-                    ],
-                    progress=(idx + 1) / len(new_playlists),
-                )
-
-    @staticmethod
-    def channel_validate(channel_id):
-        """make sure channel of playlist is there"""
-        channel = YoutubeChannel(channel_id)
-        channel.build_json(upload=True)
-
-    @staticmethod
-    def change_subscribe(playlist_id, subscribe_status):
-        """change the subscribe status of a playlist"""
-        playlist = YoutubePlaylist(playlist_id)
-        playlist.build_json()
-        playlist.json_data["playlist_subscribed"] = subscribe_status
-        playlist.upload_to_es()
-        return playlist.json_data
-
-    def find_missing(self):
-        """find videos in subscribed playlists not downloaded yet"""
-        all_playlists = [i["playlist_id"] for i in self.get_playlists()]
+    def find_missing(self) -> int:
+        """find missing"""
+        all_playlists = get_playlists(
+            subscribed_only=True, source=["playlist_id"]
+        )
         if not all_playlists:
-            return False
+            return 0
 
-        missing_videos = []
-        total = len(all_playlists)
-        for idx, playlist_id in enumerate(all_playlists):
-            playlist = YoutubePlaylist(playlist_id)
-            is_active = playlist.update_playlist()
-            if not is_active:
-                playlist.deactivate()
-                continue
-
-            playlist_entries = playlist.json_data["playlist_entries"]
-            size_limit = self.config["subscriptions"]["channel_size"]
-            if size_limit:
-                del playlist_entries[size_limit:]
-
-            to_check = [
-                i["youtube_id"]
-                for i in playlist_entries
-                if i["downloaded"] is False
-            ]
-            needs_downloading = is_missing(to_check)
-            missing_videos.extend(needs_downloading)
-
-            if not self.task:
-                continue
-
-            if self.task.is_stopped():
-                self.task.send_progress(["Received Stop signal."])
-                break
-
-            self.task.send_progress(
-                message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
-                progress=(idx + 1) / total,
+        size_limit = self.config["subscriptions"]["playlist_size"]
+        all_playlist_urls: list[ParsedURLType] = []
+        for playlist in all_playlists:
+            all_playlist_urls.append(
+                ParsedURLType(
+                    type="playlist",
+                    url=playlist["playlist_id"],
+                    vid_type=VideoTypeEnum.UNKNOWN,
+                    limit=size_limit,
+                )
             )
-            rand_sleep(self.config)
 
-        return missing_videos
+        pending_handler = PendingList(
+            youtube_ids=all_playlist_urls,
+            task=self.task,
+            auto_start=self.config["subscriptions"].get("auto_start", False),
+            flat=self.config["subscriptions"].get("extract_flat", False),
+        )
+        added = pending_handler.parse_url_list()
+
+        return added
 
 
 class SubscriptionScanner:
@@ -339,40 +129,12 @@ class SubscriptionScanner:
         if self.task:
             self.task.send_progress(["Rescanning channels and playlists."])
 
-        self.missing_videos = []
-        self.scan_channels()
+        added = 0
+        added += ChannelSubscription(task=self.task).find_missing()
         if self.task and not self.task.is_stopped():
-            self.scan_playlists()
+            added += PlaylistSubscription(task=self.task).find_missing()
 
-        return self.missing_videos
-
-    def scan_channels(self):
-        """get missing from channels"""
-        channel_handler = ChannelSubscription(task=self.task)
-        missing = channel_handler.find_missing()
-        if not missing:
-            return
-
-        for vid_id, vid_type in missing:
-            self.missing_videos.append(
-                {"type": "video", "vid_type": vid_type, "url": vid_id}
-            )
-
-    def scan_playlists(self):
-        """get missing from playlists"""
-        playlist_handler = PlaylistSubscription(task=self.task)
-        missing = playlist_handler.find_missing()
-        if not missing:
-            return
-
-        for i in missing:
-            self.missing_videos.append(
-                {
-                    "type": "video",
-                    "vid_type": VideoTypeEnum.VIDEOS.value,
-                    "url": i,
-                }
-            )
+        return added
 
 
 class SubscriptionHandler:
@@ -404,7 +166,8 @@ class SubscriptionHandler:
                     f"expected {expected_type} url but got {item.get('type')}"
                 )
 
-            PlaylistSubscription().process_url_str([item])
+            playlist = YoutubePlaylist(item["url"])
+            playlist.change_subscribe(new_subscribe_state=True)
             return
 
         if item["type"] == "video":
@@ -427,9 +190,7 @@ class SubscriptionHandler:
 
     def _subscribe(self, channel_id):
         """subscribe to channel"""
-        _ = ChannelSubscription().change_subscribe(
-            channel_id, channel_subscribed=True
-        )
+        YoutubeChannel(channel_id).change_subscribe(new_subscribe_state=True)
 
     def _notify(self, idx, item, total):
         """send notification message to redis"""

backend/download/src/thumbnails.py

@@ -265,7 +265,7 @@ class ValidatorCallback:
     def run(self):
         """run the task for page"""
         print(f"{self.index_name}: validate artwork")
-        if self.index_name == "ta_video":
+        if self.index_name in ["ta_video", "ta_download"]:
             self._validate_videos()
         elif self.index_name == "ta_channel":
             self._validate_channels()
@@ -325,6 +325,13 @@ class ThumbValidator:
             },
             "name": "ta_playlist",
         },
+        {
+            "data": {
+                "query": {"term": {"status": {"value": "pending"}}},
+                "_source": ["youtube_id", "vid_thumb_url"],
+            },
+            "name": "ta_download",
+        },
     ]
 
     def __init__(self, task=False):

backend/download/src/yt_dlp_base.py

@@ -20,7 +20,6 @@ class YtWrap:
     OBS_BASE = {
         "default_search": "ytsearch",
         "quiet": True,
-        "check_formats": "selected",
         "socket_timeout": 10,
         "extractor_retries": 3,
         "retries": 10,
@@ -66,6 +65,7 @@ class YtWrap:
 
     def download(self, url):
         """make download request"""
+        self.obs.update({"check_formats": "selected"})
         with yt_dlp.YoutubeDL(self.obs) as ydl:
             try:
                 ydl.download([url])
@@ -80,30 +80,33 @@ class YtWrap:
 
         return True, True
 
-    def extract(self, url):
-        """make extract request"""
+    def extract(self, url) -> tuple[dict | None, str | None]:
+        """
+        make extract request
+        returns response, error
+        """
         with yt_dlp.YoutubeDL(self.obs) as ydl:
             try:
                 response = ydl.extract_info(url)
             except cookiejar.LoadError as err:
                 print(f"cookie file is invalid: {err}")
-                return False
+                return None, str(err)
             except yt_dlp.utils.ExtractorError as err:
                 print(f"{url}: failed to extract: {err}, continue...")
-                return False
+                return None, str(err)
             except yt_dlp.utils.DownloadError as err:
                 if "This channel does not have a" in str(err):
-                    return False
+                    return None, None
 
                 print(f"{url}: failed to get info from youtube: {err}")
                 if "Temporary failure in name resolution" in str(err):
                     raise ConnectionError("lost the internet, abort!") from err
 
-                return False
+                return None, str(err)
 
         self._validate_cookie()
 
-        return response
+        return response, None
 
     def _validate_cookie(self):
         """check cookie and write it back for next use"""
@@ -146,7 +149,7 @@ class CookieHandler:
         AppConfig().update_config({"downloads": {"cookie_import": False}})
         print("[cookie]: revoked")
 
-    def validate(self):
+    def validate(self) -> bool:
         """validate cookie using the liked videos playlist"""
         validation = RedisArchivist().get_message_dict("cookie:valid")
         if validation:
@@ -159,8 +162,8 @@ class CookieHandler:
             "extract_flat": True,
         }
         validator = YtWrap(obs_request, self.config)
-        response = bool(validator.extract("LL"))
-        self.store_validation(response)
+        response, error = validator.extract("LL")
+        self.store_validation(bool(response))
 
         # update in redis to avoid expiring
         modified = validator.obs["cookiefile"].getvalue().strip("\x00")
@@ -173,15 +176,15 @@ class CookieHandler:
                 "status": "message:download",
                 "level": "error",
                 "title": "Cookie validation failed, exiting...",
-                "message": "",
+                "message": error,
             }
             RedisArchivist().set_message(
                 "message:download", mess_dict, expire=4
             )
             print("[cookie]: validation failed, exiting...")
 
-        print(f"[cookie]: validation success: {response}")
-        return response
+        print(f"[cookie]: validation success: {bool(response)}")
+        return bool(response)
 
     @staticmethod
     def store_validation(response):

backend/download/src/yt_dlp_handler.py

@@ -16,12 +16,12 @@ from common.src.env_settings import EnvironmentSettings
 from common.src.es_connect import ElasticWrap, IndexPaginate
 from common.src.helper import (
     get_channel_overwrites,
+    get_playlists,
     ignore_filelist,
     rand_sleep,
 )
 from common.src.ta_redis import RedisQueue
 from download.src.queue import PendingList
-from download.src.subscriptions import PlaylistSubscription
 from download.src.yt_dlp_base import YtWrap
 from playlist.src.index import YoutubePlaylist
 from video.src.comments import CommentList
@@ -360,9 +360,7 @@ class DownloadPostProcess(DownloaderBase):
 
         print("add deleted to ignore list")
         vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
-        pending = PendingList(youtube_ids=vids)
-        pending.parse_url_list()
-        _ = pending.add_to_pending(status="ignore")
+        PendingList(youtube_ids=vids).parse_url_list(status="ignore")
 
     def refresh_playlist(self) -> None:
         """match videos with playlists"""
@@ -403,8 +401,8 @@ class DownloadPostProcess(DownloaderBase):
 
     def _add_playlist_sub(self):
         """add subscribed playlists to refresh"""
-        subs = PlaylistSubscription().get_playlists()
-        to_add = [i["playlist_id"] for i in subs]
+        playlists = get_playlists(subscribed_only=True, source=["playlist_id"])
+        to_add = [i["playlist_id"] for i in playlists]
         RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
 
     def _add_channel_playlists(self):

backend/download/views.py

@@ -8,6 +8,8 @@ from common.views_base import AdminOnly, ApiBaseView
 from download.serializers import (
     AddToDownloadListSerializer,
     AddToDownloadQuerySerializer,
+    BulkUpdateDowloadDataSerializer,
+    BulkUpdateDowloadQuerySerializer,
     DownloadAggsSerializer,
     DownloadItemSerializer,
     DownloadListQuerySerializer,
@@ -15,7 +17,7 @@ from download.serializers import (
     DownloadListSerializer,
     DownloadQueueItemUpdateSerializer,
 )
-from download.src.queue import PendingInteract
+from download.src.queue_interact import PendingInteract
 from drf_spectacular.utils import OpenApiResponse, extend_schema
 from rest_framework.response import Response
 from task.tasks import download_pending, extrac_dl
@@ -65,6 +67,22 @@ class DownloadApiListView(ApiBaseView):
                 {"term": {"channel_id": {"value": filter_channel}}}
             )
 
+        vid_type_filter = validated_data.get("vid_type")
+        if vid_type_filter:
+            must_list.append(
+                {"term": {"vid_type": {"value": vid_type_filter}}}
+            )
+
+        search_query = validated_data.get("q")
+        if search_query:
+            must_list.append({"match_phrase_prefix": {"title": search_query}})
+
+        if validated_data.get("error") is not None:
+            operator = "must" if validated_data["error"] else "must_not"
+            must_list.append(
+                {"bool": {operator: [{"exists": {"field": "message"}}]}}
+            )
+
         self.data["query"] = {"bool": {"must": must_list}}
 
         self.get_document_list(request)
@@ -99,12 +117,13 @@ class DownloadApiListView(ApiBaseView):
         validated_query = query_serializer.validated_data
 
         auto_start = validated_query.get("autostart")
-        print(f"auto_start: {auto_start}")
+        flat = validated_query.get("flat", False)
+        print(f"auto_start: {auto_start}, flat: {flat}")
         to_add = validated_data["data"]
 
         pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
         url_str = " ".join(pending)
-        task = extrac_dl.delay(url_str, auto_start=auto_start)
+        task = extrac_dl.delay(url_str, auto_start=auto_start, flat=flat)
 
         message = {
             "message": "add to queue task started",
@@ -114,6 +133,42 @@ class DownloadApiListView(ApiBaseView):
 
         return Response(response_serializer.data)
 
+    @staticmethod
+    @extend_schema(
+        request=BulkUpdateDowloadDataSerializer(),
+        parameters=[BulkUpdateDowloadQuerySerializer()],
+        responses={204: OpenApiResponse(description="Status updated")},
+    )
+    def patch(request):
+        """bulk update status"""
+        data_serializer = BulkUpdateDowloadDataSerializer(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+
+        new_status = validated_data["status"]
+
+        query_serializer = BulkUpdateDowloadQuerySerializer(
+            data=request.query_params
+        )
+        query_serializer.is_valid(raise_exception=True)
+        validated_query = query_serializer.validated_data
+        status_filter = validated_query.get("filter")
+        channel = validated_query.get("channel")
+        vid_type = validated_query.get("vid_type")
+        error = validated_query.get("error")
+
+        PendingInteract(status=status_filter).update_bulk(
+            channel_id=channel,
+            vid_type=vid_type,
+            new_status=new_status,
+            error=error,
+        )
+
+        if new_status == "priority":
+            download_pending.delay(auto_only=True)
+
+        return Response(status=204)
+
     @extend_schema(
         parameters=[DownloadListQueueDeleteQuerySerializer()],
         responses={
@@ -132,9 +187,18 @@ class DownloadApiListView(ApiBaseView):
         validated_query = serializer.validated_data
 
         query_filter = validated_query["filter"]
+        channel = validated_query.get("channel")
+        vid_type = validated_query.get("vid_type")
         message = f"delete queue by status: {query_filter}"
+        if channel:
+            message += f" - filter by channel: {channel}"
+        if vid_type:
+            message += f" - filter by vid_type: {vid_type}"
+
         print(message)
-        PendingInteract(status=query_filter).delete_by_status()
+        PendingInteract(status=query_filter).delete_bulk(
+            channel_id=channel, vid_type=vid_type
+        )
 
         return Response(status=204)
 

backend/playlist/serializers.py

@@ -28,6 +28,7 @@ class PlaylistSerializer(serializers.Serializer):
     playlist_last_refresh = serializers.CharField()
     playlist_name = serializers.CharField()
     playlist_subscribed = serializers.BooleanField()
+    playlist_sort_order = serializers.ChoiceField(choices=["top", "bottom"])
     playlist_thumbnail = serializers.CharField()
     playlist_type = serializers.ChoiceField(choices=["regular", "custom"])
     _index = serializers.CharField(required=False)
@@ -68,7 +69,10 @@ class PlaylistBulkAddSerializer(serializers.Serializer):
 class PlaylistSingleUpdate(serializers.Serializer):
     """update state of single playlist"""
 
-    playlist_subscribed = serializers.BooleanField()
+    playlist_subscribed = serializers.BooleanField(required=False)
+    playlist_sort_order = serializers.ChoiceField(
+        choices=["top", "bottom"], required=False
+    )
 
 
 class PlaylistListCustomPostSerializer(serializers.Serializer):

backend/playlist/src/index.py

@@ -7,7 +7,6 @@ functionality:
 import json
 from datetime import datetime
 
-from channel.src import index as channel
 from common.src.env_settings import EnvironmentSettings
 from common.src.es_connect import ElasticWrap, IndexPaginate
 from common.src.index_generic import YouTubeItem
@@ -31,16 +30,24 @@ class YoutubePlaylist(YouTubeItem):
         self.all_members = False
         self.nav = False
 
-    def build_json(self, scrape=False):
+    def build_json(self, scrape=False, limit: int | None = None):
         """collection to create json_data"""
         self.get_from_es()
         if self.json_data:
             subscribed = self.json_data.get("playlist_subscribed")
+            playlist_sort_order = self.json_data.get("playlist_sort_order")
         else:
             subscribed = False
+            playlist_sort_order = "top"
+
+        limit_str = limit if limit is not None else ""
+        sort_order = 1 if playlist_sort_order == "top" else -1
+        playlist_items = f":{limit_str}:{sort_order}"
 
         if scrape or not self.json_data:
-            self.get_from_youtube()
+            self.get_from_youtube(
+                obs_overwrite={"playlist_items": playlist_items}
+            )
             if not self.youtube_meta:
                 self.json_data = False
                 return
@@ -49,8 +56,13 @@ class YoutubePlaylist(YouTubeItem):
             self._ensure_channel()
             ids_found = self.get_local_vids()
             self.get_entries(ids_found)
-            self.json_data["playlist_entries"] = self.all_members
-            self.json_data["playlist_subscribed"] = subscribed
+            self.json_data.update(
+                {
+                    "playlist_entries": self.all_members,
+                    "playlist_subscribed": subscribed,
+                    "playlist_sort_order": playlist_sort_order,
+                }
+            )
 
     def process_youtube_meta(self):
         """extract relevant fields from youtube"""
@@ -60,6 +72,9 @@ class YoutubePlaylist(YouTubeItem):
             print(f"{self.youtube_id}: thumbnail extraction failed")
             playlist_thumbnail = False
 
+        if not self.youtube_meta.get("channel_id"):
+            raise ValueError("Failed to extract Channel ID for Playlist")
+
         self.json_data = {
             "playlist_id": self.youtube_id,
             "playlist_active": True,
@@ -74,10 +89,24 @@ class YoutubePlaylist(YouTubeItem):
 
     def _ensure_channel(self):
         """make sure channel is indexed"""
+        from channel.src.index import YoutubeChannel
+
         channel_id = self.json_data["playlist_channel_id"]
-        channel_handler = channel.YoutubeChannel(channel_id)
+        channel_handler = YoutubeChannel(channel_id)
         channel_handler.build_json(upload=True)
 
+    def get_playlist_videos(self):
+        """get all playlist videos"""
+        data = {
+            "query": {
+                "term": {"playlist.keyword": {"value": self.youtube_id}}
+            },
+            "_source": ["youtube_id"],
+        }
+        result = IndexPaginate("ta_video", data).get_results()
+
+        return result
+
     def get_local_vids(self) -> list[str]:
         """get local video ids from youtube entries"""
         entries = self.youtube_meta["entries"]
@@ -110,6 +139,13 @@ class YoutubePlaylist(YouTubeItem):
         url = self.json_data["playlist_thumbnail"]
         ThumbManager(self.youtube_id, item_type="playlist").download(url)
 
+    def change_subscribe(self, new_subscribe_state: bool):
+        """change subscribe status"""
+        self.build_json()
+        self.json_data["playlist_subscribed"] = new_subscribe_state
+        self.upload_to_es()
+        return self.json_data
+
     def add_vids_to_playlist(self):
         """sync the playlist id to videos"""
         script = (
@@ -143,14 +179,7 @@ class YoutubePlaylist(YouTubeItem):
     def remove_vids_from_playlist(self):
         """remove playlist ids from videos if needed"""
         needed = [i["youtube_id"] for i in self.json_data["playlist_entries"]]
-        data = {
-            "query": {"match": {"playlist": self.youtube_id}},
-            "_source": ["youtube_id"],
-        }
-        data = {
-            "query": {"term": {"playlist.keyword": {"value": self.youtube_id}}}
-        }
-        result = IndexPaginate("ta_video", data).get_results()
+        result = self.get_playlist_videos()
         to_remove = [
             i["youtube_id"] for i in result if i["youtube_id"] not in needed
         ]
@@ -169,9 +198,9 @@ class YoutubePlaylist(YouTubeItem):
             if status_code == 200:
                 print(f"{self.youtube_id}: removed {video_id} from playlist")
 
-    def update_playlist(self, skip_on_empty=False):
+    def update_playlist(self, skip_on_empty=False, limit: int | None = None):
         """update metadata for playlist with data from YouTube"""
-        self.build_json(scrape=True)
+        self.build_json(scrape=True, limit=limit)
         if not self.json_data:
             # return false to deactivate
             return False
@@ -189,6 +218,15 @@ class YoutubePlaylist(YouTubeItem):
         self.get_playlist_art()
         return True
 
+    def change_sort_order(self, new_sort_order):
+        """update sort order of playlist"""
+        playlist = YoutubePlaylist(self.youtube_id)
+        playlist.build_json()
+        playlist.json_data["playlist_sort_order"] = new_sort_order
+        playlist.upload_to_es()
+
+        return playlist.json_data
+
     def build_nav(self, youtube_id):
         """find next and previous in playlist of a given youtube_id"""
         cache_root = EnvironmentSettings().get_cache_root()
@@ -287,6 +325,7 @@ class YoutubePlaylist(YouTubeItem):
         self.delete_metadata()
 
     def create(self, name):
+        """create custom playlist"""
         self.json_data = {
             "playlist_id": self.youtube_id,
             "playlist_active": False,
@@ -299,6 +338,7 @@ class YoutubePlaylist(YouTubeItem):
             "playlist_description": False,
             "playlist_thumbnail": False,
             "playlist_subscribed": False,
+            "playlist_sort_order": "top",
         }
         self.upload_to_es()
         self.get_playlist_art()

backend/playlist/src/query_building.py

@@ -45,7 +45,7 @@ class QueryBuilder:
 
         type_parsed = getattr(PlaylistTypesEnum, playlist_type.upper()).value
 
-        return {"match": {"playlist_type.keyword": type_parsed}}
+        return {"match": {"playlist_type": type_parsed}}
 
     def parse_sort(self) -> dict:
         """return sort"""

backend/playlist/tests/test_src/test_query_building.py

@@ -27,4 +27,4 @@ def test_parse_type():
         qb.parse_type("invalid")
 
     result = qb.parse_type("custom")
-    assert result == {"match": {"playlist_type.keyword": "custom"}}
+    assert result == {"match": {"playlist_type": "custom"}}

backend/playlist/views.py

@@ -7,7 +7,6 @@ from common.serializers import (
     ErrorResponseSerializer,
 )
 from common.views_base import AdminWriteOnly, ApiBaseView
-from download.src.subscriptions import PlaylistSubscription
 from drf_spectacular.utils import OpenApiResponse, extend_schema
 from playlist.serializers import (
     PlaylistBulkAddSerializer,
@@ -240,9 +239,31 @@ class PlaylistApiView(ApiBaseView):
             error = ErrorResponseSerializer({"error": "playlist not found"})
             return Response(error.data, status=404)
 
-        subscribed = validated_data["playlist_subscribed"]
-        playlist_sub = PlaylistSubscription()
-        json_data = playlist_sub.change_subscribe(playlist_id, subscribed)
+        if self.response["playlist_type"] == "custom":
+            error = ErrorResponseSerializer(
+                {"error": f"playlist with ID {playlist_id} is custom"}
+            )
+            return Response(error.data, status=400)
+
+        subscribed = validated_data.get("playlist_subscribed")
+        sort_order = validated_data.get("playlist_sort_order")
+
+        json_data = None
+        if subscribed is not None:
+            json_data = YoutubePlaylist(playlist_id).change_subscribe(
+                new_subscribe_state=subscribed
+            )
+
+        if sort_order:
+            json_data = YoutubePlaylist(playlist_id).change_sort_order(
+                new_sort_order=sort_order
+            )
+
+        if not json_data:
+            error = ErrorResponseSerializer(
+                {"error": "expect playlist_subscribed or playlist_sort_order"}
+            )
+            return Response(error.data, status=400)
 
         response_serializer = PlaylistSerializer(json_data)
         return Response(response_serializer.data)

backend/requirements-dev.txt

@@ -1,10 +1,10 @@
 -r requirements.txt
-ipython==9.2.0
+ipython==9.4.0
 pre-commit==4.2.0
 pylint-django==2.6.1
 pylint==3.3.7
 pytest-django==4.11.1
-pytest==8.3.5
-python-dotenv==1.1.0
-requirementscheck==0.0.6
-types-requests==2.32.0.20250328
+pytest==8.4.1
+python-dotenv==1.1.1
+requirementscheck==0.1.0
+types-requests==2.32.4.20250611

backend/requirements.txt

@@ -1,15 +1,15 @@
 apprise==1.9.3
-celery==5.5.2
+celery==5.5.3
 django-auth-ldap==5.2.0
-django-celery-beat==2.8.0
+django-celery-beat==2.8.1
 django-cors-headers==4.7.0
-Django==5.2.1
+Django==5.2.4
 djangorestframework==3.16.0
 drf-spectacular==0.28.0
-Pillow==11.2.1
-redis==6.0.0
-requests==2.32.3
+Pillow==11.3.0
+redis==6.2.0
+requests==2.32.4
 ryd-client==0.0.6
-uvicorn==0.34.2
+uvicorn==0.35.0
 whitenoise==6.9.0
-yt-dlp[default]==2025.4.30
+yt-dlp[default]==2025.7.21

backend/task/serializers.py

@@ -91,3 +91,12 @@ class TaskNotificationPostSerializer(serializers.Serializer):
 
     task_name = serializers.ChoiceField(choices=list(TASK_CONFIG))
     url = serializers.CharField(required=False)
+
+
+class TaskNotificationTestSerializer(serializers.Serializer):
+    """serialize task notification test POST"""
+
+    url = serializers.CharField()
+    task_name = serializers.ChoiceField(
+        choices=list(TASK_CONFIG), required=False
+    )

backend/task/src/notify.py

@@ -32,6 +32,38 @@ class Notifications:
 
         apobj.notify(body=body, title=title)
 
+    def test(self, url) -> tuple[bool, str]:
+        """send test notification"""
+        try:
+            apobj = apprise.Apprise()
+
+            if not apobj.add(url):
+                success = False
+                message = f"Invalid notification URL format: {url}"
+                return success, message
+
+            title = f"[TA] {self.task_name} process ended with SUCCESS"
+            body = "This is a test notification. Task completed successfully."
+
+            result = apobj.notify(body=body, title=title)
+
+            if result:
+                success = True
+                message = "Test notification sent successfully"
+                return success, message
+
+            success = False
+            message = (
+                "Notification failed. "
+                "Please check container logs for more information."
+            )
+            return success, message
+
+        except Exception as err:  # pylint: disable=broad-exception-caught
+            success = False
+            message = f"Notification error: {str(err)}"
+            return success, message
+
     def _build_message(
         self, task_id: str, task_title: str
     ) -> tuple[str, str | None]:

backend/task/tasks.py

@@ -39,10 +39,10 @@ class BaseTask(Task):
         RedisArchivist().set_message(key, message, expire=20)
 
     def on_success(self, retval, task_id, args, kwargs):
-        """callback task completed successfully"""
+        """callback task completed"""
         print(f"{task_id} success callback")
         message, key = self._build_message()
-        message.update({"messages": ["Task completed successfully"]})
+        message.update({"messages": ["Task completed"]})
         RedisArchivist().set_message(key, message, expire=5)
 
     def before_start(self, task_id, args, kwargs):
@@ -58,9 +58,11 @@ class BaseTask(Task):
         task_title = TASK_CONFIG.get(self.name).get("title")
         Notifications(self.name).send(task_id, task_title)
 
-    def send_progress(self, message_lines, progress=False, title=False):
+    def send_progress(
+        self, message_lines, progress=False, title=False, level="info"
+    ):
         """send progress message"""
-        message, key = self._build_message()
+        message, key = self._build_message(level=level)
         message.update(
             {
                 "messages": message_lines,
@@ -79,7 +81,7 @@ class BaseTask(Task):
         message.update({"level": level, "id": task_id})
         task_result = TaskManager().get_task(task_id)
         if task_result:
-            command = task_result.get("command", False)
+            command = task_result.get("command", None)
             message.update({"command": command})
 
         key = f"message:{message.get('group')}:{task_id.split('-')[0]}"
@@ -101,13 +103,13 @@ def update_subscribed(self):
 
     manager.init(self)
     handler = SubscriptionScanner(task=self)
-    missing_videos = handler.scan()
+    added = handler.scan()
     auto_start = handler.auto_start
-    if missing_videos:
-        print(missing_videos)
-        extrac_dl.delay(missing_videos, auto_start=auto_start)
-        message = f"Found {len(missing_videos)} videos to add to the queue."
-        return message
+    if added:
+        if auto_start:
+            download_pending.delay(auto_only=True)
+
+        return f"Found {added} videos to add to the queue."
 
     return None
 
@@ -147,7 +149,9 @@ def download_pending(self, auto_only=False):
 
 
 @shared_task(name="extract_download", bind=True, base=BaseTask)
-def extrac_dl(self, youtube_ids, auto_start=False, status="pending"):
+def extrac_dl(
+    self, youtube_ids, auto_start=False, flat=False, status="pending"
+):
     """parse list passed and add to pending"""
     TaskManager().init(self)
     if isinstance(youtube_ids, str):
@@ -155,17 +159,16 @@ def extrac_dl(self, youtube_ids, auto_start=False, status="pending"):
     else:
         to_add = youtube_ids
 
-    pending_handler = PendingList(youtube_ids=to_add, task=self)
-    pending_handler.parse_url_list()
-    videos_added = pending_handler.add_to_pending(
-        status=status, auto_start=auto_start
+    pending_handler = PendingList(
+        youtube_ids=to_add, task=self, auto_start=auto_start, flat=flat
     )
+    videos_added = pending_handler.parse_url_list(status=status)
 
     if auto_start:
         download_pending.delay(auto_only=True)
 
     if videos_added:
-        return f"added {len(videos_added)} Videos to Queue"
+        return f"added {videos_added} Videos to Queue"
 
     return None
 
@@ -259,7 +262,7 @@ def rescan_filesystem(self):
     handler = Scanner(task=self)
     handler.scan()
     handler.apply()
-    ThumbValidator(task=self).validate()
+    thumbnail_check.delay()
 
 
 @shared_task(bind=True, name="thumbnail_check", base=BaseTask)

backend/task/urls.py

@@ -34,4 +34,9 @@ urlpatterns = [
         views.ScheduleNotification.as_view(),
         name="api-schedule-notification",
     ),
+    path(
+        "notification/test/",
+        views.NotificationTestView.as_view(),
+        name="api-schedule-notification-test",
+    ),
 ]

backend/task/views.py

@@ -15,6 +15,7 @@ from task.serializers import (
     TaskIDDataSerializer,
     TaskNotificationPostSerializer,
     TaskNotificationSerializer,
+    TaskNotificationTestSerializer,
     TaskResultSerializer,
 )
 from task.src.config_schedule import CrontabValidator, ScheduleBuilder
@@ -343,3 +344,34 @@ class ScheduleNotification(ApiBaseView):
             Notifications(task_name).remove_task()
 
         return Response(status=204)
+
+
+class NotificationTestView(ApiBaseView):
+    """resolves to /api/task/notification/test/
+    POST: test notification url
+    """
+
+    @extend_schema(
+        request=TaskNotificationTestSerializer(),
+        responses={
+            200: OpenApiResponse(description="test notification sent"),
+            400: OpenApiResponse(
+                ErrorResponseSerializer(), description="bad request"
+            ),
+        },
+    )
+    def post(self, request):
+        """test notification"""
+        data_serializer = TaskNotificationTestSerializer(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+
+        url = validated_data["url"]
+        task_name = validated_data.get("task_name", "manual_test")
+
+        success, message = Notifications(task_name).test(url)
+
+        status = 200 if success else 400
+        return Response(
+            {"success": success, "message": message}, status=status
+        )

backend/user/serializers.py

@@ -31,7 +31,9 @@ class UserMeConfigSerializer(serializers.Serializer):
     page_size = serializers.IntegerField()
     sort_by = serializers.ChoiceField(choices=SortEnum.names())
     sort_order = serializers.ChoiceField(choices=OrderEnum.values())
-    view_style_home = serializers.ChoiceField(choices=["grid", "list"])
+    view_style_home = serializers.ChoiceField(
+        choices=["grid", "list", "table"]
+    )
     view_style_channel = serializers.ChoiceField(choices=["grid", "list"])
     view_style_downloads = serializers.ChoiceField(choices=["grid", "list"])
     view_style_playlist = serializers.ChoiceField(choices=["grid", "list"])

backend/video/serializers.py

@@ -12,6 +12,7 @@ class PlayerSerializer(serializers.Serializer):
     """serialize player"""
 
     watched = serializers.BooleanField()
+    watched_date = serializers.IntegerField(required=False)
     duration = serializers.IntegerField()
     duration_str = serializers.CharField()
     progress = serializers.FloatField(required=False)

backend/video/src/comments.py

@@ -79,7 +79,9 @@ class Comments:
     def get_yt_comments(self):
         """get comments from youtube"""
         yt_obs = self.build_yt_obs()
-        info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id)
+        info_json, _ = YtWrap(yt_obs, config=self.config).extract(
+            self.youtube_id
+        )
         if not info_json:
             return False, False
 

backend/video/src/constants.py

@@ -11,6 +11,9 @@ class VideoTypeEnum(enum.Enum):
     SHORTS = "shorts"
     UNKNOWN = "unknown"
 
+    def __str__(self):
+        return self.value
+
     @classmethod
     def values(cls) -> list[str]:
         """value list"""
@@ -31,6 +34,8 @@ class SortEnum(enum.Enum):
     LIKES = "stats.like_count"
     DURATION = "player.duration"
     MEDIASIZE = "media_size"
+    WIDTH = "streams.width"
+    HEIGHT = "streams.height"
 
     @classmethod
     def values(cls) -> list[str]:

backend/video/src/index.py

@@ -10,10 +10,10 @@ from datetime import datetime
 import requests
 from channel.src import index as ta_channel
 from common.src.env_settings import EnvironmentSettings
-from common.src.es_connect import ElasticWrap
 from common.src.helper import get_duration_sec, get_duration_str, randomizor
 from common.src.index_generic import YouTubeItem
 from django.conf import settings
+from download.src.thumbnails import ThumbManager
 from playlist.src import index as ta_playlist
 from ryd_client import ryd_client
 from user.src.user_config import UserConfig
@@ -173,21 +173,15 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         self._validate_id()
         # extract
         self.channel_id = self.youtube_meta["channel_id"]
-        upload_date = self.youtube_meta["upload_date"]
-        upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
-        published = upload_date_time.strftime("%Y-%m-%d")
         last_refresh = int(datetime.now().timestamp())
-        # base64_blur = ThumbManager().get_base64_blur(self.youtube_id)
-        base64_blur = False
         # build json_data basics
         self.json_data = {
             "title": self.youtube_meta["title"],
             "description": self.youtube_meta.get("description", ""),
             "category": self.youtube_meta.get("categories", []),
             "vid_thumb_url": self.youtube_meta["thumbnail"],
-            "vid_thumb_base64": base64_blur,
             "tags": self.youtube_meta.get("tags", []),
-            "published": published,
+            "published": self._build_published(),
             "vid_last_refresh": last_refresh,
             "date_downloaded": last_refresh,
             "youtube_id": self.youtube_id,
@@ -196,6 +190,18 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             "active": True,
         }
 
+    def _build_published(self):
+        """build published date or timestamp"""
+        timestamp = self.youtube_meta.get("timestamp")
+        if timestamp:
+            return timestamp
+
+        upload_date = self.youtube_meta["upload_date"]
+        upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
+        published = upload_date_time.strftime("%Y-%m-%d")
+
+        return published
+
     def _validate_id(self):
         """validate expected video ID, raise value error on mismatch"""
         remote_id = self.youtube_meta["id"]
@@ -385,12 +391,6 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
         return subtitles
 
-    def update_media_url(self):
-        """update only media_url in es for reindex channel rename"""
-        data = {"doc": {"media_url": self.json_data["media_url"]}}
-        path = f"{self.index_name}/_update/{self.youtube_id}"
-        _, _ = ElasticWrap(path).post(data=data)
-
 
 def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
     """combined classes to create new video in index"""
@@ -400,5 +400,8 @@ def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
         raise ValueError("failed to get metadata for " + youtube_id)
 
     video.check_subtitles()
+    url = video.json_data["vid_thumb_url"]
+    ThumbManager(item_id=video.youtube_id).download_video_thumb(url=url)
     video.upload_to_es()
+
     return video.json_data

backend/video/src/query_building.py

@@ -1,6 +1,7 @@
 """build query for video fetching"""
 
 from common.src.ta_redis import RedisArchivist
+from playlist.src.index import YoutubePlaylist
 from video.src.constants import OrderEnum, SortEnum, VideoTypeEnum
 
 
@@ -84,6 +85,11 @@ class QueryBuilder:
 
     def parse_sort(self) -> dict | None:
         """build sort key"""
+        playlist = self.request_params.get("playlist")
+        if playlist:
+            # overwrite sort based on idx in playlist
+            return self._get_playlist_sort(playlist_id=playlist)
+
         sort = self.request_params.get("sort")
         if not sort:
             return None
@@ -100,3 +106,39 @@ class QueryBuilder:
         order_by = getattr(OrderEnum, order.upper()).value
 
         return {"sort": [{sort_field: {"order": order_by}}]}
+
+    def _get_playlist_sort(self, playlist_id: str):
+        """get sort for playlist"""
+        playlist = YoutubePlaylist(playlist_id)
+        playlist.get_from_es()
+        if not playlist.json_data:
+            raise ValueError(f"playlist {playlist_id} not found")
+
+        sort_score = {
+            i["youtube_id"]: i["idx"]
+            for i in playlist.json_data["playlist_entries"]
+            if i["downloaded"]
+        }
+        script = (
+            "if(params.scores.containsKey(doc['youtube_id'].value)) "
+            + "{return params.scores[doc['youtube_id'].value];} "
+            + "return 100000;"
+        )
+
+        sort = {
+            "sort": [
+                {
+                    "_script": {
+                        "type": "number",
+                        "script": {
+                            "lang": "painless",
+                            "source": script,
+                            "params": {"scores": sort_score},
+                        },
+                        "order": "asc",
+                    }
+                }
+            ],
+        }
+
+        return sort

backend/video/src/subtitle.py

@@ -7,12 +7,15 @@ functionality:
 
 import json
 import os
+import re
 from datetime import datetime
+from operator import itemgetter
 
 import requests
 from common.src.env_settings import EnvironmentSettings
 from common.src.es_connect import ElasticWrap
-from common.src.helper import requests_headers
+from common.src.helper import rand_sleep, requests_headers
+from yt_dlp.utils import orderedSet_from_options
 
 
 class YoutubeSubtitle:
@@ -35,85 +38,67 @@ class YoutubeSubtitle:
             # no subtitles
             return False
 
-        relevant_subtitles = []
-        for lang in self.languages:
-            user_sub = self._get_user_subtitles(lang)
-            if user_sub:
-                relevant_subtitles.append(user_sub)
-                continue
+        available_subtitles = self._get_all_subtitles("user")
+        if self.video.config["downloads"]["subtitle_source"] == "auto":
+            for lang, auto_cap in self._get_all_subtitles("auto").items():
+                if lang not in available_subtitles:
+                    available_subtitles[lang] = auto_cap
 
-            if self.video.config["downloads"]["subtitle_source"] == "auto":
-                auto_cap = self._get_auto_caption(lang)
-                if auto_cap:
-                    relevant_subtitles.append(auto_cap)
+        all_sub_langs = tuple(available_subtitles.keys())
+        relevant_subtitles = False
+        try:
+            relevant_subtitles = [
+                available_subtitles[lang]
+                for lang in orderedSet_from_options(
+                    self.languages, {"all": all_sub_langs}, use_regex=True
+                )
+            ]
+        except re.error as e:
+            raise ValueError(f"wrong regex in subtitle config: {e.pattern}")
 
         return relevant_subtitles
 
-    def _get_auto_caption(self, lang):
-        """get auto_caption subtitles"""
-        print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
-        all_subtitles = self.video.youtube_meta.get("automatic_captions")
-
+    def _get_all_subtitles(self, source):
+        """get video subtitles or automatic captions"""
+        print(f"{self.video.youtube_id}: get {source} subtitles")
+        youtube_meta_keys = {"user": "subtitles", "auto": "automatic_captions"}
+        if not (youtube_meta_key := youtube_meta_keys.get(source, None)):
+            raise ValueError(f"unknown subtitles source: {source}")
+        all_subtitles = self.video.youtube_meta.get(youtube_meta_key)
         if not all_subtitles:
-            return False
+            return {}
 
-        video_media_url = self.video.json_data["media_url"]
-        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
-        all_formats = all_subtitles.get(lang)
-        if not all_formats:
-            return False
-
-        subtitle_json3 = [i for i in all_formats if i["ext"] == "json3"]
-        if not subtitle_json3:
-            print(f"{self.video.youtube_id}-{lang}: json3 not processed")
-            return False
-
-        subtitle = subtitle_json3[0]
-        subtitle.update(
-            {"lang": lang, "source": "auto", "media_url": media_url}
-        )
-
-        return subtitle
-
-    def _normalize_lang(self):
-        """normalize country specific language keys"""
-        all_subtitles = self.video.youtube_meta.get("subtitles")
-        if not all_subtitles:
-            return False
-
-        all_keys = list(all_subtitles.keys())
-        for key in all_keys:
-            lang = key.split("-")[0]
-            old = all_subtitles.pop(key)
+        candidate_subtitles = {}
+        for lang, all_formats in all_subtitles.items():
             if lang == "live_chat":
+                # not supported yet
                 continue
-            all_subtitles[lang] = old
 
-        return all_subtitles
+            video_media_url = self.video.json_data["media_url"]
+            media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
+            if not all_formats:
+                # no subtitles found
+                continue
 
-    def _get_user_subtitles(self, lang):
-        """get subtitles uploaded from channel owner"""
-        print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
-        all_subtitles = self._normalize_lang()
-        if not all_subtitles:
-            return False
+            subtitle_json3 = [i for i in all_formats if i["ext"] == "json3"]
+            if not subtitle_json3:
+                print(f"{self.video.youtube_id}-{lang}: json3 not processed")
+                continue
 
-        video_media_url = self.video.json_data["media_url"]
-        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
-        all_formats = all_subtitles.get(lang)
-        if not all_formats:
-            # no user subtitles found
-            return False
+            subtitle = subtitle_json3[0]
+            subtitle.update(
+                {"lang": lang, "source": source, "media_url": media_url}
+            )
+            candidate_subtitles[lang] = subtitle
 
-        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
-        subtitle.update(
-            {"lang": lang, "source": "user", "media_url": media_url}
-        )
-
-        return subtitle
+        return candidate_subtitles
 
     def download_subtitles(self, relevant_subtitles):
         """download subtitle files to archive"""
+        subtitle_list = ", ".join(map(itemgetter("lang"), relevant_subtitles))
+        print(
+            f"{self.video.youtube_id}: downloading subtitles: {subtitle_list}"
+        )
         videos_base = EnvironmentSettings.MEDIA_DIR
         indexed = []
         for subtitle in relevant_subtitles:
@@ -124,17 +109,21 @@ class YoutubeSubtitle:
                 subtitle["url"], headers=requests_headers(), timeout=30
             )
             if not response.ok:
-                print(f"{self.video.youtube_id}: failed to download subtitle")
+                subtitle_key = f"{self.video.youtube_id}-{lang}"
+                print(f"{subtitle_key}: failed to download subtitle")
                 print(response.text)
+                rand_sleep(self.video.config)
                 continue
 
             if not response.text:
-                print(f"{self.video.youtube_id}: skip empty subtitle")
+                print(f"{subtitle_key}: skip empty subtitle")
+                rand_sleep(self.video.config)
                 continue
 
             parser = SubtitleParser(response.text, lang, source)
             parser.process()
             if not parser.all_cues:
+                rand_sleep(self.video.config)
                 continue
 
             subtitle_str = parser.get_subtitle_str()
@@ -144,6 +133,7 @@ class YoutubeSubtitle:
                 self._index_subtitle(query_str)
 
             indexed.append(subtitle)
+            rand_sleep(self.video.config)
 
         return indexed
 

backend/video/tests/test_src/test_query_building.py

@@ -16,7 +16,6 @@ def test_build_data():
     qb = QueryBuilder(
         user_id=1,
         channel="test_channel",
-        playlist="test_playlist",
         watch="watched",
         type="videos",
         sort="published",

backend/video/views.py

@@ -227,7 +227,8 @@ class VideoProgressView(ApiBaseView):
             expire = False
 
         current_progress.update({"watched": watched})
-        redis_con.set_message(key, current_progress, expire=expire)
+        if position > 5:
+            redis_con.set_message(key, current_progress, expire=expire)
 
         response_serializer = PlayerSerializer(current_progress)
 

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.5'
-
 services:
   tubearchivist:
     container_name: tubearchivist
@@ -40,7 +38,7 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.18.0
+    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.18.2
     container_name: archivist-es
     restart: unless-stopped
     environment:

docker_assets/beat_auto_spawn.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# auto restart beat scheduler
+# https://github.com/celery/django-celery-beat/issues/894
+
+if [[ -n "$DJANGO_DEBUG" ]]; then
+  LOGLEVEL="DEBUG"
+else
+  LOGLEVEL="INFO"
+fi
+
+COMMAND="celery -A task beat --loglevel=$LOGLEVEL --scheduler django_celery_beat.schedulers:DatabaseScheduler"
+TIMEOUT=3600
+
+while true; do
+    echo "Starting process beat scheduler"
+
+    $COMMAND &
+    PID=$!
+
+    sleep $TIMEOUT
+
+    # Kill the process if still running
+    if kill -0 $PID 2>/dev/null; then
+        echo "Killing beat process after $TIMEOUT seconds"
+        kill $PID
+        # Wait a bit to allow graceful shutdown, then force kill if needed
+        sleep 10
+        kill -9 $PID 2>/dev/null
+    fi
+
+    echo "Restarting beat..."
+done

docker_assets/run.sh

@@ -3,6 +3,21 @@
 
 set -e
 
+if [[ -n "$DJANGO_DEBUG" ]]; then
+  LOGLEVEL="DEBUG"
+else
+  LOGLEVEL="INFO"
+fi
+
+# update yt-dlp if needed
+if [[ "${TA_AUTO_UPDATE_YTDLP,,}" =~ ^(release|nightly)$ ]]; then
+    echo "Updating yt-dlp..."
+    preflag=$([[ "${TA_AUTO_UPDATE_YTDLP,,}" == "nightly" ]] && echo "--pre" || echo "")
+    python -m pip install --target=/root/.local/bin --upgrade $preflag "yt-dlp[default]" || {
+        echo "yt-dlp update failed"
+    }
+fi
+
 # stop on pending manual migration
 python manage.py ta_stop_on_error
 
@@ -18,10 +33,11 @@ python manage.py ta_startup
 # start all tasks
 nginx &
 celery -A task.celery worker \
-    --loglevel=INFO \
+    --loglevel=$LOGLEVEL \
     --concurrency 4 \
     --max-tasks-per-child 5 \
     --max-memory-per-child 150000 &
-celery -A task beat --loglevel=INFO \
-    --scheduler django_celery_beat.schedulers:DatabaseScheduler &
+
+./beat_auto_spawn.sh &
+
 python backend_start.py

frontend/package.json

@@ -11,27 +11,27 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "dompurify": "^3.2.5",
+    "dompurify": "^3.2.6",
     "react": "^19.1.0",
     "react-dom": "^19.1.0",
-    "react-router-dom": "^7.6.0",
-    "zustand": "^5.0.4"
+    "react-router-dom": "^7.6.3",
+    "zustand": "^5.0.6"
   },
   "devDependencies": {
-    "@types/react": "^19.1.3",
-    "@types/react-dom": "^19.1.3",
-    "@typescript-eslint/eslint-plugin": "^8.32.0",
-    "@typescript-eslint/parser": "^8.32.0",
-    "@vitejs/plugin-react-swc": "^3.9.0",
-    "eslint": "^9.26.0",
+    "@types/react": "^19.1.8",
+    "@types/react-dom": "^19.1.6",
+    "@typescript-eslint/eslint-plugin": "^8.32.1",
+    "@typescript-eslint/parser": "^8.32.1",
+    "@vitejs/plugin-react-swc": "^3.10.2",
+    "eslint": "^9.30.1",
     "eslint-config-prettier": "^10.1.5",
     "eslint-plugin-react-hooks": "^5.2.0",
     "eslint-plugin-react-refresh": "^0.4.20",
-    "globals": "^16.1.0",
-    "prettier": "3.5.3",
+    "globals": "^16.3.0",
+    "prettier": "3.6.2",
     "typescript": "^5.8.3",
-    "typescript-eslint": "^8.32.0",
-    "vite": ">=6.3.5",
+    "typescript-eslint": "^8.32.1",
+    "vite": ">=7.0.0",
     "vite-plugin-checker": "^0.9.3"
   }
 }

frontend/public/img/icon-tableview.svg

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
+<svg version="1.0" width="960pt" height="960pt" viewBox="0 0 960 960" preserveAspectRatio="xMidYMid meet" xmlns="http://www.w3.org/2000/svg">
+  <g transform="translate(0,960)scale(.075,.075)">
+    <path id="path1" d="M 740 -10481 c -77 24 -152 94 -170 160 -6 23 -10 376 -10 946 l 0 910 24 50 c 28 59 66 97 121 122 38 17 268 18 5620 21 3923 1 5595 -1 5632 -8 70 -15 124 -56 157 -121 l 27 -54 0 -920 c 0 -637 -4 -932 -11 -958 -17 -58 -84 -122 -150 -141 -49 -15 -544 -16 -5635 -15 -3069 0 -5591 4 -5605 8 z M 9615 -7400 c -11 4 -31 20 -45 35 l -25 27 -3 889 c -3 978 -5 935 57 972 27 16 117 17 1253 17 1321 0 1252 3 1293 -54 20 -27 20 -43 20 -919 0 -860 -1 -893 -19 -920 -41 -60 37 -57 -1293 -56 -670 0 -1227 4 -1238 9 z M 3658 -7384 c -61 32 -58 -10 -58 959 0 973 -3 928 60 960 25 13 178 15 1250 15 1072 0 1225 -2 1250 -15 63 -32 60 13 60 -960 0 -973 3 -928 -60 -960 -25 -13 -178 -15 -1252 -15 -1062 1 -1227 3 -1250 16 z M 6551 -7374 c -17 14 -35 42 -41 62 -6 24 -10 339 -10 892 0 826 1 856 20 898 35 78 -60 73 1315 70 l 1225 -3 32 -33 33 -32 3 -895 c 2 -788 0 -899 -13 -925 -33 -64 48 -60 -1304 -60 l -1229 0 -31 26 z M 616 -7369 c -59 46 -56 0 -56 951 0 976 -4 924 70 960 33 17 113 18 1250 18 1187 0 1216 -1 1247 -20 66 -40 63 12 63 -955 0 -791 -2 -880 -16 -911 -33 -68 54 -64 -1302 -64 l -1229 0 -27 21 z M 9615 -4530 c -11 4 -31 20 -45 35 l -25 27 -3 901 c -2 891 -2 902 18 935 40 65 -32 62 1296 62 1335 0 1255 4 1292 -64 16 -29 17 -101 17 -921 0 -832 -1 -892 -18 -922 -36 -67 50 -63 -1294 -62 -670 0 -1227 4 -1238 9 z M 3664 -4514 c -68 33 -64 -26 -64 973 l 0 901 34 38 34 37 1242 0 1242 0 34 -37 34 -38 0 -900 c 0 -989 3 -942 -60 -975 -25 -13 -178 -15 -1247 -15 -1064 0 -1222 2 -1249 16 z M 6551 -4504 c -17 14 -35 42 -41 62 -14 51 -14 1733 0 1784 6 20 24 48 41 62 l 31 26 1237 0 c 1360 0 1263 5 1296 -60 23 -44 23 -1796 0 -1840 -33 -64 47 -60 -1304 -60 l -1229 0 -31 26 z M 615 -4498 c -57 44 -55 13 -55 963 0 844 1 877 19 913 11 21 31 43 46 50 20 9 323 12 1260 12 l 1233 0 31 -30 c 17 -17 33 -45 36 -63 3 -18 4 -429 3 -915 l -3 -884 -37 -34 -38 -34 -1234 0 -1233 0 -28 22 z " />
+  </g>
+</svg>

frontend/src/api/actions/deleteAppriseNotificationUrl.ts

@@ -6,10 +6,10 @@ type AppriseTaskNameType =
   | 'download_pending'
   | 'check_reindex';
 
-const deleteAppriseNotificationUrl = async (taskName: AppriseTaskNameType) => {
+const deleteAppriseNotificationUrl = async (taskName: AppriseTaskNameType, url: string) => {
   return APIClient('/api/task/notification/', {
     method: 'DELETE',
-    body: { task_name: taskName },
+    body: { task_name: taskName, url: url },
   });
 };
 

frontend/src/api/actions/deleteDownloadQueueByFilter.ts

@@ -2,9 +2,15 @@ import APIClient from '../../functions/APIClient';
 
 type FilterType = 'ignore' | 'pending';
 
-const deleteDownloadQueueByFilter = async (filter: FilterType) => {
+const deleteDownloadQueueByFilter = async (
+  filter: FilterType,
+  channel: string | null,
+  vid_type: string | null,
+) => {
   const searchParams = new URLSearchParams();
   if (filter) searchParams.append('filter', filter);
+  if (channel) searchParams.append('channel', channel);
+  if (vid_type) searchParams.append('vid_type', vid_type);
 
   return APIClient(`/api/download/?${searchParams.toString()}`, {
     method: 'DELETE',

frontend/src/api/actions/testAppriseNotificationUrl.ts

@@ -0,0 +1,16 @@
+import APIClient from '../../functions/APIClient';
+import { AppriseTaskNameType } from './createAppriseNotificationUrl';
+
+export type TestNotificationResponseType = {
+  success: boolean;
+  message: string;
+};
+
+const testAppriseNotificationUrl = async (taskName: AppriseTaskNameType, url: string) => {
+  return APIClient<TestNotificationResponseType>('/api/task/notification/test/', {
+    method: 'POST',
+    body: { task_name: taskName, url },
+  });
+};
+
+export default testAppriseNotificationUrl;

frontend/src/api/actions/updateDownloadQueue.ts

@@ -1,6 +1,6 @@
 import APIClient from '../../functions/APIClient';
 
-const updateDownloadQueue = async (youtubeIdStrings: string, autostart: boolean) => {
+const updateDownloadQueue = async (youtubeIdStrings: string, autostart: boolean, flat: boolean) => {
   const urls = [];
   const containsMultiple = youtubeIdStrings.includes('\n');
 
@@ -16,12 +16,12 @@ const updateDownloadQueue = async (youtubeIdStrings: string, autostart: boolean)
     urls.push({ youtube_id: youtubeIdStrings, status: 'pending' });
   }
 
-  let params = '';
-  if (autostart) {
-    params = '?autostart=true';
-  }
+  const searchParams = new URLSearchParams();
+  if (autostart) searchParams.append('autostart', 'true');
+  if (flat) searchParams.append('flat', 'true');
+  const endpoint = `/api/download/${searchParams.toString() ? `?${searchParams.toString()}` : ''}`;
 
-  return APIClient(`/api/download/${params}`, {
+  return APIClient(endpoint, {
     method: 'POST',
     body: { data: [...urls] },
   });

frontend/src/api/actions/updateDownloadQueueByFilter.ts

@@ -0,0 +1,25 @@
+import APIClient from '../../functions/APIClient';
+
+type FilterType = 'ignore' | 'pending';
+export type DownloadQueueStatus = 'ignore' | 'pending' | 'priority' | 'clear_error';
+
+const updateDownloadQueueByFilter = async (
+  filter: FilterType,
+  channel: string | null,
+  vid_type: string | null,
+  error: string | null,
+  status: DownloadQueueStatus,
+) => {
+  const searchParams = new URLSearchParams();
+  if (filter) searchParams.append('filter', filter);
+  if (channel) searchParams.append('channel', channel);
+  if (vid_type) searchParams.append('vid_type', vid_type);
+  if (error) searchParams.append('error', error);
+
+  return APIClient(`/api/download/?${searchParams.toString()}`, {
+    method: 'PATCH',
+    body: { status: status },
+  });
+};
+
+export default updateDownloadQueueByFilter;

frontend/src/api/actions/updatePlaylistSortOrder.ts

@@ -0,0 +1,10 @@
+import APIClient from '../../functions/APIClient';
+
+const updatePlaylistSortOrder = async (playlistId: string, newSortOrder: 'top' | 'bottom') => {
+  return APIClient(`/api/playlist/${playlistId}/`, {
+    method: 'POST',
+    body: { playlist_sort_order: newSortOrder },
+  });
+};
+
+export default updatePlaylistSortOrder;

frontend/src/api/actions/updateUserConfig.ts

@@ -1,5 +1,6 @@
-import { SortByType, SortOrderType, ViewLayoutType } from '../../pages/Home';
+import { ViewStylesType } from '../../configuration/constants/ViewStyle';
 import APIClient from '../../functions/APIClient';
+import { SortByType, SortOrderType } from '../loader/loadVideoListByPage';
 
 export type ColourVariants =
   | 'dark.css'
@@ -8,6 +9,14 @@ export type ColourVariants =
   | 'midnight.css'
   | 'custom.css';
 
+export const ColourConstant = {
+  Dark: 'dark.css',
+  Light: 'light.css',
+  Matrix: 'matrix.css',
+  Midnight: 'midnight.css',
+  Custom: 'custom.css',
+};
+
 export const FileSizeUnits = {
   Binary: 'binary',
   Metric: 'metric',
@@ -18,10 +27,10 @@ export type UserConfigType = {
   page_size: number;
   sort_by: SortByType;
   sort_order: SortOrderType;
-  view_style_home: ViewLayoutType;
-  view_style_channel: ViewLayoutType;
-  view_style_downloads: ViewLayoutType;
-  view_style_playlist: ViewLayoutType;
+  view_style_home: ViewStylesType;
+  view_style_channel: ViewStylesType;
+  view_style_downloads: ViewStylesType;
+  view_style_playlist: ViewStylesType;
   grid_items: number;
   hide_watched: boolean;
   file_size_unit: 'binary' | 'metric';

frontend/src/api/loader/loadAppsettingsConfig.ts

@@ -5,7 +5,9 @@ export type AppSettingsConfigType = {
     channel_size: number | null;
     live_channel_size: number | null;
     shorts_channel_size: number | null;
+    playlist_size: number | null;
     auto_start: boolean;
+    extract_flat: boolean;
   };
   downloads: {
     limit_speed: number | null;

frontend/src/api/loader/loadChannelNav.ts

@@ -5,6 +5,7 @@ export type ChannelNavResponseType = {
   has_shorts: boolean;
   has_playlists: boolean;
   has_pending: boolean;
+  has_ignored: boolean;
 };
 
 const loadChannelNav = async (youtubeChannelId: string) => {

frontend/src/api/loader/loadDownloadQueue.ts

@@ -1,11 +1,21 @@
 import APIClient from '../../functions/APIClient';
 import { DownloadResponseType } from '../../pages/Download';
 
-const loadDownloadQueue = async (page: number, channelId: string | null, showIgnored: boolean) => {
+const loadDownloadQueue = async (
+  page: number,
+  channelId: string | null,
+  vid_type: string | null,
+  errorFilterFromUrl: string | null,
+  showIgnored: boolean,
+  search: string,
+) => {
   const searchParams = new URLSearchParams();
 
   if (page) searchParams.append('page', page.toString());
   if (channelId) searchParams.append('channel', channelId);
+  if (vid_type) searchParams.append('vid_type', vid_type);
+  if (search) searchParams.append('q', encodeURIComponent(search));
+  if (errorFilterFromUrl !== null) searchParams.append('error', errorFilterFromUrl);
   searchParams.append('filter', showIgnored ? 'ignore' : 'pending');
 
   const endpoint = `/api/download/${searchParams.toString() ? `?${searchParams.toString()}` : ''}`;

frontend/src/api/loader/loadPlaylistById.ts

@@ -14,6 +14,7 @@ export type PlaylistType = {
   playlist_channel_id: string;
   playlist_description: string;
   playlist_entries: PlaylistEntryType[];
+  playlist_sort_order: 'top' | 'bottom';
   playlist_id: string;
   playlist_last_refresh: string;
   playlist_name: string;

frontend/src/api/loader/loadVideoListByPage.ts

@@ -1,4 +1,4 @@
-import { ConfigType, SortByType, SortOrderType, VideoType } from '../../pages/Home';
+import { ConfigType, VideoType } from '../../pages/Home';
 import { PaginationType } from '../../components/Pagination';
 import APIClient from '../../functions/APIClient';
 
@@ -8,9 +8,41 @@ export type VideoListByFilterResponseType = {
   paginate?: PaginationType;
 };
 
-type WatchTypes = 'watched' | 'unwatched' | 'continue';
+export type SortByType =
+  | 'published'
+  | 'downloaded'
+  | 'views'
+  | 'likes'
+  | 'duration'
+  | 'mediasize'
+  | 'width'
+  | 'height';
+export const SortByEnum = {
+  Published: 'published',
+  Downloaded: 'downloaded',
+  Views: 'views',
+  Likes: 'likes',
+  Duration: 'duration',
+  'Media Size': 'mediasize',
+  Width: 'width',
+  Height: 'height',
+};
+
+export type SortOrderType = 'asc' | 'desc';
+export const SortOrderEnum = {
+  Asc: 'asc',
+  Desc: 'desc',
+};
+
 export type VideoTypes = 'videos' | 'streams' | 'shorts';
 
+export type WatchTypes = 'watched' | 'unwatched' | 'continue';
+export const WatchTypesEnum = {
+  Watched: 'watched',
+  Unwatched: 'unwatched',
+  Continue: 'continue',
+};
+
 type FilterType = {
   page?: number;
   playlist?: string;

frontend/src/components/Button.tsx

@@ -9,6 +9,7 @@ export interface ButtonProps {
   children?: string | ReactNode | ReactNode[];
   value?: string;
   title?: string;
+  disabled?: boolean;
   onClick?: () => void;
 }
 
@@ -21,6 +22,7 @@ const Button = ({
   children,
   value,
   title,
+  disabled,
   onClick,
 }: ButtonProps) => {
   return (
@@ -31,6 +33,7 @@ const Button = ({
       type={type}
       value={value}
       title={title}
+      disabled={disabled}
       onClick={onClick}
     >
       {label}

frontend/src/components/ChannelList.tsx

@@ -16,7 +16,7 @@ type ChannelListProps = {
 
 const ChannelList = ({ channelList, refreshChannelList }: ChannelListProps) => {
   const { userConfig } = useUserConfigStore();
-  const viewLayout = userConfig.view_style_channel;
+  const viewStyle = userConfig.view_style_channel;
 
   if (!channelList || channelList.length === 0) {
     return <p>No channels found.</p>;
@@ -26,8 +26,8 @@ const ChannelList = ({ channelList, refreshChannelList }: ChannelListProps) => {
     <>
       {channelList.map(channel => {
         return (
-          <div key={channel.channel_id} className={`channel-item ${viewLayout}`}>
-            <div className={`channel-banner ${viewLayout}`}>
+          <div key={channel.channel_id} className={`channel-item ${viewStyle}`}>
+            <div className={`channel-banner ${viewStyle}`}>
               <Link to={Routes.Channel(channel.channel_id)}>
                 <ChannelBanner
                   channelId={channel.channel_id}
@@ -35,7 +35,7 @@ const ChannelList = ({ channelList, refreshChannelList }: ChannelListProps) => {
                 />
               </Link>
             </div>
-            <div className={`info-box info-box-2 ${viewLayout}`}>
+            <div className={`info-box info-box-2 ${viewStyle}`}>
               <div className="info-box-item">
                 <div className="round-img">
                   <Link to={Routes.Channel(channel.channel_id)}>

frontend/src/components/CommentBox.tsx

@@ -37,9 +37,10 @@ export type CommentsType = {
 
 type CommentBoxProps = {
   comment: CommentsType;
+  onTimestampClick?: (seconds: number) => void;
 };
 
-const CommentBox = ({ comment }: CommentBoxProps) => {
+const CommentBox = ({ comment, onTimestampClick }: CommentBoxProps) => {
   const [showSubComments, setShowSubComments] = useState(false);
 
   const hasSubComments =
@@ -51,7 +52,7 @@ const CommentBox = ({ comment }: CommentBoxProps) => {
         {comment.comment_author}
       </h3>
       <p>
-        <Linkify>{comment.comment_text}</Linkify>
+        <Linkify onTimestampClick={onTimestampClick}>{comment.comment_text}</Linkify>
       </p>
 
       <div className="comment-meta">
@@ -95,7 +96,7 @@ const CommentBox = ({ comment }: CommentBoxProps) => {
               comment.comment_replies?.map(comment => {
                 return (
                   <Fragment key={comment.comment_id}>
-                    <CommentBox comment={comment} />
+                    <CommentBox comment={comment} onTimestampClick={onTimestampClick} />
                   </Fragment>
                 );
               })}

frontend/src/components/DownloadListItem.tsx

@@ -16,14 +16,14 @@ type DownloadListItemProps = {
 
 const DownloadListItem = ({ download, setRefresh }: DownloadListItemProps) => {
   const { userConfig } = useUserConfigStore();
-  const view = userConfig.view_style_downloads;
+  const viewStyle = userConfig.view_style_downloads;
   const showIgnored = userConfig.show_ignored_only;
 
   const [hideDownload, setHideDownload] = useState(false);
 
   return (
-    <div className={`video-item ${view}`} id={`dl-${download.youtube_id}`}>
-      <div className={`video-thumb-wrap ${view}`}>
+    <div className={`video-item ${viewStyle}`} id={`dl-${download.youtube_id}`}>
+      <div className={`video-thumb-wrap ${viewStyle}`}>
         <div className="video-thumb">
           <VideoThumbnail videoThumbUrl={download.vid_thumb_url} />
 
@@ -39,7 +39,7 @@ const DownloadListItem = ({ download, setRefresh }: DownloadListItemProps) => {
         </div>
       </div>
 
-      <div className={`video-desc ${view}`}>
+      <div className={`video-desc ${viewStyle}`}>
         <div>
           {download.channel_indexed && (
             <Link to={Routes.Channel(download.channel_id)}>{download.channel_name}</Link>
@@ -47,14 +47,19 @@ const DownloadListItem = ({ download, setRefresh }: DownloadListItemProps) => {
 
           {!download.channel_indexed && <span>{download.channel_name}</span>}
 
-          <a href={`https://www.youtube.com/watch?v=${download.youtube_id}`} target="_blank">
+          <a
+            href={`https://www.youtube.com/watch?v=${download.youtube_id}`}
+            target="_blank"
+            rel="noopener noreferrer"
+          >
             <h3>{download.title}</h3>
           </a>
         </div>
 
         <p>
-          Published: {formatDate(download.published)} | Duration: {download.duration} |{' '}
-          {download.youtube_id}
+          {download.published && <span>Published: {formatDate(download.published)} | </span>}
+          <span>Duration: {download.duration} | </span>
+          <span>{download.youtube_id}</span>
         </p>
 
         {download.message && <p className="danger-zone">{download.message}</p>}

frontend/src/components/Filterbar.tsx

@@ -1,24 +1,45 @@
-import { useState } from 'react';
+import { useEffect, useState } from 'react';
 import iconSort from '/img/icon-sort.svg';
 import iconAdd from '/img/icon-add.svg';
 import iconSubstract from '/img/icon-substract.svg';
 import iconGridView from '/img/icon-gridview.svg';
 import iconListView from '/img/icon-listview.svg';
-import { SortByType, SortOrderType } from '../pages/Home';
+import iconTableView from '/img/icon-tableview.svg';
 import { useUserConfigStore } from '../stores/UserConfigStore';
-import { ViewStyles } from '../configuration/constants/ViewStyle';
+import { ViewStyleNamesType, ViewStylesEnum } from '../configuration/constants/ViewStyle';
 import updateUserConfig, { UserConfigType } from '../api/actions/updateUserConfig';
+import {
+  SortByEnum,
+  SortByType,
+  SortOrderEnum,
+  SortOrderType,
+} from '../api/loader/loadVideoListByPage';
 
 type FilterbarProps = {
   hideToggleText: string;
-  viewStyleName: string;
+  viewStyle: ViewStyleNamesType;
   showSort?: boolean;
 };
 
-const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: FilterbarProps) => {
+const Filterbar = ({ hideToggleText, viewStyle, showSort = true }: FilterbarProps) => {
   const { userConfig, setUserConfig } = useUserConfigStore();
+
   const [showHidden, setShowHidden] = useState(false);
-  const isGridView = userConfig.view_style_home === ViewStyles.grid;
+
+  const currentViewStyle = userConfig[viewStyle];
+  const isGridView = currentViewStyle === ViewStylesEnum.Grid;
+
+  useEffect(() => {
+    if (!showSort) {
+      return;
+    }
+
+    if (currentViewStyle === ViewStylesEnum.Table) {
+      setShowHidden(true);
+    } else {
+      setShowHidden(false);
+    }
+  }, [currentViewStyle, showSort]);
 
   const handleUserConfigUpdate = async (config: Partial<UserConfigType>) => {
     const updatedUserConfig = await updateUserConfig(config);
@@ -55,7 +76,7 @@ const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: Filterbar
         </div>
       </div>
 
-      {showHidden && showSort && (
+      {showHidden && (
         <div className="sort">
           <div id="form">
             <span>Sort by:</span>
@@ -67,12 +88,9 @@ const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: Filterbar
                 handleUserConfigUpdate({ sort_by: event.target.value as SortByType });
               }}
             >
-              <option value="published">date published</option>
-              <option value="downloaded">date downloaded</option>
-              <option value="views">views</option>
-              <option value="likes">likes</option>
-              <option value="duration">duration</option>
-              <option value="mediasize">media size</option>
+              {Object.entries(SortByEnum).map(([key, value]) => {
+                return <option value={value}>{key}</option>;
+              })}
             </select>
             <select
               name="sort_order"
@@ -82,19 +100,20 @@ const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: Filterbar
                 handleUserConfigUpdate({ sort_order: event.target.value as SortOrderType });
               }}
             >
-              <option value="asc">asc</option>
-              <option value="desc">desc</option>
+              {Object.entries(SortOrderEnum).map(([key, value]) => {
+                return <option value={value}>{key}</option>;
+              })}
             </select>
           </div>
         </div>
       )}
       <div className="view-icons">
-        {setShowHidden && showSort && (
+        {showSort && (
           <img
             src={iconSort}
             alt="sort-icon"
             onClick={() => {
-              setShowHidden?.(!showHidden);
+              setShowHidden(!showHidden);
             }}
             id="animate-icon"
           />
@@ -125,17 +144,24 @@ const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: Filterbar
         <img
           src={iconGridView}
           onClick={() => {
-            handleUserConfigUpdate({ [viewStyleName]: 'grid' });
+            handleUserConfigUpdate({ [viewStyle]: ViewStylesEnum.Grid });
           }}
           alt="grid view"
         />
         <img
           src={iconListView}
           onClick={() => {
-            handleUserConfigUpdate({ [viewStyleName]: 'list' });
+            handleUserConfigUpdate({ [viewStyle]: ViewStylesEnum.List });
           }}
           alt="list view"
         />
+        <img
+          src={iconTableView}
+          onClick={() => {
+            handleUserConfigUpdate({ [viewStyle]: ViewStylesEnum.Table });
+          }}
+          alt="table view"
+        />
       </div>
     </div>
   );

frontend/src/components/InputConfig.tsx

@@ -1,4 +1,5 @@
 import { useState } from 'react';
+import LoadingIndicator from './LoadingIndicator';
 
 type InputTextProps = {
   type: 'text' | 'number';
@@ -51,13 +52,7 @@ const InputConfig = ({ type, name, value, setValue, oldValue, updateCallback }:
           </>
         )}
         {oldValue !== null && <button onClick={() => handleUpdate(name, null)}>reset</button>}
-        {loading && (
-          <>
-            <div className="lds-ring" style={{ color: 'var(--accent-font-dark)' }}>
-              <div />
-            </div>
-          </>
-        )}
+        {loading && <LoadingIndicator />}
         {success && <span>✅</span>}
       </div>
     </div>

frontend/src/components/Linkify.tsx

@@ -3,17 +3,61 @@ import DOMPurify from 'dompurify';
 type LinkifyProps = {
   children: string;
   ignoreLineBreak?: boolean;
+  onTimestampClick?: (seconds: number) => void;
 };
 
 // source: https://www.js-craft.io/blog/react-detect-url-text-convert-link/
-const Linkify = ({ children, ignoreLineBreak = false }: LinkifyProps) => {
+const Linkify = ({ children, ignoreLineBreak = false, onTimestampClick }: LinkifyProps) => {
   const isUrl = (word: string) => {
     const urlPattern = /(https?:\/\/[^\s]+)/g;
     return word.match(urlPattern);
   };
 
+  const isTimestamp = (word: string) => {
+    const timestampPattern = /^(\d{1,}:)?(\d{1,2}):(\d{1,2})$/;
+    return word.match(timestampPattern);
+  };
+
+  const parseTimestamp = (timestamp: string): number => {
+    const parts = timestamp.split(':').map(part => parseInt(part, 10));
+
+    if (parts.length === 2) {
+      // MM:SS format
+      const [minutes, seconds] = parts;
+      return minutes * 60 + seconds;
+    }
+
+    if (parts.length === 3) {
+      // HH:MM:SS format
+      const [hours, minutes, seconds] = parts;
+      return hours * 3600 + minutes * 60 + seconds;
+    }
+
+    return 0;
+  };
+
   const addMarkup = (word: string) => {
-    return isUrl(word) ? `<a href="${word}">${word}</a>` : word;
+    if (isUrl(word)) {
+      return `<a href="${word}" rel="noopener noreferrer" target="_blank">${word}</a>`;
+    }
+
+    if (isTimestamp(word) && onTimestampClick) {
+      const seconds = parseTimestamp(word);
+      return `<span class="timestamp-link" data-timestamp="${seconds}">${word}</span>`;
+    }
+
+    return word;
+  };
+
+  const handleClick = (event: React.MouseEvent<HTMLSpanElement>) => {
+    const target = event.target as HTMLElement;
+    if (target.classList.contains('timestamp-link') && onTimestampClick) {
+      const timestamp = target.getAttribute('data-timestamp');
+      if (timestamp) {
+        const seconds = parseInt(timestamp, 10);
+        onTimestampClick(seconds);
+      }
+    }
   };
 
   let workingText = children;
@@ -26,9 +70,11 @@ const Linkify = ({ children, ignoreLineBreak = false }: LinkifyProps) => {
 
   const formatedWords = words.map(w => addMarkup(w));
 
-  const html = DOMPurify.sanitize(formatedWords.join(' '));
+  const html = DOMPurify.sanitize(formatedWords.join(' '), {
+    ADD_ATTR: ['target'],
+  });
 
-  return <span dangerouslySetInnerHTML={{ __html: html }} />;
+  return <span dangerouslySetInnerHTML={{ __html: html }} onClick={handleClick} />;
 };
 
 export default Linkify;

frontend/src/components/LoadingIndicator.tsx

@@ -0,0 +1,9 @@
+const LoadingIndicator = () => {
+  return (
+    <div className="lds-ring" style={{ color: 'var(--accent-font-dark)' }}>
+      <div />
+    </div>
+  );
+};
+
+export default LoadingIndicator;

frontend/src/components/Pagination.tsx

@@ -84,129 +84,136 @@ const Pagination = ({ pagination, setPage }: Props) => {
       <br />
       {totalHits > 1 && (
         <>
-          {currentPage > 1 && (
-            <>
-              <Link
-                to={`${Routes.Home}?${params}`}
-                className="pagination-item"
-                onClick={event => {
-                  event.preventDefault();
-                  setPage(0);
-                }}
-              >
-                First
-              </Link>{' '}
-            </>
-          )}
-
-          {prev_pages !== false &&
-            prev_pages &&
-            prev_pages.map((page: number) => {
-              if (hasParams) {
-                return (
-                  <Fragment key={page}>
-                    <Link
-                      to={`${Routes.Home}?page=${page}&${params}`}
-                      className="pagination-item"
-                      onClick={event => {
-                        event.preventDefault();
-                        setPage(page);
-                      }}
-                    >
-                      {page}
-                    </Link>{' '}
-                  </Fragment>
-                );
-              } else {
-                return (
-                  <Fragment key={page}>
-                    <Link
-                      to={`${Routes.Home}?page=${page}`}
-                      className="pagination-item"
-                      onClick={event => {
-                        event.preventDefault();
-                        setPage(page);
-                      }}
-                    >
-                      {page}
-                    </Link>{' '}
-                  </Fragment>
-                );
-              }
-            })}
-
-          {currentPage > 0 && <span>{`< Page ${currentPage} `}</span>}
-
-          {next_pages && next_pages.length > 0 && (
-            <>
-              <span>{'>'}</span>{' '}
-              {next_pages.map(page => {
-                if (hasParams) {
-                  return (
-                    <Fragment key={page}>
-                      <a
-                        className="pagination-item"
-                        href={`?page=${page}&${params}`}
-                        onClick={event => {
-                          event.preventDefault();
-                          setPage(page);
-                        }}
-                      >
-                        {page}
-                      </a>{' '}
-                    </Fragment>
-                  );
-                } else {
-                  return (
-                    <Fragment key={page}>
-                      <a
-                        className="pagination-item"
-                        href={`?page=${page}`}
-                        onClick={event => {
-                          event.preventDefault();
-                          setPage(page);
-                        }}
-                      >
-                        {page}
-                      </a>{' '}
-                    </Fragment>
-                  );
-                }
-              })}
-            </>
-          )}
-
-          {lastPage > 0 && (
-            <>
-              {hasParams && (
-                <a
+          <span>
+            {currentPage > 1 && (
+              <>
+                <Link
+                  to={`${Routes.Home}?${params}`}
                   className="pagination-item"
-                  href={`?page=${lastPage}&${params}`}
                   onClick={event => {
                     event.preventDefault();
-                    setPage(lastPage || 0);
+                    setPage(0);
                   }}
                 >
-                  {hasMaxHits && `Max (${lastPage})`}
-                  {!hasMaxHits && `Last (${lastPage})`}
-                </a>
+                  First
+                </Link>{' '}
+              </>
+            )}
+
+            {prev_pages !== false && prev_pages && (
+              <>
+                {prev_pages.map((page: number) => {
+                  if (hasParams) {
+                    return (
+                      <Fragment key={page}>
+                        <Link
+                          to={`${Routes.Home}?page=${page}&${params}`}
+                          className="pagination-item"
+                          onClick={event => {
+                            event.preventDefault();
+                            setPage(page);
+                          }}
+                        >
+                          {page}
+                        </Link>{' '}
+                      </Fragment>
+                    );
+                  } else {
+                    return (
+                      <Fragment key={page}>
+                        <Link
+                          to={`${Routes.Home}?page=${page}`}
+                          className="pagination-item"
+                          onClick={event => {
+                            event.preventDefault();
+                            setPage(page);
+                          }}
+                        >
+                          {page}
+                        </Link>{' '}
+                      </Fragment>
+                    );
+                  }
+                })}
+              </>
+            )}
+          </span>
+          {currentPage > 0 && (
+            <span>{`< Page ${currentPage} ${next_pages && next_pages.length > 0 ? '>' : ''}`}</span>
+          )}
+          <span>
+            <>
+              {next_pages && next_pages.length > 0 && (
+                <>
+                  {next_pages.map(page => {
+                    if (hasParams) {
+                      return (
+                        <Fragment key={page}>
+                          <a
+                            className="pagination-item"
+                            href={`?page=${page}&${params}`}
+                            onClick={event => {
+                              event.preventDefault();
+                              setPage(page);
+                            }}
+                          >
+                            {page}
+                          </a>{' '}
+                        </Fragment>
+                      );
+                    } else {
+                      return (
+                        <Fragment key={page}>
+                          <a
+                            className="pagination-item"
+                            href={`?page=${page}`}
+                            onClick={event => {
+                              event.preventDefault();
+                              setPage(page);
+                            }}
+                          >
+                            {page}
+                          </a>{' '}
+                        </Fragment>
+                      );
+                    }
+                  })}
+                </>
               )}
 
-              {!hasParams && (
-                <a
-                  className="pagination-item"
-                  href={`?page=${lastPage}`}
-                  onClick={event => {
-                    event.preventDefault();
-                    setPage(lastPage || 0);
-                  }}
-                >
-                  {hasMaxHits && `Max (${lastPage})`}
-                  {!hasMaxHits && `Last (${lastPage})`}
-                </a>
+              {lastPage > 0 && (
+                <>
+                  {hasParams && (
+                    <a
+                      className="pagination-item"
+                      href={`?page=${lastPage}&${params}`}
+                      onClick={event => {
+                        event.preventDefault();
+                        setPage(lastPage || 0);
+                      }}
+                    >
+                      {hasMaxHits && `Max (${lastPage})`}
+                      {!hasMaxHits && `Last (${lastPage})`}
+                    </a>
+                  )}
+
+                  {!hasParams && (
+                    <a
+                      className="pagination-item"
+                      href={`?page=${lastPage}`}
+                      onClick={event => {
+                        event.preventDefault();
+                        setPage(lastPage || 0);
+                      }}
+                    >
+                      {hasMaxHits && `Max (${lastPage})`}
+                      {!hasMaxHits && `Last (${lastPage})`}
+                    </a>
+                  )}
+                </>
               )}
             </>
-          )}
+          </span>
         </>
       )}
     </div>

frontend/src/components/PlaylistList.tsx

@@ -14,7 +14,7 @@ type PlaylistListProps = {
 
 const PlaylistList = ({ playlistList, setRefresh }: PlaylistListProps) => {
   const { userConfig } = useUserConfigStore();
-  const viewLayout = userConfig.view_style_playlist;
+  const viewStyle = userConfig.view_style_playlist;
 
   if (!playlistList || playlistList.length === 0) {
     return <p>No playlists found.</p>;
@@ -24,7 +24,7 @@ const PlaylistList = ({ playlistList, setRefresh }: PlaylistListProps) => {
     <>
       {playlistList.map((playlist: PlaylistType) => {
         return (
-          <div key={playlist.playlist_id} className={`playlist-item ${viewLayout}`}>
+          <div key={playlist.playlist_id} className={`playlist-item ${viewStyle}`}>
             <div className="playlist-thumbnail">
               <Link to={Routes.Playlist(playlist.playlist_id)}>
                 <PlaylistThumbnail
@@ -33,7 +33,7 @@ const PlaylistList = ({ playlistList, setRefresh }: PlaylistListProps) => {
                 />
               </Link>
             </div>
-            <div className={`playlist-desc ${viewLayout}`}>
+            <div className={`playlist-desc ${viewStyle}`}>
               {playlist.playlist_type != 'custom' && (
                 <Link to={Routes.Channel(playlist.playlist_channel_id)}>
                   <h3>{playlist.playlist_channel}</h3>

frontend/src/components/SearchExampleQueries.tsx

@@ -95,8 +95,8 @@ const SearchExampleQueries = () => {
               <span>full:</span> — search in video subtitles
               <ul>
                 <li>
-                  <span>lang:</span> — subtitles language (use two-letter ISO country code, same as
-                  the one from settings page)
+                  <span>lang:</span> — subtitles language (use two-letter ISO 639 language code,
+                  same as the one from settings page)
                 </li>
                 <li>
                   <span>source:</span>

frontend/src/components/VideoList.tsx

@@ -1,9 +1,11 @@
-import { VideoType, ViewLayoutType } from '../pages/Home';
+import { ViewStylesEnum, ViewStylesType } from '../configuration/constants/ViewStyle';
+import { VideoType } from '../pages/Home';
 import VideoListItem from './VideoListItem';
+import VideoListItemTable from './VideoListItemTable';
 
 type VideoListProps = {
   videoList: VideoType[] | undefined;
-  viewLayout: ViewLayoutType;
+  viewStyle: ViewStylesType;
   playlistId?: string;
   showReorderButton?: boolean;
   refreshVideoList: (refresh: boolean) => void;
@@ -11,7 +13,7 @@ type VideoListProps = {
 
 const VideoList = ({
   videoList,
-  viewLayout,
+  viewStyle,
   playlistId,
   showReorderButton = false,
   refreshVideoList,
@@ -20,6 +22,10 @@ const VideoList = ({
     return <p>No videos found.</p>;
   }
 
+  if (viewStyle === ViewStylesEnum.Table) {
+    return <VideoListItemTable videoList={videoList} viewStyle={viewStyle} />;
+  }
+
   return (
     <>
       {videoList.map(video => {
@@ -27,7 +33,7 @@ const VideoList = ({
           <VideoListItem
             key={video.youtube_id}
             video={video}
-            viewLayout={viewLayout}
+            viewStyle={viewStyle}
             playlistId={playlistId}
             showReorderButton={showReorderButton}
             refreshVideoList={refreshVideoList}