update installation wiki links and pages

Changed:
- Switched search form to API
- Additional validations for RYD
- Additional log output for curl ES startup check

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+github: bbilly1
+ko_fi: bbilly1
+custom: https://paypal.me/bbilly1

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -0,0 +1,53 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to help improve this project!
+
+  - type: checkboxes
+    id: latest
+    attributes:
+      label: Latest and Greatest
+      options:
+        - label: I'm running the latest version of Tube Archivist and have read the [release notes](https://github.com/tubearchivist/tubearchivist/releases/latest).
+          required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: On what operating system are you running Tube Archivist on?
+      placeholder: ex. Linux, Unraid, Synology...
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: Your Bug Report
+      value: "## Describe the bug\n\n## Steps To Reproduce\n\n## Expected behavior"
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Anything else?
+      description: |
+        Links? References? Anything that will give us more context about the issue you are encountering!
+
+        Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -0,0 +1,37 @@
+name: Feature Request
+description: Create a new feature request
+title: "[Feature Request]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to help improve this project!
+
+  - type: checkboxes
+    id: already
+    attributes:
+      label: Already implemented?
+      options:
+        - label: I have read through the [wiki](https://github.com/tubearchivist/tubearchivist/wiki).
+          required: true
+        - label: I understand the [scope](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) of this project and am aware of the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations).
+          required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Your Feature Request
+      value: "## Is your feature request related to a problem? Please describe.\n\n## Describe the solution you'd like\n\n## Additional context"
+      placeholder: Tell us what you see!
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: help
+    attributes:
+      label: Your help is needed!
+      description: This project is ambitious as it is, please contribute.
+      options:
+        - label: Yes I can help with this feature request!
+          required: false

.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml

@@ -0,0 +1,45 @@
+name: Installation Help
+description: There is an issue with installing or configuring Tube Archivist
+title: "[Installation]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This is most likely not the best place to ask for installation help. There is a helpful community on [Discord](https://github.com/tubearchivist/tubearchivist#connect).
+
+  - type: checkboxes
+    id: instructions
+    attributes:
+      label: Installation instructions
+      options:
+        - label: I have read and understand the [installation instructions](https://github.com/tubearchivist/tubearchivist#installing-and-updating).
+          required: true
+        - label: My issue is not described in the [potential pitfalls](https://github.com/tubearchivist/tubearchivist#potential-pitfalls) section.
+          required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: On what operating system do you want to install Tube Archivist on?
+      placeholder: ex. Linux, Unraid, Synology...
+    validations:
+      required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Your installation or setup question
+      placeholder: Describe your issue
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+    validations:
+      required: true

.github/workflows/lint_python.yml

@@ -11,7 +11,7 @@ jobs:
                          flake8-comprehensions isort
       - run: black --check --diff --line-length 79 .
       - run: codespell
-      - run: flake8 . --count --max-complexity=12 --max-line-length=79
+      - run: flake8 . --count --max-complexity=10 --max-line-length=79
                       --show-source --statistics
       - run: isort --check-only --line-length 79 --profile black .
       # - run: pip install -r tubearchivist/requirements.txt

CONTRIBUTING.md

@@ -5,13 +5,13 @@ If you haven't already, the best place to start is the README. This will give yo
 
 ## Report a bug
 
-If you notice something is not working as expected, check to see if it has been previously reported in the [open issues](https://github.com/bbilly1/tubearchivist/issues).
+If you notice something is not working as expected, check to see if it has been previously reported in the [open issues](https://github.com/tubearchivist/tubearchivist/issues).
 If it has not yet been disclosed, go ahead and create an issue.  
 If the issue doesn't move forward due to a lack of response, I assume it's solved and will close it after some time to keep the list fresh. 
 
 ## Wiki
 
-The wiki is where all user functions are explained in detail. These pages are mirrored into the **docs** folder of the repo. This allows for pull requests and all other features like regular code. Make any changes there, and I'll sync them with the wiki tab.
+The wiki is where all user functions are documented in detail. These pages are mirrored into the **docs** folder of the repo. This allows for pull requests and all other features like regular code. Make any changes there, and I'll sync them with the wiki tab.
 
 ## Development Environment
 
@@ -19,24 +19,42 @@ I have learned the hard way, that working on a dockerized application outside of
 
 This is my setup I have landed on, YMMV:
 - Clone the repo, work on it with your favorite code editor in your local filesystem. *testing* branch is the where all the changes are happening, might be unstable and is WIP.
-- Then I have a VM on KVM hypervisor running standard Ubuntu Server LTS with docker installed. The VM keeps my projects separate and offers convenient snapshot functionality. The VM also offers ways to simulate lowend environments by limiting CPU cores and memory. But you could also just run docker on your host system.
+- Then I have a VM running standard Ubuntu Server LTS with docker installed. The VM keeps my projects separate and offers convenient snapshot functionality. The VM also offers ways to simulate lowend environments by limiting CPU cores and memory. You can use this [Ansible Docker Ubuntu](https://github.com/bbilly1/ansible-playbooks) playbook to get started quickly. But you could also just run docker on your host system.
 - The `Dockerfile` is structured in a way that the actual application code is in the last layer so rebuilding the image with only code changes utilizes the build cache for everything else and will just take a few seconds.
 - Take a look at the `deploy.sh` file. I have my local DNS resolve `tubearchivist.local` to the IP of the VM for convenience. To deploy the latest changes and rebuild the application to the testing VM run:
 ```bash
 ./deploy.sh test
 ```
-- The command above will also copy the file `tubarchivist/testing.sh` into the working folder of the container. Running this script will install additional debugging tools I regularly use in testing. 
+- The command above will call the docker build command with `--build-arg INSTALL_DEBUG=1` to install additional useful debug tools.
 - The `test` argument takes another optional argument to build for a specific architecture valid options are: `amd64`, `arm64` and `multi`, default is `amd64`.
-- This `deploy.sh` file is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking. 
+- This `deploy.sh` script is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking. 
 
 ## Working with Elasticsearch
-Additionally to the required services as listed in the example docker-compose file, the **Dev Tools** of [Kibana](https://www.elastic.co/guide/en/kibana/current/docker.html) are invaluable for running and testing Elasticsearch queries.  
+Additionally to the required services as listed in the example docker-compose file, the **Dev Tools** of [Kibana](https://www.elastic.co/guide/en/kibana/current/docker.html) are invaluable for running and testing Elasticsearch queries. 
 
-If you want to run queries in on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.
+**Quick start**  
+Generate your access token in Elasitcsearch:
+```bash
+bin/elasticsearch-service-tokens create elastic/kibana kibana
+```
+
+Example docker compose, use same version as for Elasticsearch:
+```yml
+kibana:
+  image: docker.elastic.co/kibana/kibana:0.0.0
+  container_name: kibana
+  environment:
+    - "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
+    - "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
+  ports:
+    - "5601:5601"
+```
+
+If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.
 
 ## Implementing a new feature
 
-Do you see anything on the roadmap that you would like to take a closer look at but you are not sure, what's the best way to tackle that? Or anything not on there yet you'd like to implement but are not sure how? Open up an issue and we try to find a solution together.
+Do you see anything on the roadmap that you would like to take a closer look at but you are not sure, what's the best way to tackle that? Or anything not on there yet you'd like to implement but are not sure how? Reach out on Discord and we'll look into it together.
 
 ## Making changes
 
@@ -53,10 +71,10 @@ If you want to see what's in your container, checkout the matching release tag.
 
 ## Code formatting and linting
 
-To keep things clean and consistent for everybody, there is a github action setup to lint and check the changes. You can test your code locally first if you want. For example if you made changes in the **download** module, run
+To keep things clean and consistent for everybody, there is a github action setup to lint and check the changes. You can test your code locally first if you want. For example if you made changes in the **video** module, run
 
 ```shell
-./deploy.sh validate tubearchivist/home/src/download.py
+./deploy.sh validate tubearchivist/home/src/index/video.py
 ```
 
-to validate your changes. If you omit the path, all the project files will get checked. This is subject to change as the codebase improves. 
+to validate your changes. If you omit the path, all the project files will get checked. This is subject to change as the codebase improves. 

Dockerfile

@@ -1,16 +1,36 @@
-# build the tube archivist image from default python slim image
+# multi stage to build tube archivist
+# first stage to build python wheel, copy into final image
 
-FROM python:3.10.2-slim-bullseye
+
+# First stage to build python wheel
+FROM python:3.10.5-slim-bullseye AS builder
 ARG TARGETPLATFORM
 
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+
+# install requirements
+COPY ./tubearchivist/requirements.txt /requirements.txt
+RUN pip install --user -r requirements.txt
+
+# build final image
+FROM python:3.10.5-slim-bullseye as tubearchivist
+
+ARG TARGETPLATFORM
+ARG INSTALL_DEBUG
+
 ENV PYTHONUNBUFFERED 1
 
+# copy build requirements
+COPY --from=builder /root/.local /root/.local
+ENV PATH=/root/.local/bin:$PATH
+
 # install distro packages needed
 RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
-    build-essential \
     nginx \
     atomicparsley \
-    curl && rm -rf /var/lib/apt/lists/*
+    curl \
+    xz-utils && rm -rf /var/lib/apt/lists/*
 
 # get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
@@ -26,17 +46,19 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
         apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
     ; fi
 
-# make folders
-RUN mkdir /cache
-RUN mkdir /youtube
-RUN mkdir /app
+# install debug tools for testing environment
+RUN if [ "$INSTALL_DEBUG" ] ; then \
+        apt-get -y update && apt-get -y install --no-install-recommends \
+        vim htop bmon net-tools iputils-ping procps \
+        && pip install --user ipython \
+    ; fi
 
-# install python dependencies
-COPY ./tubearchivist/requirements.txt /requirements.txt
-RUN pip install --no-cache-dir -r requirements.txt --src /usr/local/src
+# make folders
+RUN mkdir /cache /youtube /app
 
 # copy config files
-COPY docker_assets/nginx.conf /etc/nginx/conf.d/
+COPY docker_assets/nginx.conf /etc/nginx/sites-available/default
+RUN sed -i 's/^user www\-data\;$/user root\;/' /etc/nginx/nginx.conf
 
 # copy application into container
 COPY ./tubearchivist /app

README.md

@@ -1,13 +1,19 @@
 ![Tube Archivist](assets/tube-archivist-banner.jpg?raw=true "Tube Archivist Banner")  
 
-<center><h1>Your self hosted YouTube media server</h1></center>
+<h1 align="center">Your self hosted YouTube media server</h1>
+<div align="center">
+<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-docker.png" alt="tubearchivist-docker" title="Tube Archivist Docker Pulls" height="50" width="200"/></a>
+<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-star.png" alt="tubearchivist-github-star" title="Tube Archivist GitHub Stars" height="50" width="200"/></a>
+<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-forks.png" alt="tubearchivist-github-forks" title="Tube Archivist GitHub Forks" height="50" width="200"/></a>
+</div>
 
 ## Table of contents:
-* [Wiki](https://github.com/bbilly1/tubearchivist/wiki) for a detailed documentation, with [FAQ](https://github.com/bbilly1/tubearchivist/wiki/FAQ)
+* [Wiki](https://github.com/tubearchivist/tubearchivist/wiki) with [FAQ](https://github.com/tubearchivist/tubearchivist/wiki/FAQ)
 * [Core functionality](#core-functionality)
 * [Screenshots](#screenshots)
 * [Problem Tube Archivist tries to solve](#problem-tube-archivist-tries-to-solve)
 * [Connect](#connect)
+* [Extended Universe](#extended-universe)
 * [Installing and updating](#installing-and-updating)
 * [Getting Started](#getting-started)
 * [Potential pitfalls](#potential-pitfalls)
@@ -24,6 +30,9 @@
 * Play videos
 * Keep track of viewed and unviewed videos
 
+## Tube Archivist on YouTube
+[![ibracorp-youtube-video-thumb](assets/tube-archivist-ibracorp-O8H8Z01c0Ys.jpg)](https://www.youtube.com/watch?v=O8H8Z01c0Ys)
+
 ## Screenshots
 ![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")  
 *Home Page*
@@ -44,8 +53,12 @@
 Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface.
 
 ## Connect
-- [Discord](https://discord.gg/AFwz8nE7BK): Connect with us on our brand new Discord server.
-- [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our brand new Subreddit.
+- [Discord](https://discord.gg/AFwz8nE7BK): Connect with us on our Discord server.
+- [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit.
+
+## Extended Universe
+- [Browser Extension](https://github.com/tubearchivist/browser-extension) Tube Archivist Companion, for [Firefox](https://addons.mozilla.org/addon/tubearchivist-companion/) and [Chrome](https://chrome.google.com/webstore/detail/tubearchivist-companion/jjnkmicfnfojkkgobdfeieblocadmcie)
+- [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.
 
 ## Installing and updating
 Take a look at the example `docker-compose.yml` file provided. Use the *latest* or the named semantic version tag. The *unstable* tag is for intermediate testing and as the name implies, is **unstable** and not be used on your main installation but in a [testing environment](CONTRIBUTING.md).  
@@ -59,6 +72,7 @@ The main Python application that displays and serves your video collection, buil
   - And another volume to save application data at **/cache**.
   - The environment variables `ES_URL` and `REDIS_HOST` are needed to tell Tube Archivist where Elasticsearch and Redis respectively are located.
   - The environment variables `HOST_UID` and `HOST_GID` allows Tube Archivist to `chown` the video files to the main host system user instead of the container user. Those two variables are optional, not setting them will disable that functionality. That might be needed if the underlying filesystem doesn't support `chown` like *NFS*. 
+  - Set the environment variable `TA_HOST` to match with the system running Tube Archivist. This can be a domain like *example.com*, a subdomain like *ta.example.com* or an IP address like *192.168.1.20*, add without the protocol and without the port. You can add multiple hostnames separated with a space.
   - Change the environment variables `TA_USERNAME` and `TA_PASSWORD` to create the initial credentials. 
   - `ELASTIC_PASSWORD` is for the password for Elasticsearch. The environment variable `ELASTIC_USER` is optional, should you want to change the username from the default *elastic*.
   - For the scheduler to know what time it is, set your timezone with the `TZ` environment variable, defaults to *UTC*.
@@ -72,7 +86,23 @@ Should that not be an option, the Tube Archivist container takes these two addit
 
 Changing any of these two environment variables will change the files *nginx.conf* and *uwsgi.ini* at startup using `sed` in your container.
 
+### LDAP Authentication
+You can configure LDAP with the following environment variables:
+
+ - `TA_LDAP` (ex: `true`) Set to anything besides empty string to use LDAP authentication **instead** of local user authentication.
+ - `TA_LDAP_SERVER_URI` (ex: `ldap://ldap-server:389`) Set to the uri of your LDAP server.
+ - `TA_LDAP_BIND_DN` (ex: `uid=search-user,ou=users,dc=your-server`) DN of the user that is able to perform searches on your LDAP account.
+ - `TA_LDAP_BIND_PASSWORD` (ex: `yoursecretpassword`) Password for the search user.
+ - `TA_LDAP_USER_BASE` (ex: `ou=users,dc=your-server`) Search base for user filter.
+ - `TA_LDAP_USER_FILTER` (ex: `(objectClass=user)`) Filter for valid users. Login usernames are automatically matched using `uid` and does not need to be specified in this filter.
+
+When LDAP authentication is enabled, django passwords (e.g. the password defined in TA_PASSWORD), will not allow you to login, only the LDAP server is used.
+
 ### Elasticsearch
+**Note**: Tube Archivist depends on Elasticsearch 8. 
+
+Use `bbilly1/tubearchivist-es` to automatically get the recommended version, or use the official image with the version tag in the docker-compose file.
+
 Stores video meta data and makes everything searchable. Also keeps track of the download queue.
   - Needs to be accessible over the default port `9200`
   - Needs a volume at **/usr/share/elasticsearch/data** to store data
@@ -93,13 +123,12 @@ For some architectures it might be required to run Redis JSON on a nonstandard p
 ### Updating Tube Archivist
 You will see the current version number of **Tube Archivist** in the footer of the interface so you can compare it with the latest release to make sure you are running the *latest and greatest*.  
 * There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.  
-* All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned.
+* All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version.
 
 ### Alternative installation instructions:
 - **arm64**: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, you can use `bbilly1/rejson`, an unofficial rebuild for arm64.
-- **Synology**: There is a [discussion thread](https://github.com/bbilly1/tubearchivist/discussions/48) with Synology installation instructions.
-- **Unraid**: The three containers needed are all in the Community Applications. First install `TubeArchivist RedisJSON` followed by `TubeArchivist ES`, and finally you can install `TubeArchivist`. If you have unraid specific issues, report those to the [support thread](https://forums.unraid.net/topic/114073-support-crocs-tube-archivist/ "support thread").
 - **Helm Chart**: There is a Helm Chart available at https://github.com/insuusvenerati/helm-charts. Mostly self-explanatory but feel free to ask questions in the discord / subreddit.
+- **Wiki**: There are additional helpful installation instructions in the [wiki](https://github.com/tubearchivist/tubearchivist/wiki/Installation) for Unraid, Truenas and Synology.
 
 
 ## Potential pitfalls
@@ -117,15 +146,17 @@ To apply the change permanently depends on your host operating system:
 - On any other platform look up in the documentation on how to pass kernel parameters.
 
 ### Permissions for elasticsearch
-If you see a message similar to `AccessDeniedException[/usr/share/elasticsearch/data/nodes]` when initially starting elasticsearch, that means the container is not allowed to write files to the volume.  
-That's most likely the case when you run `docker-compose` as an unprivileged user. To fix that issue, shutdown the container and on your host machine run:
+If you see a message similar to `failed to obtain node locks, tried [/usr/share/elasticsearch/data]` and `maybe these locations are not writable` when initially starting elasticsearch, that probably means the container is not allowed to write files to the volume.  
+To fix that issue, shutdown the container and on your host machine run:
 ```
-chown 1000:0 /path/to/mount/point
+chown 1000:0 -R /path/to/mount/point
 ```
-This will match the permissions with the **UID** and **GID** of elasticsearch within the container and should fix the issue.
+This will match the permissions with the **UID** and **GID** of elasticsearch process within the container and should fix the issue.
 
 ### Disk usage
-The Elasticsearch index will turn to *read only* if the disk usage of the container goes above 95% until the usage drops below 90% again. Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
+The Elasticsearch index will turn to *read only* if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`, [link](https://github.com/tubearchivist/tubearchivist#disk-usage).  
+
+Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
 
 ## Getting Started
 1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS and some other browsers a compatible format must be specified. For example:
@@ -138,24 +169,30 @@ bestvideo[VCODEC=avc1]+bestaudio[ACODEC=mp4a]/mp4
 5. Enjoy your archived collection!
   
 ## Roadmap
-We have come far, nonetheless we are not short of ideas on how to improve and extend this project:
+We have come far, nonetheless we are not short of ideas on how to improve and extend this project. Issues waiting for you to be tackled in no particular order:
 
 - [ ] User roles
 - [ ] Podcast mode to serve channel as mp3
 - [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage
-- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications
-- [ ] Add [SponsorBlock](https://sponsor.ajay.app/) integration
-- [ ] Implement per channel settings
-- [ ] User created playlists
-- [ ] Auto play or play next link
-- [ ] SSO / LDAP support
+- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/tubearchivist/tubearchivist/issues/97))
+- [ ] User created playlists, random and repeat controls ([#108](https://github.com/tubearchivist/tubearchivist/issues/108), [#220](https://github.com/tubearchivist/tubearchivist/issues/220))
+- [ ] Auto play or play next link ([#226](https://github.com/tubearchivist/tubearchivist/issues/226))
 - [ ] Show similar videos on video page
 - [ ] Multi language support
 - [ ] Show total video downloaded vs total videos available in channel
-- [ ] Make items in grid row configurable
 - [ ] Add statistics of index
+- [ ] Download speed schedule ([#198](https://github.com/tubearchivist/tubearchivist/issues/198))
+- [ ] Implement complete offline media file import from json file ([#138](https://github.com/tubearchivist/tubearchivist/issues/138))
+- [ ] Auto ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163))
+- [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144))
+- [ ] Download video comments
 
 Implemented:
+- [X] Filter and query in search form, search by url query [2022-07-23]
+- [X] Make items in grid row configurable to use more of the screen [2022-06-04]
+- [X] Add passing browser cookies to yt-dlp [2022-05-08]
+- [X] Add [SponsorBlock](https://sponsor.ajay.app/) integration [2022-04-16]
+- [X] Implement per channel settings [2022-03-26]
 - [X] Subtitle download & indexing [2022-02-13]
 - [X] Fancy advanced unified search interface [2022-01-08]
 - [X] Auto rescan and auto download on a schedule [2021-12-17]
@@ -174,13 +211,22 @@ Implemented:
 ## Known limitations
 - Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
 - Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
-- For now this is meant to be run in a trusted network environment. Not everything is properly authenticated.
 - There is currently no flexibility in naming of the media files.
 
 
 ## Donate
 The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.  
 Second best way to support the development is to provide for caffeinated beverages:
+* [GitHub Sponsor](https://github.com/sponsors/bbilly1) become a sponsor here on GitHub
 * [Paypal.me](https://paypal.me/bbilly1) for a one time coffee
 * [Paypal Subscription](https://www.paypal.com/webapps/billing/plans/subscribe?plan_id=P-03770005GR991451KMFGVPMQ) for a monthly coffee
 * [ko-fi.com](https://ko-fi.com/bbilly1) for an alternative platform
+
+
+## Sponsor
+Big thank you to [Digitalocean](https://www.digitalocean.com/) for generously donating credit for the tubearchivist.com VPS and buildserver. 
+<p>
+  <a href="https://www.digitalocean.com/">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/PoweredByDO/DO_Powered_by_Badge_blue.svg" width="201px">
+  </a>
+</p>

browser_extension/README.md

@@ -1,26 +0,0 @@
-# Tube Archivist Companion
-A browser extension to directly add videos from YouTube to Tube Archivist.
-
-## MVP or better *bearly viable product*
-This is a proof of concept with the following functionality:
-- Add your Tube Archivist connection details in the addon popup
-- Inject a download button into youtube search results page
-- Clicking the button will automatically add the video to the your download queue
-
-## Test this extension
-- Firefox
-  - Open `about:debugging#/runtime/this-firefox`
-  - Click on *Load Temporary Add-on*
-  - Select the *manifest.json* file to load the addon. 
-- Chrome / Chromium
-  - Open `chrome://extensions/`
-  - Toggle *Developer mode* on top right
-  - Click on *Load unpacked*
-  - Open the folder containing the *manifest.json* file.
-
-## Help needed
-This is only minimally useful in this state. Join us on our Discord and please help us improve that.
-
-## Note:
-- For mysterious reasons sometimes the download buttons will only load when refreshing the YouTube search page and not on first load... Hence: Help needed!
-- For your testing environment only for now: Point the extension to the newest *unstable* build.

browser_extension/extension/background.js

@@ -1,94 +0,0 @@
-/*
-extension background script listening for events
-*/
-
-console.log("running background.js");
-
-let browserType = getBrowser();
-
-
-// boilerplate to dedect browser type api
-function getBrowser() {
-    if (typeof chrome !== "undefined") {
-        if (typeof browser !== "undefined") {
-            console.log("detected firefox");
-            return browser;
-        } else {
-            console.log("detected chrome");
-            return chrome;
-        }
-    } else {
-        console.log("failed to dedect browser");
-        throw "browser detection error"
-    };
-}
-
-
-// send post request to API backend
-async function sendPayload(url, token, payload) {
-
-    const rawResponse = await fetch(url, {
-        method: "POST",
-        headers: {
-            "Accept": "application/json",
-            "Content-Type": "application/json",
-            "Authorization": token,
-            "mode": "no-cors"
-        },
-        body: JSON.stringify(payload)
-    });
-
-    const content = await rawResponse.json();
-    return content;
-}
-
-
-// read access storage and send
-function forwardRequest(payload) {
-
-    console.log("running forwardRequest");
-
-    function onGot(item) {
-        console.log(item.access);
-
-        const url = `${item.access.url}:${item.access.port}/api/download/`;
-        console.log(`sending to ${url}`);
-        const token = `Token ${item.access.apiKey}`;
-
-        sendPayload(url, token, payload).then(content => {
-            console.log(content);
-        })
-
-    };
-
-    function onError(error) {
-        console.local("failed to get access details");
-        console.log(`Error: ${error}`);
-    };
-
-    browserType.storage.local.get("access", function(result) {
-        onGot(result)
-    });
-
-}
-
-
-// listen for messages
-browserType.runtime.onMessage.addListener(
-    function(request, sender, sendResponse) {
-        console.log("responding from background.js listener");
-        console.log(JSON.stringify(request));
-        if (request.download) {
-            console.log("found new download task");
-            let payload = {
-                "data": [
-                    {
-                        "youtube_id": request.download["videoId"],
-                        "status": "pending",
-                    }
-                ]
-            }
-            forwardRequest(payload);
-        };
-    }
-);

browser_extension/extension/images/logo.svg

@@ -1,102 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 26.0.2, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 350 75" style="enable-background:new 0 0 350 75;" xml:space="preserve">
-<style type="text/css">
-	.st0{fill:#FFFFFF;}
-	.st1{fill:#2C9D8A;}
-</style>
-<g id="Layer_1_00000061458428874772671790000018384579824765206440_">
-	<g id="archivist">
-		<path class="st0" d="M218.5,15.2v4.6c-0.7-0.5-1.5-0.8-2.5-0.8c-1.8,0-3,1.4-3.9,3.1v8.9h-4v-16h4v3c1.1-1.9,2.6-3.3,4.5-3.3
-			C217.3,14.7,217.9,14.9,218.5,15.2z"/>
-		<path class="st0" d="M233.6,16.3V20c-1.1-0.8-2.4-1.4-4.4-1.4c-2.4,0-4.5,1.9-4.5,4.5c0,2.6,1.9,4.5,4.5,4.5c2,0,3.3-0.6,4.4-1.4
-			v3.7c-0.9,0.8-2.3,1.6-4.7,1.6c-4.8,0-8.4-3.9-8.4-8.4c0-4.5,3.6-8.5,8.4-8.5C231.3,14.6,232.7,15.6,233.6,16.3z"/>
-		<path class="st0" d="M251.8,21.2v9.8h-4l0-9.3c0-2.2-1.1-3.2-2.9-3.2c-1.4,0-2.7,1.1-3.6,2.3v10.1h-4V8.2h4v9.1
-			c1-1.3,2.7-2.7,4.8-2.7C249.3,14.6,251.8,16.9,251.8,21.2z"/>
-		<path class="st0" d="M262.1,31.1h-4V18.5h-2.5v-3.4h6.5V31.1z M256.9,10.1c0-1.5,1.2-2.6,2.6-2.6s2.7,1.2,2.7,2.6
-			c0,1.5-1.2,2.7-2.7,2.7S256.9,11.6,256.9,10.1z"/>
-		<path class="st0" d="M272.2,31.1l-7.5-16h4.7l4,9.9l4.2-9.9h4.7l-7.6,16H272.2z"/>
-		<path class="st0" d="M290.8,31.1h-4V18.5h-2.5v-3.4h6.5V31.1z M285.6,10.1c0-1.5,1.2-2.6,2.6-2.6c1.5,0,2.7,1.2,2.7,2.6
-			c0,1.5-1.2,2.7-2.7,2.7C286.8,12.8,285.6,11.6,285.6,10.1z"/>
-		<path class="st0" d="M305.5,16.1v3.5c-1-0.7-2.5-1.5-4.1-1.5c-1.5,0-2.5,0.8-2.5,1.7c0,0.7,0.4,1.2,1.8,1.7l1.8,0.6
-			c2.5,0.7,3.6,2.2,3.6,4.2c0,2.9-2.8,5.1-6.2,5.1c-2.2,0-4.4-0.8-5.5-1.6v-3.5c1.2,0.8,3.4,1.6,4.9,1.6c1.6,0,2.5-0.6,2.5-1.6
-			c0-0.8-0.7-1.2-2.1-1.6l-2-0.7c-2.4-0.8-3.2-2.3-3.2-4.3c0-3,2.5-5.1,6.1-5.1C302.7,14.6,304.5,15.3,305.5,16.1z"/>
-		<path class="st0" d="M319.8,15.1v3.4h-4.4v6.9c0,1.4,0.7,2.3,2,2.3c1.1,0,1.9-0.3,2.6-0.7v3.2c-0.8,0.8-1.9,1.4-3.8,1.4
-			c-2.9,0-4.8-2-4.8-5.2v-8h-2.9v-2.1c2.1-0.7,4.2-2.7,5.2-6h1.7v4.7H319.8z"/>
-		<polygon id="A" class="st0" points="183.9,31.1 192.7,8 196.6,8 205.4,31.1 201,31.1 194.7,13.5 188.3,31.1 		"/>
-		<path class="st1" d="M197.7,26.2l-1.8,1.6v-5.2v-0.7v-1.8c0-0.8-0.6-1.4-1.3-1.4s-1.3,0.6-1.3,1.4v1.8v0.7v5.2l-1.8-1.6l-1.2,1.2
-			l4.3,3.9l4.3-3.9L197.7,26.2z"/>
-	</g>
-	<g id="tube">
-		<g id="T_00000039852614045324270430000005353782644941079486_">
-			<polygon class="st0" points="94.7,8.3 94.7,12.3 101.3,12.3 101.3,18.4 105.5,18.4 105.5,12.3 111.9,12.3 111.9,8.3 			"/>
-			<rect x="101.3" y="22.6" class="st0" width="4.1" height="9"/>
-			<path class="st1" d="M105.8,21.7H101c-0.7,0-1.3-0.6-1.3-1.3l0,0c0-0.7,0.6-1.3,1.3-1.3h4.8c0.7,0,1.3,0.6,1.3,1.3l0,0
-				C107.1,21.1,106.5,21.7,105.8,21.7z"/>
-		</g>
-		<path class="st0" d="M127.4,24.5c0,4.2-3,6.8-7.1,6.8c-4.1,0-7.1-2.6-7.1-6.8v-9.4h4l0,8.7c0,2.4,1.3,3.7,3.1,3.7
-			c1.8,0,3.1-1.2,3.1-3.7v-8.7h4V24.5z"/>
-		<path class="st0" d="M136,17.3c1-1.8,2.8-3.1,5.2-3.1c3.9,0,7.3,3,7.3,8.4c0,5.4-3.4,8.7-7.5,8.7c-1.8,0-3.7-1-4.9-2.3v1.8h-4.1
-			V7.7h4.1V17.3z M136,22.9v2.3c1.2,1.2,2.5,2.1,4.1,2.1c2.5,0,4.1-2.3,4.1-4.7c0-2.4-1.6-4.3-4-4.3C138,18.3,135.9,20,136,22.9z"/>
-		<path class="st0" d="M167,24.1h-11.7c0.4,2.2,2.3,3.7,4.8,3.7c2.4,0,3.9-1,4.9-1.6v3.7c-0.9,0.7-2.6,1.6-5.1,1.6
-			c-4.8,0-8.7-3.5-8.7-8.4c0-4.7,3.6-8.7,8.2-8.7c4.8,0,7.6,3.3,7.6,8.1V24.1z M163.1,21.5c-0.2-2.3-2-3.5-3.8-3.5
-			c-2.1,0-3.5,1.4-3.9,3.5H163.1z"/>
-	</g>
-	<g id="icon">
-		<path class="st1" d="M27.6,33.7l1,0.4l0.5-2l-6.7-2l-0.7,2l0.9,0.4C20.9,44.4,25,54.5,34,61.9l2.9-4
-			C29.6,51.4,25.8,43.2,27.6,33.7z"/>
-		<path class="st1" d="M41.2,60.5l0.6-0.8L40,58.4L35.7,64l1.8,1.3l0.6-0.7c10.7,5.7,21.9,4.7,31.8-1.5L66.8,59
-			C58.3,63.9,49.9,65.3,41.2,60.5z"/>
-		<path class="st1" d="M70.8,55.6l-0.7-0.7l-1.8,1.3l4.3,5.9l1.8-1.3L73.9,60c8.7-8.5,11.2-18.7,8.4-30l-4.9,1.4
-			C79.3,40.9,78,48.7,70.8,55.6z"/>
-		<path class="st1" d="M75.5,26.5l-0.9,0.4l0.6,2.4l7.2-2.1l-0.7-2.4l-0.9,0.3c-5.7-10.7-15-17.4-26.6-17.9v5.4
-			C63.8,13.4,70.9,17.6,75.5,26.5z"/>
-		<path class="st1" d="M49,12.8l0.1,0.9l2.3-0.1v-7h-2.3v0.9C40,8.8,28.6,14.8,24,27.6l4.9,1.4C32.4,19,40.2,14.7,49,12.8z"/>
-		<polygon id="A_00000111884739883312047940000001222474654974762168_" class="st0" points="48,54 60.6,20.9 66.1,20.9 78.8,54 
-			72.5,54 63.4,28.8 54.3,54 		"/>
-		<g id="T">
-			<polygon class="st0" points="27.6,20.4 27.6,26.1 37.1,26.1 37.1,35 43.1,35 43.1,26.1 52.4,26.1 52.4,20.4 			"/>
-			<rect x="37.1" y="41" class="st0" width="6" height="13"/>
-		</g>
-		<path class="st1" d="M43.5,39.7h-6.9c-1,0-1.9-0.8-1.9-1.9l0,0c0-1,0.8-1.9,1.9-1.9h6.9c1,0,1.9,0.8,1.9,1.9l0,0
-			C45.4,38.9,44.5,39.7,43.5,39.7z"/>
-		<path class="st1" d="M67.8,47l-2.5,2.3v-7.5v-1v-2.5c0-1.1-0.8-2-1.9-2s-1.9,0.9-1.9,2v2.5v1v7.5L58.9,47l-1.7,1.7l6.2,5.6
-			l6.1-5.6L67.8,47z"/>
-	</g>
-</g>
-<g id="Layer_2">
-	<g>
-		<path class="st0" d="M142.1,58c0,6-4.3,9.6-10.1,9.6c-5.8,0-10.1-3.6-10.1-9.6V44.7h5.7l0,12.3c0,3.4,1.8,5.2,4.4,5.2
-			c2.6,0,4.4-1.7,4.4-5.2V44.7h5.7V58z"/>
-		<path class="st0" d="M154.1,48.5c1.3-2.4,3.8-4.3,7.2-4.3c5.4,0,10,4.1,10,11.6c0,7.4-4.6,12-10.4,12c-2.4,0-5.1-1.3-6.8-3.2V67
-			h-5.6V35.2h5.6V48.5z M154.1,56.1v3.2c1.6,1.7,3.4,2.9,5.6,2.9c3.5,0,5.7-3.2,5.7-6.5c0-3.4-2.2-6-5.6-6
-			C156.9,49.7,154.1,52.1,154.1,56.1z"/>
-		<path class="st0" d="M190.4,46.2v5c-1.4-0.9-3.5-2-5.7-2c-2.1,0-3.4,1.1-3.4,2.4c0,1,0.6,1.7,2.6,2.3l2.5,0.8c3.4,1,5.1,3,5.1,5.9
-			c0,4.1-3.9,7.2-8.7,7.2c-3,0-6.1-1.1-7.6-2.3v-4.9c1.7,1.1,4.7,2.3,6.9,2.3c2.3,0,3.5-0.8,3.5-2.3c0-1.2-1-1.6-3-2.3l-2.7-1
-			c-3.4-1.1-4.5-3.2-4.5-6.1c0-4.2,3.5-7.1,8.6-7.1C186.6,44.1,189.2,45.1,190.4,46.2z"/>
-		<path class="st0" d="M212.9,46.5v5.2c-1.5-1.1-3.4-2-6.1-2c-3.4,0-6.2,2.7-6.2,6.3s2.6,6.3,6.2,6.3c2.7,0,4.6-0.8,6.1-1.9v5.1
-			c-1.2,1.1-3.3,2.3-6.6,2.3c-6.7,0-11.7-5.4-11.7-11.7s5-11.9,11.7-11.9C209.7,44.1,211.7,45.5,212.9,46.5z"/>
-		<path class="st0" d="M232.5,44.9v6.4c-1-0.7-2.1-1.1-3.4-1.1c-2.6,0-4.2,1.9-5.5,4.4V67h-5.6V44.7h5.6v4.2
-			c1.5-2.7,3.6-4.6,6.3-4.6C230.9,44.3,231.7,44.5,232.5,44.9z"/>
-		<path class="st0" d="M244.7,67h-5.6V49.5h-3.5v-4.8h9.1V67z M237.5,37.8c0-2,1.6-3.7,3.7-3.7s3.7,1.6,3.7,3.7s-1.7,3.7-3.7,3.7
-			S237.5,39.8,237.5,37.8z"/>
-		<path class="st0" d="M257,48.5c1.3-2.4,3.8-4.3,7.2-4.3c5.4,0,10,4.1,10,11.6c0,7.4-4.6,12-10.4,12c-2.4,0-5.1-1.3-6.8-3.2V67
-			h-5.6V35.2h5.6V48.5z M257,56.1v3.2c1.6,1.7,3.4,2.9,5.6,2.9c3.5,0,5.7-3.2,5.7-6.5c0-3.4-2.2-6-5.6-6
-			C259.8,49.7,257,52.1,257,56.1z"/>
-		<path class="st0" d="M299.8,57.6h-16.2c0.6,3.1,3.1,5,6.5,5c3.3,0,5.4-1.4,6.8-2.3v5.1c-1.2,0.9-3.6,2.3-7,2.3
-			c-6.6,0-12-4.9-12-11.6c0-6.5,5-12,11.3-12c6.6,0,10.5,4.6,10.5,11.1V57.6z M294.4,54c-0.2-3.2-2.7-4.8-5.3-4.8
-			c-2.9,0-4.8,1.9-5.4,4.8H294.4z"/>
-		<path class="st0" d="M319.5,44.9v6.4c-1-0.7-2.1-1.1-3.4-1.1c-2.6,0-4.2,1.9-5.5,4.4V67H305V44.7h5.6v4.2c1.5-2.7,3.6-4.6,6.3-4.6
-			C317.9,44.3,318.7,44.5,319.5,44.9z"/>
-	</g>
-	<g>
-		<path class="st0" d="M102.6,54l2.4,0.9v-6.1c0-0.2,0-0.3,0.1-0.5c-1.2-0.6-1.9-1.4-1.9-2.6c0-2.3,2.6-3.2,4.9-3.2
-			c3.1,0,5.4,1.3,7.5,2.7v-6.1c-1.9-1.2-4.8-2.3-8.3-2.3c-5.5,0-10.5,3.5-10.5,9C96.7,49.3,98,52.3,102.6,54z"/>
-		<path class="st0" d="M111.1,51l-1.9-0.8v6.6c0.8,0.5,1.3,1.2,1.3,2.1c0,1.9-2,2.9-4.5,2.9c-3.5,0-6.7-1.5-9.3-3.4v6.1
-			c2.4,1.7,5.7,3.1,9.2,3.1c6.1,0,11.2-3.8,11.2-9C117.2,55.3,115.3,52.8,111.1,51z"/>
-	</g>
-</g>
-<path class="st1" d="M105.5,56.1v-6.9c0-1,0.8-1.9,1.6-1.9l0,0c0.9,0,1.6,0.9,1.6,1.9v6.9c0,1-0.8,1.9-1.6,1.9l0,0
-	C106.2,58,105.5,57.1,105.5,56.1z"/>
-</svg>

browser_extension/extension/images/question.svg

@@ -1,16 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 26.0.2, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 374.4 375.1" style="enable-background:new 0 0 374.4 375.1;" xml:space="preserve">
-<style type="text/css">
-	.st0{fill:#FFFFFF;}
-</style>
-<path class="st0" d="M187.5,0C84.2-0.1,0.1,83.8,0,187.2C-0.2,290.8,83.7,375,187.1,375.1c103.3,0,187.2-83.9,187.3-187.4
-	C374.5,84.3,290.7,0.2,187.5,0z M187.9,304.5c-16,0-29.2-13.3-29.2-29.3c0.1-16,13.2-29,29.1-29.1c16,0,29.2,13.3,29.2,29.3
-	C216.9,291.4,203.8,304.5,187.9,304.5z M238.2,195.7c-4.8,2.9-9.7,5.8-14.5,8.7c-9.1,5.5-13.6,13.6-13.9,24.2c0,0.8,0,1.7,0,2.7
-	c-14.6,0-29,0-43.8,0c0.1-2.8,0.2-5.6,0.4-8.5c0.8-10.6,3.2-20.9,8.4-30.3c5.9-11,14.9-18.9,25.5-25.1c4.1-2.4,8.4-4.7,12.5-7.1
-	c12.7-7.7,13.1-19.1,6.2-29.9c-6.5-10.3-16.3-14.9-28.1-15.7c-8-0.5-15.7,0.7-22.6,4.9c-10.1,6.2-15,15.6-16.9,26.9
-	c-0.1,0.4-0.1,0.9-0.2,1.6c-14.5-1.9-28.9-3.8-43.9-5.8c1.4-5.4,2.5-10.8,4.2-15.9c5.2-14.9,13.6-27.8,25.8-38
-	c11.1-9.3,23.9-14.7,38.2-16.6c2.3-0.3,4.6-0.6,6.9-1c3.2,0,6.4,0,9.6,0c4.4,0.6,8.9,1.1,13.3,1.9c29.8,5.9,55.4,28.1,61.4,59.3
-	C271.5,157.5,260.7,182.2,238.2,195.7z"/>
-</svg>

browser_extension/extension/images/social/discord.svg

@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 26.0.2, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 384 384" style="enable-background:new 0 0 384 384;" xml:space="preserve">
-<style type="text/css">
-	.st0{fill:#FFFFFF;}
-</style>
-<g>
-	<path class="st0" d="M191.57,379.5C88.15,379.44,4.32,295.25,4.46,191.57C4.6,88.18,88.66,4.28,191.95,4.42
-		c103.29,0.14,187.02,84.23,186.93,187.72C378.8,295.6,294.86,379.55,191.57,379.5z M244.55,271.24
-		c4.89,5.91,9.44,11.19,13.73,16.69c1.8,2.32,3.72,3.11,6.64,2.93c28.39-1.75,53.17-11.56,72.96-32.55c1.12-1.19,2.08-3.2,1.97-4.76
-		c-0.79-11.71-1.27-23.48-2.97-35.07c-5.07-34.65-16.23-67.36-32.74-98.23c-0.73-1.36-1.88-2.67-3.15-3.55
-		c-19.34-13.43-40.61-21.62-64.16-23.63c-3.19-0.27-5.5,0.48-6.89,3.09c10.27,3.97,20.59,7.65,30.65,11.94
-		c10.1,4.32,19.64,9.74,28.21,16.92c-64.82-29.49-129.53-29.49-194.34,0.02c17.77-13.75,38.02-22.07,59.27-28.39
-		c-1.74-3.15-4.12-3.86-7.32-3.58c-23.15,2.02-44.13,9.94-63.2,23.08c-1.76,1.21-3.51,2.96-4.4,4.86
-		c-5.36,11.39-11.02,22.69-15.57,34.41c-12.22,31.43-19.35,64-19.75,97.86c-0.02,1.71,0.88,3.85,2.07,5.1
-		c18.74,19.74,41.98,29.72,68.88,32.19c5.73,0.53,9.64-0.46,12.96-5.26c3.36-4.86,7.35-9.28,11.2-14.05
-		c-22.46-8.05-39.37-18.53-42.93-26.17c63.97,36.11,128.17,36.13,192.51-0.02C276.69,258.45,261.71,265.95,244.55,271.24z"/>
-	<path class="st0" d="M237.28,232.74c-14.34-0.02-26.07-12-25.9-26.45c0.17-14.67,11.86-26.25,26.56-26.33
-		c13.63-0.07,25.18,12,25.21,26.35C263.18,220.98,251.66,232.76,237.28,232.74z"/>
-	<path class="st0" d="M146.11,232.74c-14.35,0.05-25.9-11.7-25.92-26.38c-0.02-14.3,11.56-26.46,25.16-26.41
-		c14.85,0.06,26.74,11.96,26.62,26.66C171.84,221.04,160.32,232.69,146.11,232.74z"/>
-</g>
-</svg>

browser_extension/extension/images/social/github.svg

@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 26.0.2, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 384 384" style="enable-background:new 0 0 384 384;" xml:space="preserve">
-<style type="text/css">
-	.st0{fill:#FFFFFF;}
-</style>
-<g>
-	<path class="st0" d="M244.51,375.15c0-10.59-0.31-21.12,0.08-31.61c0.5-13.65,0.02-26.89-10.41-37.88
-		c8.18-2.06,15.59-3.84,22.95-5.83c3.51-0.95,6.95-2.22,10.37-3.47c27.26-9.99,40.88-31.15,45.79-58.73
-		c2.71-15.21,4.11-30.82-0.74-45.88c-2.65-8.23-6.01-16.69-11.08-23.5c-3.83-5.14-3.43-9.05-2.23-14.27
-		c2.86-12.48,0.71-24.71-2.64-36.79c-0.99-3.58-3.09-4.99-6.88-4.6c-11.45,1.18-21.48,6.12-31.34,11.6
-		c-1.42,0.79-2.97,1.46-4.16,2.52c-5.55,4.99-11.4,5.13-18.58,3.42c-25.28-6.04-50.67-5.14-75.85,1.12
-		c-3.59,0.89-6.27,0.27-9.31-1.86c-11.08-7.73-22.87-14.02-36.37-16.31c-0.13-0.02-0.26-0.07-0.4-0.09
-		c-7.12-1.23-8.12-0.76-10.11,6.16c-3.51,12.23-5.18,24.6-1.55,37.14c1.14,3.94,0.44,7.16-2.01,10.61
-		c-23.14,32.46-16.36,80.65,0.11,103.41c12.16,16.81,29.02,26.2,48.63,31.06c5.49,1.36,11.11,2.15,17.1,3.28
-		c-0.24,0.79-0.43,1.81-0.85,2.71c-3.15,6.91-6.27,13.83-9.6,20.65c-0.67,1.38-2.06,3.14-3.34,3.35c-7.3,1.18-14.65,2.46-22.02,2.7
-		c-6.5,0.21-12.09-3.32-16.37-8c-5.11-5.59-9.65-11.73-14.19-17.83c-6.07-8.15-14.2-12.42-24.05-13.3c-2.22-0.2-4.58,1.12-7.99,2.05
-		c2.69,3.23,4.46,6.17,6.96,8.2c10.32,8.35,17.42,19.1,22.26,31.24c4.72,11.85,14.1,16.55,25.31,19.27
-		c9.24,2.24,18.51,1.5,27.83,0.78c6.07-0.47,6.09-0.22,6.09,5.98c0.01,5.58,0,11.17,0,16.81C86.7,368.22,7.24,301.96,4.67,198.28
-		C2.19,97.88,77.65,13.69,175.94,5.34c103.69-8.81,192.95,68.52,202.14,171.23C387.22,278.72,317.17,355.78,244.51,375.15z"/>
-</g>
-</svg>

browser_extension/extension/images/social/reddit.svg

@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 26.0.2, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 384 384" style="enable-background:new 0 0 384 384;" xml:space="preserve">
-<style type="text/css">
-	.st0{fill:#FFFFFF;}
-</style>
-<g>
-	<path class="st0" d="M240.82,274c-3.86,3.78-8.45,6.33-13.43,8.26c-9.77,3.79-19.98,5.34-30.38,5.85
-		c-5.42,0.26-10.85,0.12-16.26-0.44c-5.89-0.61-11.72-1.56-17.42-3.18c-7.65-2.17-14.86-5.18-20.68-10.86
-		c-0.98-0.96-2.26-1.44-3.64-1.48c-2.72-0.08-4.71,1.17-5.94,3.55c-1.14,2.2-0.8,4.87,0.94,6.7c1.28,1.35,2.7,2.59,4.17,3.73
-		c5.88,4.57,12.58,7.49,19.65,9.63c11.11,3.37,22.52,4.66,36.04,4.85c2.63-0.18,7.2-0.42,11.76-0.82c6.75-0.6,13.38-1.86,19.88-3.8
-		c6.56-1.95,12.83-4.56,18.42-8.57c1.94-1.4,3.82-2.93,5.52-4.61c2.44-2.4,2.35-6.62,0.01-8.95
-		C247.1,271.52,243.37,271.51,240.82,274z"/>
-	<path class="st0" d="M192.39,4.72C89.03,4.5,5.07,88.18,4.86,191.61c-0.21,103.43,83.4,187.46,186.75,187.67
-		c103.35,0.21,187.31-83.46,187.53-186.89C379.36,88.96,295.74,4.93,192.39,4.72z M327.66,225.47c0.14,1.32,0.27,2.64,0.41,3.96
-		c0.53,4.98,0.43,9.96-0.23,14.91c-2.24,16.85-10.03,30.98-21.45,43.28c-8.94,9.62-19.42,17.22-30.95,23.42
-		c-13.96,7.52-28.79,12.64-44.3,15.87c-8.46,1.76-17,2.91-25.62,3.51c-6.58,0.46-13.16,0.6-19.75,0.39
-		c-6.8-0.22-13.58-0.8-20.34-1.72c-11.93-1.64-23.6-4.38-34.96-8.39c-14.15-4.99-27.38-11.73-39.34-20.86
-		c-9.74-7.44-18.15-16.12-24.63-26.59c-5.11-8.26-8.64-17.14-10.2-26.75c-1.1-6.76-1.23-13.55-0.14-20.33c0.03-0.21,0-0.44,0-0.7
-		c-10.51-5.25-17.19-13.54-19.32-25.14c-1.71-9.32,0.32-17.97,5.8-25.72c11.73-16.6,35.97-19.53,51.41-4.96
-		c27.77-19.23,59-27.5,92.46-28.92c0.46-2.12,0.92-4.2,1.36-6.28c3.46-16.26,6.92-32.53,10.37-48.79
-		c1.82-8.57,3.63-17.14,5.43-25.71c0.26-1.23,0.59-2.42,1.44-3.4c1.68-1.92,3.83-2.34,6.2-1.85c6.31,1.3,12.62,2.66,18.92,3.99
-		c10.35,2.19,20.7,4.39,31.05,6.59c2,0.42,4,0.84,6.09,1.28c3.7-6.86,9.29-11.3,16.92-12.93c5.54-1.19,10.89-0.41,15.96,2.14
-		c10.03,5.06,15.32,16.56,12.65,27.42c-2.74,11.12-12.68,18.73-24.01,18.65c-11.69-0.08-23.34-9.38-24.16-23.21
-		c-1.43-0.6-49.41-10.81-50.5-10.7c-5.15,24.12-10.32,48.27-15.51,72.57c0.64,0.04,1.07,0.07,1.51,0.09
-		c19.74,0.93,38.91,4.68,57.37,11.79c11.05,4.25,21.5,9.64,31.21,16.44c0.29,0.2,0.6,0.38,0.94,0.59
-		c8.62-7.75,18.62-11.01,30.07-8.89c9.47,1.76,16.97,6.78,22.22,14.87c5.67,8.73,7.06,18.26,4.21,28.25
-		C343.42,213.65,337.03,220.82,327.66,225.47z"/>
-	<path class="st0" d="M162.26,218.68c0.08-13.49-10.62-24.33-24.09-24.41c-13.86,0-24.09,11.31-24.22,23.89
-		c-0.13,12.52,9.75,24.24,23.96,24.42C151.22,242.73,162.19,231.9,162.26,218.68z"/>
-	<path class="st0" d="M245.16,194.56c-13.42,0.03-24.09,10.83-24.11,24.24c-0.02,12.58,10.14,24.01,24.17,24.05
-		c13.33,0.04,24.16-10.89,24.12-24.26C269.32,205.3,258.48,194.52,245.16,194.56z"/>
-</g>
-</svg>

browser_extension/extension/index.html

@@ -1,93 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>TubeArchivist Companion</title>
-    <link rel="stylesheet" href="https://fonts.googleapi.com/css?family=Sen" type="text/css">
-    <style>
-        body {
-            font-family: Sen-Regular, sans-serif;
-            background-color: #00202f;
-            color: #97d4c8;
-        }
-        .container {
-            padding: 10px;
-            min-width: 300px;
-        }
-        hr {
-            height: 1px;
-            color: white;
-            background-color: white;
-        }
-        .login-form {
-            display: grid;
-            grid-template-columns: 1fr 1fr;
-        }
-        .login-form label,
-        .login-form input {
-            margin: 3px 0;
-        }
-        .submit button {
-            margin: 10px auto 15px auto;
-            display: block;
-            border-radius: 0;
-            padding: 5px 13px;
-            border: none;
-            cursor: pointer;
-            background-color: #259485;
-            color: #ffffff;
-        }
-        .submit button:hover {
-            background-color: #97d4c8;
-            transform: scale(1.05);
-            color: #00202f;
-        }
-        .icons {
-            display: flex;
-            grid-template-columns: 1fr 1fr;
-            justify-content: space-between;
-        }
-        .icons img {
-            width: 25px;
-        }
-    </style>
-</head>
-<body>
-    <div class="container">
-        <img src="/images/logo.svg" alt="ta-logo">
-        <hr>
-        <form class="login-form">
-            <label for="url">Tube Archivist Url:</label>
-            <input type="text" id="url" name="url">
-            <label for="port">Tube Archivist Port:</label>
-            <input type="text" id="port" name="port">
-            <label for="api-key">API key:</label>
-            <input type="text" id="api-key" name="api-key">
-        </form>
-        <div class="submit">
-            <button onclick="storeLogin()" id="save-login">Save</button>
-        </div>
-        <div class="icons">
-            <div>
-                <a href="https://www.reddit.com/r/TubeArchivist/" target="_blank">
-                    <img src="/images/social/reddit.svg" alt="reddit-icon">
-                </a>
-                <a href="https://discord.gg/AFwz8nE7BK" target="_blank">
-                    <img src="/images/social/discord.svg" alt="discord-icon">
-                </a>
-                <a href="https://github.com/bbilly1/tubearchivist" target="_blank">
-                    <img src="/images/social/github.svg" alt="github-icon">
-                </a>
-            </div>
-            <div>
-                <a href="#">
-                    <img src="/images/question.svg" alt="question-icon">
-                </a>
-            </div>
-        </div>
-    </div>
-    <script type="text/javascript" src="popup.js"></script>
-</body>
-</html>

browser_extension/extension/manifest.json

@@ -1,25 +0,0 @@
-{
-    "manifest_version": 2,
-    "name": "TubeArchivist Companion",
-    "description": "Interact with your selhosted TA server.",
-    "version": "0.0.1",
-    "icons": {
-        "128": "/images/icon128.png"
-    },
-    "browser_action": {
-        "default_icon": "/images/icon.png",
-        "default_popup": "index.html"
-    },
-    "permissions": [
-        "storage"
-    ],
-    "content_scripts": [
-        {
-            "matches": ["https://www.youtube.com/results*"],
-            "js": ["script.js"]
-        }
-    ],
-    "background": {
-        "scripts": ["background.js"]
-    }
-}

browser_extension/extension/popup.js

@@ -1,63 +0,0 @@
-/*
-Loaded into popup index.html
-*/
-
-let browserType = getBrowser();
-
-// boilerplate to dedect browser type api
-function getBrowser() {
-    if (typeof chrome !== "undefined") {
-        if (typeof browser !== "undefined") {
-            console.log("detected firefox");
-            return browser;
-        } else {
-            console.log("detected chrome");
-            return chrome;
-        }
-    } else {
-        console.log("failed to dedect browser");
-        throw "browser detection error"
-    };
-}
-
-// store access details
-document.getElementById("save-login").addEventListener("click", function () {
-    console.log("save form");
-    let toStore = {
-        "access": {
-            "url": document.getElementById("url").value,
-            "port": document.getElementById("port").value,
-            "apiKey": document.getElementById("api-key").value
-        }
-    };
-    console.log(toStore);
-    browserType.storage.local.set(toStore, function() {
-        console.log("Stored connection details: " + JSON.stringify(toStore));
-    });
-})
-
-// fill in form
-document.addEventListener("DOMContentLoaded", async () => {
-
-    console.log("executing dom loader");
-
-    function onGot(item) {
-        if (!item.access) {
-            console.log("no access details found");
-            return
-        }
-        console.log(item.access);
-        document.getElementById("url").value = item.access.url;
-        document.getElementById("port").value = item.access.port;
-        document.getElementById("api-key").value = item.access.apiKey;
-    };
-    
-    function onError(error) {
-        console.log(`Error: ${error}`);
-    };
-
-    browserType.storage.local.get("access", function(result) {
-        onGot(result)
-    });
-
-})

browser_extension/extension/script.js

@@ -1,72 +0,0 @@
-/*
-content script running on youtube.com
-*/
-
-console.log("running script.js");
-
-let browserType = getBrowser();
-
-setTimeout(function(){
-    console.log("running setimeout")
-    linkFinder();
-    return false;
-}, 2000);
-
-
-// boilerplate to dedect browser type api
-function getBrowser() {
-    if (typeof chrome !== "undefined") {
-        if (typeof browser !== "undefined") {
-            console.log("detected firefox");
-            return browser;
-        } else {
-            console.log("detected chrome");
-            return chrome;
-        }
-    } else {
-        console.log("failed to dedect browser");
-        throw "browser detection error"
-    };
-}
-
-
-// event handler for download task
-function addToDownload(videoId) {
-
-    console.log(`downloading ${videoId}`);
-    let payload = {
-        "download": {
-            "videoId": videoId
-        }
-    };
-
-    browserType.runtime.sendMessage(payload);
-
-}
-
-
-// find relevant links to add a button to
-function linkFinder() {
-
-    console.log("running link finder");
-
-    var allLinks = document.links;
-    for (let i = 0; i < allLinks.length; i++) {
-        
-        const linkItem = allLinks[i];
-        const linkDest = linkItem.getAttribute("href");
-
-        if (linkDest.startsWith("/watch?v=") && linkItem.id == "video-title") {
-            var dlButton = document.createElement("button");
-            dlButton.innerText = "download";
-            var videoId = linkDest.split("=")[1];
-            dlButton.setAttribute("data-id", videoId);
-            dlButton.setAttribute("id", "ta-dl-" + videoId);
-            dlButton.onclick = function(event) {
-                var videoId = this.getAttribute("data-id");
-                addToDownload(videoId);
-            };
-            linkItem.parentElement.appendChild(dlButton);
-        }
-    }
-}

deploy.sh

@@ -12,7 +12,7 @@
 # docker buildx inspect --bootstrap
 
 # more details:
-# https://github.com/bbilly1/tubearchivist/issues/6
+# https://github.com/tubearchivist/tubearchivist/issues/6
 
 set -e
 
@@ -33,7 +33,7 @@ function sync_blackhole {
         . -e ssh "$host":tubearchivist
 
     echo "$PASS" | ssh "$host" 'sudo -S docker buildx build --platform linux/amd64 -t bbilly1/tubearchivist:latest tubearchivist --load 2>/dev/null'
-    echo "$PASS" | ssh "$host" 'sudo -S docker-compose up -d 2>/dev/null'
+    echo "$PASS" | ssh "$host" 'sudo -S docker compose up -d 2>/dev/null'
 
 }
 
@@ -43,7 +43,10 @@ function sync_test {
     # pass argument to build for specific platform
 
     host="tubearchivist.local"
+    # make base folder
+    ssh "$host" "mkdir -p docker"
 
+    # copy project files to build image
     rsync -a --progress --delete-after \
         --exclude ".git" \
         --exclude ".gitignore" \
@@ -52,8 +55,8 @@ function sync_test {
         --exclude "db.sqlite3" \
         . -e ssh "$host":tubearchivist
 
-    # uncomment or copy your own docker-compose file
-    # rsync -r --progress --delete docker-compose.yml -e ssh "$host":docker
+    # copy default docker-compose file if not exist
+    rsync --progress --ignore-existing docker-compose.yml -e ssh "$host":docker
 
     if [[ $1 = "amd64" ]]; then
         platform="linux/amd64"
@@ -65,11 +68,8 @@ function sync_test {
         platform="linux/amd64"
     fi
 
-    ssh "$host" "docker buildx build --platform $platform -t bbilly1/tubearchivist:latest tubearchivist --load"
-    ssh "$host" 'docker-compose -f docker/docker-compose.yml up -d'
-
-    ssh "$host" 'docker cp tubearchivist/tubearchivist/testing.sh tubearchivist:/app/testing.sh'
-    ssh "$host" 'docker exec tubearchivist chmod +x /app/testing.sh'
+    ssh "$host" "docker buildx build --build-arg INSTALL_DEBUG=1 --platform $platform -t bbilly1/tubearchivist:latest tubearchivist --load"
+    ssh "$host" 'docker compose -f docker/docker-compose.yml up -d'
 
 }
 
@@ -91,7 +91,7 @@ function validate {
     echo "running codespell"
     codespell --skip="./.git" "$check_path"
     echo "running flake8"
-    flake8 "$check_path" --count --max-complexity=12 --max-line-length=79 \
+    flake8 "$check_path" --count --max-complexity=10 --max-line-length=79 \
         --show-source --statistics
     echo "running isort"
     isort --check-only --diff --profile black -l 79 "$check_path"
@@ -100,6 +100,34 @@ function validate {
 }
 
 
+# update latest tag compatible es for set and forget
+function sync_latest_es {
+
+    VERSION=$(grep "bbilly1/tubearchivist-es" docker-compose.yml | awk '{print $NF}')
+    printf "\nsync new ES version %s\nContinue?\n" "$VERSION"
+    read -rn 1
+
+    if [[ $(systemctl is-active docker) != 'active' ]]; then
+        echo "starting docker"
+        sudo systemctl start docker
+    fi
+
+    sudo docker image pull docker.elastic.co/elasticsearch/elasticsearch:"$VERSION"
+
+    sudo docker tag \
+        docker.elastic.co/elasticsearch/elasticsearch:"$VERSION" \
+        bbilly1/tubearchivist-es
+
+    sudo docker tag \
+        docker.elastic.co/elasticsearch/elasticsearch:"$VERSION" \
+        bbilly1/tubearchivist-es:"$VERSION"
+
+    sudo docker push bbilly1/tubearchivist-es
+    sudo docker push bbilly1/tubearchivist-es:"$VERSION"
+
+}
+
+
 # publish unstable tag to docker
 function sync_unstable {
 
@@ -116,6 +144,7 @@ function sync_unstable {
 }
 
 
+# new function, sync only tag, build with build server
 function sync_docker {
 
     # check things
@@ -124,16 +153,40 @@ function sync_docker {
         return
     fi
 
+    echo "latest tags:"
+    git tag | tail -n 5 | sort -r
+
+    printf "\ncreate new version:\n"
+    read -r VERSION
+
+    echo "push new tag: $VERSION?"
+    read -rn 1
+
+    # create release tag
+    echo "commits since last version:"
+    git log "$(git describe --tags --abbrev=0)"..HEAD --oneline
+    git tag -a "$VERSION" -m "new release version $VERSION"
+    git push origin "$VERSION"
+
+}
+
+
+# old builder, sync tag, build and push locally
+function sync_docker_old {
+
+    # check things
+    if [[ $(git branch --show-current) != 'master' ]]; then
+        echo 'you are not on master, dummy!'
+        return
+    fi
+
     if [[ $(systemctl is-active docker) != 'active' ]]; then
         echo "starting docker"
         sudo systemctl start docker
     fi
 
     echo "latest tags:"
-    git tag
-
-    echo "latest docker images:"
-    sudo docker image ls bbilly1/tubearchivist
+    git tag | tail -n 5 | sort -r
 
     printf "\ncreate new version:\n"
     read -r VERSION
@@ -151,28 +204,27 @@ function sync_docker {
     echo "commits since last version:"
     git log "$(git describe --tags --abbrev=0)"..HEAD --oneline
     git tag -a "$VERSION" -m "new release version $VERSION"
-    git push all "$VERSION"
+    git push origin "$VERSION"
 
 }
 
 
-# check package versions in requirements.txt for updates
-python version_check.py
-
-
 if [[ $1 == "blackhole" ]]; then
     sync_blackhole
 elif [[ $1 == "test" ]]; then
     sync_test "$2"
 elif [[ $1 == "validate" ]]; then
+    # check package versions in requirements.txt for updates
+    python version_check.py
     validate "$2"
 elif [[ $1 == "docker" ]]; then
     sync_docker
-    sync_unstable
 elif [[ $1 == "unstable" ]]; then
     sync_unstable
+elif [[ $1 == "es" ]]; then
+    sync_latest_es
 else
-    echo "valid options are: blackhole | test | validate | docker | unstable"
+    echo "valid options are: blackhole | test | validate | docker | unstable | es"
 fi
 
 

docker-compose.yml

@@ -3,29 +3,30 @@ version: '3.3'
 services:
   tubearchivist:
     container_name: tubearchivist
-    restart: always
-    image: bbilly1/tubearchivist:latest
+    restart: unless-stopped
+    image: bbilly1/tubearchivist
     ports:
       - 8000:8000
     volumes:
       - media:/youtube
       - cache:/cache
     environment:
-      - ES_URL=http://archivist-es:9200
-      - REDIS_HOST=archivist-redis
+      - ES_URL=http://archivist-es:9200     # needs protocol e.g. http and port
+      - REDIS_HOST=archivist-redis          # don't add protocol
       - HOST_UID=1000
       - HOST_GID=1000
-      - TA_USERNAME=tubearchivist
-      - TA_PASSWORD=verysecret
-      - ELASTIC_PASSWORD=verysecret
-      - TZ=America/New_York
+      - TA_HOST=tubearchivist.local         # set your host name
+      - TA_USERNAME=tubearchivist           # your initial TA credentials
+      - TA_PASSWORD=verysecret              # your initial TA credentials
+      - ELASTIC_PASSWORD=verysecret         # set password for Elasticsearch
+      - TZ=America/New_York                 # set your time zone
     depends_on:
       - archivist-es
       - archivist-redis
   archivist-redis:
-    image: redislabs/rejson:latest      # For arm64 just update this line with bbilly1/rejson:latest
+    image: redislabs/rejson                 # for arm64 use bbilly1/rejson
     container_name: archivist-redis
-    restart: always
+    restart: unless-stopped
     expose:
       - "6379"
     volumes:
@@ -33,12 +34,12 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.3.3
     container_name: archivist-es
-    restart: always
+    restart: unless-stopped
     environment:
       - "xpack.security.enabled=true"
-      - "ELASTIC_PASSWORD=verysecret"
+      - "ELASTIC_PASSWORD=verysecret"       # matching Elasticsearch password
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     ulimits:
@@ -46,7 +47,7 @@ services:
         soft: -1
         hard: -1
     volumes:
-      - es:/usr/share/elasticsearch/data
+      - es:/usr/share/elasticsearch/data    # check for permission error when using bind mount, see readme
     expose:
       - "9200"
 

docker_assets/nginx.conf

@@ -1,30 +1,33 @@
 server {
 
     listen 8000;
-
+    
     location /cache/videos/ {
+        auth_request /api/ping/;
         alias /cache/videos/;
     }
-
+    
     location /cache/channels/ {
+        auth_request /api/ping/;
         alias /cache/channels/;
     }
-
+    
     location /cache/playlists/ {
+        auth_request /api/ping/;
         alias /cache/playlists/;
     }
-
+    
     location /media/ {
+        auth_request /api/ping/;
         alias /youtube/;
-
         types {
             text/vtt vtt;
         }
     }
-
+    
     location / {
         include uwsgi_params;
         uwsgi_pass localhost:8080;
     }
-
+    
 }

docker_assets/run.sh

@@ -5,21 +5,23 @@ if [[ -z "$ELASTIC_USER" ]]; then
     export ELASTIC_USER=elastic
 fi
 
-ENV_VARS=("TA_USERNAME" "TA_PASSWORD" "ELASTIC_PASSWORD" "ELASTIC_USER")
-for each in "${ENV_VARS[@]}"; do
-    if ! [[ -v $each ]]; then
-        echo "missing environment variable $each"
-        exit 1
-    fi
-done
+cachedir=/cache
+[[ -d $cachedir ]] || cachedir=.
+lockfile=${cachedir}/initsu.lock
+
+required="Missing required environment variable"
+[[ -f $lockfile ]] || : "${TA_USERNAME:?$required}"
+: "${TA_PASSWORD:?$required}"
+: "${ELASTIC_PASSWORD:?$required}"
+: "${TA_HOST:?$required}"
 
 # ugly nginx and uwsgi port overwrite with env vars
 if [[ -n "$TA_PORT" ]]; then
-    sed -i "s/8000/$TA_PORT/g" /etc/nginx/conf.d/nginx.conf
+    sed -i "s/8000/$TA_PORT/g" /etc/nginx/sites-available/default
 fi
 
 if [[ -n "$TA_UWSGI_PORT" ]]; then
-    sed -i "s/8080/$TA_UWSGI_PORT/g" /etc/nginx/conf.d/nginx.conf
+    sed -i "s/8080/$TA_UWSGI_PORT/g" /etc/nginx/sites-available/default
     sed -i "s/8080/$TA_UWSGI_PORT/g" /app/uwsgi.ini
 fi
 
@@ -31,6 +33,7 @@ until curl -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL" -fs; do
     if [[ $counter -eq 12 ]]; then
         # fail after 2 min
         echo "failed to connect to elastic search, exiting..."
+        curl -v -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL"?pretty
         exit 1
     fi
     sleep 10
@@ -39,12 +42,27 @@ done
 # start python application
 python manage.py makemigrations
 python manage.py migrate
-export DJANGO_SUPERUSER_PASSWORD=$TA_PASSWORD && \
-    python manage.py createsuperuser --noinput --name "$TA_USERNAME"
+
+if [[ -f $lockfile ]]; then
+    echo -e "\e[33;1m[WARNING]\e[0m This is not the first run! Skipping" \
+        "superuser creation.\nTo force it, remove $lockfile"
+else
+    export DJANGO_SUPERUSER_PASSWORD=$TA_PASSWORD
+    output="$(python manage.py createsuperuser --noinput --name "$TA_USERNAME" 2>&1)"
+
+    case "$output" in
+        *"Superuser created successfully"*)
+            echo "$output" && touch $lockfile ;;
+        *"That name is already taken."*)
+            echo "Superuser already exists. Creation will be skipped on next start."
+            touch $lockfile ;;
+        *) echo "$output" && exit 1
+    esac
+fi
 
 python manage.py collectstatic --noinput -c
 nginx &
 celery -A home.tasks worker --loglevel=INFO &
 celery -A home beat --loglevel=INFO \
-    -s "${BEAT_SCHEDULE_PATH:-/cache/celerybeat-schedule}" &
+    -s "${BEAT_SCHEDULE_PATH:-${cachedir}/celerybeat-schedule}" &
 uwsgi --ini uwsgi.ini

docker_assets/uwsgi.ini

@@ -5,4 +5,7 @@ pidfile = /tmp/project-master.pid
 vacuum = True
 max-requests = 5000
 socket = :8080
-buffer-size = 8192
+buffer-size = 8192
+log-5xx = true
+log-4xx = true
+disable-logging = true

docs/Channels.md

@@ -17,10 +17,18 @@ The **Subscribe to Channels** button <img src="assets/icon-add.png?raw=true" alt
 You can search your indexed channels by clicking on the search icon <img src="assets/icon-search.png?raw=true" alt="search icon" width="20px" style="margin:0 5px;">. This will open a dedicated page.
 
 ## Channel Detail
-Each channel will get a dedicated channel detail page accessible at `/channel/<channel-id>/` of your Tube Archivist. This page shows all the videos you have downloaded from this channel plus additional metadata. 
+Each channel will get a dedicated channel detail page accessible at `/channel/<channel-id>/` of your Tube Archivist. This page shows all the videos you have downloaded from this channel.
+
 - If you are subscribed to the channel, an *Unsubscribe* button will show, else the *Subscribe* button will show.
-- You can *Show* the channel description, that matches with the *About* tab on YouTube.
 - The **Mark as Watched** button will mark all videos of this channel as watched.
+
+Additionally there is a *Channel Playlist* page, accessible at `/channel/<channel-id>/playlist/` to show all indexed playlists from this channel. 
+
+On the *Channel About* page you can see additional metadata.
 - The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata additionally this will also delete playlists metadata belonging to that channel.
-- The button **Find Playlists** will go out to youtube and look for playlists belonging to this channel and index all with at least one video downloaded. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.
-- The button **Show Playlists** will go to the [playlists](Playlists) page and filter the list to only show playlists from this channel.
+
+The channel customize form gives options to change settings on a per channel basis. Any configurations here will overwrite your configurations from the [settings](Settings) page.
+- **Download Format**: Overwrite the download quality for videos from this channel.
+- **Auto Delete**: Automatically delete watched videos from this channel after selected days.
+- **Index Playlists**: Automatically add all Playlists with at least a video downloaded to your index. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.
+- **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. Customize per channel: You can *disable* or *enable* SponsorBlock for certain channels only to overwrite the behavior set on the [Settings](settings) page. Selecting *unset* will remove the overwrite and your setting will fall back to the default on the settings page.

docs/Downloads.md

@@ -33,11 +33,9 @@ Every video in the download queue has two buttons:
 - **Ignore**: This will remove that video from the download queue and this video will not get added again, even when you **Rescan Subscriptions**.
 - **Download now**: This will give priority to this video. If the download process is already running, the prioritized video will get downloaded as soon as the current video is finished. If there is no download process running, this will start downloading this single video and stop after that.  
 
-The button **Delete all queued** will delete all pending videos from the download queue.
-
 You can flip the view by activating **Show Only Ignored Videos**. This will show all videos you have previously *ignored*.  
 Every video in the ignored list has two buttons:
 - **Forget**: This will delete the item form the ignored list.
 - **Add to Queue**: This will add the ignored video back to the download queue.  
 
-The button **Delete all ignored** will delete all videos you have previously ignored.
+You can delete your download queue from the [Settings](Settings#actions) page.

docs/FAQ.md

@@ -15,7 +15,7 @@ Defining the scope is important for the success of any project:
 - A too narrow scope will make this project uninteresting and will exclude audiences that could also benefit from this project.
 - Not defining a scope will easily lead to misunderstandings and false hopes of where this project tries to go.
 
-Of course this is subject to change, as this project continues to grow and more people contribute.
+Of course this is subject to change: The scope can be expanded as this project continues to grow and more people contribute.
 
 ## 2. Emby/Plex/Jellyfin/Kodi integrations
 Although there are similarities between these excellent projects and Tube Archivist, they have a very different use case. Trying to fit the metadata relations and database structure of a YouTube archival project into these media servers that specialize in Movies and TV shows is always going to be limiting.

docs/Home.md

@@ -9,6 +9,8 @@ Table of contents:
 * [Settings](Settings): All the configuration options
 * [Video](Video): All details of a single video and playlist navigation.
 * [Users](Users): User management admin interface
+* [Search](Search): Search your archive
+* [Installation](Installation): Detailed installation instructions for various platforms.
 
 ## Getting Started
 1. [Subscribe](Channels#channels-overview) to some of your favourite YouTube channels.
@@ -27,4 +29,4 @@ Table of contents:
 
 An empty checkbox icon <img src="assets/icon-unseen.png?raw=true" alt="unseen icon" width="20px" style="margin:0 5px;"> will show for videos you haven't marked as watched. Click on it and the icon will change to a filled checkbox <img src="assets/icon-seen.png?raw=true" alt="seen icon" width="20px" style="margin:0 5px;"> indicating it as watched - click again to revert.
 
-When available the <img src="assets/icon-gridview.png?raw=true" alt="gridview icon" width="20px" style="margin:0 5px;"> gridview icon will display the list in a grid, the <img src="assets/icon-listview.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> listview icon will arrange the items in a list. The sort icon <img src="assets/icon-sort.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> will open additional sort options.
+When available the <img src="assets/icon-gridview.png?raw=true" alt="gridview icon" width="20px" style="margin:0 5px;"> gridview icon will display the list in a grid. A grid row holds 3 items by default, use the <img src="assets/icon-add.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> icon to add more or the <img src="assets/icon-substract.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> icon to remove items per row, depending on your screen size. The <img src="assets/icon-listview.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> listview icon will arrange the items in a list. The sort icon <img src="assets/icon-sort.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> will open additional sort options.

docs/Installation.md

@@ -0,0 +1,255 @@
+# Detailed Installation Instructions for Various Platforms
+
+## Table of Contents
+- [Unraid](#unraid)
+- [Truenas Scale](#truenas-scale)
+- [Synology](#synology)
+
+These are beginners guides installation instructions for additional platforms generously provided by users of these platforms. When in doubt, verify the details with the [Readme](https://github.com/tubearchivist/tubearchivist#installing-and-updating). If you see any issues here while using these instructions, please contribute. 
+
+## Unraid
+
+Tube Archivist, and all if it's dependencies are located in the [community applications](https://forums.unraid.net/topic/38582-plug-in-community-applications/) store. The three containers you will need are as follows:
+
+-   **TubeArchivist-RedisJSON**: This container acts as a cache and temporary link between the application and the file system. Used to store and display messages and configuration variables.
+-   **TubeArchivist-ES**: ElasticSearch stores video meta data and makes everything searchable. Also keeps track of the download queue.
+-   **TubeArchivist**: Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface.
+
+### Step 1: Install  `TubeArchivist-RedisJSON`
+
+![enter image description here](https://i.imgur.com/ycAqFRU.png)
+This is the easiest container to setup of the thee, just make sure that you do not have any port conflicts, and that your `/data` is mounted to the correct path. The other containers will map to the same directory.
+
+If you need to install `TubeArchivist-RedisJSON`on a different port, you'll have to follow [these steps](https://github.com/tubearchivist/tubearchivist#redis-on-a-custom-port) later on when installing the `TubeArchivist` container
+
+
+### Step 2: Install  `TubeArchivist-ES`
+![enter image description here](https://i.imgur.com/o6tsTdt.png)
+ElasticSeach is also pretty easy to setup. Again, make sure you have no port conflicts, make sure that you mapped `/usr/share/elasticsearch/data` to the same directory as `RedisJSON`, and make sure to change the default password to something more secure. 
+
+There is three additional settings in the "show more settings" area, but leave those as they are.
+
+
+### Step 3: Install  `TubeArchivist`
+
+![enter image description here](https://i.imgur.com/dwSCfgO.png)
+It's finally time to set up TubeArchivist!
+
+ - `Port:`Again, make sure that you have no port conflicts on 8000.
+   
+ - `Youtube Media Path:` is where you'll download all of your videos to.
+   Make sure that this is an empty directory to not cause confusion when
+   starting the application. If you have existing videos that you'd like
+   to import into Tube Archivist, please checkout the [settings
+   wiki.](https://github.com/tubearchivist/tubearchivist/wiki/Settings#manual-media-files-import)
+   
+   
+- `Appdata:` This should be the same base path as the other two containers.
+   
+ - `TA Username:`This will be your username for TubeArchivist.
+   
+ - `TA Password:`This will be your password for TubeArchivist.
+   
+ - `Redis` This will be JUST the ip address of your redis container
+
+ - `ElasticSearch Password:`This is the password you defined in the `TubeArchivist-ES` container.
+ - `ElasticSearch:` This seems to cause some confusion, but it's a pretty simple step, just replace the IP and Port to match you `TubeArchivist-ES` container.
+
+ (example: if your IP is 192.168.1.15, the value should be http://192.168.1.15:9200)
+
+ - `Time Zone:` This is an important step for your scheduler, to find your timezone, use a site like [TimeZoneConverter](http://www.timezoneconverter.com/cgi-bin/findzone.tzc) 
+
+### From there, you should be able to start up your containers and you're good to go!
+If you're still having trouble, join us on [discord](https://discord.gg/AFwz8nE7BK) and come to the #unraid channel.
+
+<br />
+<br />
+
+## Truenas Scale 
+
+Truenas Scale can be a bit confusing, with its k3s kubernetes implementation. 
+
+However, there is a step by step guide available for it's users here:
+
+https://heavysetup.info/applications/tube-archivist/dataset/
+
+- Ensure you are navigating the columns under `Tube Archivist` on the left hand side of the screen
+
+<br />
+<br />
+
+## Synology
+
+There are several different methods to install TubeArchivist on Synology platforms. This will focus on the available `docker` package and `docker-compose` implementations.
+
+### Prepare Directories/Folders
+Before we setup TubeArchivist, we need to setup the directories/folders. You are assumed to be logged into the Synology NAS.
+#### 1. Docker Base Folder
+   1. Open the `File Station` utility.
+   2. Click on the **Create🔽** button and choose *Create New Shared Folder*.
+   3. **Name** the folder "Docker".
+   4. Add a **Description**.
+   5. Select the **Volume Location**. 
+      > Note: By default, this will be where all data is stored. Change the folders as best meets your requirements.
+   6. Select the appropriate options from the remaining checkbox configurations.
+![Synology - Create Docker Folder](assets/Synology_0.2.0_Docker-Folder-Create.png)
+   7. Click the **Next** button.
+   8. If you are going to **Encrypt** your folder, check the appropriate box and provide the Encryption Key and its confirmation.
+   9. Click the **Next** button.
+   10. On the **Advanced Settings** page, you can select the *Enable data checksum for advanced data integrity* setting. This may cause a performance impact, but will allow for potential file self-healing. **This cannot be changed later.**
+       > Note: This is not recommended, as we will be hosting databases within this folder.
+   11. If you are enabling a quota for how large the folder can get, you can select the *Enabled shared folder quota* setting and choose the maximum size this folder can grow. This can be changed later.
+   12. Click the **Next** button.
+   13. Confirm the settings, then click the **Apply** button. This will create the folder.
+#### 2. TubeArchivist Base Folder
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Click on the `Create🔽` button and choose *create Folder*.
+   4. **Name** the folder "TubeArchivist".
+#### 3. Redis Data
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Select the "TubeArchivist" folder beneath "Docker".
+   4. Click on the `Create🔽` button and choose *create Folder*.
+   5. **Name** the folder "redis".
+#### 4. Elastic Search Data
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Select the "TubeArchivist" folder beneath "Docker".
+   4. Click on the `Create🔽` button and choose *create Folder*.
+   5. **Name** the folder "es".
+#### 5. TubeArchivist Cache
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Select the "TubeArchivist" folder beneath "Docker".
+   4. Click on the `Create🔽` button and choose *create Folder*.
+   5. **Name** the folder "cache".
+#### 6. TubeArchivist Output
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Select the "TubeArchivist" folder beneath "Docker".
+   4. Click on the `Create🔽` button and choose *create Folder*.
+   5. **Name** the folder "media".
+#### 7. Confirm Folder Structure
+Once all of the folders have been created, it should have a folder structure within Docker\TubeArchivist that includes "cache", "es", "media", and "redis" folders.
+![Synology - Docker Folder Structure](assets/Synology_0.2.0_Docker-Folder-Structure.png)
+
+#### 8. Change Permissions - CLI Required
+> If you do not have SSH access enabled for CLI, [enable it](https://kb.synology.com/en-sg/DSM/tutorial/How_to_login_to_DSM_with_root_permission_via_SSH_Telnet) before continuing.
+   1. Open the SSH connection to the Synology. Login as your primary `Admin` user, or the user that was enabled for SSH access.
+   2. Elevate your access to `root`. Steps are provided [here](https://kb.synology.com/en-sg/DSM/tutorial/How_to_login_to_DSM_with_root_permission_via_SSH_Telnet).
+   3. Change directories to the **Volume** where the "Docker" folder resides.
+      </br>Example: `cd /volume1`
+   4. Change directories to the "Docker" folder.
+      </br>Example: `cd Docker`
+   5. Change directories to the "TubeArchivist" folder.
+      </br>Example: `cd TubeArchivist`
+   6. Change the owner of the "redis" folder. *If correct, this does not have an output.*
+      </br>Example: `chown 999:100 redis`
+   7. Change the owner of the "es" folder. *If correct, this does not have an output.*
+      </br>Example: `chown 1000:1000 es`
+   8. Confirm that the folders have the correct permissions.
+      </br>Example: `ls -hl`
+![Synology - Docker Folder Permissions Command](assets/Synology_0.2.0_Docker-Folder-Permissions-Commands.png)
+   9. Logout from root.
+      </br>Example: `logout`
+   10. Disconnect from the SSH connection.
+      </br>Example: `exit`
+### Docker Setup
+1. Install the `Docker` Synology Package.
+  1. Log in to your Synology NAS.
+  2. Open the `Package Center` utility.
+  3. Search for `Docker`.
+  4. Click `Install`.
+
+![Synology - Install Docker Utility](assets/Synology_0.2.0_Docker-Install.png)
+
+2. After `Docker` is installed, open the `Docker` utility.
+3. Go to the `Registry` tab.
+4. Search for the following `images` and download them. Follow the recommended versions for each of the images.
+  - `redislabs/rejson`
+![Synology - Redis Image Search](assets/Synology_0.2.0_Docker-Redis-Search.png)
+  - `bbilly1/tubearchivist-es`
+![Synology - ElasticSearch Image Search](assets/Synology_0.2.0_Docker-ES-Search.png)
+  - `bbilly1/tubearchivist`
+![Synology - TubeArchivist Image Search](assets/Synology_0.2.0_Docker-TA-Search.png)
+
+
+
+5. Go to the `Image` tab. From here, create an container based on each image with the associated configurations below.
+  - ElasticSearch
+    1. Select the associated image.
+    2. Click the **Launch** button in the top.
+    3. Edit the **Container Name** to be "tubearchivist-es".
+    4. Click on the **Advanced Settings** button.
+    5. In the **Advanced Settings** tab, check the box for `Enable auto-restart`.
+    6. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/es" folder, then type in `/usr/share/elasticsearch/data` for the mount path.
+    7. In the **Network** tab, leave the default `bridge` Network (unless you have a specific Network design that you know how to implement).
+    8. In the **Port Settings** tab, replace the "Auto" entry under **Local Port** with the port that will be used to connect to ElasticSearch (default is 9200).
+    9. In the **Port Settings** tab, select the entryline for port 9300 and **➖ delete** the line. It is not needed for this container.
+    10. The **Links** tab does not require configuration for this container.
+    11. In the **Environment** tab, add in the following ElasticSearch specific environment variables that may apply.
+        - "discovery.type=single-node"
+        - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+        - "UID=1000"
+        - "GID=0"
+        - "xpack.security.enabled=true"
+        - "ELASTIC_PASSWORD=verysecret" 
+          > Do not use the default password as it is very insecure.
+        ![Synology - ElasticSearch Environment Configurations](assets/Synology_0.2.0_Docker-ES-Env-Conf.png)
+    12. Click on the **Apply** button.
+    13. Back on the **Create Container** screen, click the **Next** button.
+    14. Review the settings to confirm, then click the **Apply** button.
+  - Redis
+    1. Select the associated image.
+    2. Click the **Launch** button in the top.
+    3. Edit the **Container Name** to be "tubearchivist-redis".
+    4. Click on the **Advanced Settings** button.
+    5. In the **Advanced Settings** tab, check the box for `Enable auto-restart`.
+    6. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/redis" folder, then type in `/data` for the mount path.
+    7. In the **Network** tab, leave the default `bridge` Network (unless you have a specific Network design that you know how to implement).
+    8. In the **Port Settings** tab, replace the "Auto" entry under **Local Port** with the port that will be used to connect to Redis (default is 6379).
+    9. In the **Links** tab, select the "tubearchivist-es" container from the **Container Name** dropdown and provide it the same alias, "tubearchivist-es".
+    10. In the **Environment** tab, add in any Redis specific environment variables that may apply (none by default).
+    11. Click on the **Apply** button.
+    12. Back on the **Create Container** screen, click the **Next** button.
+    13. Review the settings to confirm, then click the **Apply** button.
+  
+  - TubeArchivist
+    1. Select the associated image.
+    2. Click the **Launch** button in the top.
+    3. Edit the **Container Name** to be "tubearchivist".
+    4. Click on the **Advanced Settings** button.
+    5. In the **Advanced Settings** tab, check the box for `Enable auto-restart`.
+    6. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/cache" folder, then type in `/cache` for the mount path.
+    7. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/media" folder, then type in `/youtube` for the mount path.
+    8. In the **Network** tab, leave the default `bridge` Network (unless you have a specific Network design that you know how to implement).
+    9. In the **Port Settings** tab, replace the "Auto" entry under **Local Port** with the port that will be used to connect to TubeArchivist (default is 8000).
+    10. In the **Links** tab, select the "tubearchivist-es" container from the **Container Name** dropdown and provide it the same alias, "tubearchivist-es".
+    11. In the **Links** tab, select the "tubearchivist-redis" container from the **Container Name** dropdown and provide it the same alias, "tubearchivist-redis".
+    12. In the **Environment** tab, add in the following TubeArchivist specific environment variables that may apply. **Change the variables as-is appropriate to your use case. Follow the [README section](https://github.com/tubearchivist/tubearchivist#tube-archivist) for details on what to set each variable.**
+        - "TA_HOST=synology.local"
+        - "ES_URL=http://tubearchivist-es:9200"
+        - "REDIS_HOST=tubearchivist-redis"
+        - "HOST_UID=1000"
+        - "HOST_GID=1000"
+        - "TA_USERNAME=tubearchivist"
+        - "TA_PASSWORD=verysecret"
+        - "ELASTIC_PASSWORD=verysecret"
+        - "TZ=America/New_York"
+          > Do not use the default password as it is very insecure.
+          > Ensure that ELASTIC_PASSWORD matches the password used on the tubearchivist-es container.
+        ![Synology - TubeArchivist Environment Configurations](assets/Synology_0.2.0_Docker-TA-Env-Conf.png)
+    13. Click on the **Apply** button.
+    14. Back on the **Create Container** screen, click the **Next** button.
+    15. Review the settings to confirm, then click the **Apply** button.
+6. After the containers have been configured and started, you can go to the **Container** tab and monitor the containers.
+7. To review the logs to ensure that the system has started successfully, select the "tubearchivist" container and click on the **Details** button. In the new window, go to the **Log** tab. Monitor the logs until either an error occurs or the message `celery@tubearchivist ready.` is in the logs. This may take a few minutes, especially for a first time setup.
+   > Note: Synology Docker presents the logs in a pagination format. If you are not seeing the logs update, check if there are additional pages.
+8. After it has started, go to the location in the `TA_HOST`. This should give you the standard TubeArchivist login screen.
+<!-- 
+### Docker-Compose Setup -->
+<!-- This section is a Work In Progress -->
+
+### From there, you should be able to start up your containers and you're good to go!
+If you're still having trouble, join us on [discord](https://discord.gg/AFwz8nE7BK) and come to the #synology channel.

docs/Search.md

@@ -0,0 +1,54 @@
+# Search Page
+Accessible at `/search/` of your **Tube Archivist**, search your archive for Videos, Channels and Playlists - or even full text search throughout your indexed subtitles.
+
+- All your queries are case insensitive and are normalized to lowercase.
+- All your queries are analyzed for the english language, this means *singular*, *plural* and word variations like *-ing*, *-ed*, *-able* etc are treated as synonyms.
+- Fuzzy search is activated for all your searches. This can catch typos in your queries or in the matching documents with one to two letters difference, depending on the query length.
+- All text searches are ranked, meaning the better a match the higher ranked the result. Unless otherwise stated, queries with multiple words are processed with the `and` operator, meaning all words need to match so each word will narrow down the result.
+- This will return 30 results per query, pagination is not implemented yet.
+
+Just start typing to start a *simple* search or start your query with a primary keyword to search for a specific type and narrow down the result with secondary keywords. Secondary keywords can be in any order. Use *yes* or *no* for boolean values.
+
+## Simple
+Start your query without a keyword to make a simple query. This will search in *video titles*, *channel names* and *playlist titles* and will return matching videos, channels and playlists. Keyword searches will return more results in a particular category due to the fact that more fields are searched for matches.
+
+## Video
+Start your query with the primary keyword `video:` to search for videos only. This will search through the *video titles*, *tags* and *category* fields. Narrow your search down with secondary keywords:
+- `channel:` search for videos matching the channel name.
+- `active:` is a boolean value, to search for videos that are still active on youtube or that are not active any more.
+
+**Example**:
+- `video:learn python channel:corey shafer active:yes`: This will return all videos with the term *Learn Python* from the channel *Corey Shafer* that are still *Active* on YouTube.
+- `video: channel:tom scott active:no`: Note the omitted term after the primary key, this will show all videos from the channel *Tom Scott* that are no longer active on YouTube.
+
+## Channel
+Start with the `channel:` primary keyword to search for channels matching your query. This will search through the *channel name* and *channel description* fields. Narrow your search down with secondary keywords:
+- `subscribed:` is a boolean value, search for channels that you are subscribed to or not.
+- `active:` is a boolean value, to search for channels that are still active on YouTube or that are no longer active.
+
+**Example**:
+- `channel:linux subscribed:yes`: Search for channels with the term *Linux* that you are subscribed to.
+- `channel: active:no`: Note the omitted term after the primary key, this will return all channels that are no longer active on YouTube.
+
+## Playlist
+Start your query with the primary keyword `playlist:` to search for playlists only. This will search through the *playlist title* and *playlist description* fields. Narrow down your search with these secondary keywords:
+- `subscribed`: is a boolean value, search for playlists that you are subscribed to or not.
+- `active:` is a boolean value, to search for playlists that are still active on YouTube or that are no longer active.
+
+**Example**:
+- `playlist:backend engineering subscribed:yes`: Search for playlists about *Backend Engineering* that you are subscribed to.
+- `playlist: active:yes subscribed:yes`: Note the omitted primary search term, this will return all playlists active on YouTube that you are subscribed to.
+- `playlist:html css active:yes`: Search for playlists containing *HTML CSS* that are still active on YouTube.
+
+## Full
+Start a full text search by beginning your query with the primary keyword `full:`. This will search through your indexed Subtitles showing segments with possible matches. This will only show any results if you have activated *subtitle download and index* on the settings page. The operator for full text searches is `or` meaning when searching for multiple words not all words need to match, but additional words will change the ranking of the result, the more words match and the better they match, the higher ranked the result. The matching words will get highlighted in the text preview.
+
+Clicking the play button on the thumbnail will open the inplace player at the timestamp from where the segment starts. Same when clicking the video title, this will open the video page and put the player at the segment timestamp. This will overwrite any previous playback position.
+
+Narrow down your search with these secondary keywords:
+- `lang`: Search for matches only within a language. Use the same two letter ISO country code as you have set on the settings page.
+- `source`: Can either be *auto* to search through auto generated subtitles only or *user* to search through user uploaded subtitles only.
+
+**Example**:
+- `full:contribute to open source lang:en` search for subtitle segments matching with the words *Contribute to Open Source* in the language *en*.
+- `full:flight simulator cockpit source:user` to search for the words *Flight Simulator Cockpit* from *user* uploaded subtitle segments.

docs/Settings.md

@@ -32,10 +32,37 @@ Additional settings passed to yt-dlp.
 - **Source Settings**: User created subtitles are provided from the uploader and are usually the video script. Auto generated is from YouTube, quality varies, particularly for auto translated tracks.
 - **Index Settings**: Enabling subtitle indexing will add the lines to Elasticsearch and will make subtitles searchable. This will increase the index size and is not recommended on low-end hardware.
 
+## Cookie
+Importing your YouTube Cookie into Tube Archivist allows yt-dlp to bypass age restrictions, gives access to private videos and your *watch later* or *liked videos*.
+
+### Security concerns
+Cookies are used to store your session and contain your access token to your google account, this information can be used to take over your account. Treat that data with utmost care as you would any other password or credential. *Tube Archivist* stores your cookie in Redis and will automatically append it to yt-dlp for every request.
+
+### Auto import
+Easiest way to import your cookie is to use the **Tube Archivist Companion** [browser extension](https://github.com/tubearchivist/browser-extension) for Firefox and Chrome.
+
+### Alternative Manual Export your cookie
+- Install **Cookies.txt** addon for [chrome](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid) or [firefox](https://addons.mozilla.org/firefox/addon/cookies-txt).
+- Visit YouTube and login with whichever YouTube account you wish to use to generate the cookies.
+- Click on the extension icon in the toolbar - it will drop down showing the active cookies for YT.
+- Click Export to export the cookies, filename is by default *cookies.google.txt*.
+
+### Alternative Manual Import your cookie
+Place the file *cookies.google.txt* into the *cache/import* folder of Tube Archivist and enable the cookie import. Once you click on *Update Application Configurations* to save your changes, your cookie will get imported and stored internally.
+
+Once imported, a **Validate Cookie File** button will show, where you can confirm if your cookie is working or not.
+
+### Use your cookie
+Once imported, additionally to the advantages above, your [Watch Later](https://www.youtube.com/playlist?list=WL) and [Liked Videos](https://www.youtube.com/playlist?list=LL) become a regular playlist you can download and subscribe to as any other [playlist](Playlists).
+
+### Limitation
+There is only one cookie per Tube Archivist instance, this will be shared between all users.
+
 ## Integrations
 All third party integrations of TubeArchivist will **always** be *opt in*.
-- **API**: Your access token for the Tube Archivist API. 
+- **API**: Your access token for the Tube Archivist API.
 - **returnyoutubedislike.com**: This will get return dislikes and average ratings for each video by integrating with the API from [returnyoutubedislike.com](https://www.returnyoutubedislike.com/).
+- **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. If a video doesn't have timestamps, or has unlocked timestamps, use the browser addon to contribute to this excellent project. Can also be activated and deactivated as a per [channel overwrite](Settings#channel-customize).
 - **Cast**: Enabling the cast integration in the settings page will load an additional JS library from **Google**.
     * Requirements
         - HTTPS
@@ -55,7 +82,7 @@ Examples:
 - **0 15 \***: Run task every day at 15:00 in the afternoon.
 - **30 8 \*/2**: Run task every second day of the week (Sun, Tue, Thu, Sat) at 08:30 in the morning.
 - **0 \*/3,8-17 \***: Execute every hour divisible by 3, and every hour during office hours (8 in the morning - 5 in the afternoon).
-- **0 8,16 \***: Execute every day at 8 in the morning and at 4 in the afternoon. 
+- **0 8,16 \***: Execute every day at 8 in the morning and at 4 in the afternoon.
 - **auto**: Sensible default.
 - **0**: (zero), deactivate that task.
 
@@ -71,9 +98,9 @@ That's the equivalent task as run from the downloads page looking through your c
 Start downloading all videos currently in the download queue.
 
 ## Refresh Metadata
-Rescan videos, channels and playlists on youtube and update metadata periodically. This will also deactivate an item and exclude it from future refreshes if the link on YouTube is no longer available. This task is meant to be run once per day, set your schedule accordingly. 
+Rescan videos, channels and playlists on youtube and update metadata periodically. This will also refresh your subtitles based on your current settings. If an item is no longer available on YouTube, this will deactivate it and exclude it from future refreshes. This task is meant to be run once per day, set your schedule accordingly.
 
-The field **Refresh older than x days** takes a number where TubeArchivist will consider an item as *outdated*. This value is used to calculate how many items need to be refreshed today based on the total indexed. This will spread out the requests to YouTube. Sensible value here is **90** days. 
+The field **Refresh older than x days** takes a number where TubeArchivist will consider an item as *outdated*. This value is used to calculate how many items need to be refreshed today based on the total indexed. This will spread out the requests to YouTube. Sensible value here is **90** days.
 
 ## Thumbnail check
 This will check if all expected thumbnails are there and will delete any artwork without matching video.
@@ -85,25 +112,36 @@ Create a zip file of the metadata and select **Max auto backups to keep** to aut
 # Actions
 Additional database functionality.
 
+## Delete download queue
+The button **Delete all queued** will delete all pending videos from the download queue. The button **Delete all ignored** will delete all videos you have previously ignored.
+
 ## Manual Media Files Import
-So far this depends on the video you are trying to import to be still available on YouTube to get the metadata. Add the files you'd like to import to the */cache/import* folder. Then start the process from the settings page *Manual Media Files Import*. Make sure to follow one of the two methods below.
+Add the files you'd like to import to the */cache/import* folder. Only add files, don't add subdirectories. All files you are adding, need to have the same *base name* as the media file. Then start the process from the settings page *Manual Media Files Import*.
+
+Valid media extensions are *.mp4*, *.mkv* or *.webm*. If you have other file extensions or incompatible codecs, convert them first to mp4. **Tube Archivist** can identify the videos with one of the following methods.
 
 ### Method 1:
-Add a matching *.json* file with the media file. Both files need to have the same base name, for example:
-- For the media file: \<base-name>.mp4
-- For the JSON file: \<base-name>.info.json
-- Alternate JSON file: \<base-name>.json
+Add a matching *.info.json* file with the media file. Both files need to have the same base name, for example:
+- For the media file: `<base-name>.mp4`
+- For the JSON file: `<base-name>.info.json`
 
-**Tube Archivist** then looks for the 'id' key within the JSON file to identify the video.
+The import process then looks for the 'id' key within the JSON file to identify the video.
 
 ### Method 2:
 Detect the YouTube ID from filename, this accepts the default yt-dlp naming convention for file names like:
-- \<base-name>[\<youtube-id>].mp4
+- `<base-name>[<youtube-id>].mp4`
 - The YouTube ID in square brackets at the end of the filename is the crucial part.
 
+### Offline import:
+If the video you are trying to import is not available on YouTube any more, **Tube Archivist** can import the required metadata:
+- The file `<base-name>.info.json` is required to extract the required information.
+- Add the thumbnail as `<base-name>.<ext>`, where valid file extensions are *.jpg*, *.png* or *.webp*. If there is no thumbnail file, **Tube Archivist** will try to extract it from the media file or will fallback to a default thumbnail.
+- Add subtitles as `<base-name>.<lang>.vtt` where *lang* is the two letter ISO country code. This will archive all subtitle files you add to the import folder, independent from your configurations. Subtitles can be archived and used in the player, but they can't be indexed or made searchable due to the fact, that they have a very different structure than the subtitles as **Tube Archivist** needs them.
+- For videos, where the whole channel is not available any more, you can add the `<channel-id>.info.json` file as generated by *youtube-dl/yt-dlp* to get the full metadata. Alternatively **Tube Archivist** will extract as much info as possible from the video info.json file. 
+
 ### Some notes:
 - This will **consume** the files you put into the import folder: Files will get converted to mp4 if needed (this might take a long time...) and moved to the archive, *.json* files will get deleted upon completion to avoid having duplicates on the next run.
-- There should be no subdirectories added to */cache/import*, only video files. If your existing video library has video files inside subdirectories, you can get all the files into one directory by running `find ./ -mindepth 2 -type f -exec mv '{}' . \;` from the top-level directory of your existing video library. You can also delete any remaining empty subdirectories with `find ./ -mindepth 1 -type d -delete`.
+- For best file transcoding quality, convert your media files with desired settings first before importing.
 - Maybe start with a subset of your files to import to make sure everything goes well...
 - Follow the logs to monitor progress and errors: `docker-compose logs -f tubearchivist`.
 
@@ -111,7 +149,7 @@ Detect the YouTube ID from filename, this accepts the default yt-dlp naming conv
 This will write or overwrite all thumbnails in the media file using the downloaded thumbnail. This is only necessary if you didn't download the files with the option *Embed Thumbnail* enabled or want to make sure all media files get the newest thumbnail. Follow the docker-compose logs to monitor progress.
 
 ## Backup Database
-This will backup your metadata into a zip file. The file will get stored at *cache/backup* and will contain the necessary files to restore the Elasticsearch index formatted **nd-json** files plus a complete export of the index in a set of conventional **json** files.  
+This will backup your metadata into a zip file. The file will get stored at *cache/backup* and will contain the necessary files to restore the Elasticsearch index formatted **nd-json** files.  
 
 BE AWARE: This will **not** backup any media files, just the metadata from the Elasticsearch.
 
@@ -129,5 +167,6 @@ This function will go through all your media files and looks at the whole index
 - Should the filename not match with the indexed media url, this will rename the video files correctly and update the index with the new link.
 - When you delete media files from the filesystem outside of the Tube Archivist interface, this will delete leftover metadata from the index.
 - When you have media files that are not indexed yet, this will grab the metadata from YouTube like it was a newly downloaded video. This can be useful when restoring from an older backup file with missing metadata but already downloaded mediafiles. NOTE: This only works if the media files are named in the same convention as Tube Archivist does, particularly the YouTube ID needs to be at the same index in the filename, alternatively see above for *Manual Media Files Import*.
+-This will also check all of your thumbnails and download any that are missing.
 
 BE AWARE: There is no undo.

tubearchivist/api/README.md

@@ -1,6 +1,40 @@
 # TubeArchivist API
 Documentation of available API endpoints.  
-**Note: This is very early alpha and will change!**
+
+Note:  
+- This is very early stages and will change!
+- Check the commit history to see if a documented feature is already in your release
+
+## Table of contents
+- [Authentication](#authentication)
+- [Pagination](#pagination)
+
+**Video**
+- [Video List](#video-list-view)
+- [Video Single](#video-item-view)
+- [Video Single Progress](#video-progress-view)
+- [Video Single Sponsorblock](#sponsor-block-view) WIP
+
+**Channel**
+- [Channel List](#channel-list-view)
+- [Channel Single](#channel-item-view)
+- [Channel Video List](#channel-videos-view)
+
+**Playlist**
+- [Playlist List](#playlist-list-view)
+- [Playlist Single](#playlist-item-view)
+- [Playlist Videos List](#playlist-videos-view)
+
+**Download queue**
+- [Download Queue List](#download-queue-list-view)
+- [Download Queue Single](#download-queue-item-view)
+
+**Additional**
+- [Login](#login-view)
+- [Task](#task-view) WIP
+- [Cookie](#cookie-view)
+- [Search](#search-view)
+- [Ping](#ping-view)
 
 ## Authentication
 API token will get automatically created, accessible on the settings page. Token needs to be passed as an authorization header with every request. Additionally session based authentication is enabled too: When you are logged into your TubeArchivist instance, you'll have access to the api in the browser for testing.
@@ -20,6 +54,22 @@ headers = {"Authorization": "Token xxxxxxxxxx"}
 response = requests.get(url, headers=headers)
 ```
 
+## Pagination
+The list views return a paginate object with the following keys:
+- page_size: *int* current page size set in config
+- page_from: *int* first result idx
+- prev_pages: *array of ints* of previous pages, if available
+- current_page: *int* current page from query
+- max_hits: *bool* if max of 10k results is reached
+- last_page: *int* of last page link
+- next_pages: *array of ints* of next pages
+- total_hits: *int* total results
+
+Pass page number as a query parameter: `page=2`. Defaults to *0*, `page=1` is redundant and falls back to *0*. If a page query doesn't return any results, you'll get `HTTP 404 Not Found`.
+
+## Video List View
+/api/video/
+
 ## Video Item View
 /api/video/\<video_id>/
 
@@ -49,6 +99,43 @@ POST /api/video/\<video_id>/progress
 ### Delete player position of video
 DELETE /api/video/\<video_id>/progress  
 
+
+## Sponsor Block View
+/api/video/\<video_id>/sponsor/
+
+Integrate with sponsorblock
+
+### Get list of segments
+GET /api/video/\<video_id>/sponsor/
+
+
+### Vote on existing segment
+**This only simulates the request**  
+POST /api/video/\<video_id>/sponsor/
+```json
+{
+    "vote": {
+        "uuid": "<uuid>",
+        "yourVote": 1
+    }
+}
+```
+yourVote needs to be *int*: 0 for downvote, 1 for upvote, 20 to undo vote
+
+### Create new segment
+**This only simulates the request**  
+POST /api/video/\<video_id>/sponsor/
+```json
+{
+    "segment": {
+        "startTime": 5,
+        "endTime": 10
+    }
+}
+```
+Timestamps either *int* or *float*, end time can't be before start time.
+
+
 ## Channel List View
 /api/channel/
 
@@ -65,11 +152,23 @@ POST /api/channel/
 ## Channel Item View
 /api/channel/\<channel_id>/
 
-## Playlists Item View
+## Channel Videos View
+/api/channel/\<channel_id>/video/
+
+## Playlist List View
+/api/playlist/
+
+## Playlist Item View
 /api/playlist/\<playlist_id>/
 
+## Playlist Videos View
+/api/playlist/\<playlist_id>/video/
+
 ## Download Queue List View
-/api/download/
+GET /api/download/
+
+Parameter:
+- filter: pending, ignore
 
 ### Add list of videos to download queue
 POST /api/download/
@@ -81,5 +180,132 @@ POST /api/download/
 }
 ```
 
+### Delete download queue items by filter
+DELETE /api/download/?filter=ignore  
+DELETE /api/download/?filter=pending
+
 ## Download Queue Item View
-/api/download/\<video_id>/
+GET /api/download/\<video_id>/  
+POST /api/download/\<video_id>/
+
+Ignore video in download queue:
+```json
+{
+    "status": "ignore"
+}
+```
+
+Add to queue previously ignored video:
+```json
+{
+    "status": "pending"
+}
+```
+
+DELETE /api/download/\<video_id>/  
+Forget or delete from download queue
+
+## Login View
+Return token and user ID for username and password:  
+POST /api/login
+```json
+{
+    "username": "tubearchivist",
+    "password": "verysecret"
+}
+```
+
+after successful login returns 
+```json
+{
+    "token": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+    "user_id": 1
+}
+```
+
+## Task View
+GET /api/task/
+POST /api/task/
+
+Check if there is an ongoing task:
+GET /api/task/
+
+Returns:
+```json
+{
+    "rescan": false,
+    "downloading": false
+}
+```
+
+Start a background task  
+POST /api/task/
+```json
+{
+    "run": "task_name"
+}
+```
+
+List of valid task names:
+- **download_pending**: Start the download queue
+- **rescan_pending**: Rescan your subscriptions
+
+
+## Cookie View
+Check your youtube cookie settings, *status* turns to `true` if cookie has been validated.  
+GET /api/cookie/
+```json
+{
+    "cookie_enabled": true,
+    "status": true,
+    "validated": <timestamp>,
+    "validated_str": "timestamp"
+}
+```
+
+POST /api/cookie/  
+Send empty post request to validate cookie.
+```json
+{
+    "cookie_validated": true
+}
+```
+
+PUT /api/cookie/  
+Send put request containing the cookie as a string:
+```json
+{
+    "cookie": "your-cookie-as-string"
+}
+```
+Imports and validates cookie, returns on success:
+```json
+{
+    "cookie_import": "done",
+    "cookie_validated": true
+}
+```
+Or returns status code 400 on failure:
+```json
+{
+    "cookie_import": "fail",
+    "cookie_validated": false
+}
+```
+
+## Search View
+GET /api/search/?query=\<query>
+
+Returns search results from your query.
+
+## Ping View
+Validate your connection with the API  
+GET /api/ping
+
+When valid returns message with user id: 
+```json
+{
+    "response": "pong",
+    "user": 1
+}
+```

tubearchivist/api/src/search_processor.py

@@ -0,0 +1,125 @@
+"""
+Functionality:
+- processing search results for frontend
+- this is duplicated code from home.src.frontend.searching.SearchHandler
+"""
+
+import urllib.parse
+
+from home.src.download.thumbnails import ThumbManager
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import date_praser
+
+
+class SearchProcess:
+    """process search results"""
+
+    CONFIG = AppConfig().config
+    CACHE_DIR = CONFIG["application"]["cache_dir"]
+
+    def __init__(self, response):
+        self.response = response
+        self.processed = False
+
+    def process(self):
+        """detect type and process"""
+        if "_source" in self.response.keys():
+            # single
+            self.processed = self._process_result(self.response)
+
+        elif "hits" in self.response.keys():
+            # multiple
+            self.processed = []
+            all_sources = self.response["hits"]["hits"]
+            for result in all_sources:
+                self.processed.append(self._process_result(result))
+
+        return self.processed
+
+    def _process_result(self, result):
+        """detect which type of data to process"""
+        index = result["_index"]
+        processed = False
+        if index == "ta_video":
+            processed = self._process_video(result["_source"])
+        if index == "ta_channel":
+            processed = self._process_channel(result["_source"])
+        if index == "ta_playlist":
+            processed = self._process_playlist(result["_source"])
+        if index == "ta_download":
+            processed = self._process_download(result["_source"])
+
+        return processed
+
+    @staticmethod
+    def _process_channel(channel_dict):
+        """run on single channel"""
+        channel_id = channel_dict["channel_id"]
+        art_base = f"/cache/channels/{channel_id}"
+        date_str = date_praser(channel_dict["channel_last_refresh"])
+        channel_dict.update(
+            {
+                "channel_last_refresh": date_str,
+                "channel_banner_url": f"{art_base}_banner.jpg",
+                "channel_thumb_url": f"{art_base}_thumb.jpg",
+                "channel_tvart_url": False,
+            }
+        )
+
+        return dict(sorted(channel_dict.items()))
+
+    def _process_video(self, video_dict):
+        """run on single video dict"""
+        video_id = video_dict["youtube_id"]
+        media_url = urllib.parse.quote(video_dict["media_url"])
+        vid_last_refresh = date_praser(video_dict["vid_last_refresh"])
+        published = date_praser(video_dict["published"])
+        vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
+        channel = self._process_channel(video_dict["channel"])
+
+        if "subtitles" in video_dict:
+            for idx, _ in enumerate(video_dict["subtitles"]):
+                url = video_dict["subtitles"][idx]["media_url"]
+                video_dict["subtitles"][idx]["media_url"] = f"/media/{url}"
+
+        video_dict.update(
+            {
+                "channel": channel,
+                "media_url": f"/media/{media_url}",
+                "vid_last_refresh": vid_last_refresh,
+                "published": published,
+                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+            }
+        )
+
+        return dict(sorted(video_dict.items()))
+
+    @staticmethod
+    def _process_playlist(playlist_dict):
+        """run on single playlist dict"""
+        playlist_id = playlist_dict["playlist_id"]
+        playlist_last_refresh = date_praser(
+            playlist_dict["playlist_last_refresh"]
+        )
+        playlist_dict.update(
+            {
+                "playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg",
+                "playlist_last_refresh": playlist_last_refresh,
+            }
+        )
+
+        return dict(sorted(playlist_dict.items()))
+
+    def _process_download(self, download_dict):
+        """run on single download item"""
+        video_id = download_dict["youtube_id"]
+        vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
+        published = date_praser(download_dict["published"])
+
+        download_dict.update(
+            {
+                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+                "published": published,
+            }
+        )
+        return dict(sorted(download_dict.items()))

tubearchivist/api/src/task_processor.py

@@ -0,0 +1,54 @@
+"""
+Functionality:
+- process tasks from API
+- validate
+- handover to celery
+"""
+
+from home.src.ta.ta_redis import RedisArchivist
+from home.tasks import download_pending, update_subscribed
+
+
+class TaskHandler:
+    """handle tasks from api"""
+
+    def __init__(self, data):
+        self.data = data
+
+    def run_task(self):
+        """map data and run"""
+        task_name = self.data["run"]
+        try:
+            to_run = self.exec_map(task_name)
+        except KeyError as err:
+            print(f"invalid task name {task_name}")
+            raise ValueError from err
+
+        response = to_run()
+        response.update({"task": task_name})
+        return response
+
+    def exec_map(self, task_name):
+        """map dict key and return function to execute"""
+        exec_map = {
+            "download_pending": self._download_pending,
+            "rescan_pending": self._rescan_pending,
+        }
+
+        return exec_map[task_name]
+
+    @staticmethod
+    def _rescan_pending():
+        """look for new items in subscribed channels"""
+        print("rescan subscribed channels")
+        update_subscribed.delay()
+        return {"success": True}
+
+    @staticmethod
+    def _download_pending():
+        """start the download queue"""
+        print("download pending")
+        running = download_pending.delay()
+        print("set task id: " + running.id)
+        RedisArchivist().set_message("dl_queue_id", running.id)
+        return {"success": True}

tubearchivist/api/urls.py

@@ -2,16 +2,33 @@
 
 from api.views import (
     ChannelApiListView,
+    ChannelApiVideoView,
     ChannelApiView,
+    CookieView,
     DownloadApiListView,
     DownloadApiView,
+    LoginApiView,
+    PingView,
+    PlaylistApiListView,
+    PlaylistApiVideoView,
     PlaylistApiView,
+    SearchView,
+    TaskApiView,
+    VideoApiListView,
     VideoApiView,
     VideoProgressView,
+    VideoSponsorView,
 )
 from django.urls import path
 
 urlpatterns = [
+    path("ping/", PingView.as_view(), name="ping"),
+    path("login/", LoginApiView.as_view(), name="api-login"),
+    path(
+        "video/",
+        VideoApiListView.as_view(),
+        name="api-video-list",
+    ),
     path(
         "video/<slug:video_id>/",
         VideoApiView.as_view(),
@@ -22,6 +39,11 @@ urlpatterns = [
         VideoProgressView.as_view(),
         name="api-video-progress",
     ),
+    path(
+        "video/<slug:video_id>/sponsor/",
+        VideoSponsorView.as_view(),
+        name="api-video-sponsor",
+    ),
     path(
         "channel/",
         ChannelApiListView.as_view(),
@@ -32,11 +54,26 @@ urlpatterns = [
         ChannelApiView.as_view(),
         name="api-channel",
     ),
+    path(
+        "channel/<slug:channel_id>/video/",
+        ChannelApiVideoView.as_view(),
+        name="api-channel-video",
+    ),
+    path(
+        "playlist/",
+        PlaylistApiListView.as_view(),
+        name="api-playlist-list",
+    ),
     path(
         "playlist/<slug:playlist_id>/",
         PlaylistApiView.as_view(),
         name="api-playlist",
     ),
+    path(
+        "playlist/<slug:playlist_id>/video/",
+        PlaylistApiVideoView.as_view(),
+        name="api-playlist-video",
+    ),
     path(
         "download/",
         DownloadApiListView.as_view(),
@@ -47,4 +84,19 @@ urlpatterns = [
         DownloadApiView.as_view(),
         name="api-download",
     ),
+    path(
+        "task/",
+        TaskApiView.as_view(),
+        name="api-task",
+    ),
+    path(
+        "cookie/",
+        CookieView.as_view(),
+        name="api-cookie",
+    ),
+    path(
+        "search/",
+        SearchView.as_view(),
+        name="api-search",
+    ),
 ]

tubearchivist/api/views.py

@@ -1,15 +1,23 @@
 """all API views"""
 
-import requests
-from home.src.download.thumbnails import ThumbManager
+from api.src.search_processor import SearchProcess
+from api.src.task_processor import TaskHandler
+from home.src.download.queue import PendingInteract
+from home.src.download.yt_dlp_base import CookieHandler
+from home.src.es.connect import ElasticWrap
+from home.src.frontend.searching import SearchForm
+from home.src.index.generic import Pagination
+from home.src.index.video import SponsorBlock
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import UrlListParser
-from home.src.ta.ta_redis import RedisArchivist
+from home.src.ta.ta_redis import RedisArchivist, RedisQueue
 from home.tasks import extrac_dl, subscribe_to
 from rest_framework.authentication import (
     SessionAuthentication,
     TokenAuthentication,
 )
+from rest_framework.authtoken.models import Token
+from rest_framework.authtoken.views import ObtainAuthToken
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 from rest_framework.views import APIView
@@ -21,67 +29,54 @@ class ApiBaseView(APIView):
     authentication_classes = [SessionAuthentication, TokenAuthentication]
     permission_classes = [IsAuthenticated]
     search_base = False
+    data = False
 
     def __init__(self):
         super().__init__()
-        self.response = {"data": False}
+        self.response = {"data": False, "config": AppConfig().config}
+        self.data = {"query": {"match_all": {}}}
         self.status_code = False
         self.context = False
-        self.default_conf = AppConfig().config
-
-    def config_builder(self):
-        """build confic context"""
-        self.context = {
-            "es_url": self.default_conf["application"]["es_url"],
-            "es_auth": self.default_conf["application"]["es_auth"],
-        }
-        self.response["config"] = self.default_conf
+        self.pagination_handler = False
 
     def get_document(self, document_id):
         """get single document from es"""
-        es_url = self.context["es_url"]
-        url = f"{es_url}{self.search_base}{document_id}"
-        print(url)
-        response = requests.get(url, auth=self.context["es_auth"])
+        path = f"{self.search_base}{document_id}"
+        print(path)
+        response, status_code = ElasticWrap(path).get()
         try:
-            self.response["data"] = response.json()["_source"]
+            self.response["data"] = SearchProcess(response).process()
         except KeyError:
             print(f"item not found: {document_id}")
             self.response["data"] = False
-        self.status_code = response.status_code
+        self.status_code = status_code
 
-    def process_keys(self):
-        """process keys for frontend"""
-        all_keys = self.response["data"].keys()
-        if "media_url" in all_keys:
-            media_url = self.response["data"]["media_url"]
-            self.response["data"]["media_url"] = f"/media/{media_url}"
-        if "vid_thumb_url" in all_keys:
-            youtube_id = self.response["data"]["youtube_id"]
-            vid_thumb_url = ThumbManager().vid_thumb_path(youtube_id)
-            cache_dir = self.default_conf["application"]["cache_dir"]
-            new_thumb = f"{cache_dir}/{vid_thumb_url}"
-            self.response["data"]["vid_thumb_url"] = new_thumb
-        if "subtitles" in all_keys:
-            all_subtitles = self.response["data"]["subtitles"]
-            for idx, _ in enumerate(all_subtitles):
-                url = self.response["data"]["subtitles"][idx]["media_url"]
-                new_url = f"/media/{url}"
-                self.response["data"]["subtitles"][idx]["media_url"] = new_url
+    def initiate_pagination(self, request):
+        """set initial pagination values"""
+        user_id = request.user.id
+        page_get = int(request.GET.get("page", 0))
+        self.pagination_handler = Pagination(page_get, user_id)
+        self.data.update(
+            {
+                "size": self.pagination_handler.pagination["page_size"],
+                "from": self.pagination_handler.pagination["page_from"],
+            }
+        )
 
-    def get_paginate(self):
-        """add pagination detail to response"""
-        self.response["paginate"] = False
-
-    def get_document_list(self, data):
+    def get_document_list(self, request):
         """get a list of results"""
-        es_url = self.context["es_url"]
-        url = f"{es_url}{self.search_base}"
-        print(url)
-        response = requests.get(url, json=data, auth=self.context["es_auth"])
-        all_hits = response.json()["hits"]["hits"]
-        self.response["data"] = [i["_source"] for i in all_hits]
-        self.status_code = response.status_code
+        print(self.search_base)
+        self.initiate_pagination(request)
+        es_handler = ElasticWrap(self.search_base)
+        response, status_code = es_handler.get(data=self.data)
+        self.response["data"] = SearchProcess(response).process()
+        if self.response["data"]:
+            self.status_code = status_code
+        else:
+            self.status_code = 404
+
+        self.pagination_handler.validate(response["hits"]["total"]["value"])
+        self.response["paginate"] = self.pagination_handler.pagination
 
 
 class VideoApiView(ApiBaseView):
@@ -89,17 +84,30 @@ class VideoApiView(ApiBaseView):
     GET: returns metadata dict of video
     """
 
-    search_base = "/ta_video/_doc/"
+    search_base = "ta_video/_doc/"
 
     def get(self, request, video_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(video_id)
-        self.process_keys()
         return Response(self.response, status=self.status_code)
 
 
+class VideoApiListView(ApiBaseView):
+    """resolves to /api/video/
+    GET: returns list of videos
+    """
+
+    search_base = "ta_video/_search/"
+
+    def get(self, request):
+        """get request"""
+        self.data.update({"sort": [{"published": {"order": "desc"}}]})
+        self.get_document_list(request)
+
+        return Response(self.response)
+
+
 class VideoProgressView(ApiBaseView):
     """resolves to /api/video/<video_id>/
     handle progress status for video
@@ -124,7 +132,7 @@ class VideoProgressView(ApiBaseView):
         position = request.data.get("position", 0)
         key = f"{request.user.id}:progress:{video_id}"
         message = {"position": position, "youtube_id": video_id}
-        RedisArchivist().set_message(key, message, expire=False)
+        RedisArchivist().set_message(key, message)
         self.response = request.data
 
         return Response(self.response)
@@ -138,17 +146,65 @@ class VideoProgressView(ApiBaseView):
         return Response(self.response)
 
 
+class VideoSponsorView(ApiBaseView):
+    """resolves to /api/video/<video_id>/sponsor/
+    handle sponsor block integration
+    """
+
+    search_base = "ta_video/_doc/"
+
+    def get(self, request, video_id):
+        """get sponsor info"""
+        # pylint: disable=unused-argument
+
+        self.get_document(video_id)
+        sponsorblock = self.response["data"].get("sponsorblock")
+
+        return Response(sponsorblock)
+
+    def post(self, request, video_id):
+        """post verification and timestamps"""
+        if "segment" in request.data:
+            response, status_code = self._create_segment(request, video_id)
+        elif "vote" in request.data:
+            response, status_code = self._vote_on_segment(request)
+
+        return Response(response, status=status_code)
+
+    @staticmethod
+    def _create_segment(request, video_id):
+        """create segment in API"""
+        start_time = request.data["segment"]["startTime"]
+        end_time = request.data["segment"]["endTime"]
+        response, status_code = SponsorBlock(request.user.id).post_timestamps(
+            video_id, start_time, end_time
+        )
+
+        return response, status_code
+
+    @staticmethod
+    def _vote_on_segment(request):
+        """validate on existing segment"""
+        user_id = request.user.id
+        uuid = request.data["vote"]["uuid"]
+        vote = request.data["vote"]["yourVote"]
+        response, status_code = SponsorBlock(user_id).vote_on_segment(
+            uuid, vote
+        )
+
+        return response, status_code
+
+
 class ChannelApiView(ApiBaseView):
     """resolves to /api/channel/<channel_id>/
     GET: returns metadata dict of channel
     """
 
-    search_base = "/ta_channel/_doc/"
+    search_base = "ta_channel/_doc/"
 
     def get(self, request, channel_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(channel_id)
         return Response(self.response, status=self.status_code)
 
@@ -159,15 +215,14 @@ class ChannelApiListView(ApiBaseView):
     POST: edit a list of channels
     """
 
-    search_base = "/ta_channel/_search/"
+    search_base = "ta_channel/_search/"
 
     def get(self, request):
-        # pylint: disable=unused-argument
         """get request"""
-        data = {"query": {"match_all": {}}}
-        self.config_builder()
-        self.get_document_list(data)
-        self.get_paginate()
+        self.get_document_list(request)
+        self.data.update(
+            {"sort": [{"channel_name.keyword": {"order": "asc"}}]}
+        )
 
         return Response(self.response)
 
@@ -189,51 +244,139 @@ class ChannelApiListView(ApiBaseView):
         return Response(data)
 
 
+class ChannelApiVideoView(ApiBaseView):
+    """resolves to /api/channel/<channel-id>/video
+    GET: returns a list of videos of channel
+    """
+
+    search_base = "ta_video/_search/"
+
+    def get(self, request, channel_id):
+        """handle get request"""
+        self.data.update(
+            {
+                "query": {
+                    "term": {"channel.channel_id": {"value": channel_id}}
+                },
+                "sort": [{"published": {"order": "desc"}}],
+            }
+        )
+        self.get_document_list(request)
+
+        return Response(self.response, status=self.status_code)
+
+
+class PlaylistApiListView(ApiBaseView):
+    """resolves to /api/playlist/
+    GET: returns list of indexed playlists
+    """
+
+    search_base = "ta_playlist/_search/"
+
+    def get(self, request):
+        """handle get request"""
+        self.data.update(
+            {"sort": [{"playlist_name.keyword": {"order": "asc"}}]}
+        )
+        self.get_document_list(request)
+        return Response(self.response)
+
+
 class PlaylistApiView(ApiBaseView):
     """resolves to /api/playlist/<playlist_id>/
     GET: returns metadata dict of playlist
     """
 
-    search_base = "/ta_playlist/_doc/"
+    search_base = "ta_playlist/_doc/"
 
     def get(self, request, playlist_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(playlist_id)
         return Response(self.response, status=self.status_code)
 
 
+class PlaylistApiVideoView(ApiBaseView):
+    """resolves to /api/playlist/<playlist_id>/video
+    GET: returns list of videos in playlist
+    """
+
+    search_base = "ta_video/_search/"
+
+    def get(self, request, playlist_id):
+        """handle get request"""
+        self.data["query"] = {
+            "term": {"playlist.keyword": {"value": playlist_id}}
+        }
+        self.data.update({"sort": [{"published": {"order": "desc"}}]})
+
+        self.get_document_list(request)
+        return Response(self.response, status=self.status_code)
+
+
 class DownloadApiView(ApiBaseView):
     """resolves to /api/download/<video_id>/
     GET: returns metadata dict of an item in the download queue
+    POST: update status of item to pending or ignore
+    DELETE: forget from download queue
     """
 
-    search_base = "/ta_download/_doc/"
+    search_base = "ta_download/_doc/"
+    valid_status = ["pending", "ignore"]
 
     def get(self, request, video_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(video_id)
         return Response(self.response, status=self.status_code)
 
+    def post(self, request, video_id):
+        """post to video to change status"""
+        item_status = request.data["status"]
+        if item_status not in self.valid_status:
+            message = f"{video_id}: invalid status {item_status}"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        print(f"{video_id}: change status to {item_status}")
+        PendingInteract(video_id=video_id, status=item_status).update_status()
+        RedisQueue().clear_item(video_id)
+
+        return Response(request.data)
+
+    @staticmethod
+    def delete(request, video_id):
+        # pylint: disable=unused-argument
+        """delete single video from queue"""
+        print(f"{video_id}: delete from queue")
+        PendingInteract(video_id=video_id).delete_item()
+
+        return Response({"success": True})
+
 
 class DownloadApiListView(ApiBaseView):
     """resolves to /api/download/
     GET: returns latest videos in the download queue
     POST: add a list of videos to download queue
+    DELETE: remove items based on query filter
     """
 
-    search_base = "/ta_download/_search/"
+    search_base = "ta_download/_search/"
+    valid_filter = ["pending", "ignore"]
 
     def get(self, request):
-        # pylint: disable=unused-argument
         """get request"""
-        data = {"query": {"match_all": {}}}
-        self.config_builder()
-        self.get_document_list(data)
-        self.get_paginate()
+        query_filter = request.GET.get("filter", False)
+        self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
+        if query_filter:
+            if query_filter not in self.valid_filter:
+                message = f"invalid url query filder: {query_filter}"
+                print(message)
+                return Response({"message": message}, status=400)
+
+            self.data["query"] = {"term": {"status": {"value": query_filter}}}
+
+        self.get_document_list(request)
         return Response(self.response)
 
     @staticmethod
@@ -260,3 +403,146 @@ class DownloadApiListView(ApiBaseView):
         extrac_dl.delay(youtube_ids)
 
         return Response(data)
+
+    def delete(self, request):
+        """delete download queue"""
+        query_filter = request.GET.get("filter", False)
+        if query_filter not in self.valid_filter:
+            message = f"invalid url query filter: {query_filter}"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        message = f"delete queue by status: {query_filter}"
+        print(message)
+        PendingInteract(status=query_filter).delete_by_status()
+
+        return Response({"message": message})
+
+
+class PingView(ApiBaseView):
+    """resolves to /api/ping/
+    GET: test your connection
+    """
+
+    @staticmethod
+    def get(request):
+        """get pong"""
+        data = {"response": "pong", "user": request.user.id}
+        return Response(data)
+
+
+class LoginApiView(ObtainAuthToken):
+    """resolves to /api/login/
+    POST: return token and username after successful login
+    """
+
+    def post(self, request, *args, **kwargs):
+        """post data"""
+        # pylint: disable=no-member
+        serializer = self.serializer_class(
+            data=request.data, context={"request": request}
+        )
+        serializer.is_valid(raise_exception=True)
+        user = serializer.validated_data["user"]
+        token, _ = Token.objects.get_or_create(user=user)
+
+        print(f"returning token for user with id {user.pk}")
+
+        return Response({"token": token.key, "user_id": user.pk})
+
+
+class TaskApiView(ApiBaseView):
+    """resolves to /api/task/
+    GET: check if ongoing background task
+    POST: start a new background task
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        response = {"rescan": False, "downloading": False}
+        for key in response.keys():
+            response[key] = RedisArchivist().is_locked(key)
+
+        return Response(response)
+
+    def post(self, request):
+        """handle post request"""
+
+        data = request.data
+        print(data)
+        response = TaskHandler(data).run_task()
+
+        return Response(response)
+
+
+class CookieView(ApiBaseView):
+    """resolves to /api/cookie/
+    GET: check if cookie is enabled
+    POST: verify validity of cookie
+    PUT: import cookie
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        config = AppConfig().config
+        valid = RedisArchivist().get_message("cookie:valid")
+        response = {"cookie_enabled": config["downloads"]["cookie_import"]}
+        response.update(valid)
+
+        return Response(response)
+
+    @staticmethod
+    def post(request):
+        """handle post request"""
+        # pylint: disable=unused-argument
+        config = AppConfig().config
+        validated = CookieHandler(config).validate()
+
+        return Response({"cookie_validated": validated})
+
+    @staticmethod
+    def put(request):
+        """handle put request"""
+        # pylint: disable=unused-argument
+        config = AppConfig().config
+        cookie = request.data.get("cookie")
+        if not cookie:
+            message = "missing cookie key in request data"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        print(f"cookie preview:\n\n{cookie[:300]}")
+        handler = CookieHandler(config)
+        handler.set_cookie(cookie)
+        validated = handler.validate()
+        if not validated:
+            handler.revoke()
+            message = {"cookie_import": "fail", "cookie_validated": validated}
+            print(f"cookie: {message}")
+            return Response({"message": message}, status=400)
+
+        message = {"cookie_import": "done", "cookie_validated": validated}
+        return Response(message)
+
+
+class SearchView(ApiBaseView):
+    """resolves to /api/search/
+    GET: run a search with the string in the ?query parameter
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request
+        search through all indexes"""
+        search_query = request.GET.get("query", None)
+        if search_query is None:
+            return Response(
+                {"message": "no search query specified"}, status=400
+            )
+        print("searching for: " + search_query)
+        search_results = SearchForm().multi_search(search_query)
+        return Response(search_results)

tubearchivist/config/settings.py

@@ -14,7 +14,9 @@ import hashlib
 from os import environ, path
 from pathlib import Path
 
+import ldap
 from corsheaders.defaults import default_headers
+from django_auth_ldap.config import LDAPSearch
 from home.src.ta.config import AppConfig
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -30,7 +32,14 @@ SECRET_KEY = PW_HASH.hexdigest()
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = bool(environ.get("DJANGO_DEBUG"))
 
-ALLOWED_HOSTS = ["*"]
+ALLOWED_HOSTS = [i.strip() for i in environ.get("TA_HOST").split()]
+
+CSRF_TRUSTED_ORIGINS = []
+for host in ALLOWED_HOSTS:
+    if host.startswith("http://") or host.startswith("https://"):
+        CSRF_TRUSTED_ORIGINS.append(host)
+    else:
+        CSRF_TRUSTED_ORIGINS.append(f"http://{host}")
 
 
 # Application definition
@@ -83,6 +92,35 @@ TEMPLATES = [
 
 WSGI_APPLICATION = "config.wsgi.application"
 
+if bool(environ.get("TA_LDAP")):
+    # pylint: disable=global-at-module-level
+    global AUTH_LDAP_SERVER_URI
+    AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
+
+    global AUTH_LDAP_BIND_DN
+    AUTH_LDAP_BIND_DN = environ.get("TA_LDAP_BIND_DN")
+
+    global AUTH_LDAP_BIND_PASSWORD
+    AUTH_LDAP_BIND_PASSWORD = environ.get("TA_LDAP_BIND_PASSWORD")
+
+    global AUTH_LDAP_USER_SEARCH
+    # pylint: disable=no-member
+    AUTH_LDAP_USER_SEARCH = LDAPSearch(
+        environ.get("TA_LDAP_USER_BASE"),
+        ldap.SCOPE_SUBTREE,
+        "(&(uid=%(user)s)" + environ.get("TA_LDAP_USER_FILTER") + ")",
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP
+    AUTH_LDAP_USER_ATTR_MAP = {
+        "username": "uid",
+        "first_name": "givenName",
+        "last_name": "sn",
+        "email": "mail",
+    }
+
+    global AUTHENTICATION_BACKENDS
+    AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
 
 # Database
 # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
@@ -146,8 +184,21 @@ LOGOUT_REDIRECT_URL = "/login/"
 
 # Cors needed for browser extension
 # background.js makes the request so HTTP_ORIGIN will be from extension
-CORS_ALLOWED_ORIGIN_REGEXES = [r"moz-extension://*", r"chrome-extension://*"]
+if environ.get("DISABLE_CORS"):
+    # disable cors
+    CORS_ORIGIN_ALLOW_ALL = True
+else:
+    CORS_ALLOWED_ORIGIN_REGEXES = [
+        r"moz-extension://*",
+        r"chrome-extension://*",
+    ]
+    CORS_ALLOWED_ORIGINS = ["http://localhost:3000"]
+
 
 CORS_ALLOW_HEADERS = list(default_headers) + [
     "mode",
 ]
+
+# TA application settings
+TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
+TA_VERSION = "v0.2.1"

tubearchivist/home/apps.py

@@ -1,53 +1,114 @@
 """handle custom startup functions"""
 
 import os
+import sys
 
 from django.apps import AppConfig
+from home.src.es.connect import ElasticWrap
 from home.src.es.index_setup import index_check
 from home.src.ta.config import AppConfig as ArchivistConfig
 from home.src.ta.ta_redis import RedisArchivist
 
 
-def sync_redis_state():
-    """make sure redis gets new config.json values"""
-    print("sync redis")
-    config_handler = ArchivistConfig()
-    config_handler.load_new_defaults()
+class StartupCheck:
+    """checks to run at application startup"""
 
+    MIN_MAJOR, MAX_MAJOR = 8, 8
+    MIN_MINOR = 0
 
-def make_folders():
-    """make needed cache folders here so docker doesn't mess it up"""
-    folders = [
-        "download",
-        "channels",
-        "videos",
-        "playlists",
-        "import",
-        "backup",
-    ]
-    config = ArchivistConfig().config
-    cache_dir = config["application"]["cache_dir"]
-    for folder in folders:
-        folder_path = os.path.join(cache_dir, folder)
-        try:
-            os.makedirs(folder_path)
-        except FileExistsError:
-            continue
+    def __init__(self):
+        self.config_handler = ArchivistConfig()
+        self.redis_con = RedisArchivist()
+        self.has_run = self.get_has_run()
 
+    def run(self):
+        """run all startup checks"""
+        print("run startup checks")
+        self.es_version_check()
+        self.release_lock()
+        index_check()
+        self.sync_redis_state()
+        self.set_redis_conf()
+        self.make_folders()
+        self.set_has_run()
 
-def release_lock():
-    """make sure there are no leftover locks set in redis on container start"""
-    all_locks = [
-        "manual_import",
-        "downloading",
-        "dl_queue",
-        "dl_queue_id",
-        "rescan",
-    ]
-    for lock in all_locks:
-        response = RedisArchivist().del_message(lock)
-        if response:
-            print("deleted leftover key from redis: " + lock)
+    def get_has_run(self):
+        """validate if check has already executed"""
+        return self.redis_con.get_message("startup_check")
+
+    def set_has_run(self):
+        """startup checks run"""
+        message = {"status": True}
+        self.redis_con.set_message("startup_check", message, expire=120)
+
+    def sync_redis_state(self):
+        """make sure redis gets new config.json values"""
+        print("sync redis")
+        self.config_handler.load_new_defaults()
+
+    def set_redis_conf(self):
+        """set conf values for redis"""
+        self.redis_con.conn.config_set("timeout", 3600)
+
+    def make_folders(self):
+        """make needed cache folders here so docker doesn't mess it up"""
+        folders = [
+            "download",
+            "channels",
+            "videos",
+            "playlists",
+            "import",
+            "backup",
+        ]
+        cache_dir = self.config_handler.config["application"]["cache_dir"]
+        for folder in folders:
+            folder_path = os.path.join(cache_dir, folder)
+            try:
+                os.makedirs(folder_path)
+            except FileExistsError:
+                continue
+
+    def release_lock(self):
+        """make sure there are no leftover locks set in redis"""
+        all_locks = [
+            "startup_check",
+            "manual_import",
+            "downloading",
+            "dl_queue",
+            "dl_queue_id",
+            "rescan",
+            "run_backup",
+        ]
+        for lock in all_locks:
+            response = self.redis_con.del_message(lock)
+            if response:
+                print("deleted leftover key from redis: " + lock)
+
+    def is_invalid(self, version):
+        """return true if es version is invalid, false if ok"""
+        major, minor = [int(i) for i in version.split(".")[:2]]
+        if not self.MIN_MAJOR <= major <= self.MAX_MAJOR:
+            return True
+
+        if minor >= self.MIN_MINOR:
+            return False
+
+        return True
+
+    def es_version_check(self):
+        """check for minimal elasticsearch version"""
+        response, _ = ElasticWrap("/").get()
+        version = response["version"]["number"]
+        invalid = self.is_invalid(version)
+
+        if invalid:
+            print(
+                "required elasticsearch version: "
+                + f"{self.MIN_MAJOR}.{self.MIN_MINOR}"
+            )
+            sys.exit(1)
+
+        print("elasticsearch version check passed")
 
 
 class HomeConfig(AppConfig):
@@ -57,7 +118,9 @@ class HomeConfig(AppConfig):
     name = "home"
 
     def ready(self):
-        release_lock()
-        index_check()
-        sync_redis_state()
-        make_folders()
+        startup = StartupCheck()
+        if startup.has_run["status"]:
+            print("startup checks run in other thread")
+            return
+
+        startup.run()

tubearchivist/home/config.json

@@ -8,7 +8,8 @@
         "home": "grid",
         "channel": "list",
         "downloads": "list",
-        "playlist": "grid"
+        "playlist": "grid",
+        "grid_items": 3
     },
     "subscriptions": {
         "auto_search": false,
@@ -26,14 +27,15 @@
         "subtitle": false,
         "subtitle_source": false,
         "subtitle_index": false,
+        "cookie_import": false,
         "throttledratelimit": false,
-        "integrate_ryd": false
+        "integrate_ryd": false,
+        "integrate_sponsorblock": false
     },
     "application": {
         "app_root": "/app",
         "cache_dir": "/cache",
         "videos": "/youtube",
-        "file_template": "%(id)s_%(title)s.mp4",
         "colors": "dark",
         "enable_cast": false
     },

tubearchivist/home/src/download/queue.py

@@ -5,36 +5,144 @@ Functionality:
 """
 
 import json
-import os
 from datetime import datetime
 
-import requests
-import yt_dlp
-from home.src.download.subscriptions import ChannelSubscription
-from home.src.es.connect import IndexPaginate
+from home.src.download.subscriptions import (
+    ChannelSubscription,
+    PlaylistSubscription,
+)
+from home.src.download.thumbnails import ThumbManager
+from home.src.download.yt_dlp_base import YtWrap
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import DurationConverter, ignore_filelist
+from home.src.ta.helper import DurationConverter
 from home.src.ta.ta_redis import RedisArchivist
 
 
-class PendingList:
-    """manage the pending videos list"""
-
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
-    VIDEOS = CONFIG["application"]["videos"]
+class PendingIndex:
+    """base class holding all export methods"""
 
     def __init__(self):
-        self.all_channel_ids = False
-        self.all_downloaded = False
-        self.missing_from_playlists = []
+        self.all_pending = False
+        self.all_ignored = False
+        self.all_videos = False
+        self.all_channels = False
+        self.channel_overwrites = False
+        self.video_overwrites = False
+        self.to_skip = False
 
-    def parse_url_list(self, youtube_ids):
+    def get_download(self):
+        """get a list of all pending videos in ta_download"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"timestamp": {"order": "asc"}}],
+        }
+        all_results = IndexPaginate("ta_download", data).get_results()
+
+        self.all_pending = []
+        self.all_ignored = []
+        self.to_skip = []
+
+        for result in all_results:
+            self.to_skip.append(result["youtube_id"])
+            if result["status"] == "pending":
+                self.all_pending.append(result)
+            elif result["status"] == "ignore":
+                self.all_ignored.append(result)
+
+    def get_indexed(self):
+        """get a list of all videos indexed"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        self.all_videos = IndexPaginate("ta_video", data).get_results()
+        for video in self.all_videos:
+            self.to_skip.append(video["youtube_id"])
+
+    def get_channels(self):
+        """get a list of all channels indexed"""
+        self.all_channels = []
+        self.channel_overwrites = {}
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"channel_id": {"order": "asc"}}],
+        }
+        channels = IndexPaginate("ta_channel", data).get_results()
+
+        for channel in channels:
+            channel_id = channel["channel_id"]
+            self.all_channels.append(channel_id)
+            if channel.get("channel_overwrites"):
+                self.channel_overwrites.update(
+                    {channel_id: channel.get("channel_overwrites")}
+                )
+
+        self._map_overwrites()
+
+    def _map_overwrites(self):
+        """map video ids to channel ids overwrites"""
+        self.video_overwrites = {}
+        for video in self.all_pending:
+            video_id = video["youtube_id"]
+            channel_id = video["channel_id"]
+            overwrites = self.channel_overwrites.get(channel_id, False)
+            if overwrites:
+                self.video_overwrites.update({video_id: overwrites})
+
+
+class PendingInteract:
+    """interact with items in download queue"""
+
+    def __init__(self, video_id=False, status=False):
+        self.video_id = video_id
+        self.status = status
+
+    def delete_item(self):
+        """delete single item from pending"""
+        path = f"ta_download/_doc/{self.video_id}"
+        _, _ = ElasticWrap(path).delete(refresh=True)
+
+    def delete_by_status(self):
+        """delete all matching item by status"""
+        data = {"query": {"term": {"status": {"value": self.status}}}}
+        path = "ta_download/_delete_by_query"
+        _, _ = ElasticWrap(path).post(data=data)
+
+    def update_status(self):
+        """update status field of pending item"""
+        data = {"doc": {"status": self.status}}
+        path = f"ta_download/_update/{self.video_id}"
+        _, _ = ElasticWrap(path).post(data=data)
+
+
+class PendingList(PendingIndex):
+    """manage the pending videos list"""
+
+    yt_obs = {
+        "default_search": "ytsearch",
+        "quiet": True,
+        "check_formats": "selected",
+        "noplaylist": True,
+        "writethumbnail": True,
+        "simulate": True,
+        "socket_timeout": 3,
+    }
+
+    def __init__(self, youtube_ids=False):
+        super().__init__()
+        self.config = AppConfig().config
+        self.youtube_ids = youtube_ids
+        self.to_skip = False
+        self.missing_videos = False
+
+    def parse_url_list(self):
         """extract youtube ids from list"""
-        missing_videos = []
-        for entry in youtube_ids:
+        self.missing_videos = []
+        self.get_download()
+        self.get_indexed()
+        for entry in self.youtube_ids:
             # notify
             mess_dict = {
                 "status": "message:add",
@@ -42,126 +150,119 @@ class PendingList:
                 "title": "Adding to download queue.",
                 "message": "Extracting lists",
             }
-            RedisArchivist().set_message("message:add", mess_dict)
-            # extract
-            url = entry["url"]
-            url_type = entry["type"]
-            if url_type == "video":
-                missing_videos.append(url)
-            elif url_type == "channel":
-                video_results = ChannelSubscription().get_last_youtube_videos(
-                    url, limit=False
-                )
-                youtube_ids = [i[0] for i in video_results]
-                missing_videos = missing_videos + youtube_ids
-            elif url_type == "playlist":
-                self.missing_from_playlists.append(entry)
-                playlist = YoutubePlaylist(url)
-                playlist.build_json()
-                video_results = playlist.json_data.get("playlist_entries")
-                youtube_ids = [i["youtube_id"] for i in video_results]
-                missing_videos = missing_videos + youtube_ids
+            RedisArchivist().set_message("message:add", mess_dict, expire=True)
+            self._process_entry(entry)
 
-        return missing_videos
+    def _process_entry(self, entry):
+        """process single entry from url list"""
+        if entry["type"] == "video":
+            self._add_video(entry["url"])
+        elif entry["type"] == "channel":
+            self._parse_channel(entry["url"])
+        elif entry["type"] == "playlist":
+            self._parse_playlist(entry["url"])
+            PlaylistSubscription().process_url_str([entry], subscribed=False)
+        else:
+            raise ValueError(f"invalid url_type: {entry}")
 
-    def add_to_pending(self, missing_videos, ignore=False):
-        """build the bulk json data from pending"""
-        # check if channel is indexed
-        channel_handler = ChannelSubscription()
-        all_indexed = channel_handler.get_channels(subscribed_only=False)
-        self.all_channel_ids = [i["channel_id"] for i in all_indexed]
-        # check if already there
-        self.all_downloaded = self.get_all_downloaded()
+    def _add_video(self, url):
+        """add video to list"""
+        if url not in self.missing_videos and url not in self.to_skip:
+            self.missing_videos.append(url)
+        else:
+            print(f"{url}: skipped adding already indexed video to download.")
 
-        bulk_list, all_videos_added = self.build_bulk(missing_videos, ignore)
-        # add last newline
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-        headers = {"Content-type": "application/x-ndjson"}
-        url = self.ES_URL + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=self.ES_AUTH
+    def _parse_channel(self, url):
+        """add all videos of channel to list"""
+        video_results = ChannelSubscription().get_last_youtube_videos(
+            url, limit=False
         )
-        if not request.ok:
-            print(request)
-            raise ValueError("failed to add video to download queue")
+        youtube_ids = [i[0] for i in video_results]
+        for video_id in youtube_ids:
+            self._add_video(video_id)
 
-        return all_videos_added
+    def _parse_playlist(self, url):
+        """add all videos of playlist to list"""
+        playlist = YoutubePlaylist(url)
+        playlist.build_json()
+        video_results = playlist.json_data.get("playlist_entries")
+        youtube_ids = [i["youtube_id"] for i in video_results]
+        for video_id in youtube_ids:
+            self._add_video(video_id)
 
-    def build_bulk(self, missing_videos, ignore=False):
-        """build the bulk lists"""
+    def add_to_pending(self, status="pending"):
+        """add missing videos to pending list"""
+        self.get_channels()
         bulk_list = []
-        all_videos_added = []
 
-        for idx, youtube_id in enumerate(missing_videos):
-            # check if already downloaded
-            if youtube_id in self.all_downloaded:
+        for idx, youtube_id in enumerate(self.missing_videos):
+            video_details = self.get_youtube_details(youtube_id)
+            if not video_details:
                 continue
 
-            video = self.get_youtube_details(youtube_id)
-            # skip on download error
-            if not video:
-                continue
-
-            channel_indexed = video["channel_id"] in self.all_channel_ids
-            video["channel_indexed"] = channel_indexed
-            if ignore:
-                video["status"] = "ignore"
-            else:
-                video["status"] = "pending"
+            video_details["status"] = status
             action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
             bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(video))
-            all_videos_added.append((youtube_id, video["vid_thumb_url"]))
-            # notify
-            progress = f"{idx + 1}/{len(missing_videos)}"
-            mess_dict = {
-                "status": "message:add",
-                "level": "info",
-                "title": "Adding new videos to download queue.",
-                "message": "Progress: " + progress,
-            }
-            if idx + 1 == len(missing_videos):
-                RedisArchivist().set_message(
-                    "message:add", mess_dict, expire=4
-                )
-            else:
-                RedisArchivist().set_message("message:add", mess_dict)
-            if idx + 1 % 25 == 0:
-                print("adding to queue progress: " + progress)
+            bulk_list.append(json.dumps(video_details))
 
-        return bulk_list, all_videos_added
+            url = video_details["vid_thumb_url"]
+            ThumbManager(youtube_id).download_video_thumb(url)
 
-    @staticmethod
-    def get_youtube_details(youtube_id):
-        """get details from youtubedl for single pending video"""
-        obs = {
-            "default_search": "ytsearch",
-            "quiet": True,
-            "check_formats": "selected",
-            "noplaylist": True,
-            "writethumbnail": True,
-            "simulate": True,
+            self._notify_add(idx)
+
+        if bulk_list:
+            # add last newline
+            bulk_list.append("\n")
+            query_str = "\n".join(bulk_list)
+            _, _ = ElasticWrap("_bulk").post(query_str, ndjson=True)
+
+    def _notify_add(self, idx):
+        """send notification for adding videos to download queue"""
+        progress = f"{idx + 1}/{len(self.missing_videos)}"
+        mess_dict = {
+            "status": "message:add",
+            "level": "info",
+            "title": "Adding new videos to download queue.",
+            "message": "Progress: " + progress,
         }
-        try:
-            vid = yt_dlp.YoutubeDL(obs).extract_info(youtube_id)
-        except yt_dlp.utils.DownloadError:
-            print("failed to extract info for: " + youtube_id)
+        if idx + 1 == len(self.missing_videos):
+            expire = 4
+        else:
+            expire = True
+
+        RedisArchivist().set_message("message:add", mess_dict, expire=expire)
+        if idx + 1 % 25 == 0:
+            print("adding to queue progress: " + progress)
+
+    def get_youtube_details(self, youtube_id):
+        """get details from youtubedl for single pending video"""
+        vid = YtWrap(self.yt_obs, self.config).extract(youtube_id)
+        if not vid:
+            return False
+
+        if vid.get("id") != youtube_id:
+            # skip premium videos with different id
+            print(f"{youtube_id}: skipping premium video, id not matching")
             return False
         # stop if video is streaming live now
         if vid["is_live"]:
             return False
-        # parse response
-        seconds = vid["duration"]
-        duration_str = DurationConverter.get_str(seconds)
+
+        return self._parse_youtube_details(vid)
+
+    def _parse_youtube_details(self, vid):
+        """parse response"""
+        vid_id = vid.get("id")
+        duration_str = DurationConverter.get_str(vid["duration"])
         if duration_str == "NA":
-            print(f"skip extracting duration for: {youtube_id}")
-        upload_date = vid["upload_date"]
-        upload_dt = datetime.strptime(upload_date, "%Y%m%d")
-        published = upload_dt.strftime("%Y-%m-%d")
+            print(f"skip extracting duration for: {vid_id}")
+        published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
+            "%Y-%m-%d"
+        )
+
         # build dict
         youtube_details = {
-            "youtube_id": youtube_id,
+            "youtube_id": vid_id,
             "channel_name": vid["channel"],
             "vid_thumb_url": vid["thumbnail"],
             "title": vid["title"],
@@ -170,94 +271,8 @@ class PendingList:
             "published": published,
             "timestamp": int(datetime.now().strftime("%s")),
         }
+        if self.all_channels:
+            youtube_details.update(
+                {"channel_indexed": vid["channel_id"] in self.all_channels}
+            )
         return youtube_details
-
-    @staticmethod
-    def get_all_pending():
-        """get a list of all pending videos in ta_download"""
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"timestamp": {"order": "asc"}}],
-        }
-        all_results = IndexPaginate("ta_download", data).get_results()
-
-        all_pending = []
-        all_ignore = []
-
-        for result in all_results:
-            if result["status"] == "pending":
-                all_pending.append(result)
-            elif result["status"] == "ignore":
-                all_ignore.append(result)
-
-        return all_pending, all_ignore
-
-    @staticmethod
-    def get_all_indexed():
-        """get a list of all videos indexed"""
-
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"published": {"order": "desc"}}],
-        }
-        all_indexed = IndexPaginate("ta_video", data).get_results()
-
-        return all_indexed
-
-    def get_all_downloaded(self):
-        """get a list of all videos in archive"""
-        channel_folders = os.listdir(self.VIDEOS)
-        all_channel_folders = ignore_filelist(channel_folders)
-        all_downloaded = []
-        for channel_folder in all_channel_folders:
-            channel_path = os.path.join(self.VIDEOS, channel_folder)
-            videos = os.listdir(channel_path)
-            all_videos = ignore_filelist(videos)
-            youtube_vids = [i[9:20] for i in all_videos]
-            for youtube_id in youtube_vids:
-                all_downloaded.append(youtube_id)
-        return all_downloaded
-
-    def delete_from_pending(self, youtube_id):
-        """delete the youtube_id from ta_download"""
-        url = f"{self.ES_URL}/ta_download/_doc/{youtube_id}"
-        response = requests.delete(url, auth=self.ES_AUTH)
-        if not response.ok:
-            print(response.text)
-
-    def delete_pending(self, status):
-        """delete download queue based on status value"""
-        data = {"query": {"term": {"status": {"value": status}}}}
-        payload = json.dumps(data)
-        url = self.ES_URL + "/ta_download/_delete_by_query"
-        headers = {"Content-type": "application/json"}
-        response = requests.post(
-            url, data=payload, headers=headers, auth=self.ES_AUTH
-        )
-        if not response.ok:
-            print(response.text)
-
-    def ignore_from_pending(self, ignore_list):
-        """build the bulk query string"""
-
-        stamp = int(datetime.now().strftime("%s"))
-        bulk_list = []
-
-        for youtube_id in ignore_list:
-            action = {"update": {"_id": youtube_id, "_index": "ta_download"}}
-            source = {"doc": {"status": "ignore", "timestamp": stamp}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(source))
-
-        # add last newline
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-
-        headers = {"Content-type": "application/x-ndjson"}
-        url = self.ES_URL + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request)
-            raise ValueError("failed to set video to ignore")

tubearchivist/home/src/download/subscriptions.py

@@ -4,8 +4,9 @@ Functionality:
 - handle playlist subscriptions
 """
 
-import yt_dlp
 from home.src.download import queue  # partial import
+from home.src.download.thumbnails import ThumbManager
+from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import IndexPaginate
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
@@ -17,10 +18,7 @@ class ChannelSubscription:
     """manage the list of channels subscribed"""
 
     def __init__(self):
-        config = AppConfig().config
-        self.es_url = config["application"]["es_url"]
-        self.es_auth = config["application"]["es_auth"]
-        self.channel_size = config["subscriptions"]["channel_size"]
+        self.config = AppConfig().config
 
     @staticmethod
     def get_channels(subscribed_only=True):
@@ -39,33 +37,27 @@ class ChannelSubscription:
 
     def get_last_youtube_videos(self, channel_id, limit=True):
         """get a list of last videos from channel"""
-        url = f"https://www.youtube.com/channel/{channel_id}/videos"
         obs = {
-            "default_search": "ytsearch",
-            "quiet": True,
             "skip_download": True,
             "extract_flat": True,
         }
         if limit:
-            obs["playlistend"] = self.channel_size
+            obs["playlistend"] = self.config["subscriptions"]["channel_size"]
 
-        try:
-            chan = yt_dlp.YoutubeDL(obs).extract_info(url, download=False)
-        except yt_dlp.utils.DownloadError:
-            print(f"{channel_id}: failed to extract videos, skipping.")
+        url = f"https://www.youtube.com/channel/{channel_id}/videos"
+        channel = YtWrap(obs, self.config).extract(url)
+        if not channel:
             return False
 
-        last_videos = [(i["id"], i["title"]) for i in chan["entries"]]
+        last_videos = [(i["id"], i["title"]) for i in channel["entries"]]
         return last_videos
 
     def find_missing(self):
         """add missing videos from subscribed channels to pending"""
         all_channels = self.get_channels()
-        pending_handler = queue.PendingList()
-        all_pending, all_ignore = pending_handler.get_all_pending()
-        all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
-        all_downloaded = pending_handler.get_all_downloaded()
-        to_ignore = all_ids + all_downloaded
+        pending = queue.PendingList()
+        pending.get_download()
+        pending.get_indexed()
 
         missing_videos = []
 
@@ -75,7 +67,7 @@ class ChannelSubscription:
 
             if last_videos:
                 for video in last_videos:
-                    if video[0] not in to_ignore:
+                    if video[0] not in pending.to_skip:
                         missing_videos.append(video[0])
             # notify
             message = {
@@ -85,11 +77,13 @@ class ChannelSubscription:
                 "message": f"Progress: {idx + 1}/{len(all_channels)}",
             }
             if idx + 1 == len(all_channels):
-                RedisArchivist().set_message(
-                    "message:rescan", message=message, expire=4
-                )
+                expire = 4
             else:
-                RedisArchivist().set_message("message:rescan", message=message)
+                expire = True
+
+            RedisArchivist().set_message(
+                "message:rescan", message=message, expire=expire
+            )
 
         return missing_videos
 
@@ -129,14 +123,16 @@ class PlaylistSubscription:
 
     def process_url_str(self, new_playlists, subscribed=True):
         """process playlist subscribe form url_str"""
-        all_indexed = queue.PendingList().get_all_indexed()
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        all_indexed = IndexPaginate("ta_video", data).get_results()
         all_youtube_ids = [i["youtube_id"] for i in all_indexed]
 
-        new_thumbs = []
         for idx, playlist in enumerate(new_playlists):
-            url_type = playlist["type"]
             playlist_id = playlist["url"]
-            if not url_type == "playlist":
+            if not playlist["type"] == "playlist":
                 print(f"{playlist_id} not a playlist, skipping...")
                 continue
 
@@ -147,8 +143,11 @@ class PlaylistSubscription:
             playlist_h.upload_to_es()
             playlist_h.add_vids_to_playlist()
             self.channel_validate(playlist_h.json_data["playlist_channel_id"])
-            thumb = playlist_h.json_data["playlist_thumbnail"]
-            new_thumbs.append((playlist_id, thumb))
+
+            url = playlist_h.json_data["playlist_thumbnail"]
+            thumb = ThumbManager(playlist_id, item_type="playlist")
+            thumb.download_playlist_thumb(url)
+
             # notify
             message = {
                 "status": "message:subplaylist",
@@ -157,16 +156,14 @@ class PlaylistSubscription:
                 "message": f"Processing {idx + 1} of {len(new_playlists)}",
             }
             RedisArchivist().set_message(
-                "message:subplaylist", message=message
+                "message:subplaylist", message=message, expire=True
             )
 
-        return new_thumbs
-
     @staticmethod
     def channel_validate(channel_id):
         """make sure channel of playlist is there"""
         channel = YoutubeChannel(channel_id)
-        channel.build_json()
+        channel.build_json(upload=True)
 
     @staticmethod
     def change_subscribe(playlist_id, subscribe_status):
@@ -179,12 +176,11 @@ class PlaylistSubscription:
     @staticmethod
     def get_to_ignore():
         """get all youtube_ids already downloaded or ignored"""
-        pending_handler = queue.PendingList()
-        all_pending, all_ignore = pending_handler.get_all_pending()
-        all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
-        all_downloaded = pending_handler.get_all_downloaded()
-        to_ignore = all_ids + all_downloaded
-        return to_ignore
+        pending = queue.PendingList()
+        pending.get_download()
+        pending.get_indexed()
+
+        return pending.to_skip
 
     def find_missing(self):
         """find videos in subscribed playlists not downloaded yet"""
@@ -212,7 +208,9 @@ class PlaylistSubscription:
                 "title": "Scanning playlists: Looking for new videos.",
                 "message": f"Progress: {idx + 1}/{len(all_playlists)}",
             }
-            RedisArchivist().set_message("message:rescan", message=message)
+            RedisArchivist().set_message(
+                "message:rescan", message=message, expire=True
+            )
 
             for video in all_missing:
                 youtube_id = video["youtube_id"]

tubearchivist/home/src/download/thumbnails.py

@@ -4,132 +4,68 @@ functionality:
 - check for missing thumbnails
 """
 
+import base64
 import os
-from collections import Counter
+from io import BytesIO
 from time import sleep
 
 import requests
 from home.src.download import queue  # partial import
-from home.src.download import subscriptions  # partial import
+from home.src.es.connect import IndexPaginate
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import ignore_filelist
-from home.src.ta.ta_redis import RedisArchivist
 from mutagen.mp4 import MP4, MP4Cover
-from PIL import Image
+from PIL import Image, ImageFile, ImageFilter
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
 
 
-class ThumbManager:
-    """handle thumbnails related functions"""
+class ThumbManagerBase:
+    """base class for thumbnail management"""
 
     CONFIG = AppConfig().config
-    MEDIA_DIR = CONFIG["application"]["videos"]
     CACHE_DIR = CONFIG["application"]["cache_dir"]
     VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
     CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
     PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
 
-    def get_all_thumbs(self):
-        """get all video artwork already downloaded"""
-        all_thumb_folders = ignore_filelist(os.listdir(self.VIDEO_DIR))
-        all_thumbs = []
-        for folder in all_thumb_folders:
-            folder_path = os.path.join(self.VIDEO_DIR, folder)
-            if os.path.isfile(folder_path):
-                self.update_path(folder)
-                all_thumbs.append(folder_path)
-                continue
-                # raise exemption here in a future version
-                # raise FileExistsError("video cache dir has files inside")
+    def __init__(self, item_id, item_type, fallback=False):
+        self.item_id = item_id
+        self.item_type = item_type
+        self.fallback = fallback
 
-            all_folder_thumbs = ignore_filelist(os.listdir(folder_path))
-            all_thumbs.extend(all_folder_thumbs)
+    def download_raw(self, url):
+        """download thumbnail for video"""
+        if not url:
+            return self.get_fallback()
 
-        return all_thumbs
+        for i in range(3):
+            try:
+                response = requests.get(url, stream=True, timeout=5)
+                if response.ok:
+                    return Image.open(response.raw)
+                if response.status_code == 404:
+                    return self.get_fallback()
 
-    def update_path(self, file_name):
-        """reorganize thumbnails into folders as update path from v0.0.5"""
-        folder_name = file_name[0].lower()
-        folder_path = os.path.join(self.VIDEO_DIR, folder_name)
-        old_file = os.path.join(self.VIDEO_DIR, file_name)
-        new_file = os.path.join(folder_path, file_name)
-        os.makedirs(folder_path, exist_ok=True)
-        os.rename(old_file, new_file)
+            except requests.exceptions.RequestException:
+                print(f"{self.item_id}: retry thumbnail download {url}")
+                sleep((i + 1) ** i)
 
-    def get_needed_thumbs(self, missing_only=False):
-        """get a list of all missing thumbnails"""
-        all_thumbs = self.get_all_thumbs()
-        all_indexed = queue.PendingList().get_all_indexed()
-        all_in_queue, all_ignored = queue.PendingList().get_all_pending()
+        return False
 
-        needed_thumbs = []
-        for video in all_indexed:
-            youtube_id = video["youtube_id"]
-            thumb_url = video["vid_thumb_url"]
-            if missing_only:
-                if youtube_id + ".jpg" not in all_thumbs:
-                    needed_thumbs.append((youtube_id, thumb_url))
-            else:
-                needed_thumbs.append((youtube_id, thumb_url))
+    def get_fallback(self):
+        """get fallback thumbnail if not available"""
+        if self.fallback:
+            img_raw = Image.open(self.fallback)
+            return img_raw
 
-        for video in all_in_queue + all_ignored:
-            youtube_id = video["youtube_id"]
-            thumb_url = video["vid_thumb_url"]
-            if missing_only:
-                if youtube_id + ".jpg" not in all_thumbs:
-                    needed_thumbs.append((youtube_id, thumb_url))
-            else:
-                needed_thumbs.append((youtube_id, thumb_url))
-
-        return needed_thumbs
-
-    def get_missing_channels(self):
-        """get all channel artwork"""
-        all_channel_art = os.listdir(self.CHANNEL_DIR)
-        files = [i[0:24] for i in all_channel_art]
-        cached_channel_ids = [k for (k, v) in Counter(files).items() if v > 1]
-        channel_sub = subscriptions.ChannelSubscription()
-        channels = channel_sub.get_channels(subscribed_only=False)
-
-        missing_channels = []
-        for channel in channels:
-            channel_id = channel["channel_id"]
-            if channel_id not in cached_channel_ids:
-                channel_banner = channel["channel_banner_url"]
-                channel_thumb = channel["channel_thumb_url"]
-                missing_channels.append(
-                    (channel_id, channel_thumb, channel_banner)
-                )
-
-        return missing_channels
-
-    def get_missing_playlists(self):
-        """get all missing playlist artwork"""
-        all_downloaded = ignore_filelist(os.listdir(self.PLAYLIST_DIR))
-        all_ids_downloaded = [i.replace(".jpg", "") for i in all_downloaded]
-        playlist_sub = subscriptions.PlaylistSubscription()
-        playlists = playlist_sub.get_playlists(subscribed_only=False)
-
-        missing_playlists = []
-        for playlist in playlists:
-            playlist_id = playlist["playlist_id"]
-            if playlist_id not in all_ids_downloaded:
-                playlist_thumb = playlist["playlist_thumbnail"]
-                missing_playlists.append((playlist_id, playlist_thumb))
-
-        return missing_playlists
-
-    def get_raw_img(self, img_url, thumb_type):
-        """get raw image from youtube and handle 404"""
-        try:
-            app_root = self.CONFIG["application"]["app_root"]
-        except KeyError:
-            # lazy keyerror fix to not have to deal with a strange startup
-            # racing contition between the threads in HomeConfig.ready()
-            app_root = "/app"
+        app_root = self.CONFIG["application"]["app_root"]
         default_map = {
             "video": os.path.join(
                 app_root, "static/img/default-video-thumb.jpg"
             ),
+            "playlist": os.path.join(
+                app_root, "static/img/default-video-thumb.jpg"
+            ),
             "icon": os.path.join(
                 app_root, "static/img/default-channel-icon.jpg"
             ),
@@ -137,152 +73,261 @@ class ThumbManager:
                 app_root, "static/img/default-channel-banner.jpg"
             ),
         }
-        if img_url:
-            try:
-                response = requests.get(img_url, stream=True)
-            except ConnectionError:
-                sleep(5)
-                response = requests.get(img_url, stream=True)
-            if not response.ok and not response.status_code == 404:
-                print("retry thumbnail download for " + img_url)
-                sleep(5)
-                response = requests.get(img_url, stream=True)
-        else:
-            response = False
-        if not response or response.status_code == 404:
-            # use default
-            img_raw = Image.open(default_map[thumb_type])
-        else:
-            # use response
-            img_obj = response.raw
-            img_raw = Image.open(img_obj)
+
+        img_raw = Image.open(default_map[self.item_type])
 
         return img_raw
 
-    def download_vid(self, missing_thumbs, notify=True):
-        """download all missing thumbnails from list"""
-        print(f"downloading {len(missing_thumbs)} thumbnails")
-        for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs):
-            folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower())
-            thumb_path = os.path.join(
-                self.CACHE_DIR, self.vid_thumb_path(youtube_id)
-            )
 
-            os.makedirs(folder_path, exist_ok=True)
-            img_raw = self.get_raw_img(thumb_url, "video")
+class ThumbManager(ThumbManagerBase):
+    """handle thumbnails related functions"""
 
-            width, height = img_raw.size
-            if not width / height == 16 / 9:
-                new_height = width / 16 * 9
-                offset = (height - new_height) / 2
-                img_raw = img_raw.crop((0, offset, width, height - offset))
-            img_raw.convert("RGB").save(thumb_path)
+    def __init__(self, item_id, item_type="video", fallback=False):
+        super().__init__(item_id, item_type, fallback=fallback)
 
-            progress = f"{idx + 1}/{len(missing_thumbs)}"
-            if notify:
-                mess_dict = {
-                    "status": "message:add",
-                    "level": "info",
-                    "title": "Processing Videos",
-                    "message": "Downloading Thumbnails, Progress: " + progress,
-                }
-                if idx + 1 == len(missing_thumbs):
-                    RedisArchivist().set_message(
-                        "message:add", mess_dict, expire=4
-                    )
-                else:
-                    RedisArchivist().set_message("message:add", mess_dict)
+    def download(self, url):
+        """download thumbnail"""
+        print(f"{self.item_id}: download {self.item_type} thumbnail")
+        if self.item_type == "video":
+            self.download_video_thumb(url)
+        elif self.item_type == "channel":
+            self.download_channel_art(url)
+        elif self.item_type == "playlist":
+            self.download_playlist_thumb(url)
 
-            if idx + 1 % 25 == 0:
-                print("thumbnail progress: " + progress)
+    def delete(self):
+        """delete thumbnail file"""
+        print(f"{self.item_id}: delete {self.item_type} thumbnail")
+        if self.item_type == "video":
+            self.delete_video_thumb()
+        elif self.item_type == "channel":
+            self.delete_channel_thumb()
+        elif self.item_type == "playlist":
+            self.delete_playlist_thumb()
 
-    def download_chan(self, missing_channels):
-        """download needed artwork for channels"""
-        print(f"downloading {len(missing_channels)} channel artwork")
-        for channel in missing_channels:
-            channel_id, channel_thumb, channel_banner = channel
+    def download_video_thumb(self, url, skip_existing=False):
+        """pass url for video thumbnail"""
+        folder_path = os.path.join(self.VIDEO_DIR, self.item_id[0].lower())
+        thumb_path = self.vid_thumb_path(absolute=True)
 
-            thumb_path = os.path.join(
-                self.CHANNEL_DIR, channel_id + "_thumb.jpg"
-            )
-            img_raw = self.get_raw_img(channel_thumb, "icon")
-            img_raw.convert("RGB").save(thumb_path)
+        if skip_existing and os.path.exists(thumb_path):
+            return
 
-            banner_path = os.path.join(
-                self.CHANNEL_DIR, channel_id + "_banner.jpg"
-            )
-            img_raw = self.get_raw_img(channel_banner, "banner")
-            img_raw.convert("RGB").save(banner_path)
+        os.makedirs(folder_path, exist_ok=True)
+        img_raw = self.download_raw(url)
+        width, height = img_raw.size
 
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Processing Channels",
-                "message": "Downloading Channel Art.",
-            }
-            RedisArchivist().set_message("message:download", mess_dict)
+        if not width / height == 16 / 9:
+            new_height = width / 16 * 9
+            offset = (height - new_height) / 2
+            img_raw = img_raw.crop((0, offset, width, height - offset))
 
-    def download_playlist(self, missing_playlists):
-        """download needed artwork for playlists"""
-        print(f"downloading {len(missing_playlists)} playlist artwork")
-        for playlist in missing_playlists:
-            playlist_id, playlist_thumb_url = playlist
-            thumb_path = os.path.join(self.PLAYLIST_DIR, playlist_id + ".jpg")
-            img_raw = self.get_raw_img(playlist_thumb_url, "video")
-            img_raw.convert("RGB").save(thumb_path)
+        img_raw.convert("RGB").save(thumb_path)
 
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Processing Playlists",
-                "message": "Downloading Playlist Art.",
-            }
-            RedisArchivist().set_message("message:download", mess_dict)
-
-    @staticmethod
-    def vid_thumb_path(youtube_id):
+    def vid_thumb_path(self, absolute=False):
         """build expected path for video thumbnail from youtube_id"""
-        folder_name = youtube_id[0].lower()
+        folder_name = self.item_id[0].lower()
         folder_path = os.path.join("videos", folder_name)
-        thumb_path = os.path.join(folder_path, youtube_id + ".jpg")
+        thumb_path = os.path.join(folder_path, f"{self.item_id}.jpg")
+        if absolute:
+            thumb_path = os.path.join(self.CACHE_DIR, thumb_path)
+
         return thumb_path
 
-    def delete_vid_thumb(self, youtube_id):
+    def download_channel_art(self, urls, skip_existing=False):
+        """pass tuple of channel thumbnails"""
+        channel_thumb, channel_banner = urls
+        self._download_channel_thumb(channel_thumb, skip_existing)
+        self._download_channel_banner(channel_banner, skip_existing)
+
+    def _download_channel_thumb(self, channel_thumb, skip_existing):
+        """download channel thumbnail"""
+
+        thumb_path = os.path.join(
+            self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg"
+        )
+        self.item_type = "icon"
+
+        if skip_existing and os.path.exists(thumb_path):
+            return
+
+        img_raw = self.download_raw(channel_thumb)
+        img_raw.convert("RGB").save(thumb_path)
+
+    def _download_channel_banner(self, channel_banner, skip_existing):
+        """download channel banner"""
+
+        banner_path = os.path.join(
+            self.CHANNEL_DIR, self.item_id + "_banner.jpg"
+        )
+        self.item_type = "banner"
+        if skip_existing and os.path.exists(banner_path):
+            return
+
+        img_raw = self.download_raw(channel_banner)
+        img_raw.convert("RGB").save(banner_path)
+
+    def download_playlist_thumb(self, url, skip_existing=False):
+        """pass thumbnail url"""
+        thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
+        if skip_existing and os.path.exists(thumb_path):
+            return
+
+        img_raw = self.download_raw(url)
+        img_raw.convert("RGB").save(thumb_path)
+
+    def delete_video_thumb(self):
         """delete video thumbnail if exists"""
-        thumb_path = self.vid_thumb_path(youtube_id)
+        thumb_path = self.vid_thumb_path()
         to_delete = os.path.join(self.CACHE_DIR, thumb_path)
         if os.path.exists(to_delete):
             os.remove(to_delete)
 
-    def delete_chan_thumb(self, channel_id):
+    def delete_channel_thumb(self):
         """delete all artwork of channel"""
-        thumb = os.path.join(self.CHANNEL_DIR, channel_id + "_thumb.jpg")
-        banner = os.path.join(self.CHANNEL_DIR, channel_id + "_banner.jpg")
+        thumb = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg")
+        banner = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_banner.jpg")
         if os.path.exists(thumb):
             os.remove(thumb)
         if os.path.exists(banner):
             os.remove(banner)
 
-    def cleanup_downloaded(self):
-        """find downloaded thumbnails without video indexed"""
-        all_thumbs = self.get_all_thumbs()
-        all_indexed = self.get_needed_thumbs()
-        all_needed_thumbs = [i[0] + ".jpg" for i in all_indexed]
-        for thumb in all_thumbs:
-            if thumb not in all_needed_thumbs:
-                # cleanup
-                youtube_id = thumb.rstrip(".jpg")
-                self.delete_vid_thumb(youtube_id)
+    def delete_playlist_thumb(self):
+        """delete playlist thumbnail"""
+        thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
+        if os.path.exists(thumb_path):
+            os.remove(thumb_path)
+
+    def get_vid_base64_blur(self):
+        """return base64 encoded placeholder"""
+        file_path = os.path.join(self.CACHE_DIR, self.vid_thumb_path())
+        img_raw = Image.open(file_path)
+        img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20))
+        img_blur = img_raw.filter(ImageFilter.BLUR)
+        buffer = BytesIO()
+        img_blur.save(buffer, format="JPEG")
+        img_data = buffer.getvalue()
+        img_base64 = base64.b64encode(img_data).decode()
+        data_url = f"data:image/jpg;base64,{img_base64}"
+
+        return data_url
+
+
+class ValidatorCallback:
+    """handle callback validate thumbnails page by page"""
+
+    def __init__(self, source, index_name):
+        self.source = source
+        self.index_name = index_name
+
+    def run(self):
+        """run the task for page"""
+        print(f"{self.index_name}: validate artwork")
+        if self.index_name == "ta_video":
+            self._validate_videos()
+        elif self.index_name == "ta_channel":
+            self._validate_channels()
+        elif self.index_name == "ta_playlist":
+            self._validate_playlists()
+
+    def _validate_videos(self):
+        """check if video thumbnails are correct"""
+        for video in self.source:
+            url = video["_source"]["vid_thumb_url"]
+            handler = ThumbManager(video["_source"]["youtube_id"])
+            handler.download_video_thumb(url, skip_existing=True)
+
+    def _validate_channels(self):
+        """check if all channel artwork is there"""
+        for channel in self.source:
+            urls = (
+                channel["_source"]["channel_thumb_url"],
+                channel["_source"]["channel_banner_url"],
+            )
+            handler = ThumbManager(channel["_source"]["channel_id"])
+            handler.download_channel_art(urls, skip_existing=True)
+
+    def _validate_playlists(self):
+        """check if all playlist artwork is there"""
+        for playlist in self.source:
+            url = playlist["_source"]["playlist_thumbnail"]
+            handler = ThumbManager(playlist["_source"]["playlist_id"])
+            handler.download_playlist_thumb(url, skip_existing=True)
+
+
+class ThumbValidator:
+    """validate thumbnails"""
+
+    def download_missing(self):
+        """download all missing artwork"""
+        self.download_missing_videos()
+        self.download_missing_channels()
+        self.download_missing_playlists()
+
+    def download_missing_videos(self):
+        """get all missing video thumbnails"""
+        data = {
+            "query": {"term": {"active": {"value": True}}},
+            "sort": [{"youtube_id": {"order": "asc"}}],
+            "_source": ["vid_thumb_url", "youtube_id"],
+        }
+        paginate = IndexPaginate(
+            "ta_video", data, size=5000, callback=ValidatorCallback
+        )
+        _ = paginate.get_results()
+
+    def download_missing_channels(self):
+        """get all missing channel thumbnails"""
+        data = {
+            "query": {"term": {"channel_active": {"value": True}}},
+            "sort": [{"channel_id": {"order": "asc"}}],
+            "_source": {
+                "excludes": ["channel_description", "channel_overwrites"]
+            },
+        }
+        paginate = IndexPaginate(
+            "ta_channel", data, callback=ValidatorCallback
+        )
+        _ = paginate.get_results()
+
+    def download_missing_playlists(self):
+        """get all missing playlist artwork"""
+        data = {
+            "query": {"term": {"playlist_active": {"value": True}}},
+            "sort": [{"playlist_id": {"order": "asc"}}],
+            "_source": ["playlist_id", "playlist_thumbnail"],
+        }
+        paginate = IndexPaginate(
+            "ta_playlist", data, callback=ValidatorCallback
+        )
+        _ = paginate.get_results()
+
+
+class ThumbFilesystem:
+    """filesystem tasks for thumbnails"""
+
+    CONFIG = AppConfig().config
+    CACHE_DIR = CONFIG["application"]["cache_dir"]
+    MEDIA_DIR = CONFIG["application"]["videos"]
+    VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
+
+    def sync(self):
+        """embed thumbnails to mediafiles"""
+        video_list = self.get_thumb_list()
+        self._embed_thumbs(video_list)
 
     def get_thumb_list(self):
         """get list of mediafiles and matching thumbnails"""
-        all_indexed = queue.PendingList().get_all_indexed()
+        pending = queue.PendingList()
+        pending.get_download()
+        pending.get_indexed()
+
         video_list = []
-        for video in all_indexed:
-            youtube_id = video["youtube_id"]
+        for video in pending.all_videos:
+            video_id = video["youtube_id"]
             media_url = os.path.join(self.MEDIA_DIR, video["media_url"])
             thumb_path = os.path.join(
-                self.CACHE_DIR, self.vid_thumb_path(youtube_id)
+                self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path()
             )
             video_list.append(
                 {
@@ -294,7 +339,7 @@ class ThumbManager:
         return video_list
 
     @staticmethod
-    def write_all_thumbs(video_list):
+    def _embed_thumbs(video_list):
         """rewrite the thumbnail into media file"""
 
         counter = 1
@@ -312,15 +357,3 @@ class ThumbManager:
             if counter % 50 == 0:
                 print(f"thumbnail write progress {counter}/{len(video_list)}")
             counter = counter + 1
-
-
-def validate_thumbnails():
-    """check if all thumbnails are there and organized correctly"""
-    handler = ThumbManager()
-    thumbs_to_download = handler.get_needed_thumbs(missing_only=True)
-    handler.download_vid(thumbs_to_download)
-    missing_channels = handler.get_missing_channels()
-    handler.download_chan(missing_channels)
-    missing_playlists = handler.get_missing_playlists()
-    handler.download_playlist(missing_playlists)
-    handler.cleanup_downloaded()

tubearchivist/home/src/download/yt_dlp_base.py

@@ -0,0 +1,156 @@
+"""
+functionality:
+- base class to make all calls to yt-dlp
+- handle yt-dlp errors
+"""
+
+import os
+from datetime import datetime
+from http import cookiejar
+from io import StringIO
+
+import yt_dlp
+from home.src.ta.ta_redis import RedisArchivist
+
+
+class YtWrap:
+    """wrap calls to yt"""
+
+    OBS_BASE = {
+        "default_search": "ytsearch",
+        "quiet": True,
+        "check_formats": "selected",
+        "socket_timeout": 2,
+    }
+
+    def __init__(self, obs_request, config=False):
+        self.obs_request = obs_request
+        self.config = config
+        self.build_obs()
+
+    def build_obs(self):
+        """build yt-dlp obs"""
+        self.obs = self.OBS_BASE.copy()
+        self.obs.update(self.obs_request)
+        if self.config:
+            self.add_cookie()
+
+    def add_cookie(self):
+        """add cookie if enabled"""
+        if self.config["downloads"]["cookie_import"]:
+            cookie_io = CookieHandler(self.config).get()
+            self.obs["cookiefile"] = cookie_io
+
+    def download(self, url):
+        """make download request"""
+        with yt_dlp.YoutubeDL(self.obs) as ydl:
+            try:
+                ydl.download([url])
+            except yt_dlp.utils.DownloadError:
+                print(f"{url}: failed to download.")
+                return False
+
+        return True
+
+    def extract(self, url):
+        """make extract request"""
+        try:
+            response = yt_dlp.YoutubeDL(self.obs).extract_info(url)
+        except cookiejar.LoadError:
+            print("cookie file is invalid")
+            return False
+        except (yt_dlp.utils.ExtractorError, yt_dlp.utils.DownloadError):
+            print(f"{url}: failed to get info from youtube")
+            return False
+
+        return response
+
+
+class CookieHandler:
+    """handle youtube cookie for yt-dlp"""
+
+    def __init__(self, config):
+        self.cookie_io = False
+        self.config = config
+
+    def get(self):
+        """get cookie io stream"""
+        cookie = RedisArchivist().get_message("cookie")
+        self.cookie_io = StringIO(cookie)
+        return self.cookie_io
+
+    def import_cookie(self):
+        """import cookie from file"""
+        cache_path = self.config["application"]["cache_dir"]
+        import_path = os.path.join(cache_path, "import", "cookies.google.txt")
+
+        try:
+            with open(import_path, encoding="utf-8") as cookie_file:
+                cookie = cookie_file.read()
+        except FileNotFoundError as err:
+            print(f"cookie: {import_path} file not found")
+            raise err
+
+        self.set_cookie(cookie)
+
+        os.remove(import_path)
+        print("cookie: import successful")
+
+    def set_cookie(self, cookie):
+        """set cookie str and activate in cofig"""
+        RedisArchivist().set_message("cookie", cookie)
+        path = ".downloads.cookie_import"
+        RedisArchivist().set_message("config", True, path=path)
+        self.config["downloads"]["cookie_import"] = True
+        print("cookie: activated and stored in Redis")
+
+    @staticmethod
+    def revoke():
+        """revoke cookie"""
+        RedisArchivist().del_message("cookie")
+        RedisArchivist().del_message("cookie:valid")
+        RedisArchivist().set_message(
+            "config", False, path=".downloads.cookie_import"
+        )
+        print("cookie: revoked")
+
+    def validate(self):
+        """validate cookie using the liked videos playlist"""
+        print("validating cookie")
+        obs_request = {
+            "skip_download": True,
+            "extract_flat": True,
+        }
+        validator = YtWrap(obs_request, self.config)
+        response = bool(validator.extract("LL"))
+        self.store_validation(response)
+
+        # update in redis to avoid expiring
+        modified = validator.obs["cookiefile"].getvalue()
+        if modified:
+            RedisArchivist().set_message("cookie", modified)
+
+        if not response:
+            mess_dict = {
+                "status": "message:download",
+                "level": "error",
+                "title": "Cookie validation failed, exiting...",
+                "message": "",
+            }
+            RedisArchivist().set_message(
+                "message:download", mess_dict, expire=4
+            )
+            print("cookie validation failed, exiting...")
+
+        return response
+
+    @staticmethod
+    def store_validation(response):
+        """remember last validation"""
+        now = datetime.now()
+        message = {
+            "status": response,
+            "validated": int(now.strftime("%s")),
+            "validated_str": now.strftime("%Y-%m-%d %H:%M"),
+        }
+        RedisArchivist().set_message("cookie:valid", message)

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -9,13 +9,11 @@ functionality:
 import os
 import shutil
 from datetime import datetime
-from time import sleep
 
-import requests
-import yt_dlp
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import PlaylistSubscription
-from home.src.es.connect import IndexPaginate
+from home.src.download.yt_dlp_base import CookieHandler, YtWrap
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo, index_new_video
@@ -24,14 +22,135 @@ from home.src.ta.helper import clean_string, ignore_filelist
 from home.src.ta.ta_redis import RedisArchivist, RedisQueue
 
 
+class DownloadPostProcess:
+    """handle task to run after download queue finishes"""
+
+    def __init__(self, download):
+        self.download = download
+        self.now = int(datetime.now().strftime("%s"))
+        self.pending = False
+
+    def run(self):
+        """run all functions"""
+        self.pending = PendingList()
+        self.pending.get_download()
+        self.pending.get_channels()
+        self.pending.get_indexed()
+        self.auto_delete_all()
+        self.auto_delete_overwrites()
+        self.validate_playlists()
+
+    def auto_delete_all(self):
+        """handle auto delete"""
+        autodelete_days = self.download.config["downloads"]["autodelete_days"]
+        if not autodelete_days:
+            return
+
+        print(f"auto delete older than {autodelete_days} days")
+        now_lte = self.now - autodelete_days * 24 * 60 * 60
+        data = {
+            "query": {"range": {"player.watched_date": {"lte": now_lte}}},
+            "sort": [{"player.watched_date": {"order": "asc"}}],
+        }
+        self._auto_delete_watched(data)
+
+    def auto_delete_overwrites(self):
+        """handle per channel auto delete from overwrites"""
+        for channel_id, value in self.pending.channel_overwrites.items():
+            if "autodelete_days" in value:
+                autodelete_days = value.get("autodelete_days")
+                print(f"{channel_id}: delete older than {autodelete_days}d")
+                now_lte = self.now - autodelete_days * 24 * 60 * 60
+                must_list = [
+                    {"range": {"player.watched_date": {"lte": now_lte}}},
+                    {"term": {"channel.channel_id": {"value": channel_id}}},
+                ]
+                data = {
+                    "query": {"bool": {"must": must_list}},
+                    "sort": [{"player.watched_date": {"order": "desc"}}],
+                }
+                self._auto_delete_watched(data)
+
+    @staticmethod
+    def _auto_delete_watched(data):
+        """delete watched videos after x days"""
+        to_delete = IndexPaginate("ta_video", data).get_results()
+        if not to_delete:
+            return
+
+        for video in to_delete:
+            youtube_id = video["youtube_id"]
+            print(f"{youtube_id}: auto delete video")
+            YoutubeVideo(youtube_id).delete_media_file()
+
+        print("add deleted to ignore list")
+        vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
+        pending = PendingList(youtube_ids=vids)
+        pending.parse_url_list()
+        pending.add_to_pending(status="ignore")
+
+    def validate_playlists(self):
+        """look for playlist needing to update"""
+        for id_c, channel_id in enumerate(self.download.channels):
+            channel = YoutubeChannel(channel_id)
+            overwrites = self.pending.channel_overwrites.get(channel_id, False)
+            if overwrites and overwrites.get("index_playlists"):
+                # validate from remote
+                channel.index_channel_playlists()
+                continue
+
+            # validate from local
+            playlists = channel.get_indexed_playlists()
+            all_channel_playlist = [i["playlist_id"] for i in playlists]
+            self._validate_channel_playlist(all_channel_playlist, id_c)
+
+    def _validate_channel_playlist(self, all_channel_playlist, id_c):
+        """scan channel for playlist needing update"""
+        all_youtube_ids = [i["youtube_id"] for i in self.pending.all_videos]
+        for id_p, playlist_id in enumerate(all_channel_playlist):
+            playlist = YoutubePlaylist(playlist_id)
+            playlist.all_youtube_ids = all_youtube_ids
+            playlist.build_json(scrape=True)
+            if not playlist.json_data:
+                playlist.deactivate()
+
+            playlist.add_vids_to_playlist()
+            playlist.upload_to_es()
+            self._notify_playlist_progress(all_channel_playlist, id_c, id_p)
+
+    def _notify_playlist_progress(self, all_channel_playlist, id_c, id_p):
+        """notify to UI"""
+        title = (
+            "Processing playlists for channels: "
+            + f"{id_c + 1}/{len(self.download.channels)}"
+        )
+        message = f"Progress: {id_p + 1}/{len(all_channel_playlist)}"
+        key = "message:download"
+        mess_dict = {
+            "status": key,
+            "level": "info",
+            "title": title,
+            "message": message,
+        }
+        if id_p + 1 == len(all_channel_playlist):
+            expire = 4
+        else:
+            expire = True
+
+        RedisArchivist().set_message(key, mess_dict, expire=expire)
+
+
 class VideoDownloader:
     """
     handle the video download functionality
     if not initiated with list, take from queue
     """
 
+    MSG = "message:download"
+
     def __init__(self, youtube_id_list=False):
         self.obs = False
+        self.video_overwrites = False
         self.youtube_id_list = youtube_id_list
         self.config = AppConfig().config
         self._build_obs()
@@ -39,7 +158,9 @@ class VideoDownloader:
 
     def run_queue(self):
         """setup download queue in redis loop until no more items"""
-        queue = RedisQueue("dl_queue")
+        self._setup_queue()
+
+        queue = RedisQueue()
 
         limit_queue = self.config["downloads"]["limit_count"]
         if limit_queue:
@@ -50,57 +171,78 @@ class VideoDownloader:
             if not youtube_id:
                 break
 
-            try:
-                self._dl_single_vid(youtube_id)
-            except yt_dlp.utils.DownloadError:
-                print("failed to download " + youtube_id)
+            success = self._dl_single_vid(youtube_id)
+            if not success:
                 continue
-            vid_dict = index_new_video(youtube_id)
+
+            vid_dict = index_new_video(
+                youtube_id, video_overwrites=self.video_overwrites
+            )
             self.channels.add(vid_dict["channel"]["channel_id"])
+            mess_dict = {
+                "status": self.MSG,
+                "level": "info",
+                "title": "Moving....",
+                "message": "Moving downloaded file to storage folder",
+            }
+            RedisArchivist().set_message(self.MSG, mess_dict)
+
             self.move_to_archive(vid_dict)
+            mess_dict = {
+                "status": self.MSG,
+                "level": "info",
+                "title": "Completed",
+                "message": "",
+            }
+            RedisArchivist().set_message(self.MSG, mess_dict, expire=10)
             self._delete_from_pending(youtube_id)
 
-        autodelete_days = self.config["downloads"]["autodelete_days"]
-        if autodelete_days:
-            print(f"auto delete older than {autodelete_days} days")
-            self.auto_delete_watched(autodelete_days)
+        # post processing
+        self._add_subscribed_channels()
+        DownloadPostProcess(self).run()
 
-    @staticmethod
-    def add_pending():
+    def _setup_queue(self):
+        """setup required and validate"""
+        if self.config["downloads"]["cookie_import"]:
+            valid = CookieHandler(self.config).validate()
+            if not valid:
+                return
+
+        pending = PendingList()
+        pending.get_download()
+        pending.get_channels()
+        self.video_overwrites = pending.video_overwrites
+
+    def add_pending(self):
         """add pending videos to download queue"""
         mess_dict = {
-            "status": "message:download",
+            "status": self.MSG,
             "level": "info",
             "title": "Looking for videos to download",
             "message": "Scanning your download queue.",
         }
-        RedisArchivist().set_message("message:download", mess_dict)
-        all_pending, _ = PendingList().get_all_pending()
-        to_add = [i["youtube_id"] for i in all_pending]
+        RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
+        pending = PendingList()
+        pending.get_download()
+        to_add = [i["youtube_id"] for i in pending.all_pending]
         if not to_add:
             # there is nothing pending
             print("download queue is empty")
             mess_dict = {
-                "status": "message:download",
+                "status": self.MSG,
                 "level": "error",
                 "title": "Download queue is empty",
                 "message": "Add some videos to the queue first.",
             }
-            RedisArchivist().set_message("message:download", mess_dict)
+            RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
             return
 
-        queue = RedisQueue("dl_queue")
-        queue.add_list(to_add)
+        RedisQueue().add_list(to_add)
 
-    @staticmethod
-    def _progress_hook(response):
+    def _progress_hook(self, response):
         """process the progress_hooks from yt_dlp"""
-        # title
-        path = os.path.split(response["filename"])[-1][12:]
-        filename = os.path.splitext(os.path.splitext(path)[0])[0]
-        filename_clean = filename.replace("_", " ")
-        title = "Downloading: " + filename_clean
-        # message
+        title = "Downloading: " + response["info_dict"]["title"]
+
         try:
             percent = response["_percent_str"]
             size = response["_total_bytes_str"]
@@ -109,13 +251,14 @@ class VideoDownloader:
             message = f"{percent} of {size} at {speed} - time left: {eta}"
         except KeyError:
             message = "processing"
+
         mess_dict = {
-            "status": "message:download",
+            "status": self.MSG,
             "level": "info",
             "title": title,
             "message": message,
         }
-        RedisArchivist().set_message("message:download", mess_dict)
+        RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
 
     def _build_obs(self):
         """collection to build all obs passed to yt-dlp"""
@@ -128,11 +271,9 @@ class VideoDownloader:
         self.obs = {
             "default_search": "ytsearch",
             "merge_output_format": "mp4",
-            "restrictfilenames": True,
             "outtmpl": (
                 self.config["application"]["cache_dir"]
-                + "/download/"
-                + self.config["application"]["file_template"]
+                + "/download/%(id)s.mp4"
             ),
             "progress_hooks": [self._progress_hook],
             "noprogress": True,
@@ -142,6 +283,7 @@ class VideoDownloader:
             "writethumbnail": False,
             "noplaylist": True,
             "check_formats": "selected",
+            "socket_timeout": 3,
         }
 
     def _build_obs_user(self):
@@ -169,6 +311,17 @@ class VideoDownloader:
                     "add_metadata": True,
                 }
             )
+            postprocessors.append(
+                {
+                    "key": "MetadataFromField",
+                    "formats": [
+                        "%(title)s:%(meta_title)s",
+                        "%(uploader)s:%(meta_artist)s",
+                        ":(?P<album>)",
+                    ],
+                    "when": "pre_process",
+                }
+            )
 
         if self.config["downloads"]["add_thumbnail"]:
             postprocessors.append(
@@ -181,22 +334,30 @@ class VideoDownloader:
 
         self.obs["postprocessors"] = postprocessors
 
+    def get_format_overwrites(self, youtube_id):
+        """get overwrites from single video"""
+        overwrites = self.video_overwrites.get(youtube_id, False)
+        if overwrites:
+            return overwrites.get("download_format", False)
+
+        return False
+
     def _dl_single_vid(self, youtube_id):
         """download single video"""
+        obs = self.obs.copy()
+        format_overwrite = self.get_format_overwrites(youtube_id)
+        if format_overwrite:
+            obs["format"] = format_overwrite
+
         dl_cache = self.config["application"]["cache_dir"] + "/download/"
 
         # check if already in cache to continue from there
         all_cached = ignore_filelist(os.listdir(dl_cache))
         for file_name in all_cached:
             if youtube_id in file_name:
-                self.obs["outtmpl"] = os.path.join(dl_cache, file_name)
-        with yt_dlp.YoutubeDL(self.obs) as ydl:
-            try:
-                ydl.download([youtube_id])
-            except yt_dlp.utils.DownloadError:
-                print("retry failed download: " + youtube_id)
-                sleep(10)
-                ydl.download([youtube_id])
+                obs["outtmpl"] = os.path.join(dl_cache, file_name)
+
+        success = YtWrap(obs, self.config).download(youtube_id)
 
         if self.obs["writethumbnail"]:
             # webp files don't get cleaned up automatically
@@ -206,6 +367,8 @@ class VideoDownloader:
                 file_path = os.path.join(dl_cache, file_name)
                 os.remove(file_path)
 
+        return success
+
     def move_to_archive(self, vid_dict):
         """move downloaded video from cache to archive"""
         videos = self.config["application"]["videos"]
@@ -227,21 +390,18 @@ class VideoDownloader:
         for file_str in all_cached:
             if vid_dict["youtube_id"] in file_str:
                 old_file = file_str
-        old_file_path = os.path.join(cache_dir, "download", old_file)
-        new_file_path = os.path.join(videos, vid_dict["media_url"])
+        old_path = os.path.join(cache_dir, "download", old_file)
+        new_path = os.path.join(videos, vid_dict["media_url"])
         # move media file and fix permission
-        shutil.move(old_file_path, new_file_path)
+        shutil.move(old_path, new_path, copy_function=shutil.copyfile)
         if host_uid and host_gid:
-            os.chown(new_file_path, host_uid, host_gid)
+            os.chown(new_path, host_uid, host_gid)
 
-    def _delete_from_pending(self, youtube_id):
+    @staticmethod
+    def _delete_from_pending(youtube_id):
         """delete downloaded video from pending index if its there"""
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        url = f"{es_url}/ta_download/_doc/{youtube_id}"
-        response = requests.delete(url, auth=es_auth)
-        if not response.ok and not response.status_code == 404:
-            print(response.text)
+        path = f"ta_download/_doc/{youtube_id}"
+        _, _ = ElasticWrap(path).delete()
 
     def _add_subscribed_channels(self):
         """add all channels subscribed to refresh"""
@@ -254,63 +414,3 @@ class VideoDownloader:
             self.channels.add(channel_id)
 
         return
-
-    def validate_playlists(self):
-        """look for playlist needing to update"""
-        print("sync playlists")
-        self._add_subscribed_channels()
-        all_indexed = PendingList().get_all_indexed()
-        all_youtube_ids = [i["youtube_id"] for i in all_indexed]
-        for id_c, channel_id in enumerate(self.channels):
-            playlists = YoutubeChannel(channel_id).get_indexed_playlists()
-            all_playlist_ids = [i["playlist_id"] for i in playlists]
-            for id_p, playlist_id in enumerate(all_playlist_ids):
-                playlist = YoutubePlaylist(playlist_id)
-                playlist.all_youtube_ids = all_youtube_ids
-                playlist.build_json(scrape=True)
-                if not playlist.json_data:
-                    playlist.deactivate()
-
-                playlist.add_vids_to_playlist()
-                playlist.upload_to_es()
-
-                # notify
-                title = (
-                    "Processing playlists for channels: "
-                    + f"{id_c + 1}/{len(self.channels)}"
-                )
-                message = f"Progress: {id_p + 1}/{len(all_playlist_ids)}"
-                mess_dict = {
-                    "status": "message:download",
-                    "level": "info",
-                    "title": title,
-                    "message": message,
-                }
-                if id_p + 1 == len(all_playlist_ids):
-                    RedisArchivist().set_message(
-                        "message:download", mess_dict, expire=4
-                    )
-                else:
-                    RedisArchivist().set_message("message:download", mess_dict)
-
-    @staticmethod
-    def auto_delete_watched(autodelete_days):
-        """delete watched videos after x days"""
-        now = int(datetime.now().strftime("%s"))
-        now_lte = now - autodelete_days * 24 * 60 * 60
-        data = {
-            "query": {"range": {"player.watched_date": {"lte": now_lte}}},
-            "sort": [{"player.watched_date": {"order": "asc"}}],
-        }
-        all_to_delete = IndexPaginate("ta_video", data).get_results()
-        all_youtube_ids = [i["youtube_id"] for i in all_to_delete]
-        if not all_youtube_ids:
-            return
-
-        for youtube_id in all_youtube_ids:
-            print(f"autodelete {youtube_id}")
-            YoutubeVideo(youtube_id).delete_media_file()
-
-        print("add deleted to ignore list")
-        pending_handler = PendingList()
-        pending_handler.add_to_pending(all_youtube_ids, ignore=True)

tubearchivist/home/src/es/connect.py

@@ -75,8 +75,10 @@ class ElasticWrap:
 
         return response.json(), response.status_code
 
-    def delete(self, data=False):
+    def delete(self, data=False, refresh=False):
         """delete document from es"""
+        if refresh:
+            self.url = f"{self.url}/?refresh=true"
         if data:
             response = requests.delete(self.url, json=data, auth=self.auth)
         else:
@@ -89,15 +91,22 @@ class ElasticWrap:
 
 
 class IndexPaginate:
-    """use search_after to go through whole index"""
+    """use search_after to go through whole index
+    kwargs:
+    - size: int, overwrite DEFAULT_SIZE
+    - keep_source: bool, keep _source key from es resutls
+    - callback: obj, Class with run method collback for every loop
+    """
 
     DEFAULT_SIZE = 500
 
-    def __init__(self, index_name, data, size=False):
+    def __init__(self, index_name, data, **kwargs):
         self.index_name = index_name
         self.data = data
         self.pit_id = False
-        self.size = size
+        self.size = kwargs.get("size")
+        self.keep_source = kwargs.get("keep_source")
+        self.callback = kwargs.get("callback")
 
     def get_results(self):
         """get all results"""
@@ -119,24 +128,34 @@ class IndexPaginate:
             print(self.data)
             raise ValueError("missing sort key in data")
 
-        size = self.size or self.DEFAULT_SIZE
-
-        self.data["size"] = size
+        self.data["size"] = self.size or self.DEFAULT_SIZE
         self.data["pit"] = {"id": self.pit_id, "keep_alive": "10m"}
 
     def run_loop(self):
         """loop through results until last hit"""
         all_results = []
+        counter = 0
         while True:
             response, _ = ElasticWrap("_search").get(data=self.data)
             all_hits = response["hits"]["hits"]
             if all_hits:
                 for hit in all_hits:
-                    source = hit["_source"]
-                    search_after = hit["sort"]
-                    all_results.append(source)
+                    if self.keep_source:
+                        source = hit
+                    else:
+                        source = hit["_source"]
+
+                    if not self.callback:
+                        all_results.append(source)
+
+                if self.callback:
+                    self.callback(all_hits, self.index_name).run()
+                    if counter % 10 == 0:
+                        print(f"{self.index_name}: processing page {counter}")
+                    counter = counter + 1
+
                 # update search_after with last hit data
-                self.data["search_after"] = search_after
+                self.data["search_after"] = all_hits[-1]["sort"]
             else:
                 break
 

tubearchivist/home/src/es/index_mapping.json

@@ -39,6 +39,22 @@
                 "channel_last_refresh": {
                     "type": "date",
                     "format": "epoch_second"
+                },
+                "channel_overwrites": {
+                    "properties": {
+                        "download_format": {
+                            "type": "text"
+                        },
+                        "autodelete_days": {
+                            "type": "long"
+                        },
+                        "index_playlists": {
+                            "type": "boolean"
+                        },
+                        "integrate_sponsorblock": {
+                            "type" : "boolean"
+                        }
+                    }
                 }
             },
             "expected_set": {
@@ -60,6 +76,10 @@
                     "type": "text",
                     "index": false
                 },
+                "vid_thumb_base64": {
+                    "type": "text",
+                    "index": false
+                },
                 "date_downloaded": {
                     "type": "date"
                 },
@@ -102,6 +122,22 @@
                         "channel_last_refresh": {
                             "type": "date",
                             "format": "epoch_second"
+                        },
+                        "channel_overwrites": {
+                            "properties": {
+                                "download_format": {
+                                    "type": "text"
+                                },
+                                "autodelete_days": {
+                                    "type": "long"
+                                },
+                                "index_playlists": {
+                                    "type": "boolean"
+                                },
+                                "integrate_sponsorblock": {
+                                    "type" : "boolean"
+                                }
+                            }
                         }
                     }
                 },
@@ -157,6 +193,22 @@
                         }
                     }
                 },
+                "stats" : {
+                    "properties" : {
+                        "average_rating" : {
+                            "type" : "float"
+                        },
+                        "dislike_count" : {
+                            "type" : "long"
+                        },
+                        "like_count" : {
+                            "type" : "long"
+                        },
+                        "view_count" : {
+                            "type" : "long"
+                        }
+                    }
+                },
                 "subtitles": {
                     "properties": {
                         "ext": {
@@ -182,6 +234,44 @@
                             "index": false
                         }
                     }
+                },
+                "sponsorblock": {
+                    "properties": {
+                        "last_refresh": {
+                            "type": "date"
+                        },
+                        "has_unlocked": {
+                            "type": "boolean"
+                        },
+                        "is_enabled": {
+                            "type": "boolean"
+                        },
+                        "segments" : {
+                            "properties" : {
+                                "UUID" : {
+                                    "type": "keyword"
+                                },
+                                "actionType" : {
+                                    "type": "keyword"
+                                },
+                                "category" : {
+                                    "type": "keyword"
+                                },
+                                "locked" : {
+                                    "type" : "short"
+                                },
+                                "segment" : {
+                                    "type" : "float"
+                                },
+                                "videoDuration" : {
+                                    "type" : "float"
+                                },
+                                "votes" : {
+                                    "type" : "long"
+                                }
+                            }
+                        }
+                    }
                 }
             },
             "expected_set": {

tubearchivist/home/src/es/index_setup.py

@@ -10,7 +10,7 @@ import os
 import zipfile
 from datetime import datetime
 
-import requests
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import ignore_filelist
 
@@ -20,11 +20,6 @@ class ElasticIndex:
     handle mapping and settings on elastic search for a given index
     """
 
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
-    HEADERS = {"Content-type": "application/json"}
-
     def __init__(self, index_name, expected_map, expected_set):
         self.index_name = index_name
         self.expected_map = expected_map
@@ -33,15 +28,9 @@ class ElasticIndex:
 
     def index_exists(self):
         """check if index already exists and return mapping if it does"""
-        index_name = self.index_name
-        url = f"{self.ES_URL}/ta_{index_name}"
-        response = requests.get(url, auth=self.ES_AUTH)
-        exists = response.ok
-
-        if exists:
-            details = response.json()[f"ta_{index_name}"]
-        else:
-            details = False
+        response, status_code = ElasticWrap(f"ta_{self.index_name}").get()
+        exists = status_code == 200
+        details = response.get(f"ta_{self.index_name}", False)
 
         return exists, details
 
@@ -65,14 +54,15 @@ class ElasticIndex:
 
     def validate_mappings(self):
         """check if all mappings are as expected"""
-
-        expected_map = self.expected_map
         now_map = self.details["mappings"]["properties"]
 
-        for key, value in expected_map.items():
+        for key, value in self.expected_map.items():
             # nested
             if list(value.keys()) == ["properties"]:
                 for key_n, value_n in value["properties"].items():
+                    if key not in now_map:
+                        print(key_n, value_n)
+                        return True
                     if key_n not in now_map[key]["properties"].keys():
                         print(key_n, value_n)
                         return True
@@ -110,125 +100,61 @@ class ElasticIndex:
 
     def rebuild_index(self):
         """rebuild with new mapping"""
-        # backup
         self.reindex("backup")
-        # delete original
         self.delete_index(backup=False)
-        # create new
         self.create_blank()
         self.reindex("restore")
-        # delete backup
         self.delete_index()
 
     def reindex(self, method):
         """create on elastic search"""
-        index_name = self.index_name
         if method == "backup":
-            source = f"ta_{index_name}"
-            destination = f"ta_{index_name}_backup"
+            source = f"ta_{self.index_name}"
+            destination = f"ta_{self.index_name}_backup"
         elif method == "restore":
-            source = f"ta_{index_name}_backup"
-            destination = f"ta_{index_name}"
+            source = f"ta_{self.index_name}_backup"
+            destination = f"ta_{self.index_name}"
 
-        query = {"source": {"index": source}, "dest": {"index": destination}}
-        data = json.dumps(query)
-        url = self.ES_URL + "/_reindex?refresh=true"
-        response = requests.post(
-            url=url, data=data, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not response.ok:
-            print(response.text)
+        data = {"source": {"index": source}, "dest": {"index": destination}}
+        _, _ = ElasticWrap("_reindex?refresh=true").post(data=data)
 
     def delete_index(self, backup=True):
         """delete index passed as argument"""
+        path = f"ta_{self.index_name}"
         if backup:
-            url = f"{self.ES_URL}/ta_{self.index_name}_backup"
-        else:
-            url = f"{self.ES_URL}/ta_{self.index_name}"
-        response = requests.delete(url, auth=self.ES_AUTH)
-        if not response.ok:
-            print(response.text)
+            path = path + "_backup"
+
+        _, _ = ElasticWrap(path).delete()
 
     def create_blank(self):
         """apply new mapping and settings for blank new index"""
-        expected_map = self.expected_map
-        expected_set = self.expected_set
-        # stich payload
-        payload = {}
-        if expected_set:
-            payload.update({"settings": expected_set})
-        if expected_map:
-            payload.update({"mappings": {"properties": expected_map}})
-        # create
-        url = f"{self.ES_URL}/ta_{self.index_name}"
-        data = json.dumps(payload)
-        response = requests.put(
-            url=url, data=data, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not response.ok:
-            print(response.text)
+        data = {}
+        if self.expected_set:
+            data.update({"settings": self.expected_set})
+        if self.expected_map:
+            data.update({"mappings": {"properties": self.expected_map}})
+
+        _, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
 
 
-class ElasticBackup:
-    """dump index to nd-json files for later bulk import"""
+class BackupCallback:
+    """handle backup ndjson writer as callback for IndexPaginate"""
 
-    def __init__(self, index_config, reason):
-        self.config = AppConfig().config
-        self.index_config = index_config
-        self.reason = reason
+    def __init__(self, source, index_name):
+        self.source = source
+        self.index_name = index_name
         self.timestamp = datetime.now().strftime("%Y%m%d")
-        self.backup_files = []
 
-    def get_all_documents(self, index_name):
-        """export all documents of a single index"""
-        headers = {"Content-type": "application/json"}
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        # get PIT ID
-        url = f"{es_url}/ta_{index_name}/_pit?keep_alive=1m"
-        response = requests.post(url, auth=es_auth)
-        json_data = json.loads(response.text)
-        pit_id = json_data["id"]
-        # build query
-        data = {
-            "query": {"match_all": {}},
-            "size": 100,
-            "pit": {"id": pit_id, "keep_alive": "1m"},
-            "sort": [{"_id": {"order": "asc"}}],
-        }
-        query_str = json.dumps(data)
-        url = es_url + "/_search"
-        # loop until nothing left
-        all_results = []
-        while True:
-            response = requests.get(
-                url, data=query_str, headers=headers, auth=es_auth
-            )
-            json_data = json.loads(response.text)
-            all_hits = json_data["hits"]["hits"]
-            if all_hits:
-                for hit in all_hits:
-                    search_after = hit["sort"]
-                    all_results.append(hit)
-                # update search_after with last hit data
-                data["search_after"] = search_after
-                query_str = json.dumps(data)
-            else:
-                break
-        # clean up PIT
-        query_str = json.dumps({"id": pit_id})
-        requests.delete(
-            es_url + "/_pit", data=query_str, headers=headers, auth=es_auth
-        )
+    def run(self):
+        """run the junk task"""
+        file_content = self._build_bulk()
+        self._write_es_json(file_content)
 
-        return all_results
-
-    @staticmethod
-    def build_bulk(all_results):
+    def _build_bulk(self):
         """build bulk query data from all_results"""
         bulk_list = []
 
-        for document in all_results:
+        for document in self.source:
             document_id = document["_id"]
             es_index = document["_index"]
             action = {"index": {"_index": es_index, "_id": document_id}}
@@ -242,70 +168,72 @@ class ElasticBackup:
 
         return file_content
 
-    def write_es_json(self, file_content, index_name):
+    def _write_es_json(self, file_content):
         """write nd-json file for es _bulk API to disk"""
-        cache_dir = self.config["application"]["cache_dir"]
-        file_name = f"es_{index_name}-{self.timestamp}.json"
+        cache_dir = AppConfig().config["application"]["cache_dir"]
+        file_name = f"es_{self.index_name.lstrip('ta_')}-{self.timestamp}.json"
         file_path = os.path.join(cache_dir, "backup", file_name)
-        with open(file_path, "w", encoding="utf-8") as f:
+        with open(file_path, "a+", encoding="utf-8") as f:
             f.write(file_content)
 
-        self.backup_files.append(file_path)
 
-    def write_ta_json(self, all_results, index_name):
-        """write generic json file to disk"""
-        cache_dir = self.config["application"]["cache_dir"]
-        file_name = f"ta_{index_name}-{self.timestamp}.json"
-        file_path = os.path.join(cache_dir, "backup", file_name)
-        to_write = [i["_source"] for i in all_results]
-        file_content = json.dumps(to_write)
-        with open(file_path, "w", encoding="utf-8") as f:
-            f.write(file_content)
+class ElasticBackup:
+    """dump index to nd-json files for later bulk import"""
 
-        self.backup_files.append(file_path)
+    def __init__(self, index_config, reason):
+        self.config = AppConfig().config
+        self.cache_dir = self.config["application"]["cache_dir"]
+        self.timestamp = datetime.now().strftime("%Y%m%d")
+        self.index_config = index_config
+        self.reason = reason
+
+    @staticmethod
+    def backup_index(index_name):
+        """export all documents of a single index"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"_doc": {"order": "desc"}}],
+        }
+        paginate = IndexPaginate(
+            f"ta_{index_name}", data, keep_source=True, callback=BackupCallback
+        )
+        _ = paginate.get_results()
 
     def zip_it(self):
         """pack it up into single zip file"""
-        cache_dir = self.config["application"]["cache_dir"]
         file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
-        backup_folder = os.path.join(cache_dir, "backup")
-        backup_file = os.path.join(backup_folder, file_name)
+        folder = os.path.join(self.cache_dir, "backup")
 
-        with zipfile.ZipFile(
-            backup_file, "w", compression=zipfile.ZIP_DEFLATED
-        ) as zip_f:
-            for backup_file in self.backup_files:
+        to_backup = []
+        for file in os.listdir(folder):
+            if file.endswith(".json"):
+                to_backup.append(os.path.join(folder, file))
+
+        backup_file = os.path.join(folder, file_name)
+
+        comp = zipfile.ZIP_DEFLATED
+        with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
+            for backup_file in to_backup:
                 zip_f.write(backup_file, os.path.basename(backup_file))
 
         # cleanup
-        for backup_file in self.backup_files:
+        for backup_file in to_backup:
             os.remove(backup_file)
 
     def post_bulk_restore(self, file_name):
         """send bulk to es"""
-        cache_dir = self.config["application"]["cache_dir"]
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        headers = {"Content-type": "application/x-ndjson"}
-        file_path = os.path.join(cache_dir, file_name)
-
+        file_path = os.path.join(self.cache_dir, file_name)
         with open(file_path, "r", encoding="utf-8") as f:
-            query_str = f.read()
+            data = f.read()
 
-        if not query_str.strip():
+        if not data.strip():
             return
 
-        url = es_url + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=es_auth
-        )
-        if not request.ok:
-            print(request.text)
+        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
 
     def get_all_backup_files(self):
         """build all available backup files for view"""
-        cache_dir = self.config["application"]["cache_dir"]
-        backup_dir = os.path.join(cache_dir, "backup")
+        backup_dir = os.path.join(self.cache_dir, "backup")
         backup_files = os.listdir(backup_dir)
         all_backup_files = ignore_filelist(backup_files)
         all_available_backups = [
@@ -336,8 +264,7 @@ class ElasticBackup:
 
     def unpack_zip_backup(self, filename):
         """extract backup zip and return filelist"""
-        cache_dir = self.config["application"]["cache_dir"]
-        backup_dir = os.path.join(cache_dir, "backup")
+        backup_dir = os.path.join(self.cache_dir, "backup")
         file_path = os.path.join(backup_dir, filename)
 
         with zipfile.ZipFile(file_path, "r") as z:
@@ -348,9 +275,7 @@ class ElasticBackup:
 
     def restore_json_files(self, zip_content):
         """go through the unpacked files and restore"""
-
-        cache_dir = self.config["application"]["cache_dir"]
-        backup_dir = os.path.join(cache_dir, "backup")
+        backup_dir = os.path.join(self.cache_dir, "backup")
 
         for json_f in zip_content:
 
@@ -364,14 +289,13 @@ class ElasticBackup:
             self.post_bulk_restore(file_name)
             os.remove(file_name)
 
-    def index_exists(self, index_name):
+    @staticmethod
+    def index_exists(index_name):
         """check if index already exists to skip"""
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        url = f"{es_url}/ta_{index_name}"
-        response = requests.get(url, auth=es_auth)
+        _, status_code = ElasticWrap(f"ta_{index_name}").get()
+        exists = status_code == 200
 
-        return response.ok
+        return exists
 
     def rotate_backup(self):
         """delete old backups if needed"""
@@ -386,8 +310,7 @@ class ElasticBackup:
             print("no backup files to rotate")
             return
 
-        cache_dir = self.config["application"]["cache_dir"]
-        backup_dir = os.path.join(cache_dir, "backup")
+        backup_dir = os.path.join(self.cache_dir, "backup")
 
         all_to_delete = auto[rotate:]
         for to_delete in all_to_delete:
@@ -459,12 +382,10 @@ def backup_all_indexes(reason):
 
     for index in backup_handler.index_config:
         index_name = index["index_name"]
+        print(f"backup: export in progress for {index_name}")
         if not backup_handler.index_exists(index_name):
             continue
-        all_results = backup_handler.get_all_documents(index_name)
-        file_content = backup_handler.build_bulk(all_results)
-        backup_handler.write_es_json(file_content, index_name)
-        backup_handler.write_ta_json(all_results, index_name)
+        backup_handler.backup_index(index_name)
 
     backup_handler.zip_it()
 

tubearchivist/home/src/frontend/api_calls.py

@@ -4,12 +4,11 @@ Functionality:
 - called via user input
 """
 
-from home.src.download.queue import PendingList
+from home.src.download.queue import PendingInteract
 from home.src.download.subscriptions import (
     ChannelSubscription,
     PlaylistSubscription,
 )
-from home.src.frontend.searching import SearchForm
 from home.src.frontend.watched import WatchState
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
@@ -19,7 +18,6 @@ from home.src.ta.ta_redis import RedisArchivist, RedisQueue
 from home.tasks import (
     download_pending,
     download_single,
-    extrac_dl,
     index_channel_playlists,
     kill_dl,
     re_sync_thumbs,
@@ -55,6 +53,7 @@ class PostData:
             "watched": self._watched,
             "un_watched": self._un_watched,
             "change_view": self._change_view,
+            "change_grid": self._change_grid,
             "rescan_pending": self._rescan_pending,
             "ignore": self._ignore,
             "dl_pending": self._dl_pending,
@@ -74,7 +73,6 @@ class PostData:
             "db-backup": self._db_backup,
             "db-restore": self._db_restore,
             "fs-rescan": self._fs_rescan,
-            "multi_search": self._multi_search,
             "delete-video": self._delete_video,
             "delete-channel": self._delete_channel,
             "delete-playlist": self._delete_playlist,
@@ -98,7 +96,18 @@ class PostData:
         origin, new_view = self.exec_val.split(":")
         key = f"{self.current_user}:view:{origin}"
         print(f"change view: {key} to {new_view}")
-        RedisArchivist().set_message(key, {"status": new_view}, expire=False)
+        RedisArchivist().set_message(key, {"status": new_view})
+        return {"success": True}
+
+    def _change_grid(self):
+        """process change items in grid"""
+        grid_items = int(self.exec_val)
+        grid_items = max(grid_items, 3)
+        grid_items = min(grid_items, 7)
+
+        key = f"{self.current_user}:grid_items"
+        print(f"change grid items: {grid_items}")
+        RedisArchivist().set_message(key, {"status": grid_items})
         return {"success": True}
 
     @staticmethod
@@ -110,12 +119,11 @@ class PostData:
 
     def _ignore(self):
         """ignore from download queue"""
-        id_to_ignore = self.exec_val
-        print("ignore video " + id_to_ignore)
-        handler = PendingList()
-        handler.ignore_from_pending([id_to_ignore])
+        video_id = self.exec_val
+        print(f"{video_id}: ignore video from download queue")
+        PendingInteract(video_id=video_id, status="ignore").update_status()
         # also clear from redis queue
-        RedisQueue("dl_queue").clear_item(id_to_ignore)
+        RedisQueue().clear_item(video_id)
         return {"success": True}
 
     @staticmethod
@@ -124,8 +132,8 @@ class PostData:
         print("download pending")
         running = download_pending.delay()
         task_id = running.id
-        print("set task id: " + task_id)
-        RedisArchivist().set_message("dl_queue_id", task_id, expire=False)
+        print(f"{task_id}: set task id")
+        RedisArchivist().set_message("dl_queue_id", task_id)
         return {"success": True}
 
     def _queue_handler(self):
@@ -133,7 +141,7 @@ class PostData:
         to_execute = self.exec_val
         if to_execute == "stop":
             print("stopping download queue")
-            RedisQueue("dl_queue").clear()
+            RedisQueue().clear()
         elif to_execute == "kill":
             task_id = RedisArchivist().get_message("dl_queue_id")
             if not isinstance(task_id, str):
@@ -147,7 +155,7 @@ class PostData:
     def _unsubscribe(self):
         """unsubscribe from channels or playlists"""
         id_unsub = self.exec_val
-        print("unsubscribe from " + id_unsub)
+        print(f"{id_unsub}: unsubscribe")
         to_unsub_list = UrlListParser(id_unsub).process_list()
         for to_unsub in to_unsub_list:
             unsub_type = to_unsub["type"]
@@ -168,7 +176,7 @@ class PostData:
     def _subscribe(self):
         """subscribe to channel or playlist, called from js buttons"""
         id_sub = self.exec_val
-        print("subscribe to " + id_sub)
+        print(f"{id_sub}: subscribe")
         subscribe_to.delay(id_sub)
         return {"success": True}
 
@@ -177,11 +185,11 @@ class PostData:
         sort_order = {"status": self.exec_val}
         if self.exec_val in ["asc", "desc"]:
             RedisArchivist().set_message(
-                f"{self.current_user}:sort_order", sort_order, expire=False
+                f"{self.current_user}:sort_order", sort_order
             )
         else:
             RedisArchivist().set_message(
-                f"{self.current_user}:sort_by", sort_order, expire=False
+                f"{self.current_user}:sort_by", sort_order
             )
         return {"success": True}
 
@@ -190,7 +198,7 @@ class PostData:
         key = f"{self.current_user}:hide_watched"
         message = {"status": bool(int(self.exec_val))}
         print(f"toggle {key}: {message}")
-        RedisArchivist().set_message(key, message, expire=False)
+        RedisArchivist().set_message(key, message)
         return {"success": True}
 
     def _show_subed_only(self):
@@ -198,17 +206,17 @@ class PostData:
         key = f"{self.current_user}:show_subed_only"
         message = {"status": bool(int(self.exec_val))}
         print(f"toggle {key}: {message}")
-        RedisArchivist().set_message(key, message, expire=False)
+        RedisArchivist().set_message(key, message)
         return {"success": True}
 
     def _dlnow(self):
         """start downloading single vid now"""
         youtube_id = self.exec_val
-        print("downloading: " + youtube_id)
+        print(f"{youtube_id}: downloading now")
         running = download_single.delay(youtube_id=youtube_id)
         task_id = running.id
         print("set task id: " + task_id)
-        RedisArchivist().set_message("dl_queue_id", task_id, expire=False)
+        RedisArchivist().set_message("dl_queue_id", task_id)
         return {"success": True}
 
     def _show_ignored_only(self):
@@ -217,33 +225,28 @@ class PostData:
         key = f"{self.current_user}:show_ignored_only"
         value = {"status": show_value}
         print(f"Filter download view ignored only: {show_value}")
-        RedisArchivist().set_message(key, value, expire=False)
+        RedisArchivist().set_message(key, value)
         return {"success": True}
 
     def _forget_ignore(self):
         """delete from ta_download index"""
-        youtube_id = self.exec_val
-        print("forgetting from download index: " + youtube_id)
-        PendingList().delete_from_pending(youtube_id)
+        video_id = self.exec_val
+        print(f"{video_id}: forget from download")
+        PendingInteract(video_id=video_id).delete_item()
         return {"success": True}
 
     def _add_single(self):
         """add single youtube_id to download queue"""
-        youtube_id = self.exec_val
-        print("add vid to dl queue: " + youtube_id)
-        PendingList().delete_from_pending(youtube_id)
-        youtube_ids = UrlListParser(youtube_id).process_list()
-        extrac_dl.delay(youtube_ids)
+        video_id = self.exec_val
+        print(f"{video_id}: add single vid to download queue")
+        PendingInteract(video_id=video_id, status="pending").update_status()
         return {"success": True}
 
     def _delete_queue(self):
         """delete download queue"""
         status = self.exec_val
         print("deleting from download queue: " + status)
-        if status == "pending":
-            PendingList().delete_pending("pending")
-        elif status == "ignore":
-            PendingList().delete_pending("ignore")
+        PendingInteract(status=status).delete_by_status()
         return {"success": True}
 
     @staticmethod
@@ -281,13 +284,6 @@ class PostData:
         rescan_filesystem.delay()
         return {"success": True}
 
-    def _multi_search(self):
-        """search through all indexes"""
-        search_query = self.exec_val
-        print("searching for: " + search_query)
-        search_results = SearchForm().multi_search(search_query)
-        return search_results
-
     def _delete_video(self):
         """delete media file, metadata and thumb"""
         youtube_id = self.exec_val

tubearchivist/home/src/frontend/forms.py

@@ -62,6 +62,12 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable ryd integration"),
     ]
 
+    SP_CHOICES = [
+        ("", "-- change sponsorblock integrations"),
+        ("0", "disable sponsorblock integration"),
+        ("1", "enable sponsorblock integration"),
+    ]
+
     CAST_CHOICES = [
         ("", "-- change Cast integration --"),
         ("0", "disable Cast"),
@@ -80,6 +86,12 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable subtitle index"),
     ]
 
+    COOKIE_IMPORT_CHOICES = [
+        ("", "-- change cookie settings"),
+        ("0", "disable cookie"),
+        ("1", "enable cookie"),
+    ]
+
     subscriptions_channel_size = forms.IntegerField(required=False)
     downloads_limit_count = forms.IntegerField(required=False)
     downloads_limit_speed = forms.IntegerField(required=False)
@@ -100,9 +112,15 @@ class ApplicationSettingsForm(forms.Form):
     downloads_subtitle_index = forms.ChoiceField(
         widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False
     )
+    downloads_cookie_import = forms.ChoiceField(
+        widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False
+    )
     downloads_integrate_ryd = forms.ChoiceField(
         widget=forms.Select, choices=RYD_CHOICES, required=False
     )
+    downloads_integrate_sponsorblock = forms.ChoiceField(
+        widget=forms.Select, choices=SP_CHOICES, required=False
+    )
     application_enable_cast = forms.ChoiceField(
         widget=forms.Select, choices=CAST_CHOICES, required=False
     )
@@ -141,12 +159,14 @@ class MultiSearchForm(forms.Form):
 class AddToQueueForm(forms.Form):
     """text area form to add to downloads"""
 
+    HELP_TEXT = "Enter at least one video, channel or playlist id/URL here..."
+
     vid_url = forms.CharField(
         label=False,
         widget=forms.Textarea(
             attrs={
                 "rows": 4,
-                "placeholder": "Enter Video Urls or IDs here...",
+                "placeholder": HELP_TEXT,
             }
         ),
     )
@@ -178,3 +198,29 @@ class SubscribeToPlaylistForm(forms.Form):
             }
         ),
     )
+
+
+class ChannelOverwriteForm(forms.Form):
+    """custom overwrites for channel settings"""
+
+    PLAYLIST_INDEX = [
+        ("", "-- change playlist index --"),
+        ("0", "Disable playlist index"),
+        ("1", "Enable playlist index"),
+    ]
+
+    SP_CHOICES = [
+        ("", "-- change sponsorblock integrations"),
+        ("disable", "disable sponsorblock integration"),
+        ("1", "enable sponsorblock integration"),
+        ("0", "unset sponsorblock integration"),
+    ]
+
+    download_format = forms.CharField(label=False, required=False)
+    autodelete_days = forms.IntegerField(label=False, required=False)
+    index_playlists = forms.ChoiceField(
+        widget=forms.Select, choices=PLAYLIST_INDEX, required=False
+    )
+    integrate_sponsorblock = forms.ChoiceField(
+        widget=forms.Select, choices=SP_CHOICES, required=False
+    )

tubearchivist/home/src/frontend/searching.py

@@ -119,7 +119,7 @@ class SearchHandler:
 
         if "vid_thumb_url" in hit_keys:
             youtube_id = hit["source"]["youtube_id"]
-            thumb_path = ThumbManager().vid_thumb_path(youtube_id)
+            thumb_path = ThumbManager(youtube_id).vid_thumb_path()
             hit["source"]["vid_thumb_url"] = thumb_path
 
         if "channel_last_refresh" in hit_keys:
@@ -136,6 +136,11 @@ class SearchHandler:
                 date_str = datetime.strftime(date_refresh, "%d %b, %Y")
                 hit["source"]["channel"]["channel_last_refresh"] = date_str
 
+        if "subtitle_fragment_id" in hit_keys:
+            youtube_id = hit["source"]["youtube_id"]
+            thumb_path = ThumbManager(youtube_id).vid_thumb_path()
+            hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
+
         return hit
 
 
@@ -146,38 +151,12 @@ class SearchForm:
 
     def multi_search(self, search_query):
         """searching through index"""
-        path = "ta_video,ta_channel,ta_playlist/_search"
-        data = {
-            "size": 30,
-            "query": {
-                "multi_match": {
-                    "query": search_query,
-                    "type": "bool_prefix",
-                    "operator": "and",
-                    "fuzziness": "auto",
-                    "fields": [
-                        "category",
-                        "channel_description",
-                        "channel_name._2gram",
-                        "channel_name._3gram",
-                        "channel_name.search_as_you_type",
-                        "playlist_description",
-                        "playlist_name._2gram",
-                        "playlist_name._3gram",
-                        "playlist_name.search_as_you_type",
-                        "tags",
-                        "title._2gram",
-                        "title._3gram",
-                        "title.search_as_you_type",
-                    ],
-                }
-            },
-        }
-        look_up = SearchHandler(path, config=self.CONFIG, data=data)
+        path, query, query_type = SearchParser(search_query).run()
+        look_up = SearchHandler(path, config=self.CONFIG, data=query)
         search_results = look_up.get_data()
         all_results = self.build_results(search_results)
 
-        return {"results": all_results}
+        return {"results": all_results, "queryType": query_type}
 
     @staticmethod
     def build_results(search_results):
@@ -185,6 +164,7 @@ class SearchForm:
         video_results = []
         channel_results = []
         playlist_results = []
+        fulltext_results = []
         if search_results:
             for result in search_results:
                 if result["_index"] == "ta_video":
@@ -193,11 +173,337 @@ class SearchForm:
                     channel_results.append(result)
                 elif result["_index"] == "ta_playlist":
                     playlist_results.append(result)
+                elif result["_index"] == "ta_subtitle":
+                    fulltext_results.append(result)
 
         all_results = {
             "video_results": video_results,
             "channel_results": channel_results,
             "playlist_results": playlist_results,
+            "fulltext_results": fulltext_results,
         }
 
         return all_results
+
+
+class SearchParser:
+    """handle structured searches"""
+
+    def __init__(self, search_query):
+        self.query_words = search_query.lower().split()
+        self.query_map = False
+        self.append_to = "term"
+
+    def run(self):
+        """collection, return path and query dict for es"""
+        print(f"query words: {self.query_words}")
+        query_type = self._find_map()
+        self._run_words()
+        self._delete_unset()
+        self._match_data_types()
+
+        path, query = QueryBuilder(self.query_map, query_type).run()
+
+        return path, query, query_type
+
+    def _find_map(self):
+        """find query in keyword map"""
+        first_word = self.query_words[0]
+        key_word_map = self._get_map()
+
+        if ":" in first_word:
+            index_match, query_string = first_word.split(":")
+            if index_match in key_word_map:
+                self.query_map = key_word_map.get(index_match)
+                self.query_words[0] = query_string
+                return index_match
+
+        self.query_map = key_word_map.get("simple")
+        print(f"query_map: {self.query_map}")
+
+        return "simple"
+
+    @staticmethod
+    def _get_map():
+        """return map to build on"""
+        return {
+            "simple": {
+                "index": "ta_video,ta_channel,ta_playlist",
+                "term": [],
+            },
+            "video": {
+                "index": "ta_video",
+                "term": [],
+                "channel": [],
+                "active": [],
+            },
+            "channel": {
+                "index": "ta_channel",
+                "term": [],
+                "active": [],
+                "subscribed": [],
+            },
+            "playlist": {
+                "index": "ta_playlist",
+                "term": [],
+                "active": [],
+                "subscribed": [],
+            },
+            "full": {
+                "index": "ta_subtitle",
+                "term": [],
+                "lang": [],
+                "source": [],
+            },
+        }
+
+    def _run_words(self):
+        """append word by word"""
+        for word in self.query_words:
+            if ":" in word:
+                keyword, search_string = word.split(":")
+                if keyword in self.query_map:
+                    self.append_to = keyword
+                    word = search_string
+
+            if word:
+                self.query_map[self.append_to].append(word)
+
+    def _delete_unset(self):
+        """delete unset keys"""
+        new_query_map = {}
+        for key, value in self.query_map.items():
+            if value:
+                new_query_map.update({key: value})
+        self.query_map = new_query_map
+
+    def _match_data_types(self):
+        """match values with data types"""
+        for key, value in self.query_map.items():
+            if key in ["term", "channel"]:
+                self.query_map[key] = " ".join(self.query_map[key])
+            if key in ["active", "subscribed"]:
+                self.query_map[key] = "yes" in value
+
+
+class QueryBuilder:
+    """build query for ES from form data"""
+
+    def __init__(self, query_map, query_type):
+        self.query_map = query_map
+        self.query_type = query_type
+
+    def run(self):
+        """build query"""
+        path = self._build_path()
+        query = self.build_query()
+        print(f"es path: {path}")
+        print(f"query: {query}")
+
+        return path, query
+
+    def _build_path(self):
+        """build es index search path"""
+        return f"{self.query_map.get('index')}/_search"
+
+    def build_query(self):
+        """build query based on query_type"""
+
+        exec_map = {
+            "simple": self._build_simple,
+            "video": self._build_video,
+            "channel": self._build_channel,
+            "playlist": self._build_playlist,
+            "full": self._build_fulltext,
+        }
+
+        build_must_list = exec_map[self.query_type]
+
+        if self.query_type == "full":
+            query = build_must_list()
+        else:
+            query = {
+                "size": 30,
+                "query": {"bool": {"must": build_must_list()}},
+            }
+
+        return query
+
+    def _build_simple(self):
+        """build simple cross index query"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": "auto",
+                        "operator": "and",
+                        "fields": [
+                            "channel_name._2gram",
+                            "channel_name._3gram",
+                            "channel_name.search_as_you_type",
+                            "playlist_name._2gram",
+                            "playlist_name._3gram",
+                            "playlist_name.search_as_you_type",
+                            "title._2gram",
+                            "title._3gram",
+                            "title.search_as_you_type",
+                        ],
+                    }
+                }
+            )
+
+        return must_list
+
+    def _build_video(self):
+        """build video query"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": "auto",
+                        "operator": "and",
+                        "fields": [
+                            "title._2gram^2",
+                            "title._3gram^2",
+                            "title.search_as_you_type^2",
+                            "tags",
+                            "category",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"active": {"value": active}}})
+
+        if (channel := self.query_map.get("channel")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": channel,
+                        "type": "bool_prefix",
+                        "fuzziness": "auto",
+                        "operator": "and",
+                        "fields": [
+                            "channel.channel_name._2gram",
+                            "channel.channel_name._3gram",
+                            "channel.channel_name.search_as_you_type",
+                        ],
+                    }
+                }
+            )
+
+        return must_list
+
+    def _build_channel(self):
+        """build query for channel"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": "auto",
+                        "operator": "and",
+                        "fields": [
+                            "channel_description",
+                            "channel_name._2gram^2",
+                            "channel_name._3gram^2",
+                            "channel_name.search_as_you_type^2",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"channel_active": {"value": active}}})
+
+        if (subscribed := self.query_map.get("subscribed")) is not None:
+            must_list.append(
+                {"term": {"channel_subscribed": {"value": subscribed}}}
+            )
+
+        return must_list
+
+    def _build_playlist(self):
+        """build query for playlist"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": "auto",
+                        "operator": "and",
+                        "fields": [
+                            "playlist_description",
+                            "playlist_name._2gram^2",
+                            "playlist_name._3gram^2",
+                            "playlist_name.search_as_you_type^2",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"playlist_active": {"value": active}}})
+
+        if (subscribed := self.query_map.get("subscribed")) is not None:
+            must_list.append(
+                {"term": {"playlist_subscribed": {"value": subscribed}}}
+            )
+
+        return must_list
+
+    def _build_fulltext(self):
+        """build query for fulltext search"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "match": {
+                        "subtitle_line": {
+                            "query": term,
+                            "fuzziness": "auto",
+                        }
+                    }
+                }
+            )
+
+        if (lang := self.query_map.get("lang")) is not None:
+            must_list.append({"term": {"subtitle_lang": {"value": lang[0]}}})
+
+        if (source := self.query_map.get("source")) is not None:
+            must_list.append(
+                {"term": {"subtitle_source": {"value": source[0]}}}
+            )
+
+        query = {
+            "size": 30,
+            "_source": {"excludes": "subtitle_line"},
+            "query": {"bool": {"must": must_list}},
+            "highlight": {
+                "fields": {
+                    "subtitle_line": {
+                        "number_of_fragments": 0,
+                        "pre_tags": ['<span class="settings-current">'],
+                        "post_tags": ["</span>"],
+                    }
+                }
+            },
+        }
+
+        return query

tubearchivist/home/src/frontend/watched.py

@@ -3,22 +3,15 @@ functionality:
 - handle watched state for videos, channels and playlists
 """
 
-import json
 from datetime import datetime
 
-import requests
-from home.src.ta.config import AppConfig
+from home.src.es.connect import ElasticWrap
 from home.src.ta.helper import UrlListParser
 
 
 class WatchState:
     """handle watched checkbox for videos and channels"""
 
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
-    HEADERS = {"Content-type": "application/json"}
-
     def __init__(self, youtube_id):
         self.youtube_id = youtube_id
         self.stamp = int(datetime.now().strftime("%s"))
@@ -33,7 +26,7 @@ class WatchState:
         elif url_type == "playlist":
             self.mark_playlist_watched()
 
-        print(f"marked {self.youtube_id} as watched")
+        print(f"{self.youtube_id}: marked as watched")
 
     def mark_as_unwatched(self):
         """revert watched state to false"""
@@ -41,7 +34,7 @@ class WatchState:
         if url_type == "video":
             self.mark_vid_watched(revert=True)
 
-        print(f"revert {self.youtube_id} as unwatched")
+        print(f"{self.youtube_id}: revert as unwatched")
 
     def dedect_type(self):
         """find youtube id type"""
@@ -52,77 +45,54 @@ class WatchState:
 
     def mark_vid_watched(self, revert=False):
         """change watched status of single video"""
-        url = self.ES_URL + "/ta_video/_update/" + self.youtube_id
+        path = f"ta_video/_update/{self.youtube_id}"
         data = {
             "doc": {"player": {"watched": True, "watched_date": self.stamp}}
         }
         if revert:
             data["doc"]["player"]["watched"] = False
 
-        payload = json.dumps(data)
-        request = requests.post(
-            url, data=payload, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request.text)
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code != 200:
+            print(response)
             raise ValueError("failed to mark video as watched")
 
     def mark_channel_watched(self):
         """change watched status of every video in channel"""
+        path = "ta_video/_update_by_query"
+        must_list = [
+            {"term": {"channel.channel_id": {"value": self.youtube_id}}},
+            {"term": {"player.watched": {"value": False}}},
+        ]
         data = {
-            "query": {
-                "bool": {
-                    "must": [
-                        {
-                            "term": {
-                                "channel.channel_id": {
-                                    "value": self.youtube_id
-                                }
-                            }
-                        },
-                        {"term": {"player.watched": {"value": False}}},
-                    ]
-                }
-            },
+            "query": {"bool": {"must": must_list}},
             "script": {
                 "source": "ctx._source.player['watched'] = true",
                 "lang": "painless",
             },
         }
-        payload = json.dumps(data)
-        url = f"{self.ES_URL}/ta_video/_update_by_query"
-        request = requests.post(
-            url, data=payload, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request.text)
+
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code != 200:
+            print(response)
             raise ValueError("failed mark channel as watched")
 
     def mark_playlist_watched(self):
         """change watched state of all videos in playlist"""
+        path = "ta_video/_update_by_query"
+        must_list = [
+            {"term": {"playlist.keyword": {"value": self.youtube_id}}},
+            {"term": {"player.watched": {"value": False}}},
+        ]
         data = {
-            "query": {
-                "bool": {
-                    "must": [
-                        {
-                            "term": {
-                                "playlist.keyword": {"value": self.youtube_id}
-                            }
-                        },
-                        {"term": {"player.watched": {"value": False}}},
-                    ]
-                }
-            },
+            "query": {"bool": {"must": must_list}},
             "script": {
                 "source": "ctx._source.player['watched'] = true",
                 "lang": "painless",
             },
         }
-        payload = json.dumps(data)
-        url = f"{self.ES_URL}/ta_video/_update_by_query"
-        request = requests.post(
-            url, data=payload, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request.text)
+
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code != 200:
+            print(response)
             raise ValueError("failed mark playlist as watched")

tubearchivist/home/src/index/channel.py

@@ -10,13 +10,15 @@ import re
 from datetime import datetime
 
 import requests
-import yt_dlp
 from bs4 import BeautifulSoup
+from home.src.download import queue  # partial import
 from home.src.download.thumbnails import ThumbManager
+from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.generic import YouTubeItem
 from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.helper import clean_string
+from home.src.ta.helper import clean_string, requests_headers
+from home.src.ta.ta_redis import RedisArchivist
 
 
 class ChannelScraper:
@@ -35,6 +37,9 @@ class ChannelScraper:
         """main method to return channel dict"""
         self.get_soup()
         self._extract_yt_json()
+        if self._is_deactivated():
+            return False
+
         self._parse_channel_main()
         self._parse_channel_meta()
         return self.json_data
@@ -44,7 +49,9 @@ class ChannelScraper:
         print(f"{self.channel_id}: scrape channel data from youtube")
         url = f"https://www.youtube.com/channel/{self.channel_id}/about?hl=en"
         cookies = {"CONSENT": "YES+xxxxxxxxxxxxxxxxxxxxxxxxxxx"}
-        response = requests.get(url, cookies=cookies)
+        response = requests.get(
+            url, cookies=cookies, headers=requests_headers()
+        )
         if response.ok:
             channel_page = response.text
         else:
@@ -64,6 +71,16 @@ class ChannelScraper:
         json_raw = script_content.rstrip(";</script>")
         self.yt_json = json.loads(json_raw)
 
+    def _is_deactivated(self):
+        """check if channel is deactivated"""
+        alert_text = "This channel does not exist."
+        alerts = self.yt_json.get("alerts")
+        if alerts and alert_text in str(alerts):
+            print(f"{self.channel_id}: {alert_text}")
+            return True
+
+        return False
+
     def _parse_channel_main(self):
         """extract maintab values from scraped channel json data"""
         main_tab = self.yt_json["header"]["c4TabbedHeaderRenderer"]
@@ -149,35 +166,78 @@ class YoutubeChannel(YouTubeItem):
     es_path = False
     index_name = "ta_channel"
     yt_base = "https://www.youtube.com/channel/"
+    msg = "message:playlistscan"
 
     def __init__(self, youtube_id):
         super().__init__(youtube_id)
         self.es_path = f"{self.index_name}/_doc/{youtube_id}"
+        self.all_playlists = False
 
-    def build_json(self, upload=False):
+    def build_json(self, upload=False, fallback=False):
         """get from es or from youtube"""
         self.get_from_es()
         if self.json_data:
             return
 
-        self.get_from_youtube()
+        self.get_from_youtube(fallback)
+
         if upload:
             self.upload_to_es()
         return
 
-    def get_from_youtube(self):
+    def get_from_youtube(self, fallback=False):
         """use bs4 to scrape channel about page"""
         self.json_data = ChannelScraper(self.youtube_id).get_json()
+
+        if not self.json_data and fallback:
+            self._video_fallback(fallback)
+
         self.get_channel_art()
 
+    def _video_fallback(self, fallback):
+        """use video metadata as fallback"""
+        print(f"{self.youtube_id}: fallback to video metadata")
+        self.json_data = {
+            "channel_active": False,
+            "channel_last_refresh": int(datetime.now().strftime("%s")),
+            "channel_subs": fallback.get("channel_follower_count", 0),
+            "channel_name": fallback["uploader"],
+            "channel_banner_url": False,
+            "channel_tvart_url": False,
+            "channel_id": self.youtube_id,
+            "channel_subscribed": False,
+            "channel_description": False,
+            "channel_thumb_url": False,
+            "channel_views": 0,
+        }
+        self._info_json_fallback()
+
+    def _info_json_fallback(self):
+        """read channel info.json for additional metadata"""
+        info_json = os.path.join(
+            self.config["application"]["cache_dir"],
+            "import",
+            f"{self.youtube_id}.info.json",
+        )
+        if os.path.exists(info_json):
+            print(f"{self.youtube_id}: read info.json file")
+            with open(info_json, "r", encoding="utf-8") as f:
+                content = json.loads(f.read())
+
+            self.json_data.update(
+                {
+                    "channel_subs": content["channel_follower_count"],
+                    "channel_description": content["description"],
+                }
+            )
+
     def get_channel_art(self):
         """download channel art for new channels"""
-        channel_id = self.youtube_id
-        channel_thumb = self.json_data["channel_thumb_url"]
-        channel_banner = self.json_data["channel_banner_url"]
-        ThumbManager().download_chan(
-            [(channel_id, channel_thumb, channel_banner)]
+        urls = (
+            self.json_data["channel_thumb_url"],
+            self.json_data["channel_banner_url"],
         )
+        ThumbManager(self.youtube_id, item_type="channel").download(urls)
 
     def sync_to_videos(self):
         """sync new channel_dict to all videos of channel"""
@@ -241,21 +301,78 @@ class YoutubeChannel(YouTubeItem):
         self.delete_es_videos()
         self.del_in_es()
 
+    def index_channel_playlists(self):
+        """add all playlists of channel to index"""
+        print(f"{self.youtube_id}: index all playlists")
+        self.get_from_es()
+        channel_name = self.json_data["channel_name"]
+        mess_dict = {
+            "status": self.msg,
+            "level": "info",
+            "title": "Looking for playlists",
+            "message": f"{channel_name}: Scanning channel in progress",
+        }
+        RedisArchivist().set_message(self.msg, mess_dict, expire=True)
+        self.get_all_playlists()
+        if not self.all_playlists:
+            print(f"{self.youtube_id}: no playlists found.")
+            return
+
+        all_youtube_ids = self.get_all_video_ids()
+        for idx, playlist in enumerate(self.all_playlists):
+            self._notify_single_playlist(idx, playlist)
+            self._index_single_playlist(playlist, all_youtube_ids)
+
+    def _notify_single_playlist(self, idx, playlist):
+        """send notification"""
+        channel_name = self.json_data["channel_name"]
+        mess_dict = {
+            "status": self.msg,
+            "level": "info",
+            "title": f"{channel_name}: Scanning channel for playlists",
+            "message": f"Progress: {idx + 1}/{len(self.all_playlists)}",
+        }
+        RedisArchivist().set_message(self.msg, mess_dict, expire=True)
+        print("add playlist: " + playlist[1])
+
+    @staticmethod
+    def _index_single_playlist(playlist, all_youtube_ids):
+        """add single playlist if needed"""
+        playlist = YoutubePlaylist(playlist[0])
+        playlist.all_youtube_ids = all_youtube_ids
+        playlist.build_json()
+        if not playlist.json_data:
+            return
+
+        entries = playlist.json_data["playlist_entries"]
+        downloaded = [i for i in entries if i["downloaded"]]
+        if not downloaded:
+            return
+
+        playlist.upload_to_es()
+        playlist.add_vids_to_playlist()
+        playlist.get_playlist_art()
+
+    @staticmethod
+    def get_all_video_ids():
+        """match all playlists with videos"""
+        handler = queue.PendingList()
+        handler.get_download()
+        handler.get_indexed()
+        all_youtube_ids = [i["youtube_id"] for i in handler.all_videos]
+
+        return all_youtube_ids
+
     def get_all_playlists(self):
         """get all playlists owned by this channel"""
         url = (
             f"https://www.youtube.com/channel/{self.youtube_id}"
             + "/playlists?view=1&sort=dd&shelf_id=0"
         )
-        obs = {
-            "quiet": True,
-            "skip_download": True,
-            "extract_flat": True,
-        }
-        playlists = yt_dlp.YoutubeDL(obs).extract_info(url)
+        obs = {"skip_download": True, "extract_flat": True}
+        playlists = YtWrap(obs, self.config).extract(url)
         all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
-
-        return all_entries
+        self.all_playlists = all_entries
 
     def get_indexed_playlists(self):
         """get all indexed playlists from channel"""
@@ -267,3 +384,43 @@ class YoutubeChannel(YouTubeItem):
         }
         all_playlists = IndexPaginate("ta_playlist", data).get_results()
         return all_playlists
+
+    def get_overwrites(self):
+        """get all per channel overwrites"""
+        return self.json_data.get("channel_overwrites", False)
+
+    def set_overwrites(self, overwrites):
+        """set per channel overwrites"""
+        valid_keys = [
+            "download_format",
+            "autodelete_days",
+            "index_playlists",
+            "integrate_sponsorblock",
+        ]
+
+        to_write = self.json_data.get("channel_overwrites", {})
+        for key, value in overwrites.items():
+            if key not in valid_keys:
+                raise ValueError(f"invalid overwrite key: {key}")
+            if value == "disable":
+                to_write[key] = False
+                continue
+            if value in [0, "0"]:
+                del to_write[key]
+                continue
+            if value == "1":
+                to_write[key] = True
+                continue
+            if value:
+                to_write.update({key: value})
+
+        self.json_data["channel_overwrites"] = to_write
+
+
+def channel_overwrites(channel_id, overwrites):
+    """collection to overwrite settings per channel"""
+    channel = YoutubeChannel(channel_id)
+    channel.build_json()
+    channel.set_overwrites(overwrites)
+    channel.upload_to_es()
+    channel.sync_to_videos()

tubearchivist/home/src/index/filesystem.py

@@ -10,24 +10,25 @@ import os
 import re
 import shutil
 import subprocess
-from datetime import datetime
 
-import requests
 from home.src.download.queue import PendingList
-from home.src.download.yt_dlp_handler import VideoDownloader
+from home.src.download.thumbnails import ThumbManager
+from home.src.es.connect import ElasticWrap
 from home.src.index.reindex import Reindex
-from home.src.index.video import index_new_video
+from home.src.index.video import YoutubeVideo, index_new_video
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import clean_string, ignore_filelist
 from home.src.ta.ta_redis import RedisArchivist
+from PIL import Image, ImageFile
+from yt_dlp.utils import ISO639Utils
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
 
 
 class FilesystemScanner:
     """handle scanning and fixing from filesystem"""
 
     CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
     VIDEOS = CONFIG["application"]["videos"]
 
     def __init__(self):
@@ -59,9 +60,11 @@ class FilesystemScanner:
     def get_all_indexed():
         """get a list of all indexed videos"""
         index_handler = PendingList()
-        all_indexed_raw = index_handler.get_all_indexed()
+        index_handler.get_download()
+        index_handler.get_indexed()
+
         all_indexed = []
-        for video in all_indexed_raw:
+        for video in index_handler.all_videos:
             youtube_id = video["youtube_id"]
             media_url = video["media_url"]
             published = video["published"]
@@ -146,84 +149,153 @@ class FilesystemScanner:
             bulk_list.append(json.dumps(source))
         # add last newline
         bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-        # make the call
-        headers = {"Content-type": "application/x-ndjson"}
-        url = self.ES_URL + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request.text)
+        data = "\n".join(bulk_list)
+        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
 
     def delete_from_index(self):
         """find indexed but deleted mediafile"""
         for indexed in self.to_delete:
             youtube_id = indexed[0]
             print(f"deleting {youtube_id} from index")
-            url = self.ES_URL + "/ta_video/_doc/" + youtube_id
-            request = requests.delete(url, auth=self.ES_AUTH)
-            if not request.ok:
-                print(request.text)
+            path = f"ta_video/_doc/{youtube_id}"
+            _, _ = ElasticWrap(path).delete()
 
 
-class ManualImport:
-    """import and indexing existing video files"""
+class ImportFolderScanner:
+    """import and indexing existing video files
+    - identify all media files belonging to a video
+    - identify youtube id
+    - convert if needed
+    """
 
     CONFIG = AppConfig().config
     CACHE_DIR = CONFIG["application"]["cache_dir"]
     IMPORT_DIR = os.path.join(CACHE_DIR, "import")
 
+    EXT_MAP = {
+        "media": [".mp4", ".mkv", ".webm"],
+        "metadata": [".json"],
+        "thumb": [".jpg", ".png", ".webp"],
+        "subtitle": [".vtt"],
+    }
+
     def __init__(self):
-        self.identified = self.import_folder_parser()
+        self.to_import = False
 
-    def import_folder_parser(self):
-        """detect files in import folder"""
-        import_files = os.listdir(self.IMPORT_DIR)
-        to_import = ignore_filelist(import_files)
-        to_import.sort()
-        video_files = [i for i in to_import if not i.endswith(".json")]
+    def scan(self):
+        """scan and match media files"""
+        all_files = self.get_all_files()
+        self.match_files(all_files)
+        self.process_videos()
 
-        identified = []
+        return self.to_import
 
-        for file_path in video_files:
+    def get_all_files(self):
+        """get all files in /import"""
+        rel_paths = ignore_filelist(os.listdir(self.IMPORT_DIR))
+        all_files = [os.path.join(self.IMPORT_DIR, i) for i in rel_paths]
+        all_files.sort()
 
-            file_dict = {"video_file": file_path}
-            file_name, _ = os.path.splitext(file_path)
-
-            matching_json = [
-                i
-                for i in to_import
-                if i.startswith(file_name) and i.endswith(".json")
-            ]
-            if matching_json:
-                json_file = matching_json[0]
-                youtube_id = self.extract_id_from_json(json_file)
-                file_dict.update({"json_file": json_file})
-            else:
-                youtube_id = self.extract_id_from_filename(file_name)
-                file_dict.update({"json_file": False})
-
-            file_dict.update({"youtube_id": youtube_id})
-            identified.append(file_dict)
-
-        return identified
+        return all_files
 
     @staticmethod
-    def extract_id_from_filename(file_name):
+    def _get_template():
+        """base dict for video"""
+        return {
+            "media": False,
+            "video_id": False,
+            "metadata": False,
+            "thumb": False,
+            "subtitle": [],
+        }
+
+    def match_files(self, all_files):
+        """loop through all files, join what matches"""
+        self.to_import = []
+
+        current_video = self._get_template()
+        last_base = False
+
+        for file_path in all_files:
+            base_name_raw, ext = os.path.splitext(file_path)
+            base_name, _ = os.path.splitext(base_name_raw)
+
+            key, file_path = self._detect_type(file_path, ext)
+            if not key or not file_path:
+                continue
+
+            if base_name != last_base:
+                if last_base:
+                    self.to_import.append(current_video)
+
+                current_video = self._get_template()
+                last_base = base_name
+
+            if key == "subtitle":
+                current_video["subtitle"].append(file_path)
+            else:
+                current_video[key] = file_path
+
+        if current_video.get("media"):
+            self.to_import.append(current_video)
+
+    def _detect_type(self, file_path, ext):
+        """detect metadata type for file"""
+
+        for key, value in self.EXT_MAP.items():
+            if ext in value:
+                return key, file_path
+
+        return False, False
+
+    def process_videos(self):
+        """loop through all videos"""
+        for current_video in self.to_import:
+            if not current_video["media"]:
+                print(f"{current_video}: no matching media file found.")
+                raise ValueError
+
+            self._detect_youtube_id(current_video)
+            self._dump_thumb(current_video)
+            self._convert_thumb(current_video)
+            self._get_subtitles(current_video)
+            self._convert_video(current_video)
+
+            ManualImport(current_video, self.CONFIG).run()
+
+    def _detect_youtube_id(self, current_video):
+        """find video id from filename or json"""
+        print(current_video)
+        youtube_id = self._extract_id_from_filename(current_video["media"])
+        if youtube_id:
+            current_video["video_id"] = youtube_id
+            return
+
+        youtube_id = self._extract_id_from_json(current_video["metadata"])
+        if youtube_id:
+            current_video["video_id"] = youtube_id
+            return
+
+        print(current_video["media"])
+        raise ValueError("failed to find video id")
+
+    @staticmethod
+    def _extract_id_from_filename(file_name):
         """
         look at the file name for the youtube id
         expects filename ending in [<youtube_id>].<ext>
         """
-        id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", file_name)
+        base_name, _ = os.path.splitext(file_name)
+        id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
         if id_search:
             youtube_id = id_search.group(1)
             return youtube_id
 
-        print("failed to extract youtube id for: " + file_name)
-        raise Exception
+        print(f"id extraction failed from filename: {file_name}")
 
-    def extract_id_from_json(self, json_file):
+        return False
+
+    def _extract_id_from_json(self, json_file):
         """open json file and extract id"""
         json_path = os.path.join(self.CACHE_DIR, "import", json_file)
         with open(json_path, "r", encoding="utf-8") as f:
@@ -233,66 +305,273 @@ class ManualImport:
 
         return youtube_id
 
-    def process_import(self):
-        """go through identified media files"""
+    def _dump_thumb(self, current_video):
+        """extract embedded thumb before converting"""
+        if current_video["thumb"]:
+            return
 
-        all_videos_added = []
+        media_path = current_video["media"]
+        _, ext = os.path.splitext(media_path)
 
-        for media_file in self.identified:
-            json_file = media_file["json_file"]
-            video_file = media_file["video_file"]
-            youtube_id = media_file["youtube_id"]
+        new_path = False
+        if ext == ".mkv":
+            idx, thumb_type = self._get_mkv_thumb_stream(media_path)
+            if idx is not None:
+                new_path = self.dump_mpv_thumb(media_path, idx, thumb_type)
 
-            video_path = os.path.join(self.CACHE_DIR, "import", video_file)
+        elif ext == ".mp4":
+            thumb_type = self.get_mp4_thumb_type(media_path)
+            if thumb_type:
+                new_path = self.dump_mp4_thumb(media_path, thumb_type)
 
-            self.move_to_cache(video_path, youtube_id)
+        if new_path:
+            current_video["thumb"] = new_path
 
-            # identify and archive
-            vid_dict = index_new_video(youtube_id)
-            VideoDownloader([youtube_id]).move_to_archive(vid_dict)
-            youtube_id = vid_dict["youtube_id"]
-            thumb_url = vid_dict["vid_thumb_url"]
-            all_videos_added.append((youtube_id, thumb_url))
+    def _get_mkv_thumb_stream(self, media_path):
+        """get stream idx of thumbnail for mkv files"""
+        streams = self._get_streams(media_path)
+        attachments = [
+            i for i in streams["streams"] if i["codec_type"] == "attachment"
+        ]
 
-            # cleanup
-            if os.path.exists(video_path):
-                os.remove(video_path)
-            if json_file:
-                json_path = os.path.join(self.CACHE_DIR, "import", json_file)
-                os.remove(json_path)
+        for idx, stream in enumerate(attachments):
+            tags = stream["tags"]
+            if "mimetype" in tags and tags["filename"].startswith("cover"):
+                _, ext = os.path.splitext(tags["filename"])
+                return idx, ext
 
-        return all_videos_added
+        return None, None
 
-    def move_to_cache(self, video_path, youtube_id):
-        """move identified video file to cache, convert to mp4"""
-        file_name = os.path.split(video_path)[-1]
-        video_file, ext = os.path.splitext(file_name)
+    @staticmethod
+    def dump_mpv_thumb(media_path, idx, thumb_type):
+        """write cover to disk for mkv"""
+        _, media_ext = os.path.splitext(media_path)
+        new_path = f"{media_path.rstrip(media_ext)}{thumb_type}"
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-v",
+                "quiet",
+                f"-dump_attachment:t:{idx}",
+                new_path,
+                "-i",
+                media_path,
+            ],
+            check=False,
+        )
 
-        # make sure youtube_id is in filename
-        if youtube_id not in video_file:
-            video_file = f"{video_file}_{youtube_id}"
+        return new_path
 
-        # move, convert if needed
+    def get_mp4_thumb_type(self, media_path):
+        """detect filetype of embedded thumbnail"""
+        streams = self._get_streams(media_path)
+
+        for stream in streams["streams"]:
+            if stream["codec_name"] in ["png", "jpg"]:
+                return stream["codec_name"]
+
+        return False
+
+    def _convert_thumb(self, current_video):
+        """convert all thumbnails to jpg"""
+        if not current_video["thumb"]:
+            return
+
+        thumb_path = current_video["thumb"]
+
+        base_path, ext = os.path.splitext(thumb_path)
+        if ext == ".jpg":
+            return
+
+        new_path = f"{base_path}.jpg"
+        img_raw = Image.open(thumb_path)
+        img_raw.convert("RGB").save(new_path)
+
+        os.remove(thumb_path)
+        current_video["thumb"] = new_path
+
+    def _get_subtitles(self, current_video):
+        """find all subtitles in media file"""
+        if current_video["subtitle"]:
+            return
+
+        media_path = current_video["media"]
+        streams = self._get_streams(media_path)
+        base_path, ext = os.path.splitext(media_path)
+
+        if ext == ".webm":
+            print(f"{media_path}: subtitle extract from webm not supported")
+            return
+
+        for idx, stream in enumerate(streams["streams"]):
+            if stream["codec_type"] == "subtitle":
+                lang = ISO639Utils.long2short(stream["tags"]["language"])
+                sub_path = f"{base_path}.{lang}.vtt"
+                self._dump_subtitle(idx, media_path, sub_path)
+                current_video["subtitle"].append(sub_path)
+
+    @staticmethod
+    def _dump_subtitle(idx, media_path, sub_path):
+        """extract subtitle from media file"""
+        subprocess.run(
+            ["ffmpeg", "-i", media_path, "-map", f"0:{idx}", sub_path],
+            check=True,
+        )
+
+    @staticmethod
+    def _get_streams(media_path):
+        """return all streams from media_path"""
+        streams_raw = subprocess.run(
+            [
+                "ffprobe",
+                "-v",
+                "error",
+                "-show_streams",
+                "-print_format",
+                "json",
+                media_path,
+            ],
+            capture_output=True,
+            check=True,
+        )
+        streams = json.loads(streams_raw.stdout.decode())
+
+        return streams
+
+    @staticmethod
+    def dump_mp4_thumb(media_path, thumb_type):
+        """save cover to disk"""
+        _, ext = os.path.splitext(media_path)
+        new_path = f"{media_path.rstrip(ext)}.{thumb_type}"
+
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-i",
+                media_path,
+                "-map",
+                "0:v",
+                "-map",
+                "-0:V",
+                "-c",
+                "copy",
+                new_path,
+            ],
+            check=True,
+        )
+
+        return new_path
+
+    def _convert_video(self, current_video):
+        """convert if needed"""
+        current_path = current_video["media"]
+        base_path, ext = os.path.splitext(current_path)
         if ext == ".mp4":
-            new_file = video_file + ext
-            dest_path = os.path.join(self.CACHE_DIR, "download", new_file)
-            shutil.move(video_path, dest_path)
+            return
+
+        new_path = base_path + ".mp4"
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-i",
+                current_path,
+                new_path,
+                "-loglevel",
+                "warning",
+                "-stats",
+            ],
+            check=True,
+        )
+        current_video["media"] = new_path
+        os.remove(current_path)
+
+
+class ManualImport:
+    """import single identified video"""
+
+    def __init__(self, current_video, config):
+        self.current_video = current_video
+        self.config = config
+
+    def run(self):
+        """run all"""
+        json_data = self.index_metadata()
+        self._move_to_archive(json_data)
+        self._cleanup(json_data)
+
+    def index_metadata(self):
+        """get metadata from yt or json"""
+        video_id = self.current_video["video_id"]
+        video = YoutubeVideo(video_id)
+        video.build_json(
+            youtube_meta_overwrite=self._get_info_json(),
+            media_path=self.current_video["media"],
+        )
+        if not video.json_data:
+            print(f"{video_id}: manual import failed, and no metadata found.")
+            raise ValueError
+
+        video.check_subtitles(subtitle_files=self.current_video["subtitle"])
+        video.upload_to_es()
+
+        if video.offline_import and self.current_video["thumb"]:
+            old_path = self.current_video["thumb"]
+            new_path = ThumbManager(video_id).vid_thumb_path(absolute=True)
+            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
         else:
-            print(f"processing with ffmpeg: {video_file}")
-            new_file = video_file + ".mp4"
-            dest_path = os.path.join(self.CACHE_DIR, "download", new_file)
-            subprocess.run(
-                [
-                    "ffmpeg",
-                    "-i",
-                    video_path,
-                    dest_path,
-                    "-loglevel",
-                    "warning",
-                    "-stats",
-                ],
-                check=True,
-            )
+            url = video.json_data["vid_thumb_url"]
+            ThumbManager(video_id).download_video_thumb(url)
+
+        return video.json_data
+
+    def _get_info_json(self):
+        """read info_json from file"""
+        if not self.current_video["metadata"]:
+            return False
+
+        with open(self.current_video["metadata"], "r", encoding="utf-8") as f:
+            info_json = json.loads(f.read())
+
+        return info_json
+
+    def _move_to_archive(self, json_data):
+        """move identified media file to archive"""
+        videos = self.config["application"]["videos"]
+
+        channel, file = os.path.split(json_data["media_url"])
+        channel_folder = os.path.join(videos, channel)
+        if not os.path.exists(channel_folder):
+            os.makedirs(channel_folder)
+
+        old_path = self.current_video["media"]
+        new_path = os.path.join(channel_folder, file)
+        shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+
+        base_name, _ = os.path.splitext(new_path)
+        for old_path in self.current_video["subtitle"]:
+            lang = old_path.split(".")[-2]
+            new_path = f"{base_name}.{lang}.vtt"
+            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+
+    def _cleanup(self, json_data):
+        """cleanup leftover files"""
+        if os.path.exists(self.current_video["metadata"]):
+            os.remove(self.current_video["metadata"])
+
+        if os.path.exists(self.current_video["thumb"]):
+            os.remove(self.current_video["thumb"])
+
+        for subtitle_file in self.current_video["subtitle"]:
+            if os.path.exists(subtitle_file):
+                os.remove(subtitle_file)
+
+        channel_info = os.path.join(
+            self.config["application"]["cache_dir"],
+            "import",
+            f"{json_data['channel']['channel_id']}.info.json",
+        )
+        if os.path.exists(channel_info):
+            os.remove(channel_info)
 
 
 def scan_filesystem():
@@ -317,10 +596,7 @@ def scan_filesystem():
 
 def reindex_old_documents():
     """daily refresh of old documents"""
-    # continue if needed
-    reindex_handler = Reindex()
-    reindex_handler.check_outdated()
-    reindex_handler.reindex()
-    # set timestamp
-    now = int(datetime.now().strftime("%s"))
-    RedisArchivist().set_message("last_reindex", now, expire=False)
+    handler = Reindex()
+    handler.check_outdated()
+    handler.reindex()
+    RedisArchivist().set_message("last_reindex", handler.now)

tubearchivist/home/src/index/generic.py

@@ -5,7 +5,7 @@ functionality:
 
 import math
 
-import yt_dlp
+from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
 from home.src.ta.ta_redis import RedisArchivist
@@ -18,38 +18,22 @@ class YouTubeItem:
     index_name = False
     yt_base = False
     yt_obs = {
-        "quiet": True,
-        "default_search": "ytsearch",
         "skip_download": True,
-        "check_formats": "selected",
         "noplaylist": True,
     }
 
     def __init__(self, youtube_id):
         self.youtube_id = youtube_id
-        self.config = False
-        self.app_conf = False
-        self.youtube_meta = False
-        self.json_data = False
-        self._get_conf()
-
-    def _get_conf(self):
-        """read user conf"""
         self.config = AppConfig().config
         self.app_conf = self.config["application"]
+        self.youtube_meta = False
+        self.json_data = False
 
     def get_from_youtube(self):
         """use yt-dlp to get meta data from youtube"""
         print(f"{self.youtube_id}: get metadata from youtube")
-        try:
-            yt_item = yt_dlp.YoutubeDL(self.yt_obs)
-            response = yt_item.extract_info(self.yt_base + self.youtube_id)
-        except (
-            yt_dlp.utils.ExtractorError,
-            yt_dlp.utils.DownloadError,
-        ):
-            print(f"{self.youtube_id}: failed to get info from youtube")
-            response = False
+        url = self.yt_base + self.youtube_id
+        response = YtWrap(self.yt_obs, self.config).extract(url)
 
         self.youtube_meta = response
 
@@ -81,7 +65,7 @@ class YouTubeItem:
     def del_in_es(self):
         """delete item from elastic search"""
         print(f"{self.youtube_id}: delete from es")
-        _, _ = ElasticWrap(self.es_path).delete()
+        _, _ = ElasticWrap(self.es_path).delete(refresh=True)
 
 
 class Pagination:
@@ -133,7 +117,7 @@ class Pagination:
         """validate pagination with total_hits after making api call"""
         page_get = self.page_get
         max_pages = math.ceil(total_hits / self.page_size)
-        if total_hits > 10000:
+        if total_hits >= 10000:
             # es returns maximal 10000 results
             self.pagination["max_hits"] = True
             max_pages = max_pages - 1
@@ -147,3 +131,4 @@ class Pagination:
         ]
 
         self.pagination["next_pages"] = next_pages
+        self.pagination["total_hits"] = total_hits

tubearchivist/home/src/index/playlist.py

@@ -18,12 +18,7 @@ class YoutubePlaylist(YouTubeItem):
 
     es_path = False
     index_name = "ta_playlist"
-    yt_obs = {
-        "default_search": "ytsearch",
-        "quiet": True,
-        "skip_download": True,
-        "extract_flat": True,
-    }
+    yt_obs = {"extract_flat": True}
     yt_base = "https://www.youtube.com/playlist?list="
 
     def __init__(self, youtube_id):
@@ -46,7 +41,6 @@ class YoutubePlaylist(YouTubeItem):
             self.process_youtube_meta()
             self.get_entries()
             self.json_data["playlist_entries"] = self.all_members
-            self.get_playlist_art()
             self.json_data["playlist_subscribed"] = subscribed
 
     def process_youtube_meta(self):
@@ -86,12 +80,10 @@ class YoutubePlaylist(YouTubeItem):
 
         self.all_members = all_members
 
-    @staticmethod
-    def get_playlist_art():
+    def get_playlist_art(self):
         """download artwork of playlist"""
-        thumbnails = ThumbManager()
-        missing_playlists = thumbnails.get_missing_playlists()
-        thumbnails.download_playlist(missing_playlists)
+        url = self.json_data["playlist_thumbnail"]
+        ThumbManager(self.youtube_id, item_type="playlist").download(url)
 
     def add_vids_to_playlist(self):
         """sync the playlist id to videos"""
@@ -150,17 +142,15 @@ class YoutubePlaylist(YouTubeItem):
             previous_item = False
         else:
             previous_item = all_entries[current_idx - 1]
-            prev_thumb = ThumbManager().vid_thumb_path(
-                previous_item["youtube_id"]
-            )
-            previous_item["vid_thumb"] = prev_thumb
+            prev_id = previous_item["youtube_id"]
+            previous_item["vid_thumb"] = ThumbManager(prev_id).vid_thumb_path()
 
         if current_idx == len(all_entries) - 1:
             next_item = False
         else:
             next_item = all_entries[current_idx + 1]
-            next_thumb = ThumbManager().vid_thumb_path(next_item["youtube_id"])
-            next_item["vid_thumb"] = next_thumb
+            next_id = next_item["youtube_id"]
+            next_item["vid_thumb"] = ThumbManager(next_id).vid_thumb_path()
 
         self.nav = {
             "playlist_meta": {

tubearchivist/home/src/index/reindex.py

@@ -4,86 +4,74 @@ functionality:
 - index and update in es
 """
 
-import json
+import os
+import shutil
 from datetime import datetime
 from math import ceil
 from time import sleep
 
-import requests
 from home.src.download.queue import PendingList
-from home.src.download.subscriptions import ChannelSubscription
 from home.src.download.thumbnails import ThumbManager
+from home.src.download.yt_dlp_base import CookieHandler
+from home.src.download.yt_dlp_handler import VideoDownloader
+from home.src.es.connect import ElasticWrap
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_total_hits
 
 
 class Reindex:
     """check for outdated documents and refresh data from youtube"""
 
+    MATCH_FIELD = {
+        "ta_video": "active",
+        "ta_channel": "channel_active",
+        "ta_playlist": "playlist_active",
+    }
+    MULTIPLY = 1.2
+
     def __init__(self):
         # config
-        config = AppConfig().config
-        self.sleep_interval = config["downloads"]["sleep_interval"]
-        self.es_url = config["application"]["es_url"]
-        self.es_auth = config["application"]["es_auth"]
-        self.refresh_interval = config["scheduler"]["check_reindex_days"]
-        self.integrate_ryd = config["downloads"]["integrate_ryd"]
+        self.now = int(datetime.now().strftime("%s"))
+        self.config = AppConfig().config
+        self.interval = self.config["scheduler"]["check_reindex_days"]
         # scan
         self.all_youtube_ids = False
         self.all_channel_ids = False
         self.all_playlist_ids = False
 
-    def get_daily(self):
+    def check_cookie(self):
+        """validate cookie if enabled"""
+        if self.config["downloads"]["cookie_import"]:
+            valid = CookieHandler(self.config).validate()
+            if not valid:
+                return
+
+    def _get_daily(self):
         """get daily refresh values"""
-        total_videos = get_total_hits(
-            "ta_video", self.es_url, self.es_auth, "active"
-        )
-        video_daily = ceil(total_videos / self.refresh_interval * 1.2)
-        total_channels = get_total_hits(
-            "ta_channel", self.es_url, self.es_auth, "channel_active"
-        )
-        channel_daily = ceil(total_channels / self.refresh_interval * 1.2)
-        total_playlists = get_total_hits(
-            "ta_playlist", self.es_url, self.es_auth, "playlist_active"
-        )
-        playlist_daily = ceil(total_playlists / self.refresh_interval * 1.2)
+        total_videos = self._get_total_hits("ta_video")
+        video_daily = ceil(total_videos / self.interval * self.MULTIPLY)
+        if video_daily >= 10000:
+            video_daily = 9999
+
+        total_channels = self._get_total_hits("ta_channel")
+        channel_daily = ceil(total_channels / self.interval * self.MULTIPLY)
+        total_playlists = self._get_total_hits("ta_playlist")
+        playlist_daily = ceil(total_playlists / self.interval * self.MULTIPLY)
         return (video_daily, channel_daily, playlist_daily)
 
-    def get_outdated_vids(self, size):
-        """get daily videos to refresh"""
-        headers = {"Content-type": "application/json"}
-        now = int(datetime.now().strftime("%s"))
-        now_lte = now - self.refresh_interval * 24 * 60 * 60
-        data = {
-            "size": size,
-            "query": {
-                "bool": {
-                    "must": [
-                        {"match": {"active": True}},
-                        {"range": {"vid_last_refresh": {"lte": now_lte}}},
-                    ]
-                }
-            },
-            "sort": [{"vid_last_refresh": {"order": "asc"}}],
-            "_source": False,
-        }
-        query_str = json.dumps(data)
-        url = self.es_url + "/ta_video/_search"
-        response = requests.get(
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        all_youtube_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
-        return all_youtube_ids
+    def _get_total_hits(self, index):
+        """get total hits from index"""
+        match_field = self.MATCH_FIELD[index]
+        path = f"{index}/_search?filter_path=hits.total"
+        data = {"query": {"match": {match_field: True}}}
+        response, _ = ElasticWrap(path).post(data=data)
+        total_hits = response["hits"]["total"]["value"]
+        return total_hits
 
-    def get_unrated_vids(self):
-        """get all videos without rating if ryd integration is enabled"""
-        headers = {"Content-type": "application/json"}
+    def _get_unrated_vids(self):
+        """get max 200 videos without rating if ryd integration is enabled"""
         data = {
             "size": 200,
             "query": {
@@ -92,104 +80,78 @@ class Reindex:
                 }
             },
         }
-        query_str = json.dumps(data)
-        url = self.es_url + "/ta_video/_search"
-        response = requests.get(
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        missing_rating = [i["_id"] for i in response_dict["hits"]["hits"]]
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+
+        missing_rating = [i["_id"] for i in response["hits"]["hits"]]
         self.all_youtube_ids = self.all_youtube_ids + missing_rating
 
-    def get_outdated_channels(self, size):
-        """get daily channels to refresh"""
-        headers = {"Content-type": "application/json"}
-        now = int(datetime.now().strftime("%s"))
-        now_lte = now - self.refresh_interval * 24 * 60 * 60
+    def _get_outdated_vids(self, size):
+        """get daily videos to refresh"""
+        now_lte = self.now - self.interval * 24 * 60 * 60
+        must_list = [
+            {"match": {"active": True}},
+            {"range": {"vid_last_refresh": {"lte": now_lte}}},
+        ]
         data = {
             "size": size,
-            "query": {
-                "bool": {
-                    "must": [
-                        {"match": {"channel_active": True}},
-                        {"range": {"channel_last_refresh": {"lte": now_lte}}},
-                    ]
-                }
-            },
+            "query": {"bool": {"must": must_list}},
+            "sort": [{"vid_last_refresh": {"order": "asc"}}],
+            "_source": False,
+        }
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+
+        all_youtube_ids = [i["_id"] for i in response["hits"]["hits"]]
+        return all_youtube_ids
+
+    def _get_outdated_channels(self, size):
+        """get daily channels to refresh"""
+        now_lte = self.now - self.interval * 24 * 60 * 60
+        must_list = [
+            {"match": {"channel_active": True}},
+            {"range": {"channel_last_refresh": {"lte": now_lte}}},
+        ]
+        data = {
+            "size": size,
+            "query": {"bool": {"must": must_list}},
             "sort": [{"channel_last_refresh": {"order": "asc"}}],
             "_source": False,
         }
-        query_str = json.dumps(data)
-        url = self.es_url + "/ta_channel/_search"
-        response = requests.get(
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        all_channel_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
+        response, _ = ElasticWrap("ta_channel/_search").get(data=data)
+
+        all_channel_ids = [i["_id"] for i in response["hits"]["hits"]]
         return all_channel_ids
 
-    def get_outdated_playlists(self, size):
+    def _get_outdated_playlists(self, size):
         """get daily outdated playlists to refresh"""
-        headers = {"Content-type": "application/json"}
-        now = int(datetime.now().strftime("%s"))
-        now_lte = now - self.refresh_interval * 24 * 60 * 60
+        now_lte = self.now - self.interval * 24 * 60 * 60
+        must_list = [
+            {"match": {"playlist_active": True}},
+            {"range": {"playlist_last_refresh": {"lte": now_lte}}},
+        ]
         data = {
             "size": size,
-            "query": {
-                "bool": {
-                    "must": [
-                        {"match": {"playlist_active": True}},
-                        {"range": {"playlist_last_refresh": {"lte": now_lte}}},
-                    ]
-                }
-            },
+            "query": {"bool": {"must": must_list}},
             "sort": [{"playlist_last_refresh": {"order": "asc"}}],
             "_source": False,
         }
-        query_str = json.dumps(data)
-        url = self.es_url + "/ta_playlist/_search"
-        response = requests.get(
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        all_playlist_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
+        response, _ = ElasticWrap("ta_playlist/_search").get(data=data)
+
+        all_playlist_ids = [i["_id"] for i in response["hits"]["hits"]]
         return all_playlist_ids
 
     def check_outdated(self):
         """add missing vids and channels"""
-        video_daily, channel_daily, playlist_daily = self.get_daily()
-        self.all_youtube_ids = self.get_outdated_vids(video_daily)
-        self.all_channel_ids = self.get_outdated_channels(channel_daily)
-        self.all_playlist_ids = self.get_outdated_playlists(playlist_daily)
-        if self.integrate_ryd:
-            self.get_unrated_vids()
+        video_daily, channel_daily, playlist_daily = self._get_daily()
+        self.all_youtube_ids = self._get_outdated_vids(video_daily)
+        self.all_channel_ids = self._get_outdated_channels(channel_daily)
+        self.all_playlist_ids = self._get_outdated_playlists(playlist_daily)
 
-    def rescrape_all_channels(self):
-        """sync new data from channel to all matching videos"""
-        sleep_interval = self.sleep_interval
-        channel_sub_handler = ChannelSubscription()
-        all_channels = channel_sub_handler.get_channels(subscribed_only=False)
-        all_channel_ids = [i["channel_id"] for i in all_channels]
-
-        for channel_id in all_channel_ids:
-            channel = YoutubeChannel(channel_id)
-            subscribed = channel.json_data["channel_subscribed"]
-            channel.get_from_youtube()
-            channel.json_data["channel_subscribed"] = subscribed
-            channel.upload_to_es()
-            channel.sync_to_videos()
-
-            if sleep_interval:
-                sleep(sleep_interval)
+        integrate_ryd = self.config["downloads"]["integrate_ryd"]
+        if integrate_ryd:
+            self._get_unrated_vids()
 
     @staticmethod
-    def reindex_single_video(youtube_id):
+    def _reindex_single_video(youtube_id):
         """refresh data for single video"""
         video = YoutubeVideo(youtube_id)
 
@@ -199,6 +161,7 @@ class Reindex:
         date_downloaded = video.json_data["date_downloaded"]
         channel_dict = video.json_data["channel"]
         playlist = video.json_data.get("playlist")
+        subtitles = video.json_data.get("subtitles")
 
         # get new
         video.build_json()
@@ -206,7 +169,9 @@ class Reindex:
             video.deactivate()
             return
 
-        video.delete_subtitles()
+        video.delete_subtitles(subtitles=subtitles)
+        video.check_subtitles()
+
         # add back
         video.json_data["player"] = player
         video.json_data["date_downloaded"] = date_downloaded
@@ -216,25 +181,32 @@ class Reindex:
 
         video.upload_to_es()
 
-        thumb_handler = ThumbManager()
-        thumb_handler.delete_vid_thumb(youtube_id)
-        to_download = (youtube_id, video.json_data["vid_thumb_url"])
-        thumb_handler.download_vid([to_download], notify=False)
+        thumb_handler = ThumbManager(youtube_id)
+        thumb_handler.delete_video_thumb()
+        thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
+
         return
 
     @staticmethod
-    def reindex_single_channel(channel_id):
+    def _reindex_single_channel(channel_id):
         """refresh channel data and sync to videos"""
         channel = YoutubeChannel(channel_id)
         channel.get_from_es()
         subscribed = channel.json_data["channel_subscribed"]
+        overwrites = channel.json_data.get("channel_overwrites", False)
         channel.get_from_youtube()
+        if not channel.json_data:
+            channel.deactivate()
+            return
+
         channel.json_data["channel_subscribed"] = subscribed
+        if overwrites:
+            channel.json_data["channel_overwrites"] = overwrites
         channel.upload_to_es()
         channel.sync_to_videos()
 
     @staticmethod
-    def reindex_single_playlist(playlist_id, all_indexed_ids):
+    def _reindex_single_playlist(playlist_id, all_indexed_ids):
         """refresh playlist data"""
         playlist = YoutubePlaylist(playlist_id)
         playlist.get_from_es()
@@ -251,24 +223,91 @@ class Reindex:
 
     def reindex(self):
         """reindex what's needed"""
+        sleep_interval = self.config["downloads"]["sleep_interval"]
         # videos
         print(f"reindexing {len(self.all_youtube_ids)} videos")
         for youtube_id in self.all_youtube_ids:
-            self.reindex_single_video(youtube_id)
-            if self.sleep_interval:
-                sleep(self.sleep_interval)
+            try:
+                self._reindex_single_video(youtube_id)
+            except FileNotFoundError:
+                # handle channel name change here
+                ChannelUrlFixer(youtube_id, self.config).run()
+                self._reindex_single_video(youtube_id)
+            if sleep_interval:
+                sleep(sleep_interval)
         # channels
         print(f"reindexing {len(self.all_channel_ids)} channels")
         for channel_id in self.all_channel_ids:
-            self.reindex_single_channel(channel_id)
-            if self.sleep_interval:
-                sleep(self.sleep_interval)
+            self._reindex_single_channel(channel_id)
+            if sleep_interval:
+                sleep(sleep_interval)
         # playlist
         print(f"reindexing {len(self.all_playlist_ids)} playlists")
         if self.all_playlist_ids:
-            all_indexed = PendingList().get_all_indexed()
-            all_indexed_ids = [i["youtube_id"] for i in all_indexed]
+            handler = PendingList()
+            handler.get_download()
+            handler.get_indexed()
+            all_indexed_ids = [i["youtube_id"] for i in handler.all_videos]
             for playlist_id in self.all_playlist_ids:
-                self.reindex_single_playlist(playlist_id, all_indexed_ids)
-                if self.sleep_interval:
-                    sleep(self.sleep_interval)
+                self._reindex_single_playlist(playlist_id, all_indexed_ids)
+                if sleep_interval:
+                    sleep(sleep_interval)
+
+
+class ChannelUrlFixer:
+    """fix not matching channel names in reindex"""
+
+    def __init__(self, youtube_id, config):
+        self.youtube_id = youtube_id
+        self.config = config
+        self.video = False
+
+    def run(self):
+        """check and run if needed"""
+        print(f"{self.youtube_id}: failed to build channel path, try to fix.")
+        video_path_is, video_folder_is = self.get_as_is()
+        if not os.path.exists(video_path_is):
+            print(f"giving up reindex, video in video: {self.video.json_data}")
+            raise ValueError
+
+        _, video_folder_should = self.get_as_should()
+
+        if video_folder_is != video_folder_should:
+            self.process(video_path_is)
+        else:
+            print(f"{self.youtube_id}: skip channel url fixer")
+
+    def get_as_is(self):
+        """get video object as is"""
+        self.video = YoutubeVideo(self.youtube_id)
+        self.video.get_from_es()
+        video_path_is = os.path.join(
+            self.config["application"]["videos"],
+            self.video.json_data["media_url"],
+        )
+        video_folder_is = os.path.split(video_path_is)[0]
+
+        return video_path_is, video_folder_is
+
+    def get_as_should(self):
+        """add fresh metadata from remote"""
+        self.video.get_from_youtube()
+        self.video.add_file_path()
+
+        video_path_should = os.path.join(
+            self.config["application"]["videos"],
+            self.video.json_data["media_url"],
+        )
+        video_folder_should = os.path.split(video_path_should)[0]
+        return video_path_should, video_folder_should
+
+    def process(self, video_path_is):
+        """fix filepath"""
+        print(f"{self.youtube_id}: fixing channel rename.")
+        cache_dir = self.config["application"]["cache_dir"]
+        new_path = os.path.join(
+            cache_dir, "download", self.youtube_id + ".mp4"
+        )
+        shutil.move(video_path_is, new_path, copy_function=shutil.copyfile)
+        VideoDownloader().move_to_archive(self.video.json_data)
+        self.video.update_media_url()

tubearchivist/home/src/index/subtitle.py

@@ -0,0 +1,321 @@
+"""
+functionality:
+- download subtitles
+- parse subtitles into it's cues
+- index dubtitles
+"""
+
+import json
+import os
+from datetime import datetime
+
+import requests
+from home.src.es.connect import ElasticWrap
+from home.src.ta.helper import requests_headers
+
+
+class YoutubeSubtitle:
+    """handle video subtitle functionality"""
+
+    def __init__(self, video):
+        self.video = video
+        self.languages = False
+
+    def _sub_conf_parse(self):
+        """add additional conf values to self"""
+        languages_raw = self.video.config["downloads"]["subtitle"]
+        if languages_raw:
+            self.languages = [i.strip() for i in languages_raw.split(",")]
+
+    def get_subtitles(self):
+        """check what to do"""
+        self._sub_conf_parse()
+        if not self.languages:
+            # no subtitles
+            return False
+
+        relevant_subtitles = []
+        for lang in self.languages:
+            user_sub = self._get_user_subtitles(lang)
+            if user_sub:
+                relevant_subtitles.append(user_sub)
+                continue
+
+            if self.video.config["downloads"]["subtitle_source"] == "auto":
+                auto_cap = self._get_auto_caption(lang)
+                if auto_cap:
+                    relevant_subtitles.append(auto_cap)
+
+        return relevant_subtitles
+
+    def _get_auto_caption(self, lang):
+        """get auto_caption subtitles"""
+        print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
+        all_subtitles = self.video.youtube_meta.get("automatic_captions")
+
+        if not all_subtitles:
+            return False
+
+        video_media_url = self.video.json_data["media_url"]
+        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
+        all_formats = all_subtitles.get(lang)
+        if not all_formats:
+            return False
+
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
+        subtitle.update(
+            {"lang": lang, "source": "auto", "media_url": media_url}
+        )
+
+        return subtitle
+
+    def _normalize_lang(self):
+        """normalize country specific language keys"""
+        all_subtitles = self.video.youtube_meta.get("subtitles")
+        if not all_subtitles:
+            return False
+
+        all_keys = list(all_subtitles.keys())
+        for key in all_keys:
+            lang = key.split("-")[0]
+            old = all_subtitles.pop(key)
+            if lang == "live_chat":
+                continue
+            all_subtitles[lang] = old
+
+        return all_subtitles
+
+    def _get_user_subtitles(self, lang):
+        """get subtitles uploaded from channel owner"""
+        print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
+        all_subtitles = self._normalize_lang()
+        if not all_subtitles:
+            return False
+
+        video_media_url = self.video.json_data["media_url"]
+        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
+        all_formats = all_subtitles.get(lang)
+        if not all_formats:
+            # no user subtitles found
+            return False
+
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
+        subtitle.update(
+            {"lang": lang, "source": "user", "media_url": media_url}
+        )
+
+        return subtitle
+
+    def download_subtitles(self, relevant_subtitles):
+        """download subtitle files to archive"""
+        videos_base = self.video.config["application"]["videos"]
+        indexed = []
+        for subtitle in relevant_subtitles:
+            dest_path = os.path.join(videos_base, subtitle["media_url"])
+            source = subtitle["source"]
+            lang = subtitle.get("lang")
+            response = requests.get(
+                subtitle["url"], headers=requests_headers()
+            )
+            if not response.ok:
+                print(f"{self.video.youtube_id}: failed to download subtitle")
+                print(response.text)
+                continue
+
+            parser = SubtitleParser(response.text, lang, source)
+            parser.process()
+            if not parser.all_cues:
+                continue
+
+            subtitle_str = parser.get_subtitle_str()
+            self._write_subtitle_file(dest_path, subtitle_str)
+            if self.video.config["downloads"]["subtitle_index"]:
+                query_str = parser.create_bulk_import(self.video, source)
+                self._index_subtitle(query_str)
+
+            indexed.append(subtitle)
+
+        return indexed
+
+    @staticmethod
+    def _write_subtitle_file(dest_path, subtitle_str):
+        """write subtitle file to disk"""
+        # create folder here for first video of channel
+        os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
+        with open(dest_path, "w", encoding="utf-8") as subfile:
+            subfile.write(subtitle_str)
+
+    @staticmethod
+    def _index_subtitle(query_str):
+        """send subtitle to es for indexing"""
+        _, _ = ElasticWrap("_bulk").post(data=query_str, ndjson=True)
+
+    def delete(self, subtitles=False):
+        """delete subtitles from index and filesystem"""
+        youtube_id = self.video.youtube_id
+        videos_base = self.video.config["application"]["videos"]
+        # delete files
+        if subtitles:
+            files = [i["media_url"] for i in subtitles]
+        else:
+            if not self.video.json_data.get("subtitles"):
+                return
+
+            files = [i["media_url"] for i in self.video.json_data["subtitles"]]
+
+        for file_name in files:
+            file_path = os.path.join(videos_base, file_name)
+            try:
+                os.remove(file_path)
+            except FileNotFoundError:
+                print(f"{youtube_id}: {file_path} failed to delete")
+        # delete from index
+        path = "ta_subtitle/_delete_by_query?refresh=true"
+        data = {"query": {"term": {"youtube_id": {"value": youtube_id}}}}
+        _, _ = ElasticWrap(path).post(data=data)
+
+
+class SubtitleParser:
+    """parse subtitle str from youtube"""
+
+    def __init__(self, subtitle_str, lang, source):
+        self.subtitle_raw = json.loads(subtitle_str)
+        self.lang = lang
+        self.source = source
+        self.all_cues = False
+
+    def process(self):
+        """extract relevant que data"""
+        self.all_cues = []
+        all_events = self.subtitle_raw.get("events")
+
+        if not all_events:
+            return
+
+        if self.source == "auto":
+            all_events = self._flat_auto_caption(all_events)
+
+        for idx, event in enumerate(all_events):
+            if "dDurationMs" not in event or "segs" not in event:
+                # some events won't have a duration or segs
+                print(f"skipping subtitle event without content: {event}")
+                continue
+
+            cue = {
+                "start": self._ms_conv(event["tStartMs"]),
+                "end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
+                "text": "".join([i.get("utf8") for i in event["segs"]]),
+                "idx": idx + 1,
+            }
+            self.all_cues.append(cue)
+
+    @staticmethod
+    def _flat_auto_caption(all_events):
+        """flatten autocaption segments"""
+        flatten = []
+        for event in all_events:
+            if "segs" not in event.keys():
+                continue
+            text = "".join([i.get("utf8") for i in event.get("segs")])
+            if not text.strip():
+                continue
+
+            if flatten:
+                # fix overlapping retiming issue
+                last = flatten[-1]
+                if "dDurationMs" not in last or "segs" not in last:
+                    # some events won't have a duration or segs
+                    print(f"skipping subtitle event without content: {event}")
+                    continue
+
+                last_end = last["tStartMs"] + last["dDurationMs"]
+                if event["tStartMs"] < last_end:
+                    joined = last["segs"][0]["utf8"] + "\n" + text
+                    last["segs"][0]["utf8"] = joined
+                    continue
+
+            event.update({"segs": [{"utf8": text}]})
+            flatten.append(event)
+
+        return flatten
+
+    @staticmethod
+    def _ms_conv(ms):
+        """convert ms to timestamp"""
+        hours = str((ms // (1000 * 60 * 60)) % 24).zfill(2)
+        minutes = str((ms // (1000 * 60)) % 60).zfill(2)
+        secs = str((ms // 1000) % 60).zfill(2)
+        millis = str(ms % 1000).zfill(3)
+
+        return f"{hours}:{minutes}:{secs}.{millis}"
+
+    def get_subtitle_str(self):
+        """create vtt text str from cues"""
+        subtitle_str = f"WEBVTT\nKind: captions\nLanguage: {self.lang}"
+
+        for cue in self.all_cues:
+            stamp = f"{cue.get('start')} --> {cue.get('end')}"
+            cue_text = f"\n\n{cue.get('idx')}\n{stamp}\n{cue.get('text')}"
+            subtitle_str = subtitle_str + cue_text
+
+        return subtitle_str
+
+    def create_bulk_import(self, video, source):
+        """subtitle lines for es import"""
+        documents = self._create_documents(video, source)
+        bulk_list = []
+
+        for document in documents:
+            document_id = document.get("subtitle_fragment_id")
+            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(document))
+
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+
+        return query_str
+
+    def _create_documents(self, video, source):
+        """process documents"""
+        documents = self._chunk_list(video.youtube_id)
+        channel = video.json_data.get("channel")
+        meta_dict = {
+            "youtube_id": video.youtube_id,
+            "title": video.json_data.get("title"),
+            "subtitle_channel": channel.get("channel_name"),
+            "subtitle_channel_id": channel.get("channel_id"),
+            "subtitle_last_refresh": int(datetime.now().strftime("%s")),
+            "subtitle_lang": self.lang,
+            "subtitle_source": source,
+        }
+
+        _ = [i.update(meta_dict) for i in documents]
+
+        return documents
+
+    def _chunk_list(self, youtube_id):
+        """join cues for bulk import"""
+        chunk_list = []
+
+        chunk = {}
+        for cue in self.all_cues:
+            if chunk:
+                text = f"{chunk.get('subtitle_line')} {cue.get('text')}\n"
+                chunk["subtitle_line"] = text
+            else:
+                idx = len(chunk_list) + 1
+                chunk = {
+                    "subtitle_index": idx,
+                    "subtitle_line": cue.get("text"),
+                    "subtitle_start": cue.get("start"),
+                }
+
+            chunk["subtitle_fragment_id"] = f"{youtube_id}-{self.lang}-{idx}"
+
+            if cue["idx"] % 5 == 0:
+                chunk["subtitle_end"] = cue.get("end")
+                chunk_list.append(chunk)
+                chunk = {}
+
+        return chunk_list

tubearchivist/home/src/index/video.py

@@ -4,279 +4,113 @@ functionality:
 - index and update in es
 """
 
-import json
 import os
-import re
 from datetime import datetime
 
 import requests
+from django.conf import settings
 from home.src.es.connect import ElasticWrap
 from home.src.index import channel as ta_channel
+from home.src.index import playlist as ta_playlist
 from home.src.index.generic import YouTubeItem
-from home.src.ta.helper import DurationConverter, clean_string
+from home.src.index.subtitle import YoutubeSubtitle
+from home.src.ta.helper import DurationConverter, clean_string, randomizor
+from home.src.ta.ta_redis import RedisArchivist
 from ryd_client import ryd_client
 
 
-class YoutubeSubtitle:
-    """handle video subtitle functionality"""
+class SponsorBlock:
+    """handle sponsor block integration"""
 
-    def __init__(self, video):
-        self.video = video
-        self.languages = False
+    API = "https://sponsor.ajay.app/api"
 
-    def sub_conf_parse(self):
-        """add additional conf values to self"""
-        languages_raw = self.video.config["downloads"]["subtitle"]
-        if languages_raw:
-            self.languages = [i.strip() for i in languages_raw.split(",")]
+    def __init__(self, user_id=False):
+        self.user_id = user_id
+        self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
+        self.last_refresh = int(datetime.now().strftime("%s"))
 
-    def get_subtitles(self):
-        """check what to do"""
-        self.sub_conf_parse()
-        if not self.languages:
-            # no subtitles
-            return False
+    def get_sb_id(self):
+        """get sponsorblock userid or generate if needed"""
+        if not self.user_id:
+            print("missing request user id")
+            raise ValueError
 
-        relevant_subtitles = []
-        for lang in self.languages:
-            user_sub = self.get_user_subtitles(lang)
-            if user_sub:
-                relevant_subtitles.append(user_sub)
-                continue
+        key = f"{self.user_id}:id_sponsorblock"
+        sb_id = RedisArchivist().get_message(key)
+        if not sb_id["status"]:
+            sb_id = {"status": randomizor(32)}
+            RedisArchivist().set_message(key, sb_id)
 
-            if self.video.config["downloads"]["subtitle_source"] == "auto":
-                auto_cap = self.get_auto_caption(lang)
-                if auto_cap:
-                    relevant_subtitles.append(auto_cap)
+        return sb_id
 
-        return relevant_subtitles
+    def get_timestamps(self, youtube_id):
+        """get timestamps from the API"""
+        url = f"{self.API}/skipSegments?videoID={youtube_id}"
+        headers = {"User-Agent": self.user_agent}
+        print(f"{youtube_id}: get sponsorblock timestamps")
+        response = requests.get(url, headers=headers)
+        if not response.ok:
+            print(f"{youtube_id}: sponsorblock failed: {response.text}")
+            sponsor_dict = {
+                "last_refresh": self.last_refresh,
+                "is_enabled": True,
+                "segments": [],
+            }
+        else:
+            all_segments = response.json()
+            sponsor_dict = self._get_sponsor_dict(all_segments)
 
-    def get_auto_caption(self, lang):
-        """get auto_caption subtitles"""
-        print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
-        all_subtitles = self.video.youtube_meta.get("automatic_captions")
+        return sponsor_dict
 
-        if not all_subtitles:
-            return False
+    def _get_sponsor_dict(self, all_segments):
+        """format and process response"""
+        has_unlocked = False
+        cleaned_segments = []
+        for segment in all_segments:
+            if not segment["locked"]:
+                has_unlocked = True
+            del segment["userID"]
+            del segment["description"]
+            cleaned_segments.append(segment)
 
-        video_media_url = self.video.json_data["media_url"]
-        media_url = video_media_url.replace(".mp4", f"-{lang}.vtt")
-        all_formats = all_subtitles.get(lang)
-        if not all_formats:
-            return False
-
-        subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
-        subtitle.update(
-            {"lang": lang, "source": "auto", "media_url": media_url}
-        )
-
-        return subtitle
-
-    def _normalize_lang(self):
-        """normalize country specific language keys"""
-        all_subtitles = self.video.youtube_meta.get("subtitles")
-        if not all_subtitles:
-            return False
-
-        all_keys = list(all_subtitles.keys())
-        for key in all_keys:
-            lang = key.split("-")[0]
-            old = all_subtitles.pop(key)
-            if lang == "live_chat":
-                continue
-            all_subtitles[lang] = old
-
-        return all_subtitles
-
-    def get_user_subtitles(self, lang):
-        """get subtitles uploaded from channel owner"""
-        print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
-        all_subtitles = self._normalize_lang()
-        if not all_subtitles:
-            return False
-
-        video_media_url = self.video.json_data["media_url"]
-        media_url = video_media_url.replace(".mp4", f"-{lang}.vtt")
-        all_formats = all_subtitles.get(lang)
-        if not all_formats:
-            # no user subtitles found
-            return False
-
-        subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
-        subtitle.update(
-            {"lang": lang, "source": "user", "media_url": media_url}
-        )
-
-        return subtitle
-
-    def download_subtitles(self, relevant_subtitles):
-        """download subtitle files to archive"""
-        videos_base = self.video.config["application"]["videos"]
-        for subtitle in relevant_subtitles:
-            dest_path = os.path.join(videos_base, subtitle["media_url"])
-            source = subtitle["source"]
-            response = requests.get(subtitle["url"])
-            if not response.ok:
-                print(f"{self.video.youtube_id}: failed to download subtitle")
-                continue
-
-            parser = SubtitleParser(response.text, subtitle.get("lang"))
-            parser.process()
-            subtitle_str = parser.get_subtitle_str()
-            self._write_subtitle_file(dest_path, subtitle_str)
-            if self.video.config["downloads"]["subtitle_index"]:
-                query_str = parser.create_bulk_import(self.video, source)
-                self._index_subtitle(query_str)
-
-    @staticmethod
-    def _write_subtitle_file(dest_path, subtitle_str):
-        """write subtitle file to disk"""
-        # create folder here for first video of channel
-        os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
-        with open(dest_path, "w", encoding="utf-8") as subfile:
-            subfile.write(subtitle_str)
-
-    @staticmethod
-    def _index_subtitle(query_str):
-        """send subtitle to es for indexing"""
-        _, _ = ElasticWrap("_bulk").post(data=query_str, ndjson=True)
-
-
-class SubtitleParser:
-    """parse subtitle str from youtube"""
-
-    time_reg = r"^([0-9]{2}:?){3}\.[0-9]{3} --> ([0-9]{2}:?){3}\.[0-9]{3}"
-    stamp_reg = r"<([0-9]{2}:?){3}\.[0-9]{3}>"
-    tag_reg = r"</?c>"
-
-    def __init__(self, subtitle_str, lang):
-        self.subtitle_str = subtitle_str
-        self.lang = lang
-        self.header = False
-        self.parsed_cue_list = False
-        self.all_text_lines = False
-        self.matched = False
-
-    def process(self):
-        """collection to process subtitle string"""
-        self._parse_cues()
-        self._match_text_lines()
-        self._add_id()
-        self._timestamp_check()
-
-    def _parse_cues(self):
-        """split into cues"""
-        all_cues = self.subtitle_str.replace("\n \n", "\n").split("\n\n")
-        self.header = all_cues[0]
-        self.all_text_lines = []
-        self.parsed_cue_list = [self._cue_cleaner(i) for i in all_cues[1:]]
-
-    def _cue_cleaner(self, cue):
-        """parse single cue"""
-        all_lines = cue.split("\n")
-        cue_dict = {"lines": []}
-
-        for line in all_lines:
-            if re.match(self.time_reg, line):
-                clean = re.search(self.time_reg, line).group()
-                start, end = clean.split(" --> ")
-                cue_dict.update({"start": start, "end": end})
-            else:
-                clean = re.sub(self.stamp_reg, "", line)
-                clean = re.sub(self.tag_reg, "", clean)
-                cue_dict["lines"].append(clean)
-                if clean.strip() and clean not in self.all_text_lines[-4:]:
-                    # remove immediate duplicates
-                    self.all_text_lines.append(clean)
-
-        return cue_dict
-
-    def _match_text_lines(self):
-        """match unique text lines with timestamps"""
-
-        self.matched = []
-
-        while self.all_text_lines:
-            check = self.all_text_lines[0]
-            matches = [i for i in self.parsed_cue_list if check in i["lines"]]
-            new_cue = matches[-1]
-            new_cue["start"] = matches[0]["start"]
-
-            for line in new_cue["lines"]:
-                try:
-                    self.all_text_lines.remove(line)
-                except ValueError:
-                    continue
-
-            self.matched.append(new_cue)
-
-    def _timestamp_check(self):
-        """check if end timestamp is bigger than start timestamp"""
-        for idx, cue in enumerate(self.matched):
-            # this
-            end = int(re.sub("[^0-9]", "", cue.get("end")))
-            # next
-            try:
-                next_cue = self.matched[idx + 1]
-            except IndexError:
-                continue
-
-            start_next = int(re.sub("[^0-9]", "", next_cue.get("start")))
-            if end > start_next:
-                self.matched[idx]["end"] = next_cue.get("start")
-
-    def _add_id(self):
-        """add id to matched cues"""
-        for idx, _ in enumerate(self.matched):
-            self.matched[idx]["id"] = idx + 1
-
-    def get_subtitle_str(self):
-        """stitch cues and return processed new string"""
-        new_subtitle_str = self.header + "\n\n"
-
-        for cue in self.matched:
-            timestamp = f"{cue.get('start')} --> {cue.get('end')}"
-            lines = "\n".join(cue.get("lines"))
-            cue_text = f"{cue.get('id')}\n{timestamp}\n{lines}\n\n"
-            new_subtitle_str = new_subtitle_str + cue_text
-
-        return new_subtitle_str
-
-    def create_bulk_import(self, video, source):
-        """process matched for es import"""
-        bulk_list = []
-        channel = video.json_data.get("channel")
-
-        document = {
-            "youtube_id": video.youtube_id,
-            "title": video.json_data.get("title"),
-            "subtitle_channel": channel.get("channel_name"),
-            "subtitle_channel_id": channel.get("channel_id"),
-            "subtitle_last_refresh": int(datetime.now().strftime("%s")),
-            "subtitle_lang": self.lang,
-            "subtitle_source": source,
+        sponsor_dict = {
+            "last_refresh": self.last_refresh,
+            "has_unlocked": has_unlocked,
+            "is_enabled": True,
+            "segments": cleaned_segments,
         }
+        return sponsor_dict
 
-        for match in self.matched:
-            match_id = match.get("id")
-            document_id = f"{video.youtube_id}-{self.lang}-{match_id}"
-            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
-            document.update(
-                {
-                    "subtitle_fragment_id": document_id,
-                    "subtitle_start": match.get("start"),
-                    "subtitle_end": match.get("end"),
-                    "subtitle_index": match_id,
-                    "subtitle_line": " ".join(match.get("lines")),
-                }
-            )
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(document))
+    def post_timestamps(self, youtube_id, start_time, end_time):
+        """post timestamps to api"""
+        user_id = self.get_sb_id().get("status")
+        data = {
+            "videoID": youtube_id,
+            "startTime": start_time,
+            "endTime": end_time,
+            "category": "sponsor",
+            "userID": user_id,
+            "userAgent": self.user_agent,
+        }
+        url = f"{self.API}/skipSegments?videoID={youtube_id}"
+        print(f"post: {data}")
+        print(f"to: {url}")
 
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
+        return {"success": True}, 200
 
-        return query_str
+    def vote_on_segment(self, uuid, vote):
+        """send vote on existing segment"""
+        user_id = self.get_sb_id().get("status")
+        data = {
+            "UUID": uuid,
+            "userID": user_id,
+            "type": vote,
+        }
+        url = f"{self.API}/api/voteOnSponsorTime"
+        print(f"post: {data}")
+        print(f"to: {url}")
+
+        return {"success": True}, 200
 
 
 class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
@@ -286,28 +120,50 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
     index_name = "ta_video"
     yt_base = "https://www.youtube.com/watch?v="
 
-    def __init__(self, youtube_id):
+    def __init__(self, youtube_id, video_overwrites=False):
         super().__init__(youtube_id)
         self.channel_id = False
+        self.video_overwrites = video_overwrites
         self.es_path = f"{self.index_name}/_doc/{youtube_id}"
+        self.offline_import = False
 
-    def build_json(self):
+    def build_json(self, youtube_meta_overwrite=False, media_path=False):
         """build json dict of video"""
         self.get_from_youtube()
-        if not self.youtube_meta:
+        if not self.youtube_meta and not youtube_meta_overwrite:
             return
 
+        if not self.youtube_meta:
+            self.youtube_meta = youtube_meta_overwrite
+            self.offline_import = True
+
         self._process_youtube_meta()
         self._add_channel()
         self._add_stats()
         self.add_file_path()
-        self.add_player()
-        self._check_subtitles()
+        self.add_player(media_path)
         if self.config["downloads"]["integrate_ryd"]:
             self._get_ryd_stats()
 
+        if self._check_get_sb():
+            self._get_sponsorblock()
+
         return
 
+    def _check_get_sb(self):
+        """check if need to run sponsor block"""
+        integrate = self.config["downloads"]["integrate_sponsorblock"]
+
+        if self.video_overwrites:
+            single_overwrite = self.video_overwrites.get(self.youtube_id)
+            if not single_overwrite:
+                return integrate
+
+            if "integrate_sponsorblock" in single_overwrite:
+                return single_overwrite.get("integrate_sponsorblock")
+
+        return integrate
+
     def _process_youtube_meta(self):
         """extract relevant fields from youtube"""
         # extract
@@ -316,12 +172,15 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
         published = upload_date_time.strftime("%Y-%m-%d")
         last_refresh = int(datetime.now().strftime("%s"))
+        # base64_blur = ThumbManager().get_base64_blur(self.youtube_id)
+        base64_blur = False
         # build json_data basics
         self.json_data = {
             "title": self.youtube_meta["title"],
             "description": self.youtube_meta["description"],
             "category": self.youtube_meta["categories"],
             "vid_thumb_url": self.youtube_meta["thumbnail"],
+            "vid_thumb_base64": base64_blur,
             "tags": self.youtube_meta["tags"],
             "published": published,
             "vid_last_refresh": last_refresh,
@@ -333,7 +192,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
     def _add_channel(self):
         """add channel dict to video json_data"""
         channel = ta_channel.YoutubeChannel(self.channel_id)
-        channel.build_json(upload=True)
+        channel.build_json(upload=True, fallback=self.youtube_meta)
         self.json_data.update({"channel": channel.json_data})
 
     def _add_stats(self):
@@ -341,13 +200,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         # likes
         like_count = self.youtube_meta.get("like_count", 0)
         dislike_count = self.youtube_meta.get("dislike_count", 0)
+        average_rating = self.youtube_meta.get("average_rating", 0)
         self.json_data.update(
             {
                 "stats": {
                     "view_count": self.youtube_meta["view_count"],
                     "like_count": like_count,
                     "dislike_count": dislike_count,
-                    "average_rating": self.youtube_meta["average_rating"],
+                    "average_rating": average_rating,
                 }
             }
         )
@@ -364,22 +224,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
         raise FileNotFoundError
 
-    def add_player(self):
+    def add_player(self, media_path=False):
         """add player information for new videos"""
-        try:
-            # when indexing from download task
-            vid_path = self.build_dl_cache_path()
-        except FileNotFoundError as err:
-            # when reindexing needs to handle title rename
-            channel = os.path.split(self.json_data["media_url"])[0]
-            channel_dir = os.path.join(self.app_conf["videos"], channel)
-            all_files = os.listdir(channel_dir)
-            for file in all_files:
-                if self.youtube_id in file:
-                    vid_path = os.path.join(channel_dir, file)
-                    break
-            else:
-                raise FileNotFoundError("could not find video file") from err
+        vid_path = self._get_vid_path(media_path)
 
         duration_handler = DurationConverter()
         duration = duration_handler.get_sec(vid_path)
@@ -394,6 +241,28 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             }
         )
 
+    def _get_vid_path(self, media_path=False):
+        """get path of media file"""
+        if media_path:
+            return media_path
+
+        try:
+            # when indexing from download task
+            vid_path = self.build_dl_cache_path()
+        except FileNotFoundError as err:
+            # when reindexing needs to handle title rename
+            channel = os.path.split(self.json_data["media_url"])[0]
+            channel_dir = os.path.join(self.app_conf["videos"], channel)
+            all_files = os.listdir(channel_dir)
+            for file in all_files:
+                if self.youtube_id in file and file.endswith(".mp4"):
+                    vid_path = os.path.join(channel_dir, file)
+                    break
+            else:
+                raise FileNotFoundError("could not find video file") from err
+
+        return vid_path
+
     def add_file_path(self):
         """build media_url for where file will be located"""
         channel_name = self.json_data["channel"]["channel_name"]
@@ -412,61 +281,114 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
     def delete_media_file(self):
         """delete video file, meta data"""
+        print(f"{self.youtube_id}: delete video")
         self.get_from_es()
         video_base = self.app_conf["videos"]
-        to_del = [self.json_data.get("media_url")]
-
-        all_subtitles = self.json_data.get("subtitles")
-        if all_subtitles:
-            to_del = to_del + [i.get("media_url") for i in all_subtitles]
-
-        for media_url in to_del:
-            file_path = os.path.join(video_base, media_url)
+        media_url = self.json_data.get("media_url")
+        file_path = os.path.join(video_base, media_url)
+        try:
             os.remove(file_path)
+        except FileNotFoundError:
+            print(f"{self.youtube_id}: failed {media_url}, continue.")
 
+        self.del_in_playlists()
         self.del_in_es()
         self.delete_subtitles()
 
+    def del_in_playlists(self):
+        """remove downloaded in playlist"""
+        all_playlists = self.json_data.get("playlist")
+        if not all_playlists:
+            return
+
+        for playlist_id in all_playlists:
+            print(f"{playlist_id}: delete video {self.youtube_id}")
+            playlist = ta_playlist.YoutubePlaylist(playlist_id)
+            playlist.get_from_es()
+            entries = playlist.json_data["playlist_entries"]
+            for idx, entry in enumerate(entries):
+                if entry["youtube_id"] == self.youtube_id:
+                    playlist.json_data["playlist_entries"][idx].update(
+                        {"downloaded": False}
+                    )
+            playlist.upload_to_es()
+
+    def delete_subtitles(self, subtitles=False):
+        """delete indexed subtitles"""
+        print(f"{self.youtube_id}: delete subtitles")
+        YoutubeSubtitle(self).delete(subtitles=subtitles)
+
     def _get_ryd_stats(self):
         """get optional stats from returnyoutubedislikeapi.com"""
+        # pylint: disable=broad-except
         try:
             print(f"{self.youtube_id}: get ryd stats")
             result = ryd_client.get(self.youtube_id)
-        except requests.exceptions.ConnectionError:
-            print(f"{self.youtube_id}: failed to query ryd api, skipping")
-            return False
+        except Exception as err:
+            print(f"{self.youtube_id}: failed to query ryd api {err}")
+            return
 
         if result["status"] == 404:
-            return False
+            return
 
         dislikes = {
-            "dislike_count": result["dislikes"],
-            "average_rating": result["rating"],
+            "dislike_count": result.get("dislikes", 0),
+            "average_rating": result.get("rating", 0),
         }
         self.json_data["stats"].update(dislikes)
 
-        return True
+    def _get_sponsorblock(self):
+        """get optional sponsorblock timestamps from sponsor.ajay.app"""
+        sponsorblock = SponsorBlock().get_timestamps(self.youtube_id)
+        if sponsorblock:
+            self.json_data["sponsorblock"] = sponsorblock
 
-    def _check_subtitles(self):
+    def check_subtitles(self, subtitle_files=False):
         """optionally add subtitles"""
+        if self.offline_import and subtitle_files:
+            indexed = self._offline_subtitles(subtitle_files)
+            self.json_data["subtitles"] = indexed
+            return
+
         handler = YoutubeSubtitle(self)
         subtitles = handler.get_subtitles()
         if subtitles:
-            self.json_data["subtitles"] = subtitles
-            handler.download_subtitles(relevant_subtitles=subtitles)
+            indexed = handler.download_subtitles(relevant_subtitles=subtitles)
+            self.json_data["subtitles"] = indexed
 
-    def delete_subtitles(self):
-        """delete indexed subtitles"""
-        data = {"query": {"term": {"youtube_id": {"value": self.youtube_id}}}}
-        _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data=data)
+    def _offline_subtitles(self, subtitle_files):
+        """import offline subtitles"""
+        base_name, _ = os.path.splitext(self.json_data["media_url"])
+        subtitles = []
+        for subtitle in subtitle_files:
+            lang = subtitle.split(".")[-2]
+            subtitle_media_url = f"{base_name}.{lang}.vtt"
+            to_add = {
+                "ext": "vtt",
+                "url": False,
+                "name": lang,
+                "lang": lang,
+                "source": "file",
+                "media_url": subtitle_media_url,
+            }
+            subtitles.append(to_add)
+
+        return subtitles
+
+    def update_media_url(self):
+        """update only media_url in es for reindex channel rename"""
+        data = {"doc": {"media_url": self.json_data["media_url"]}}
+        path = f"{self.index_name}/_update/{self.youtube_id}"
+        _, _ = ElasticWrap(path).post(data=data)
 
 
-def index_new_video(youtube_id):
+def index_new_video(youtube_id, video_overwrites=False):
     """combined classes to create new video in index"""
-    video = YoutubeVideo(youtube_id)
+    video = YoutubeVideo(youtube_id, video_overwrites=video_overwrites)
     video.build_json()
     if not video.json_data:
         raise ValueError("failed to get metadata for " + youtube_id)
 
+    video.check_subtitles()
     video.upload_to_es()
     return video.json_data

tubearchivist/home/src/ta/config.py

@@ -83,33 +83,35 @@ class AppConfig:
 
     def update_config(self, form_post):
         """update config values from settings form"""
-        config = self.config
+        updated = []
         for key, value in form_post.items():
-            to_write = value[0]
-            if len(to_write):
-                if to_write == "0":
-                    to_write = False
-                elif to_write == "1":
-                    to_write = True
-                elif to_write.isdigit():
-                    to_write = int(to_write)
+            if not value and not isinstance(value, int):
+                continue
 
-                config_dict, config_value = key.split("_", maxsplit=1)
-                config[config_dict][config_value] = to_write
+            if value in ["0", 0]:
+                to_write = False
+            elif value == "1":
+                to_write = True
+            else:
+                to_write = value
 
-        RedisArchivist().set_message("config", config, expire=False)
+            config_dict, config_value = key.split("_", maxsplit=1)
+            self.config[config_dict][config_value] = to_write
+            updated.append((config_value, to_write))
+
+        RedisArchivist().set_message("config", self.config)
+        return updated
 
     @staticmethod
     def set_user_config(form_post, user_id):
         """set values in redis for user settings"""
         for key, value in form_post.items():
-            to_write = value[0]
-            if len(to_write):
-                if to_write.isdigit():
-                    to_write = int(to_write)
-                message = {"status": to_write}
-                redis_key = f"{user_id}:{key}"
-                RedisArchivist().set_message(redis_key, message, expire=False)
+            if not value:
+                continue
+
+            message = {"status": value}
+            redis_key = f"{user_id}:{key}"
+            RedisArchivist().set_message(redis_key, message)
 
     def get_colors(self):
         """overwrite config if user has set custom values"""
@@ -131,6 +133,8 @@ class AppConfig:
 
         # check for customizations
         if not redis_config:
+            config = self.get_config()
+            RedisArchivist().set_message("config", config)
             return
 
         needs_update = False
@@ -149,7 +153,7 @@ class AppConfig:
                     needs_update = True
 
         if needs_update:
-            RedisArchivist().set_message("config", redis_config, expire=False)
+            RedisArchivist().set_message("config", redis_config)
 
 
 class ScheduleBuilder:
@@ -163,6 +167,7 @@ class ScheduleBuilder:
         "run_backup": "0 18 0",
     }
     CONFIG = ["check_reindex_days", "run_backup_rotate"]
+    MSG = "message:setting"
 
     def __init__(self):
         self.config = AppConfig().config
@@ -172,64 +177,84 @@ class ScheduleBuilder:
         print("processing form, restart container for changes to take effect")
         redis_config = self.config
         for key, value in form_post.items():
-            to_check = value[0]
-            if key in self.SCHEDULES and to_check:
+            if key in self.SCHEDULES and value:
                 try:
-                    to_write = self.value_builder(key, to_check)
+                    to_write = self.value_builder(key, value)
                 except ValueError:
-                    print(f"failed: {key} {to_check}")
+                    print(f"failed: {key} {value}")
                     mess_dict = {
-                        "status": "message:setting",
+                        "status": self.MSG,
                         "level": "error",
                         "title": "Scheduler update failed.",
                         "message": "Invalid schedule input",
                     }
-                    RedisArchivist().set_message("message:setting", mess_dict)
+                    RedisArchivist().set_message(
+                        self.MSG, mess_dict, expire=True
+                    )
                     return
 
                 redis_config["scheduler"][key] = to_write
-            if key in self.CONFIG and to_check:
-                redis_config["scheduler"][key] = int(to_check)
-        RedisArchivist().set_message("config", redis_config, expire=False)
+            if key in self.CONFIG and value:
+                redis_config["scheduler"][key] = int(value)
+        RedisArchivist().set_message("config", redis_config)
         mess_dict = {
-            "status": "message:setting",
+            "status": self.MSG,
             "level": "info",
             "title": "Scheduler changed.",
             "message": "Please restart container for changes to take effect",
         }
-        RedisArchivist().set_message("message:setting", mess_dict)
+        RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
 
-    def value_builder(self, key, to_check):
+    def value_builder(self, key, value):
         """validate single cron form entry and return cron dict"""
-        print(f"change schedule for {key} to {to_check}")
-        if to_check == "0":
+        print(f"change schedule for {key} to {value}")
+        if value == "0":
             # deactivate this schedule
             return False
-        if re.search(r"[\d]{1,2}\/[\d]{1,2}", to_check):
+        if re.search(r"[\d]{1,2}\/[\d]{1,2}", value):
             # number/number cron format will fail in celery
             print("number/number schedule formatting not supported")
             raise ValueError
 
         keys = ["minute", "hour", "day_of_week"]
-        if to_check == "auto":
+        if value == "auto":
             # set to sensible default
             values = self.SCHEDULES[key].split()
         else:
-            values = to_check.split()
+            values = value.split()
 
         if len(keys) != len(values):
-            print(f"failed to parse {to_check} for {key}")
+            print(f"failed to parse {value} for {key}")
             raise ValueError("invalid input")
 
         to_write = dict(zip(keys, values))
-        try:
-            int(to_write["minute"])
-        except ValueError as error:
-            print("too frequent: only number in minutes are supported")
-            raise ValueError("invalid input") from error
+        self._validate_cron(to_write)
 
         return to_write
 
+    @staticmethod
+    def _validate_cron(to_write):
+        """validate all fields, raise value error for impossible schedule"""
+        all_hours = list(re.split(r"\D+", to_write["hour"]))
+        for hour in all_hours:
+            if hour.isdigit() and int(hour) > 23:
+                print("hour can not be greater than 23")
+                raise ValueError("invalid input")
+
+        all_days = list(re.split(r"\D+", to_write["day_of_week"]))
+        for day in all_days:
+            if day.isdigit() and int(day) > 6:
+                print("day can not be greater than 6")
+                raise ValueError("invalid input")
+
+        if not to_write["minute"].isdigit():
+            print("too frequent: only number in minutes are supported")
+            raise ValueError("invalid input")
+
+        if int(to_write["minute"]) > 59:
+            print("minutes can not be greater than 59")
+            raise ValueError("invalid input")
+
     def build_schedule(self):
         """build schedule dict as expected by app.conf.beat_schedule"""
         schedule_dict = {}

tubearchivist/home/src/ta/helper.py

@@ -3,29 +3,15 @@ Loose collection of helper functions
 - don't import AppConfig class here to avoid circular imports
 """
 
-import json
+import random
 import re
 import string
 import subprocess
 import unicodedata
+from datetime import datetime
 from urllib.parse import parse_qs, urlparse
 
-import requests
-import yt_dlp
-
-
-def get_total_hits(index, es_url, es_auth, match_field):
-    """get total hits from index"""
-    headers = {"Content-type": "application/json"}
-    data = {"query": {"match": {match_field: True}}}
-    payload = json.dumps(data)
-    url = f"{es_url}/{index}/_search?filter_path=hits.total"
-    request = requests.post(url, data=payload, headers=headers, auth=es_auth)
-    if not request.ok:
-        print(request.text)
-    total_json = json.loads(request.text)
-    total_hits = total_json["hits"]["total"]["value"]
-    return total_hits
+from home.src.download.yt_dlp_base import YtWrap
 
 
 def clean_string(file_name):
@@ -51,6 +37,86 @@ def ignore_filelist(filelist):
     return cleaned
 
 
+def randomizor(length):
+    """generate random alpha numeric string"""
+    pool = string.digits + string.ascii_letters
+    return "".join(random.choice(pool) for i in range(length))
+
+
+def requests_headers():
+    """build header with random user agent for requests outside of yt-dlp"""
+
+    chrome_versions = (
+        "90.0.4430.212",
+        "90.0.4430.24",
+        "90.0.4430.70",
+        "90.0.4430.72",
+        "90.0.4430.85",
+        "90.0.4430.93",
+        "91.0.4472.101",
+        "91.0.4472.106",
+        "91.0.4472.114",
+        "91.0.4472.124",
+        "91.0.4472.164",
+        "91.0.4472.19",
+        "91.0.4472.77",
+        "92.0.4515.107",
+        "92.0.4515.115",
+        "92.0.4515.131",
+        "92.0.4515.159",
+        "92.0.4515.43",
+        "93.0.4556.0",
+        "93.0.4577.15",
+        "93.0.4577.63",
+        "93.0.4577.82",
+        "94.0.4606.41",
+        "94.0.4606.54",
+        "94.0.4606.61",
+        "94.0.4606.71",
+        "94.0.4606.81",
+        "94.0.4606.85",
+        "95.0.4638.17",
+        "95.0.4638.50",
+        "95.0.4638.54",
+        "95.0.4638.69",
+        "95.0.4638.74",
+        "96.0.4664.18",
+        "96.0.4664.45",
+        "96.0.4664.55",
+        "96.0.4664.93",
+        "97.0.4692.20",
+    )
+    template = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        + "AppleWebKit/537.36 (KHTML, like Gecko) "
+        + f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
+    )
+
+    return {"User-Agent": template}
+
+
+def date_praser(timestamp):
+    """return formatted date string"""
+    if isinstance(timestamp, int):
+        date_obj = datetime.fromtimestamp(timestamp)
+    elif isinstance(timestamp, str):
+        date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
+
+    return datetime.strftime(date_obj, "%d %b, %Y")
+
+
+def time_parser(timestamp):
+    """return seconds from timestamp, false on empty"""
+    if not timestamp:
+        return False
+
+    if timestamp.isnumeric():
+        return int(timestamp)
+
+    hours, minutes, seconds = timestamp.split(":", maxsplit=3)
+    return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
+
+
 class UrlListParser:
     """take a multi line string and detect valid youtube ids"""
 
@@ -92,12 +158,12 @@ class UrlListParser:
         if parsed.query:
             # detect from query string
             query_parsed = parse_qs(parsed.query)
-            if "v" in query_parsed.keys():
+            if "v" in query_parsed:
                 youtube_id = query_parsed["v"][0]
                 _ = self.find_valid_id(youtube_id)
                 return youtube_id, "video"
 
-            if "list" in query_parsed.keys():
+            if "list" in query_parsed:
                 youtube_id = query_parsed["list"][0]
                 return youtube_id, "playlist"
 
@@ -107,19 +173,19 @@ class UrlListParser:
             _ = self.find_valid_id(youtube_id)
             return youtube_id, "channel"
 
-        # dedect channel with yt_dlp
+        # detect channel with yt_dlp
         youtube_id = self.extract_channel_name(parsed.geturl())
         return youtube_id, "channel"
 
     @staticmethod
     def find_valid_id(id_str):
-        """dedect valid id from length of string"""
+        """detect valid id from length of string"""
         str_len = len(id_str)
         if str_len == 11:
             id_type = "video"
         elif str_len == 24:
             id_type = "channel"
-        elif str_len in [34, 18]:
+        elif str_len in [34, 18] or id_str in ["LL", "WL"]:
             id_type = "playlist"
         else:
             # unable to parse
@@ -130,21 +196,25 @@ class UrlListParser:
     @staticmethod
     def extract_channel_name(url):
         """find channel id from channel name with yt-dlp help"""
-        obs = {
-            "default_search": "ytsearch",
-            "quiet": True,
+        obs_request = {
             "skip_download": True,
             "extract_flat": True,
             "playlistend": 0,
         }
-        url_info = yt_dlp.YoutubeDL(obs).extract_info(url, download=False)
-        try:
-            channel_id = url_info["channel_id"]
-        except KeyError as error:
-            print(f"failed to extract channel id from {url}")
-            raise ValueError from error
+        url_info = YtWrap(obs_request).extract(url)
+        channel_id = url_info.get("channel_id", False)
+        if channel_id:
+            return channel_id
 
-        return channel_id
+        url = url_info.get("url", False)
+        if url:
+            # handle old channel name redirect with url path split
+            channel_id = urlparse(url).path.strip("/").split("/")[1]
+
+            return channel_id
+
+        print(f"failed to extract channel id from {url}")
+        raise ValueError
 
 
 class DurationConverter:

tubearchivist/home/src/ta/ta_redis.py

@@ -8,15 +8,22 @@ import json
 import os
 
 import redis
-from home.src.ta.helper import ignore_filelist
 
 
-class RedisArchivist:
-    """collection of methods to interact with redis"""
+class RedisBase:
+    """connection base for redis"""
 
     REDIS_HOST = os.environ.get("REDIS_HOST")
     REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
     NAME_SPACE = "ta:"
+
+    def __init__(self):
+        self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
+
+
+class RedisArchivist(RedisBase):
+    """collection of methods to interact with redis"""
+
     CHANNELS = [
         "download",
         "add",
@@ -27,15 +34,10 @@ class RedisArchivist:
         "setting",
     ]
 
-    def __init__(self):
-        self.redis_connection = redis.Redis(
-            host=self.REDIS_HOST, port=self.REDIS_PORT
-        )
-
-    def set_message(self, key, message, expire=True):
+    def set_message(self, key, message, path=".", expire=False):
         """write new message to redis"""
-        self.redis_connection.execute_command(
-            "JSON.SET", self.NAME_SPACE + key, ".", json.dumps(message)
+        self.conn.execute_command(
+            "JSON.SET", self.NAME_SPACE + key, path, json.dumps(message)
         )
 
         if expire:
@@ -43,15 +45,11 @@ class RedisArchivist:
                 secs = 20
             else:
                 secs = expire
-            self.redis_connection.execute_command(
-                "EXPIRE", self.NAME_SPACE + key, secs
-            )
+            self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
 
     def get_message(self, key):
         """get message dict from redis"""
-        reply = self.redis_connection.execute_command(
-            "JSON.GET", self.NAME_SPACE + key
-        )
+        reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
         if reply:
             json_str = json.loads(reply)
         else:
@@ -61,7 +59,7 @@ class RedisArchivist:
 
     def list_items(self, query):
         """list all matches"""
-        reply = self.redis_connection.execute_command(
+        reply = self.conn.execute_command(
             "KEYS", self.NAME_SPACE + query + "*"
         )
         all_matches = [i.decode().lstrip(self.NAME_SPACE) for i in reply]
@@ -74,22 +72,26 @@ class RedisArchivist:
 
     def del_message(self, key):
         """delete key from redis"""
-        response = self.redis_connection.execute_command(
-            "DEL", self.NAME_SPACE + key
-        )
+        response = self.conn.execute_command("DEL", self.NAME_SPACE + key)
         return response
 
     def get_lock(self, lock_key):
         """handle lock for task management"""
-        redis_lock = self.redis_connection.lock(self.NAME_SPACE + lock_key)
+        redis_lock = self.conn.lock(self.NAME_SPACE + lock_key)
         return redis_lock
 
+    def is_locked(self, lock_key):
+        """check if lock is set"""
+        lock_name = self.NAME_SPACE + lock_key
+        lock_status = bool(self.conn.execute_command("GET", lock_name))
+        return lock_status
+
     def get_progress(self):
         """get a list of all progress messages"""
         all_messages = []
         for channel in self.CHANNELS:
             key = "message:" + channel
-            reply = self.redis_connection.execute_command(
+            reply = self.conn.execute_command(
                 "JSON.GET", self.NAME_SPACE + key
             )
             if reply:
@@ -98,41 +100,13 @@ class RedisArchivist:
 
         return all_messages
 
-    @staticmethod
-    def monitor_cache_dir(cache_dir):
-        """
-        look at download cache dir directly as alternative progress info
-        """
-        dl_cache = os.path.join(cache_dir, "download")
-        all_cache_file = os.listdir(dl_cache)
-        cache_file = ignore_filelist(all_cache_file)
-        if cache_file:
-            filename = cache_file[0][12:].replace("_", " ").split(".")[0]
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Downloading: " + filename,
-                "message": "",
-            }
-        else:
-            return False
 
-        return mess_dict
-
-
-class RedisQueue:
+class RedisQueue(RedisBase):
     """dynamically interact with the download queue in redis"""
 
-    REDIS_HOST = os.environ.get("REDIS_HOST")
-    REDIS_PORT = os.environ.get("REDIS_PORT")
-    NAME_SPACE = "ta:"
-
-    if not REDIS_PORT:
-        REDIS_PORT = 6379
-
-    def __init__(self, key):
-        self.key = self.NAME_SPACE + key
-        self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
+    def __init__(self):
+        super().__init__()
+        self.key = self.NAME_SPACE + "dl_queue"
 
     def get_all(self):
         """return all elements in list"""

tubearchivist/home/tasks.py

@@ -8,23 +8,22 @@ Functionality:
 
 import os
 
-import home.apps as startup_apps
 from celery import Celery, shared_task
+from home.apps import StartupCheck
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import (
     ChannelSubscription,
     PlaylistSubscription,
 )
-from home.src.download.thumbnails import ThumbManager, validate_thumbnails
+from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator
 from home.src.download.yt_dlp_handler import VideoDownloader
 from home.src.es.index_setup import backup_all_indexes, restore_from_backup
 from home.src.index.channel import YoutubeChannel
 from home.src.index.filesystem import (
-    ManualImport,
+    ImportFolderScanner,
     reindex_old_documents,
     scan_filesystem,
 )
-from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.config import AppConfig, ScheduleBuilder
 from home.src.ta.helper import UrlListParser
 from home.src.ta.ta_redis import RedisArchivist, RedisQueue
@@ -49,7 +48,7 @@ def update_subscribed():
         "title": "Rescanning channels and playlists.",
         "message": "Looking for new videos.",
     }
-    RedisArchivist().set_message("message:rescan", message)
+    RedisArchivist().set_message("message:rescan", message, expire=True)
 
     have_lock = False
     my_lock = RedisArchivist().get_lock("rescan")
@@ -63,9 +62,11 @@ def update_subscribed():
             missing_from_playlists = playlist_handler.find_missing()
             missing = missing_from_channels + missing_from_playlists
             if missing:
-                pending_handler = PendingList()
-                all_videos_added = pending_handler.add_to_pending(missing)
-                ThumbManager().download_vid(all_videos_added)
+                youtube_ids = [{"type": "video", "url": i} for i in missing]
+                pending_handler = PendingList(youtube_ids=youtube_ids)
+                pending_handler.parse_url_list()
+                pending_handler.add_to_pending()
+
         else:
             print("Did not acquire rescan lock.")
 
@@ -86,7 +87,6 @@ def download_pending():
             downloader = VideoDownloader()
             downloader.add_pending()
             downloader.run_queue()
-            downloader.validate_playlists()
         else:
             print("Did not acquire download lock.")
 
@@ -98,7 +98,7 @@ def download_pending():
 @shared_task
 def download_single(youtube_id):
     """start download single video now"""
-    queue = RedisQueue("dl_queue")
+    queue = RedisQueue()
     queue.add_priority(youtube_id)
     print("Added to queue with priority: " + youtube_id)
     # start queue if needed
@@ -108,13 +108,14 @@ def download_single(youtube_id):
     try:
         have_lock = my_lock.acquire(blocking=False)
         if have_lock:
+            key = "message:download"
             mess_dict = {
-                "status": "message:download",
+                "status": key,
                 "level": "info",
                 "title": "Download single video",
                 "message": "processing",
             }
-            RedisArchivist().set_message("message:download", mess_dict)
+            RedisArchivist().set_message(key, mess_dict, expire=True)
             VideoDownloader().run_queue()
         else:
             print("Download queue already running.")
@@ -128,19 +129,9 @@ def download_single(youtube_id):
 @shared_task
 def extrac_dl(youtube_ids):
     """parse list passed and add to pending"""
-    pending_handler = PendingList()
-    missing_videos = pending_handler.parse_url_list(youtube_ids)
-    all_videos_added = pending_handler.add_to_pending(missing_videos)
-    missing_playlists = pending_handler.missing_from_playlists
-
-    thumb_handler = ThumbManager()
-    if missing_playlists:
-        new_thumbs = PlaylistSubscription().process_url_str(
-            missing_playlists, subscribed=False
-        )
-        thumb_handler.download_playlist(new_thumbs)
-
-    thumb_handler.download_vid(all_videos_added)
+    pending_handler = PendingList(youtube_ids=youtube_ids)
+    pending_handler.parse_url_list()
+    pending_handler.add_to_pending()
 
 
 @shared_task(name="check_reindex")
@@ -159,10 +150,7 @@ def run_manual_import():
     try:
         have_lock = my_lock.acquire(blocking=False)
         if have_lock:
-            import_handler = ManualImport()
-            if import_handler.identified:
-                all_videos_added = import_handler.process_import()
-                ThumbManager().download_vid(all_videos_added)
+            ImportFolderScanner().scan()
         else:
             print("Did not acquire lock form import.")
 
@@ -174,8 +162,19 @@ def run_manual_import():
 @shared_task(name="run_backup")
 def run_backup(reason="auto"):
     """called from settings page, dump backup to zip file"""
-    backup_all_indexes(reason)
-    print("backup finished")
+    have_lock = False
+    my_lock = RedisArchivist().get_lock("run_backup")
+
+    try:
+        have_lock = my_lock.acquire(blocking=False)
+        if have_lock:
+            backup_all_indexes(reason)
+        else:
+            print("Did not acquire lock for backup task.")
+    finally:
+        if have_lock:
+            my_lock.release()
+            print("backup finished")
 
 
 @shared_task
@@ -191,7 +190,7 @@ def kill_dl(task_id):
         app.control.revoke(task_id, terminate=True)
 
     _ = RedisArchivist().del_message("dl_queue_id")
-    RedisQueue("dl_queue").clear()
+    RedisQueue().clear()
 
     # clear cache
     cache_dir = os.path.join(CONFIG["application"]["cache_dir"], "download")
@@ -206,28 +205,26 @@ def kill_dl(task_id):
         "title": "Canceling download process",
         "message": "Canceling download queue now.",
     }
-    RedisArchivist().set_message("message:download", mess_dict)
+    RedisArchivist().set_message("message:download", mess_dict, expire=True)
 
 
 @shared_task
 def rescan_filesystem():
     """check the media folder for mismatches"""
     scan_filesystem()
-    validate_thumbnails()
+    ThumbValidator().download_missing()
 
 
 @shared_task(name="thumbnail_check")
 def thumbnail_check():
     """validate thumbnails"""
-    validate_thumbnails()
+    ThumbValidator().download_missing()
 
 
 @shared_task
 def re_sync_thumbs():
     """sync thumbnails to mediafiles"""
-    handler = ThumbManager()
-    video_list = handler.get_thumb_list()
-    handler.write_all_thumbs(video_list)
+    ThumbFilesystem().sync()
 
 
 @shared_task
@@ -238,9 +235,7 @@ def subscribe_to(url_str):
     for item in to_subscribe_list:
         to_sub_id = item["url"]
         if item["type"] == "playlist":
-            new_thumbs = PlaylistSubscription().process_url_str([item])
-            if new_thumbs:
-                ThumbManager().download_playlist(new_thumbs)
+            PlaylistSubscription().process_url_str([item])
             continue
 
         if item["type"] == "video":
@@ -255,13 +250,14 @@ def subscribe_to(url_str):
             channel_id_sub, channel_subscribed=True
         )
         # notify for channels
+        key = "message:subchannel"
         message = {
-            "status": "message:subchannel",
+            "status": key,
             "level": "info",
             "title": "Subscribing to Channels",
             "message": f"Processing {counter} of {len(to_subscribe_list)}",
         }
-        RedisArchivist().set_message("message:subchannel", message=message)
+        RedisArchivist().set_message(key, message=message, expire=True)
         counter = counter + 1
 
 
@@ -270,62 +266,20 @@ def index_channel_playlists(channel_id):
     """add all playlists of channel to index"""
     channel = YoutubeChannel(channel_id)
     # notify
+    key = "message:playlistscan"
     mess_dict = {
-        "status": "message:playlistscan",
+        "status": key,
         "level": "info",
         "title": "Looking for playlists",
         "message": f'Scanning channel "{channel.youtube_id}" in progress',
     }
-    RedisArchivist().set_message("message:playlistscan", mess_dict)
-    all_playlists = channel.get_all_playlists()
-
-    if not all_playlists:
-        print(f"no playlists found for channel {channel_id}")
-        return
-
-    all_indexed = PendingList().get_all_indexed()
-    all_youtube_ids = [i["youtube_id"] for i in all_indexed]
-
-    for idx, (playlist_id, playlist_title) in enumerate(all_playlists):
-        # notify
-        mess_dict = {
-            "status": "message:playlistscan",
-            "level": "info",
-            "title": "Scanning channel for playlists",
-            "message": f"Progress: {idx + 1}/{len(all_playlists)}",
-        }
-        RedisArchivist().set_message("message:playlistscan", mess_dict)
-        print("add playlist: " + playlist_title)
-
-        playlist = YoutubePlaylist(playlist_id)
-        playlist.all_youtube_ids = all_youtube_ids
-        playlist.build_json()
-
-        if not playlist.json_data:
-            # skip if not available
-            continue
-
-        # don't add if no videos downloaded
-        downloaded = [
-            i
-            for i in playlist.json_data["playlist_entries"]
-            if i["downloaded"]
-        ]
-        if not downloaded:
-            continue
-
-        playlist.upload_to_es()
-        playlist.add_vids_to_playlist()
-
-    if all_playlists:
-        playlist.get_playlist_art()
-
-    return
+    RedisArchivist().set_message(key, mess_dict, expire=True)
+    channel.index_channel_playlists()
 
 
 try:
     app.conf.beat_schedule = ScheduleBuilder().build_schedule()
 except KeyError:
     # update path from v0.0.8 to v0.0.9 to load new defaults
-    startup_apps.sync_redis_state()
+    StartupCheck().sync_redis_state()
     app.conf.beat_schedule = ScheduleBuilder().build_schedule()

tubearchivist/home/templates/home/about.html

@@ -7,13 +7,13 @@
     </div>
     <div class="about-section">
         <h2>Useful Links</h2>
-        <p>This project is in active and constant development, take a look at the <a href="https://github.com/bbilly1/tubearchivist#roadmap" target="_blank">roadmap</a> for a overview.</p>
-        <p>For any questions on what a button or a function does, You can find the up-to-date user documentation on <a href="https://github.com/bbilly1/tubearchivist/wiki" target="_blank">Github</a>.</p>
-        <p>All contributions are welcome: Open an <a href="https://github.com/bbilly1/tubearchivist/issues" target="_blank">issue</a> for any bugs and errors, start a <a href="https://github.com/bbilly1/tubearchivist/discussions" target="_blank">discussion</a> for anything that will require a more indepth look. The <a href="https://github.com/bbilly1/tubearchivist/blob/master/CONTRIBUTING.md" target="_blank">contributing</a> page is a good place to get started.</p>
+        <p>This project is in active and constant development, take a look at the <a href="https://github.com/tubearchivist/tubearchivist#roadmap" target="_blank">roadmap</a> for a overview.</p>
+        <p>For any questions on what a button or a function does, You can find the up-to-date user documentation on <a href="https://github.com/tubearchivist/tubearchivist/wiki" target="_blank">Github</a>.</p>
+        <p>All contributions are welcome: Open an <a href="https://github.com/tubearchivist/tubearchivist/issues" target="_blank">issue</a> for any bugs and errors, join us on <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> to discuss details. The <a href="https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md" target="_blank">contributing</a> page is a good place to get started.</p>
     </div>
     <div class="about-section">
         <h2>Donate</h2>
-        <p>Here are <a href="https://github.com/bbilly1/tubearchivist#donate" target="_blank">some links</a>, if you want to buy the developer a coffee. Thank you for your support!</p>
+        <p>Here are <a href="https://github.com/tubearchivist/tubearchivist#donate" target="_blank">some links</a>, if you want to buy the developer a coffee. Thank you for your support!</p>
     </div>
 </div>
 {% endblock content %}

tubearchivist/home/templates/home/base.html

@@ -132,7 +132,7 @@
     </div>
     <div class="footer">
         <div class="boxed-content">
-            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist v0.1.2 </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
+            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist {{ version }} </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/tubearchivist/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
         </div>
     </div>
 </body>

tubearchivist/home/templates/home/channel.html

@@ -40,40 +40,40 @@
     <div class="channel-list {{ view_style }}">
         {% if results %}
             {% for channel in results %}
-            <div class="channel-item {{ view_style }}">
-                <div class="channel-banner {{ view_style }}">
-                    <a href="{% url 'channel_id' channel.source.channel_id %}">
-                        <img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
-                    </a>
-                </div>
-                <div class="info-box info-box-2 {{ view_style }}">
-                    <div class="info-box-item">
-                        <div class="round-img">
-                            <a href="{% url 'channel_id' channel.source.channel_id %}">
-                                <img src="/cache/channels/{{ channel.source.channel_id }}_thumb.jpg" alt="channel-thumb">
-                            </a>
-                        </div>
-                        <div>
-                            <h3><a href="{% url 'channel_id' channel.source.channel_id %}">{{ channel.source.channel_name }}</a></h3>
-                            {% if channel.source.channel_subs >= 1000000 %}
-                                <p>Subscribers: {{ channel.source.channel_subs|intword }}</p>
-                            {% else %}
-                                <p>Subscribers: {{ channel.source.channel_subs|intcomma }}</p>
-                            {% endif %}
-                        </div>
+                <div class="channel-item {{ view_style }}">
+                    <div class="channel-banner {{ view_style }}">
+                        <a href="{% url 'channel_id' channel.source.channel_id %}">
+                            <img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
+                        </a>
                     </div>
-                    <div class="info-box-item">
-                        <div>
-                            <p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
-                            {% if channel.source.channel_subscribed %}
-                                <button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
-                            {% else %}
-                                <button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
-                            {% endif %}
+                    <div class="info-box info-box-2 {{ view_style }}">
+                        <div class="info-box-item">
+                            <div class="round-img">
+                                <a href="{% url 'channel_id' channel.source.channel_id %}">
+                                    <img src="/cache/channels/{{ channel.source.channel_id }}_thumb.jpg" alt="channel-thumb">
+                                </a>
+                            </div>
+                            <div>
+                                <h3><a href="{% url 'channel_id' channel.source.channel_id %}">{{ channel.source.channel_name }}</a></h3>
+                                {% if channel.source.channel_subs >= 1000000 %}
+                                    <p>Subscribers: {{ channel.source.channel_subs|intword }}</p>
+                                {% else %}
+                                    <p>Subscribers: {{ channel.source.channel_subs|intcomma }}</p>
+                                {% endif %}
+                            </div>
+                        </div>
+                        <div class="info-box-item">
+                            <div>
+                                <p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
+                                {% if channel.source.channel_subscribed %}
+                                    <button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
+                                {% else %}
+                                    <button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
+                                {% endif %}
+                            </div>
                         </div>
                     </div>
                 </div>
-            </div>
             {% endfor %}
         {% else %}
             <h2>No channels found...</h2>

tubearchivist/home/templates/home/channel_id.html

@@ -6,7 +6,13 @@
     <div class="channel-banner">
         <a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
     </div>
-    <div class="info-box info-box-3">
+    <div class="info-box-item channel-nav">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
+        <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
+        <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+    </div>
+    <div id="notifications" data="channel_id"></div>
+    <div class="info-box info-box-2">
         <div class="info-box-item">
             <div class="round-img">
                 <a href="{% url 'channel_id' channel_info.channel_id %}">
@@ -29,43 +35,15 @@
         </div>
         <div class="info-box-item">
             <div>
-                <p>Last refreshed: {{ channel_info.channel_last_refresh }}</p>
-                {% if channel_info.channel_active %}
-                    <p>Youtube: <a href="https://www.youtube.com/channel/{{ channel_info.channel_id }}" target="_blank">Active</a></p>
-                {% else %}
-                    <p>Youtube: Deactivated</p>
-                {% endif %}
-                <button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
-                <div class="delete-confirm" id="delete-button">
-                    <span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
-                </div>
-            </div>
-        </div>
-        <div class="info-box-item">
-            <div>
-                {% if channel_info.channel_views >= 1000000 %}
-                    <p>Channel views: {{ channel_info.channel_views|intword }}</p>
-                {% elif channel_info.channel_views > 0 %}
-                    <p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
-                {% endif %}
                 {% if max_hits %}
-                    <p>Total Videos archived: {{ max_hits }}</p>
-                    <p>Watched: <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
+                    <p>Total Videos: {{ max_hits }}</p>
+                    <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button>
                 {% endif %}
-                <button title="Search for playlists on YouTube for {{ channel_info.channel_name }}" type="button" id="find-playlists-button" data-id="{{ channel_info.channel_id }}" onclick="findPlaylists(this)">Find Playlists</button>
-                <a href="/playlist/?search={{ channel_info.channel_id }}" title="Show all playlists belonging to {{ channel_info.channel_name }}"><button>Show Playlists</button></a>
             </div>
         </div>
     </div>
-    {% if channel_info.channel_description %}
-        <div class="info-box-item description-box">
-            <p>Description: <button onclick="textReveal()" id="text-reveal-button">Show</button></p>
-            <div id="text-reveal" class="description-text">
-                {{ channel_info.channel_description|linebreaks }}
-            </div>
-        </div>
-    {% endif %}
-    <div id="notifications" data="channel_id"></div>
+</div>
+<div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
     <div class="view-controls">
         <div class="toggle">
             <span>Hide watched videos:</span>
@@ -95,14 +73,24 @@
         </div>
         <div class="view-icons">
             <img src="{% static 'img/icon-sort.svg' %}" alt="sort-icon" onclick="showForm()" id="animate-icon">
+            {% if view_style == "grid" %}
+            <div class="grid-count">
+                {% if grid_items < 7 %}
+                    <img src="{% static 'img/icon-add.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"1"}}" alt="grid plus row">
+                {% endif %}
+                {% if grid_items > 3 %}
+                    <img src="{% static 'img/icon-substract.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"-1"}}" alt="grid minus row">
+                {% endif %}
+            </div>
+            {% endif %}
             <img src="{% static 'img/icon-gridview.svg' %}" onclick="changeView(this)" data-origin="home" data-value="grid" alt="grid view">
             <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="home" data-value="list" alt="list view">
         </div>
     </div>
 </div>
 <div id="player" class="player-wrapper"></div>
-<div class="boxed-content">
-    <div class="video-list {{ view_style }}">
+<div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
+    <div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
         {% if results %}
             {% for video in results %}
                 <div class="video-item {{ view_style }}">
@@ -111,9 +99,9 @@
                             <div class="video-thumb">
                                 <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -124,9 +112,9 @@
                     <div class="video-desc {{ view_style }}">
                         <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
                             {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
                             <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
                         </div>

tubearchivist/home/templates/home/channel_id_about.html

@@ -0,0 +1,112 @@
+{% extends "home/base.html" %}
+{% block content %}
+{% load static %}
+{% load humanize %}
+<div class="boxed-content">
+    <div class="channel-banner">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
+    </div>
+    <div class="info-box-item channel-nav">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
+        <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
+        <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+    </div>
+    <div class="info-box info-box-3">
+        <div class="info-box-item">
+            <div class="round-img">
+                <a href="{% url 'channel_id' channel_info.channel_id %}">
+                    <img src="{{ channel_info.channel_thumb_url }}" alt="channel-thumb">
+                </a>
+            </div>
+            <div>
+                <h3><a href="{% url 'channel_id' channel_info.channel_id %}">{{ channel_info.channel_name }}</a></h3>
+                {% if channel_info.channel_subs >= 1000000 %}
+                    <p>Subscribers: {{ channel_info.channel_subs|intword }}</p>
+                {% else %}
+                    <p>Subscribers: {{ channel_info.channel_subs|intcomma }}</p>
+                {% endif %}
+            </div>
+        </div>
+        <div class="info-box-item">
+            <div>
+                <p>Last refreshed: {{ channel_info.channel_last_refresh }}</p>
+                {% if channel_info.channel_active %}
+                    <p>Youtube: <a href="https://www.youtube.com/channel/{{ channel_info.channel_id }}" target="_blank">Active</a></p>
+                {% else %}
+                    <p>Youtube: Deactivated</p>
+                {% endif %}
+            </div>
+        </div>
+        <div class="info-box-item">
+            <div>
+                {% if channel_info.channel_views >= 1000000 %}
+                    <p>Channel views: {{ channel_info.channel_views|intword }}</p>
+                {% elif channel_info.channel_views > 0 %}
+                    <p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
+                {% endif %}
+                <button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
+                <div class="delete-confirm" id="delete-button">
+                    <span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
+                </div>
+            </div>
+        </div>
+    </div>
+    {% if channel_info.channel_description %}
+        <div class="description-box">
+            <p id="text-expand" class="description-text">
+                {{ channel_info.channel_description|linebreaksbr|urlizetrunc:50 }}
+            </p>
+            <button onclick="textExpand()" id="text-expand-button">Show more</button>
+        </div>
+    {% endif %}
+    <div class="description-box">
+        <h2>Customize {{ channel_info.channel_name }}</h2>
+    </div>
+    <div id="overwrite-form" class="info-box">
+        <div class="info-box-item">
+            <form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/about/" method="POST">
+                {% csrf_token %}
+                <div class="overwrite-form-item">
+                    <p>Download format: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.download_format %}
+                            {{ channel_info.channel_overwrites.download_format }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.download_format }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Auto delete watched videos after x days: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.autodelete_days %}
+                            {{ channel_info.channel_overwrites.autodelete_days }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.autodelete_days }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Index playlists: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.index_playlists %}
+                            {{ channel_info.channel_overwrites.index_playlists }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.index_playlists }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Enable <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a>: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.integrate_sponsorblock %}
+                            {{ channel_info.channel_overwrites.integrate_sponsorblock }}
+                        {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %}
+                            Disabled
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.integrate_sponsorblock }}<br>
+                </div>
+                <button type="submit">Save Channel Overwrites</button>
+            </form>
+        </div>
+    </div>
+</div>
+{% endblock content %}

tubearchivist/home/templates/home/channel_id_playlist.html

@@ -0,0 +1,57 @@
+{% extends "home/base.html" %}
+{% block content %}
+{% load static %}
+{% load humanize %}
+<div class="boxed-content">
+    <div class="channel-banner">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
+    </div>
+    <div class="info-box-item channel-nav">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
+        <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
+        <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+    </div>
+    <div class="view-controls">
+        <div class="toggle">
+            <span>Show subscribed only:</span>
+            <div class="toggleBox">
+                <input id="show_subed_only" onclick="toggleCheckbox(this)" type="checkbox" {% if show_subed_only %}checked{% endif %}>
+                {% if not show_subed_only %}
+                    <label for="" class="ofbtn">Off</label>
+                {% else %}
+                    <label for="" class="onbtn">On</label>
+                {% endif %}
+            </div>
+        </div>
+        <div class="view-icons">
+            <img src="{% static 'img/icon-gridview.svg' %}" onclick="changeView(this)" data-origin="playlist" data-value="grid" alt="grid view">
+            <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="playlist" data-value="list" alt="list view">
+        </div>
+    </div>
+    <div class="playlist-list {{ view_style }}">
+        {% if results %}
+            {% for playlist in results %}
+            <div class="playlist-item {{ view_style }}">
+                <div class="playlist-thumbnail">
+                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}">
+                        <img src="/cache/playlists/{{ playlist.source.playlist_id }}.jpg" alt="{{ playlist.source.playlist_id }}-thumbnail">
+                    </a>
+                </div>
+                <div class="playlist-desc {{ view_style }}">
+                    <a href="{% url 'channel_id' playlist.source.playlist_channel_id %}"><h3>{{ playlist.source.playlist_channel }}</h3></a>
+                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
+                    <p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
+                    {% if playlist.source.playlist_subscribed %}
+                        <button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
+                    {% else %}
+                        <button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
+                    {% endif %}
+                </div>
+            </div>
+            {% endfor %}
+        {% else %}
+            <h2>No playlists found...</h2>
+        {% endif %}
+    </div>
+</div>
+{% endblock content %}

tubearchivist/home/templates/home/downloads.html

@@ -45,15 +45,6 @@
             <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="downloads" data-value="list" alt="list view">
         </div>
     </div>
-    <div class="title-split">
-        {% if show_ignored_only %}
-            <h2>Ignored from download</h2>
-            <button onclick="deleteQueue(this)" data-id="ignore" title="Delete all previously ignored videos from the queue">Delete all ignored</button>
-        {% else %}
-            <h2>Download queue</h2>
-            <button onclick="deleteQueue(this)" data-id="pending" title="Delete all pending videos from the queue">Delete all queued</button>
-        {% endif %}
-    </div>
     <h3>Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %}</h3>
     <div class="dl-list {{ view_style }}">
         {% if results %}

tubearchivist/home/templates/home/home.html

@@ -1,7 +1,47 @@
 {% extends "home/base.html" %}
 {% block content %}
 {% load static %}
-<div class="boxed-content">
+<div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
+    {% if continue_vids %}
+        <div class="title-bar">
+            <h1>Continue Watching</h1>
+        </div>
+        <div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
+            {% for video in continue_vids %}
+                <div class="video-item {{ view_style }}">
+                    <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
+                        <div class="video-thumb-wrap {{ view_style }}">
+                            <div class="video-thumb">
+                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.source.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                {% else %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                {% endif %}
+                            </div>
+                            <div class="video-play">
+                                <img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
+                            </div>
+                        </div>
+                    </a>
+                    <div class="video-desc {{ view_style }}">
+                        <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
+                            {% if video.source.player.watched %}
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
+                            {% else %}
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
+                            {% endif %}
+                            <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
+                        </div>
+                        <div>
+                            <a href="{% url 'channel_id' video.source.channel.channel_id %}"><h3>{{ video.source.channel.channel_name }}</h3></a>
+                            <a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
+                        </div>
+                    </div>
+                </div>
+            {% endfor %}
+        </div>
+    {% endif %}
     <div class="title-bar">
         <h1>Recent Videos</h1>
     </div>
@@ -34,14 +74,24 @@
         </div>
         <div class="view-icons">
             <img src="{% static 'img/icon-sort.svg' %}" alt="sort-icon" onclick="showForm()" id="animate-icon">
+            {% if view_style == "grid" %}
+                <div class="grid-count">
+                    {% if grid_items < 7 %}
+                        <img src="{% static 'img/icon-add.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"1"}}" alt="grid plus row">
+                    {% endif %}
+                    {% if grid_items > 3 %}
+                        <img src="{% static 'img/icon-substract.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"-1"}}" alt="grid minus row">
+                    {% endif %}
+                </div>
+            {% endif %}
             <img src="{% static 'img/icon-gridview.svg' %}" onclick="changeView(this)" data-origin="home" data-value="grid" alt="grid view">
             <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="home" data-value="list" alt="list view">
         </div>
     </div>
 </div>
 <div id="player" class="player-wrapper"></div>
-<div class="boxed-content">
-    <div class="video-list {{ view_style }}">
+<div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
+    <div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
         {% if results %}
             {% for video in results %}
                 <div class="video-item {{ view_style }}">
@@ -50,9 +100,9 @@
                             <div class="video-thumb">
                                 <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -63,9 +113,9 @@
                     <div class="video-desc {{ view_style }}">
                         <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
                             {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
                             <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
                         </div>

tubearchivist/home/templates/home/login.html

@@ -45,7 +45,7 @@
             <input type="hidden" name="next" value="{{ request.GET.next }}" />
             <button type="submit">Login</button>
         </form>
-        <p class="login-links"><span><a href="https://github.com/bbilly1/tubearchivist" target="_blank">Github</a></span> <span><a href="https://github.com/bbilly1/tubearchivist#donate" target="_blank">Donate</a></span></p>
+        <p class="login-links"><span><a href="https://github.com/tubearchivist/tubearchivist" target="_blank">Github</a></span> <span><a href="https://github.com/tubearchivist/tubearchivist#donate" target="_blank">Donate</a></span></p>
     </div>
     <div class="footer-colors">
         <div class="col-1"></div>

tubearchivist/home/templates/home/playlist_id.html

@@ -49,20 +49,22 @@
         <div class="info-box-item">
             <div>
                 {% if max_hits %}
-                <p>Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}</p>
-                <p>Watched: <button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
+                    <p>Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}</p>
+                    <p>Watched: <button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
                 {% endif %}
             </div>
         </div>
     </div>
     {% if playlist_info.playlist_description %}
-        <div class="info-box-item description-box">
-            <p>Description: <button onclick="textReveal()" id="text-reveal-button">Show</button></p>
-            <div id="text-reveal" class="description-text">
-                {{ playlist_info.playlist_description|linebreaks }}
-            </div>
+        <div class="description-box">
+            <p id="text-expand" class="description-text">
+                {{ playlist_info.playlist_description|linebreaksbr|urlizetrunc:50 }}
+            </p>
+            <button onclick="textExpand()" id="text-expand-button">Show more</button>
         </div>
     {% endif %}
+</div>
+<div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
     <div class="view-controls">
         <div class="toggle">
             <span>Hide watched videos:</span>
@@ -76,14 +78,24 @@
             </div>
         </div>
         <div class="view-icons">
+            {% if view_style == "grid" %}
+            <div class="grid-count">
+                {% if grid_items < 7 %}
+                    <img src="{% static 'img/icon-add.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"1"}}" alt="grid plus row">
+                {% endif %}
+                {% if grid_items > 3 %}
+                    <img src="{% static 'img/icon-substract.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"-1"}}" alt="grid minus row">
+                {% endif %}
+            </div>
+            {% endif %}
             <img src="{% static 'img/icon-gridview.svg' %}" onclick="changeView(this)" data-origin="home" data-value="grid" alt="grid view">
             <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="home" data-value="list" alt="list view">
         </div>
     </div>
 </div>
 <div id="player" class="player-wrapper"></div>
-<div class="boxed-content">
-    <div class="video-list {{ view_style }}">
+<div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
+    <div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
         {% if results %}
             {% for video in results %}
                 <div class="video-item {{ view_style }}">
@@ -92,9 +104,9 @@
                             <div class="video-thumb">
                                 <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -105,9 +117,9 @@
                     <div class="video-desc {{ view_style }}">
                         <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
                             {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
                             <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
                         </div>

tubearchivist/home/templates/home/search.html

@@ -1,16 +1,17 @@
 {% extends "home/base.html" %}
 {% block content %}
 <div id="player" class="player-wrapper"></div>
-<div class="boxed-content">
+<div class="boxed-content {% if all_styles.home == "grid" %}boxed-{{ grid_items }}{% endif %}">
     <div class="title-bar">
-        <h1>Search</h1>
+        <h1>Search your Archive</h1>
     </div>
     <div class="multi-search-box">
         {{ search_form }}
+        <p>Start typing or use <span class="settings-current">video:</span>, <span class="settings-current">channel:</span>, <span class="settings-current">playlist:</span> or <span class="settings-current">full:</span> keywords for advanced queries. <a href="https://github.com/tubearchivist/tubearchivist/wiki/Search" target="_blank">Learn more</a>.</p>
     </div>
     <div class="multi-search-result">
         <h2>Video Results</h2>
-        <div id="video-results" class="video-list {{ all_styles.home }}">
+        <div id="video-results" class="video-list {{ all_styles.home }} {% if all_styles.home == "grid" %}grid-{{ grid_items }}{% endif %}">
             <p>No videos found.</p>
         </div>
     </div>
@@ -26,5 +27,11 @@
             <p>No playlists found.</p>
         </div>
     </div>
+    <div class="multi-search-result">
+        <h2>Fulltext Results</h2>
+        <div id="fulltext-results" class="video-list list">
+            <p>No fulltext results found.</p>
+        </div>
+    </div>
 </div>
 {% endblock content %}

tubearchivist/home/templates/home/settings.html

@@ -34,8 +34,8 @@
         <div class="settings-group">
             <h2 id="subscriptions">Subscriptions</h2>
             <div class="settings-item">
-                <p>Current page size: <span class="settings-current">{{ config.subscriptions.channel_size }}</span></p>
-                <i>Recent videos for channels and playlist to check when running <b>Rescan subscriptions</b>, max recommended 50.</i><br>
+                <p>YouTube page size: <span class="settings-current">{{ config.subscriptions.channel_size }}</span></p>
+                <i>Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
                 {{ app_form.subscriptions_channel_size }}
             </div>
         </div>
@@ -48,7 +48,7 @@
             </div>
             <div class="settings-item">
                 <p>Current download speed limit in KB/s: <span class="settings-current">{{ config.downloads.limit_speed }}</span></p>
-                <i>Limit download speed. 0 (zero) to deactivate, e.g. 1000 (1MB/s). Speeds are in KB/s.</i><br>
+                <i>Limit download speed. 0 (zero) to deactivate, e.g. 1000 (1MB/s). Speeds are in KB/s. Setting takes effect on new download jobs or application restart.</i><br>
                 {{ app_form.downloads_limit_speed }}
             </div>
             <div class="settings-item">
@@ -114,6 +114,20 @@
                 {{ app_form.downloads_subtitle_index }}
             </div>
         </div>
+        <div class="settings-group">
+            <h2 id="format">Cookie</h2>
+            <div class="settings-item">
+                <p>Import YouTube cookie: <span class="settings-current">{{ config.downloads.cookie_import }}</span><br></p>
+                <p>For automatic cookie import use <b>Tube Archivist Companion</b> <a href="https://github.com/tubearchivist/browser-extension" target="_blank">browser extension</a>.</p> 
+                <i>For manual cookie import, place your cookie file named <span class="settings-current">cookies.google.txt</span> in <span class="settings-current">cache/import</span> before enabling. Instructions in the <a href="https://github.com/tubearchivist/tubearchivist/wiki/Settings" target="_blank">Wiki.</a></i><br>
+                {{ app_form.downloads_cookie_import }}<br>
+                {% if config.downloads.cookie_import %}
+                    <div id="cookieMessage">
+                        <button onclick="handleCookieValidate()" type="button" id="cookieButton">Validate Cookie File</button>
+                    </div>
+                {% endif %}
+            </div>
+        </div>
         <div class="settings-group">
             <h2 id="integrations">Integrations</h2>
             <div class="settings-item">
@@ -124,10 +138,15 @@
                 </div>
             </div>
             <div class="settings-item">
-                <p>Integrate with <a href="https://returnyoutubedislike.com/">returnyoutubedislike.com</a> to get dislikes and average ratings back: <span class="settings-current">{{ config.downloads.integrate_ryd }}</span></p>
+                <p>Integrate with <a href="https://returnyoutubedislike.com/" target="_blank">returnyoutubedislike.com</a> to get dislikes and average ratings back: <span class="settings-current">{{ config.downloads.integrate_ryd }}</span></p>
                 <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
                 {{ app_form.downloads_integrate_ryd }}
             </div>
+            <div class="settings-item">
+                <p>Integrate with <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a> to get sponsored timestamps: <span class="settings-current">{{ config.downloads.integrate_sponsorblock }}</span></p>
+                <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
+                {{ app_form.downloads_integrate_sponsorblock }}
+            </div>
             <div class="settings-item">
                 <p>Current Cast integration: <span class="settings-current">{{ config.application.enable_cast }}</span></p>
                 <i>Enabling Cast will load an additional JS library from Google. HTTPS and a supported browser are required for this integration.</i><br>
@@ -251,9 +270,15 @@
     <div class="title-bar">
         <h1>Actions</h1>
     </div>
+    <div class="settings-group">
+        <h2>Delete download queue</h2>
+        <p>Delete your pending or previously ignored videos from your download queue.<p>
+        <button onclick="deleteQueue(this)" id="ignore-button" data-id="ignore" title="Delete all previously ignored videos from the queue">Delete all ignored</button>
+        <button onclick="deleteQueue(this)" id="pending-button" data-id="pending" title="Delete all pending videos from the queue">Delete all queued</button>
+    </div>
     <div class="settings-group">
         <h2>Manual media files import.</h2>
-        <p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://github.com/bbilly1/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
+        <p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://github.com/tubearchivist/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
         <div id="manual-import">
             <button onclick="manualImport()">Start import</button>
         </div>
@@ -298,7 +323,7 @@
     <div class="settings-group">
         <h2>Rescan filesystem</h2>
         <p><span class="danger-zone">Danger Zone</span>: This will delete the metadata of deleted videos from the filesystem.</p>
-        <p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://github.com/bbilly1/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
+        <p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://github.com/tubearchivist/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
         <div id="fs-rescan">
             <button onclick="fsRescan()">Rescan filesystem</button>
         </div>

tubearchivist/home/templates/home/video.html

@@ -3,6 +3,16 @@
 {% load static %}
 {% load humanize %}
 <div class="video-main"></div>
+<div class="notifications" id="notifications"></div>
+<div class="sponsorblock" id="sponsorblock">
+    {% if video.sponsorblock.is_enabled %}
+        {% if video.sponsorblock.segments|length == 0 %}
+            <h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
+        {% elif video.sponsorblock.has_unlocked %}
+            <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
+        {% endif %}
+    {% endif %}
+</div>
 <div class="boxed-content">
     <div class="title-bar">
         {% if cast %}
@@ -32,9 +42,9 @@
                 <p>Last refreshed: {{ video.vid_last_refresh }}</p>
                 <p class="video-info-watched">Watched:
                     {% if video.player.watched %}
-                        <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                        <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                     {% else %}
-                        <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                        <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                     {% endif %}
                 </p>
                 {% if video.active %}
@@ -67,11 +77,11 @@
         </div>
     </div>
     {% if video.description %}
-        <div class="info-box-item description-box">
-            <p>Description: <button onclick="textReveal()" id="text-reveal-button">Show</button></p>
-            <div id="text-reveal" class="description-text">
-                {{ video.description|linebreaks }}
-            </div>
+        <div class="description-box">
+            <p id="text-expand" class="description-text">
+                {{ video.description|linebreaksbr|urlizetrunc:50 }}
+            </p>
+            <button onclick="textExpand()" id="text-expand-button">Show more</button>
         </div>
     {% endif %}
     {% if playlist_nav %}
@@ -114,7 +124,12 @@
 </div>
 <script>
     var videoData = getVideoData('{{ video.youtube_id }}');
-    var videoProgress = getVideoProgress('{{ video.youtube_id }}').position;
+    sponsorBlock = videoData.data.sponsorblock;
+    {% if position %}
+        var videoProgress = {{ position }}
+    {% else %}
+        var videoProgress = getVideoProgress('{{ video.youtube_id }}').position;
+    {% endif %}
     window.onload = insertVideoTag(videoData, videoProgress);
 </script>
 {% endblock content %}

tubearchivist/home/urls.py

@@ -6,6 +6,8 @@ from django.contrib.auth.views import LogoutView
 from django.urls import path
 from home.views import (
     AboutView,
+    ChannelIdAboutView,
+    ChannelIdPlaylistView,
     ChannelIdView,
     ChannelView,
     DownloadView,
@@ -42,6 +44,16 @@ urlpatterns = [
         login_required(ChannelIdView.as_view()),
         name="channel_id",
     ),
+    path(
+        "channel/<slug:channel_id>/about/",
+        login_required(ChannelIdAboutView.as_view()),
+        name="channel_id_about",
+    ),
+    path(
+        "channel/<slug:channel_id>/playlist/",
+        login_required(ChannelIdPlaylistView.as_view()),
+        name="channel_id_playlist",
+    ),
     path(
         "video/<slug:video_id>/",
         login_required(VideoView.as_view()),

tubearchivist/home/views.py

@@ -8,17 +8,21 @@ import json
 import urllib.parse
 from time import sleep
 
-from django import forms
+from api.src.search_processor import SearchProcess
+from django.conf import settings
 from django.contrib.auth import login
 from django.contrib.auth.forms import AuthenticationForm
 from django.http import JsonResponse
 from django.shortcuts import redirect, render
 from django.views import View
+from home.src.download.yt_dlp_base import CookieHandler
+from home.src.es.connect import ElasticWrap
 from home.src.es.index_setup import get_available_backups
 from home.src.frontend.api_calls import PostData
 from home.src.frontend.forms import (
     AddToQueueForm,
     ApplicationSettingsForm,
+    ChannelOverwriteForm,
     CustomAuthForm,
     MultiSearchForm,
     SchedulerSettingsForm,
@@ -27,12 +31,13 @@ from home.src.frontend.forms import (
     UserSettingsForm,
 )
 from home.src.frontend.searching import SearchHandler
+from home.src.index.channel import channel_overwrites
 from home.src.index.generic import Pagination
 from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.config import AppConfig, ScheduleBuilder
-from home.src.ta.helper import UrlListParser
+from home.src.ta.helper import UrlListParser, time_parser
 from home.src.ta.ta_redis import RedisArchivist
-from home.tasks import extrac_dl, subscribe_to
+from home.tasks import extrac_dl, index_channel_playlists, subscribe_to
 from rest_framework.authtoken.models import Token
 
 
@@ -74,6 +79,15 @@ class ArchivistViewConfig(View):
 
         return view_style
 
+    def _get_grid_items(self):
+        """return items per row to show in grid view"""
+        grid_key = f"{self.user_id}:grid_items"
+        grid_items = self.user_conf.get_message(grid_key)["status"]
+        if not grid_items:
+            grid_items = self.default_conf["default_view"]["grid_items"]
+
+        return grid_items
+
     def get_all_view_styles(self):
         """get dict of all view stiles for search form"""
         all_keys = ["channel", "playlist", "home"]
@@ -109,7 +123,7 @@ class ArchivistViewConfig(View):
         """build default context for every view"""
         self.user_id = user_id
         self.user_conf = RedisArchivist()
-        self.default_conf = AppConfig().config
+        self.default_conf = AppConfig(self.user_id).config
 
         self.context = {
             "colors": self.default_conf["application"]["colors"],
@@ -117,9 +131,11 @@ class ArchivistViewConfig(View):
             "sort_by": self._get_sort_by(),
             "sort_order": self._get_sort_order(),
             "view_style": self._get_view_style(),
+            "grid_items": self._get_grid_items(),
             "hide_watched": self._get_hide_watched(),
             "show_ignored_only": self._get_show_ignore_only(),
             "show_subed_only": self._get_show_subed_only(),
+            "version": settings.TA_VERSION,
         }
 
 
@@ -175,14 +191,44 @@ class ArchivistResultsView(ArchivistViewConfig):
         if not results or not self.context["results"]:
             return
 
-        progress = {i["youtube_id"]: i["position"] for i in results}
+        self.context["continue_vids"] = self.get_in_progress(results)
+
+        in_progress = {i["youtube_id"]: i["position"] for i in results}
         for hit in self.context["results"]:
             video = hit["source"]
-            if video["youtube_id"] in progress:
-                played_sec = progress.get(video["youtube_id"])
+            if video["youtube_id"] in in_progress:
+                played_sec = in_progress.get(video["youtube_id"])
                 total = video["player"]["duration"]
+                if not total:
+                    total = played_sec * 2
                 video["player"]["progress"] = 100 * (played_sec / total)
 
+    def get_in_progress(self, results):
+        """get all videos in progress"""
+        ids = [{"match": {"youtube_id": i.get("youtube_id")}} for i in results]
+        data = {
+            "size": self.default_conf["archive"]["page_size"],
+            "query": {"bool": {"should": ids}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        search = SearchHandler(
+            "ta_video/_search", self.default_conf, data=data
+        )
+        videos = search.get_data()
+        if not videos:
+            return False
+
+        for video in videos:
+            youtube_id = video["source"]["youtube_id"]
+            matched = [i for i in results if i["youtube_id"] == youtube_id]
+            played_sec = matched[0]["position"]
+            total = video["source"]["player"]["duration"]
+            if not total:
+                total = matched[0].get("position") * 2
+            video["source"]["player"]["progress"] = 100 * (played_sec / total)
+
+        return videos
+
     def single_lookup(self, es_path):
         """retrieve a single item from url"""
         search = SearchHandler(es_path, config=self.default_conf)
@@ -232,6 +278,13 @@ class HomeView(ArchivistResultsView):
 
     def _update_view_data(self):
         """update view specific data dict"""
+        self.data["sort"].extend(
+            [
+                {"channel.channel_name.keyword": {"order": "asc"}},
+                {"title.keyword": {"order": "asc"}},
+            ]
+        )
+
         if self.context["hide_watched"]:
             self.data["query"] = {"term": {"player.watched": {"value": False}}}
         if self.search_get:
@@ -290,8 +343,11 @@ class AboutView(View):
     @staticmethod
     def get(request):
         """handle http get"""
-        colors = AppConfig(request.user.id).colors
-        context = {"title": "About", "colors": colors}
+        context = {
+            "title": "About",
+            "colors": AppConfig(request.user.id).colors,
+            "version": settings.TA_VERSION,
+        }
         return render(request, "home/about.html", context)
 
 
@@ -340,14 +396,15 @@ class DownloadView(ArchivistResultsView):
                 youtube_ids = UrlListParser(url_str).process_list()
             except ValueError:
                 # failed to process
+                key = "message:add"
                 print(f"failed to parse: {url_str}")
                 mess_dict = {
-                    "status": "message:add",
+                    "status": key,
                     "level": "error",
                     "title": "Failed to extract links.",
                     "message": "Not a video, channel or playlist ID or URL",
                 }
-                RedisArchivist().set_message("message:add", mess_dict)
+                RedisArchivist().set_message(key, mess_dict, expire=True)
                 return redirect("downloads")
 
             print(youtube_ids)
@@ -392,19 +449,121 @@ class ChannelIdView(ArchivistResultsView):
 
     def _update_view_data(self, channel_id):
         """update view specific data dict"""
-        query = {
+        self.data["query"] = {
             "bool": {
                 "must": [
                     {"term": {"channel.channel_id": {"value": channel_id}}}
                 ]
             }
         }
-        self.data["query"] = query
+        self.data["sort"].append({"title.keyword": {"order": "asc"}})
 
         if self.context["hide_watched"]:
             to_append = {"term": {"player.watched": {"value": False}}}
             self.data["query"]["bool"]["must"].append(to_append)
 
+    @staticmethod
+    def post(request, channel_id):
+        """handle post request"""
+        print(f"handle post from {channel_id}")
+        channel_overwrite_form = ChannelOverwriteForm(request.POST)
+        if channel_overwrite_form.is_valid():
+            overwrites = channel_overwrite_form.cleaned_data
+            print(f"{channel_id}: set overwrites {overwrites}")
+            channel_overwrites(channel_id, overwrites=overwrites)
+            if overwrites.get("index_playlists") == "1":
+                index_channel_playlists.delay(channel_id)
+
+        sleep(1)
+        return redirect("channel_id", channel_id, permanent=True)
+
+
+class ChannelIdAboutView(ArchivistResultsView):
+    """resolves to /channel/<channel-id>/about/
+    show metadata, handle per channel conf
+    """
+
+    view_origin = "channel"
+
+    def get(self, request, channel_id):
+        """handle get request"""
+        self.initiate_vars(request)
+
+        path = f"ta_channel/_doc/{channel_id}"
+        response, _ = ElasticWrap(path).get()
+
+        channel_info = SearchProcess(response).process()
+        channel_name = channel_info["channel_name"]
+
+        self.context.update(
+            {
+                "title": "Channel: About " + channel_name,
+                "channel_info": channel_info,
+                "channel_overwrite_form": ChannelOverwriteForm,
+            }
+        )
+
+        return render(request, "home/channel_id_about.html", self.context)
+
+    @staticmethod
+    def post(request, channel_id):
+        """handle post request"""
+        print(f"handle post from {channel_id}")
+        channel_overwrite_form = ChannelOverwriteForm(request.POST)
+        if channel_overwrite_form.is_valid():
+            overwrites = channel_overwrite_form.cleaned_data
+            print(f"{channel_id}: set overwrites {overwrites}")
+            channel_overwrites(channel_id, overwrites=overwrites)
+            if overwrites.get("index_playlists") == "1":
+                index_channel_playlists.delay(channel_id)
+
+        sleep(1)
+        return redirect("channel_id_about", channel_id, permanent=True)
+
+
+class ChannelIdPlaylistView(ArchivistResultsView):
+    """resolves to /channel/<channel-id>/playlist/
+    show all playlists of channel
+    """
+
+    view_origin = "playlist"
+    es_search = "ta_playlist/_search"
+
+    def get(self, request, channel_id):
+        """handle get request"""
+        self.initiate_vars(request)
+        self._update_view_data(channel_id)
+        self.find_results()
+
+        channel_info = self._get_channel_meta(channel_id)
+        channel_name = channel_info["channel_name"]
+        self.context.update(
+            {
+                "title": "Channel: Playlists " + channel_name,
+                "channel_info": channel_info,
+            }
+        )
+
+        return render(request, "home/channel_id_playlist.html", self.context)
+
+    def _update_view_data(self, channel_id):
+        """update view specific data dict"""
+        self.data["sort"] = [{"playlist_name.keyword": {"order": "asc"}}]
+        must_list = [{"match": {"playlist_channel_id": channel_id}}]
+
+        if self.context["show_subed_only"]:
+            must_list.append({"match": {"playlist_subscribed": True}})
+
+        self.data["query"] = {"bool": {"must": must_list}}
+
+    def _get_channel_meta(self, channel_id):
+        """get metadata for channel"""
+        path = f"ta_channel/_doc/{channel_id}"
+        response, _ = ElasticWrap(path).get()
+        channel_info = SearchProcess(response).process()
+
+        return channel_info
+
 
 class ChannelView(ArchivistResultsView):
     """resolves to /channel/
@@ -442,13 +601,14 @@ class ChannelView(ArchivistResultsView):
         """handle http post requests"""
         subscribe_form = SubscribeToChannelForm(data=request.POST)
         if subscribe_form.is_valid():
+            key = "message:subchannel"
             message = {
-                "status": "message:subchannel",
+                "status": key,
                 "level": "info",
                 "title": "Subscribing to Channels",
                 "message": "Parsing form data",
             }
-            RedisArchivist().set_message("message:subchannel", message=message)
+            RedisArchivist().set_message(key, message=message, expire=True)
             url_str = request.POST.get("subscribe")
             print(url_str)
             subscribe_to.delay(url_str)
@@ -589,15 +749,14 @@ class PlaylistView(ArchivistResultsView):
         if subscribe_form.is_valid():
             url_str = request.POST.get("subscribe")
             print(url_str)
+            key = "message:subplaylist"
             message = {
-                "status": "message:subplaylist",
+                "status": key,
                 "level": "info",
                 "title": "Subscribing to Playlists",
                 "message": "Parsing form data",
             }
-            RedisArchivist().set_message(
-                "message:subplaylist", message=message
-            )
+            RedisArchivist().set_message(key, message=message, expire=True)
             subscribe_to.delay(url_str)
 
         sleep(1)
@@ -611,7 +770,8 @@ class VideoView(View):
 
     def get(self, request, video_id):
         """get single video"""
-        colors, cast = self.read_config(user_id=request.user.id)
+        config_handler = AppConfig(request.user.id)
+        position = time_parser(request.GET.get("t"))
         path = f"ta_video/_doc/{video_id}"
         look_up = SearchHandler(path, config=False)
         video_hit = look_up.get_data()
@@ -633,8 +793,11 @@ class VideoView(View):
             "video": video_data,
             "playlist_nav": playlist_nav,
             "title": video_title,
-            "colors": colors,
-            "cast": cast,
+            "colors": config_handler.colors,
+            "cast": config_handler.config["application"]["enable_cast"],
+            "version": settings.TA_VERSION,
+            "config": config_handler.config,
+            "position": position,
         }
         return render(request, "home/video.html", context)
 
@@ -651,14 +814,6 @@ class VideoView(View):
 
         return all_navs
 
-    @staticmethod
-    def read_config(user_id):
-        """read config file"""
-        config_handler = AppConfig(user_id)
-        cast = config_handler.config["application"]["enable_cast"]
-        colors = config_handler.colors
-        return colors, cast
-
     @staticmethod
     def star_creator(rating):
         """convert rating float to stars"""
@@ -691,7 +846,10 @@ class SearchView(ArchivistResultsView):
         all_styles = self.get_all_view_styles()
         self.context.update({"all_styles": all_styles})
         self.context.update(
-            {"search_form": MultiSearchForm(initial=all_styles)}
+            {
+                "search_form": MultiSearchForm(initial=all_styles),
+                "version": settings.TA_VERSION,
+            }
         )
 
         return render(request, "home/search.html", self.context)
@@ -723,6 +881,7 @@ class SettingsView(View):
             "user_form": user_form,
             "app_form": app_form,
             "scheduler_form": scheduler_form,
+            "version": settings.TA_VERSION,
         }
 
         return render(request, "home/settings.html", context)
@@ -734,30 +893,73 @@ class SettingsView(View):
         token = Token.objects.get_or_create(user=request.user)[0]
         return token
 
-    @staticmethod
-    def post(request):
+    def post(self, request):
         """handle form post to update settings"""
+        user_form = UserSettingsForm(request.POST)
+        config_handler = AppConfig()
+        if user_form.is_valid():
+            user_form_post = user_form.cleaned_data
+            if any(user_form_post.values()):
+                config_handler.set_user_config(user_form_post, request.user.id)
 
-        form_response = forms.Form(request.POST)
-        if form_response.is_valid():
-            form_post = dict(request.POST)
-            print(form_post)
-            del form_post["csrfmiddlewaretoken"]
-            config_handler = AppConfig()
-            if "application-settings" in form_post:
-                del form_post["application-settings"]
-                config_handler.update_config(form_post)
-            elif "user-settings" in form_post:
-                del form_post["user-settings"]
-                config_handler.set_user_config(form_post, request.user.id)
-            elif "scheduler-settings" in form_post:
-                del form_post["scheduler-settings"]
-                print(form_post)
-                ScheduleBuilder().update_schedule_conf(form_post)
+        app_form = ApplicationSettingsForm(request.POST)
+        if app_form.is_valid():
+            app_form_post = app_form.cleaned_data
+            if app_form_post:
+                print(app_form_post)
+                updated = config_handler.update_config(app_form_post)
+                self.post_process_updated(updated, config_handler.config)
+
+        scheduler_form = SchedulerSettingsForm(request.POST)
+        if scheduler_form.is_valid():
+            scheduler_form_post = scheduler_form.cleaned_data
+            if any(scheduler_form_post.values()):
+                print(scheduler_form_post)
+                ScheduleBuilder().update_schedule_conf(scheduler_form_post)
 
         sleep(1)
         return redirect("settings", permanent=True)
 
+    def post_process_updated(self, updated, config):
+        """apply changes for config"""
+        if not updated:
+            return
+
+        for config_value, updated_value in updated:
+            if config_value == "cookie_import":
+                self.process_cookie(config, updated_value)
+
+    def process_cookie(self, config, updated_value):
+        """import and validate cookie"""
+        handler = CookieHandler(config)
+        if updated_value:
+            try:
+                handler.import_cookie()
+            except FileNotFoundError:
+                print("cookie: import failed, file not found")
+                handler.revoke()
+                self._fail_message("Cookie file not found.")
+                return
+
+            valid = handler.validate()
+            if not valid:
+                handler.revoke()
+                self._fail_message("Failed to validate cookie file.")
+        else:
+            handler.revoke()
+
+    @staticmethod
+    def _fail_message(message_line):
+        """notify our failure"""
+        key = "message:setting"
+        message = {
+            "status": key,
+            "level": "error",
+            "title": "Cookie import failed",
+            "message": message_line,
+        }
+        RedisArchivist().set_message(key, message=message, expire=True)
+
 
 def progress(request):
     # pylint: disable=unused-argument

tubearchivist/requirements.txt

@@ -1,12 +1,13 @@
-beautifulsoup4==4.10.0
-celery==5.2.3
-Django==4.0.2
-django-cors-headers==3.11.0
+beautifulsoup4==4.11.1
+celery==5.2.7
+Django==4.1
+django-auth-ldap==4.1.0
+django-cors-headers==3.13.0
 djangorestframework==3.13.1
-Pillow==9.0.1
-redis==4.1.4
-requests==2.27.1
-ryd-client==0.0.3
+Pillow==9.2.0
+redis==4.3.4
+requests==2.28.1
+ryd-client==0.0.6
 uWSGI==2.0.20
-whitenoise==6.0.0
-yt_dlp==2022.2.4
+whitenoise==6.2.0
+yt_dlp==2022.8.19

tubearchivist/static/cast-videos.js

@@ -1,6 +1,6 @@
 function initializeCastApi() {
     cast.framework.CastContext.getInstance().setOptions({
-        receiverApplicationId: chrome.cast.media.DEFAULT_MEDIA_RECEIVER_APP_ID, // Use built in reciver app on cast device, see https://developers.google.com/cast/docs/styled_receiver if you want to be able to add a theme, splash screen or watermark. Has a $5 one time fee.
+        receiverApplicationId: chrome.cast.media.DEFAULT_MEDIA_RECEIVER_APP_ID, // Use built in receiver app on cast device, see https://developers.google.com/cast/docs/styled_receiver if you want to be able to add a theme, splash screen or watermark. Has a $5 one time fee.
         autoJoinPolicy: chrome.cast.AutoJoinPolicy.ORIGIN_SCOPED
     });
 

tubearchivist/static/css/style.css

@@ -62,6 +62,13 @@ h3 {
     color: var(--accent-font-light);
 }
 
+h4 {
+    font-size: 0.7em;
+    margin-bottom: 7px;
+    font-family: Sen-Regular, sans-serif;
+    color: var(--accent-font-light);
+}
+
 p, i, li {
     font-family: Sen-Regular, sans-serif;
     margin-bottom: 10px;
@@ -126,6 +133,18 @@ button:hover {
     margin: 0 auto;
 }
 
+.boxed-content.boxed-4 {
+    max-width: 1200px;
+    width: 80%;
+}
+
+.boxed-content.boxed-5,
+.boxed-content.boxed-6,
+.boxed-content.boxed-7 {
+    max-width: unset;
+    width: 85%;
+}
+
 .round-img img {
     border-radius: 50%;
 }
@@ -321,7 +340,8 @@ button:hover {
     margin: 15px 0;
 }
 
-.view-icons {
+.view-icons,
+.grid-count {
     display: flex;
     justify-content: end;
 }
@@ -333,15 +353,29 @@ button:hover {
     filter: var(--img-filter);
 }
 
+.grid-count img {
+    width: 15px;
+    margin: 5px;
+    cursor: pointer;
+    filter: var(--img-filter);
+}
+
 #hidden-form {
     display: none;
 }
 
 #text-reveal {
-    height: 0px;
+    height: 0;
     overflow: hidden;
 }
 
+#text-expand {
+    overflow: hidden;
+    display: -webkit-inline-box;
+    -webkit-box-orient: vertical;
+    -webkit-line-clamp: 4;
+}
+
 
 /* video player */
 .player-wrapper {
@@ -355,6 +389,18 @@ button:hover {
     height: 100vh;
 }
 
+.notifications {
+    text-align: center;
+    width: 80%;
+    margin: auto;
+}
+
+.sponsorblock {
+    text-align: center;
+    width: 80%;
+    margin: auto;
+}
+
 .video-player video,
 .video-main video {
     max-height: 80vh;
@@ -369,17 +415,41 @@ button:hover {
     margin: 10px 10px 10px 0;
 }
 
+/* fix for safari full screen not scaling full */
+video:-webkit-full-screen {
+    max-height: unset !important;
+    max-width: unset !important;
+}
+
 /* video list */
-.video-list.grid {
+.video-list {
     display: grid;
-    grid-template-columns: 1fr 1fr 1fr;
     grid-gap: 1rem;
+    margin-top: 1rem;
+}
+
+.video-list.grid.grid-3 {
+    grid-template-columns: 1fr 1fr 1fr;
+}
+
+.video-list.grid.grid-4 {
+    grid-template-columns: 1fr 1fr 1fr 1fr;
+}
+
+.video-list.grid.grid-5 {
+    grid-template-columns: 1fr 1fr 1fr 1fr 1fr;
+}
+
+.video-list.grid.grid-6 {
+    grid-template-columns: 1fr 1fr 1fr 1fr 1fr 1fr;
+}
+
+.video-list.grid.grid-7 {
+    grid-template-columns: 1fr 1fr 1fr 1fr 1fr 1fr 1fr;
 }
 
 .video-list.list {
-    display: grid;
     grid-template-columns: unset;
-    grid-gap: 1rem;
 }
 
 .video-item {
@@ -388,7 +458,9 @@ button:hover {
 
 .video-item.list {
     display: grid;
-    grid-template-columns: 25% auto;
+    grid-template-columns: 26% auto;
+    background-color: var(--highlight-bg);
+    align-items: center;
 }
 
 .video-progress-bar {
@@ -438,8 +510,7 @@ button:hover {
 
 .video-desc.list {
     padding: 10px;
-    height: unset;
-    background-color: var(--highlight-bg);
+    height: 100%;
     display: flex;
     flex-wrap: wrap-reverse;
     align-content: center;
@@ -482,8 +553,7 @@ button:hover {
     align-items: center;
 }
 
-.unseen-icon,
-.seen-icon,
+.watch-button,
 .close-button {
     cursor: pointer;
     filter: var(--img-filter);
@@ -535,6 +605,8 @@ button:hover {
 
 .description-box {
     margin-top: 1rem;
+    padding: 15px;
+    background-color: var(--highlight-bg);
 }
 
 .info-box-3 {
@@ -565,6 +637,28 @@ button:hover {
     margin-bottom: 10px;
 }
 
+.overwrite-form {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    width: 100%;
+}
+
+.overwrite-form button {
+    width: 200px;
+}
+
+.overwrite-form-item {
+    margin-bottom: 1rem;
+}
+
+.overwrite-form-item input {
+    width: 90%;
+}
+
+.hidden-overwrite {
+    display: none;
+}
+
 /* login */
 .login-page {
     display: flex;
@@ -694,6 +788,10 @@ button:hover {
 }
 
 /* multi search page */
+.multi-search-box {
+    padding-right: 20px;
+}
+
 .multi-search-box input {
     width: 100%;
 }
@@ -739,6 +837,18 @@ button:hover {
     transform: translateX(-30%);
 }
 
+.info-box-item.channel-nav {
+    justify-content: center;
+}
+
+.info-box-item.channel-nav a {
+    padding: 0 1rem;
+}
+
+.info-box-item.channel-nav a:hover {
+    text-decoration: underline;
+}
+
 /* playlist overview page */
 .playlist-list.list {
     display: grid;
@@ -1008,11 +1118,19 @@ button:hover {
 }
 
 /* tablet */
-@media screen and (max-width: 1000px) {
-    .boxed-content {
+@media screen and (max-width: 1000px), screen and (max-height: 850px) {
+    .boxed-content,
+    .boxed-content.boxed-4,
+    .boxed-content.boxed-5,
+    .boxed-content.boxed-6,
+    .boxed-content.boxed-7 {
         width: 90%;
     }
-    .video-list.grid,
+    .video-list.grid.grid-3,
+    .video-list.grid.grid-4,
+    .video-list.grid.grid-5,
+    .video-list.grid.grid-6,
+    .video-list.grid.grid-7,
     .dl-list.grid,
     .channel-list.grid,
     .playlist-list.grid {
@@ -1042,6 +1160,9 @@ button:hover {
         position: unset;
         transform: unset;
     }
+    .grid-count {
+        display: none;
+    }
     .video-player {
         height: unset;
         padding: 20px 0
@@ -1056,12 +1177,19 @@ button:hover {
     * {
         word-wrap: anywhere;
     }
-    .video-list.grid,
+    .video-list.grid.grid-3,
+    .video-list.grid.grid-4,
+    .video-list.grid.grid-5,
+    .video-list.grid.grid-6,
+    .video-list.grid.grid-7,
     .dl-list.grid,
     .channel-list.grid,
     .video-item.list,
     .playlist-list.list,
-    .playlist-list.grid {
+    .playlist-list.grid,
+    .info-box-2,
+    .info-box-3,
+    .overwrite-form {
         grid-template-columns: 1fr;
     }
     .playlist-item.list {
@@ -1100,10 +1228,6 @@ button:hover {
     .sort select {
         margin: unset;
     }
-    .info-box-2,
-    .info-box-3 {
-        grid-template-columns: 1fr;
-    }
     .description-box {
         display: block;
     }

tubearchivist/static/img/icon-substract.svg

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="474.99609"
+   height="78.428696"
+   viewBox="0 0 125.67634 20.750926"
+   version="1.1"
+   id="svg1303"
+   inkscape:version="0.92.4 (5da689c313, 2019-01-14)"
+   sodipodi:docname="icon-add.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:dc="http://purl.org/dc/elements/1.1/">
+  <defs
+     id="defs1297" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.85859018"
+     inkscape:cx="-97.380081"
+     inkscape:cy="261.09215"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1920"
+     inkscape:window-height="1017"
+     inkscape:window-x="-8"
+     inkscape:window-y="-8"
+     inkscape:window-maximized="1"
+     showguides="true"
+     inkscape:guide-bbox="true"
+     inkscape:showpageshadow="2"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1">
+    <sodipodi:guide
+       position="-225.18364,87.524084"
+       orientation="1,0"
+       id="guide1072"
+       inkscape:locked="false" />
+  </sodipodi:namedview>
+  <metadata
+     id="metadata1300">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-3.3077777,-220.4781)">
+    <path
+       style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
+       d="m 12.291019,220.47796 c -4.9648228,-0.009 -8.9691711,3.98046 -8.9784054,8.94536 l -0.00482,2.62846 c -0.00925,4.9649 3.9804459,8.96933 8.9452674,8.97832 107.703889,0.29274 22.266017,0.0414 107.747629,0.19881 4.96484,0.009 8.96917,-3.98046 8.9784,-8.94534 l 0.005,-2.62847 c 0.009,-4.96489 -3.98037,-8.96923 -8.94525,-8.97831 -107.850822,-0.0255 -20.413204,-0.038 -107.747821,-0.19883 z"
+       id="rect1073"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="ccccccccc" />
+  </g>
+</svg>

tubearchivist/static/progress.js

@@ -7,7 +7,7 @@ checkMessages()
 
 // page map to notification status
 const messageTypes = {
-    "download": ["message:download", "message:add", "message:rescan"],
+    "download": ["message:download", "message:add", "message:rescan", "message:playlistscan"],
     "channel": ["message:subchannel"],
     "channel_id": ["message:playlistscan"],
     "playlist": ["message:subplaylist"],