bump TA_VERSION

* Update channel.html

Updated wording, removed "matching" as it's not necessary.

* Update downloads.html

.eslintrc.js

@@ -0,0 +1,17 @@
+'use strict';
+module.exports = {
+  extends: ['eslint:recommended', 'eslint-config-prettier'],
+  parserOptions: {
+    ecmaVersion: 2020,
+  },
+  env: {
+    browser: true,
+  },
+  rules: {
+    strict: ['error', 'global'],
+    'no-unused-vars': ['error', { vars: 'local' }],
+    eqeqeq: ['error', 'always', { null: 'ignore' }],
+    curly: ['error', 'multi-line'],
+    'no-var': 'error',
+  },
+};

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+github: bbilly1
+ko_fi: bbilly1
+custom: https://paypal.me/bbilly1

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -0,0 +1,53 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to help improve this project!
+
+  - type: checkboxes
+    id: latest
+    attributes:
+      label: Latest and Greatest
+      options:
+        - label: I'm running the latest version of Tube Archivist and have read the [release notes](https://github.com/tubearchivist/tubearchivist/releases/latest).
+          required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: On what operating system are you running Tube Archivist on?
+      placeholder: ex. Linux, Unraid, Synology...
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: Your Bug Report
+      value: "## Describe the bug\n\n## Steps To Reproduce\n\n## Expected behavior"
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Anything else?
+      description: |
+        Links? References? Anything that will give us more context about the issue you are encountering!
+
+        Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -0,0 +1,37 @@
+name: Feature Request
+description: Create a new feature request
+title: "[Feature Request]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to help improve this project!
+
+  - type: checkboxes
+    id: already
+    attributes:
+      label: Already implemented?
+      options:
+        - label: I have read through the [wiki](https://github.com/tubearchivist/tubearchivist/wiki).
+          required: true
+        - label: I understand the [scope](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) of this project and am aware of the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and my idea is not already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap).
+          required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Your Feature Request
+      value: "## Is your feature request related to a problem? Please describe.\n\n## Describe the solution you'd like\n\n## Additional context"
+      placeholder: Tell us what you see!
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: help
+    attributes:
+      label: Your help is needed!
+      description: This project is ambitious as it is, please contribute.
+      options:
+        - label: Yes I can help with this feature request!
+          required: false

.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml

@@ -0,0 +1,45 @@
+name: Installation Help
+description: There is an issue with installing or configuring Tube Archivist
+title: "[Installation]: "
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This is most likely not the best place to ask for installation help. There is a helpful community on [Discord](https://github.com/tubearchivist/tubearchivist#connect).
+
+  - type: checkboxes
+    id: instructions
+    attributes:
+      label: Installation instructions
+      options:
+        - label: I have read and understand the [installation instructions](https://github.com/tubearchivist/tubearchivist#installing-and-updating).
+          required: true
+        - label: My issue is not described in the [potential pitfalls](https://github.com/tubearchivist/tubearchivist#potential-pitfalls) section.
+          required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: On what operating system do you want to install Tube Archivist on?
+      placeholder: ex. Linux, Unraid, Synology...
+    validations:
+      required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Your installation or setup question
+      placeholder: Describe your issue
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+    validations:
+      required: true

.github/workflows/lint_js.yml

@@ -0,0 +1,16 @@
+name: lint_js
+
+on: [pull_request, push]
+
+jobs:
+  check:
+    name: lint_js
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      - run: npm ci
+      - run: npm run lint
+      - run: npm run format -- --check

.github/workflows/lint_python.yml

@@ -4,16 +4,21 @@ jobs:
   lint_python:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      # note: this logic is duplicated in the `validate` function in ./deploy.sh
+      # if you update this file, you should update that as well
       - run: pip install --upgrade pip wheel
       - run: pip install bandit black codespell flake8 flake8-bugbear
-                         flake8-comprehensions isort
-      - run: black --check --diff --line-length 79 .
-      - run: codespell
-      - run: flake8 . --count --max-complexity=10 --max-line-length=79
-                      --show-source --statistics
-      - run: isort --check-only --line-length 79 --profile black .
+                         flake8-comprehensions isort requests
+      - run: ./deploy.sh validate
+      # - run: black --check --diff --line-length 79 .
+      # - run: codespell --skip="./.git,./package.json,./package-lock.json,./node_modules"
+      # - run: flake8 . --count --max-complexity=10 --max-line-length=79
+      #                 --show-source --statistics
+      # - run: isort --check-only --line-length 79 --profile black .
       # - run: pip install -r tubearchivist/requirements.txt
       # - run: mkdir --parents --verbose .mypy_cache
       # - run: mypy --ignore-missing-imports --install-types --non-interactive .

.gitignore

@@ -5,4 +5,7 @@ __pycache__
 db.sqlite3
 
 # vscode custom conf
-.vscode
+.vscode
+
+# JavaScript stuff
+node_modules

CONTRIBUTING.md

@@ -15,13 +15,29 @@ The wiki is where all user functions are documented in detail. These pages are m
 
 ## Development Environment
 
-I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment.  
+I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. You might be able to run the application directly, but this document assumes you're using docker.
+
+### Instructions
+
+Set up docker on your development machine.
+
+Clone this repository.
+
+Functional changes should be made against the unstable `testing` branch, so check that branch out, then make a new branch for your work.
+
+Edit the `docker-compose.yml` file and replace the [`image: bbilly1/tubearchivist` line](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L7) with `build: .`. Also make any other changes to the environment variables and so on necessary to run the application, just like you're launching the application as normal.
+
+Run `docker compose up --build`. This will bring up the application. Kill it with `ctrl-c` or by running `docker compose down` from a new terminal window in the same directory.
+
+Make your changes locally and re-run `docker compose up --build`. The `Dockerfile` is structured in a way that the actual application code is in the last layer so rebuilding the image with only code changes utilizes the build cache for everything else and will just take a few seconds.
+
+### Develop environment inside a VM
+
+You may find it nice to run everything inside of a VM, though this is not necessary. There's a `deploy.sh` script which has some helpers for this use case. YMMV, this is what one of the developers does:
 
-This is my setup I have landed on, YMMV:
 - Clone the repo, work on it with your favorite code editor in your local filesystem. *testing* branch is the where all the changes are happening, might be unstable and is WIP.
 - Then I have a VM running standard Ubuntu Server LTS with docker installed. The VM keeps my projects separate and offers convenient snapshot functionality. The VM also offers ways to simulate lowend environments by limiting CPU cores and memory. You can use this [Ansible Docker Ubuntu](https://github.com/bbilly1/ansible-playbooks) playbook to get started quickly. But you could also just run docker on your host system.
-- The `Dockerfile` is structured in a way that the actual application code is in the last layer so rebuilding the image with only code changes utilizes the build cache for everything else and will just take a few seconds.
-- Take a look at the `deploy.sh` file. I have my local DNS resolve `tubearchivist.local` to the IP of the VM for convenience. To deploy the latest changes and rebuild the application to the testing VM run:
+- I have my local DNS resolve `tubearchivist.local` to the IP of the VM for convenience. To deploy the latest changes and rebuild the application to the testing VM run:
 ```bash
 ./deploy.sh test
 ```
@@ -30,7 +46,25 @@ This is my setup I have landed on, YMMV:
 - This `deploy.sh` script is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking. 
 
 ## Working with Elasticsearch
-Additionally to the required services as listed in the example docker-compose file, the **Dev Tools** of [Kibana](https://www.elastic.co/guide/en/kibana/current/docker.html) are invaluable for running and testing Elasticsearch queries.  
+Additionally to the required services as listed in the example docker-compose file, the **Dev Tools** of [Kibana](https://www.elastic.co/guide/en/kibana/current/docker.html) are invaluable for running and testing Elasticsearch queries. 
+
+**Quick start**  
+Generate your access token in Elasitcsearch:
+```bash
+bin/elasticsearch-service-tokens create elastic/kibana kibana
+```
+
+Example docker compose, use same version as for Elasticsearch:
+```yml
+kibana:
+  image: docker.elastic.co/kibana/kibana:0.0.0
+  container_name: kibana
+  environment:
+    - "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
+    - "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
+  ports:
+    - "5601:5601"
+```
 
 If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.
 
@@ -42,6 +76,10 @@ Do you see anything on the roadmap that you would like to take a closer look at
 
 To fix a bug or implement a feature, fork the repository and make all changes to the testing branch. When ready, create a pull request.
 
+## Making changes to the JavaScript
+
+The JavaScript does not require any build step; you just edit the files directly. However, there is config for eslint and prettier (a linter and formatter respectively); their use is recommended but not required. To use them, install `node`, run `npm i` from the root directory of this repository to install dependencies, then run `npm run lint` and `npm run format` to run eslint and prettier respectively.
+
 ## Releases
 
 There are three different docker tags:

Dockerfile

@@ -3,18 +3,18 @@
 
 
 # First stage to build python wheel
-FROM python:3.10.5-slim-bullseye AS builder
+FROM python:3.10.8-slim-bullseye AS builder
 ARG TARGETPLATFORM
 
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends build-essential gcc
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
 
 # install requirements
 COPY ./tubearchivist/requirements.txt /requirements.txt
 RUN pip install --user -r requirements.txt
 
 # build final image
-FROM python:3.10.5-slim-bullseye as tubearchivist
+FROM python:3.10.8-slim-bullseye as tubearchivist
 
 ARG TARGETPLATFORM
 ARG INSTALL_DEBUG
@@ -32,19 +32,23 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm
     curl \
     xz-utils && rm -rf /var/lib/apt/lists/*
 
-# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
-RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
+# install patched ffmpeg build, default to linux64
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
+    curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
+        | grep browser_download_url \
+        | grep ".*master.*linuxarm64.*tar.xz" \
+        | cut -d '"' -f 4 \
+        | xargs curl -L --output ffmpeg.tar.xz ; \
+    else \
     curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
         | grep browser_download_url \
         | grep ".*master.*linux64.*tar.xz" \
         | cut -d '"' -f 4 \
-        | xargs curl -L --output ffmpeg.tar.xz && \
-        tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
-        tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
-        rm ffmpeg.tar.xz \
-    ; elif [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
-        apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
-    ; fi
+        | xargs curl -L --output ffmpeg.tar.xz ; \
+    fi && \
+    tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
+    tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
+    rm ffmpeg.tar.xz
 
 # install debug tools for testing environment
 RUN if [ "$INSTALL_DEBUG" ] ; then \
@@ -54,12 +58,11 @@ RUN if [ "$INSTALL_DEBUG" ] ; then \
     ; fi
 
 # make folders
-RUN mkdir /cache
-RUN mkdir /youtube
-RUN mkdir /app
+RUN mkdir /cache /youtube /app
 
 # copy config files
 COPY docker_assets/nginx.conf /etc/nginx/sites-available/default
+RUN sed -i 's/^user www\-data\;$/user root\;/' /etc/nginx/nginx.conf
 
 # copy application into container
 COPY ./tubearchivist /app

README.md

@@ -61,7 +61,38 @@ Once your YouTube video collection grows, it becomes hard to search and find a s
 - [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.
 
 ## Installing and updating
-Take a look at the example `docker-compose.yml` file provided. Use the *latest* or the named semantic version tag. The *unstable* tag is for intermediate testing and as the name implies, is **unstable** and not be used on your main installation but in a [testing environment](CONTRIBUTING.md).  
+There's dedicated user-contributed install steps under [docs/Installation.md](./docs/Installation.md) for Unraid, Truenas and Synology which you can use instead of this section if you happen to be using one of those. Otherwise, continue on.
+
+For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation.  
+
+Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson` (an unofficial rebuild for arm64) instead of [the official one](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L27).
+
+This project requires docker. Ensure it is installed and running on your system.
+
+Save the [docker-compose.yml](./docker-compose.yml) file from this reposity somewhere permanent on your system, keeping it named `docker-compose.yml`. You'll need to refer to it whenever starting this application.
+
+Edit the following values from that file:
+- under `tubearchivist`->`environment`:
+  - `HOST_UID`: your UID, if you want TubeArchivist to create files with your UID. Remove if you are OK with files being owned by the the container user.
+  - `HOST_GID`: as above but GID.
+  - `TA_HOST`: change it to the address of the machine you're running this on. This can be an IP address or a domain name.
+  - `TA_PASSWORD`: pick a password to use when logging in.
+  - `ELASTIC_PASSWORD`: pick a password for the elastic service. You won't need to type this yourself.
+  - `TZ`: your time zone. If you don't know yours, you can look it up [here](https://www.timezoneconverter.com/cgi-bin/findzone/findzone).
+- under `archivist-es`->`environment`:
+  - `"ELASTIC_PASSWORD=verysecret"`: change `verysecret` to match the ELASTIC_PASSWORD you picked above.
+
+By default Docker will store all data, including downloaded data, in its own data-root directory (which you can find by running `docker info` and looking for the "Docker Root Dir"). If you want to use other locations, you can replace the `media:`, `cache:`, `redis:`, and `es:` volume names with absolute paths; if you do, remove them from the `volumes:` list at the bottom of the file.
+
+From a terminal, `cd` into the directory you saved the `docker-compose.yml` file in and run `docker compose up --detach`. The first time you do this it will download the appropriate images, which can take a minute.
+
+You can follow the logs with `docker compose logs -f`. Once it's ready it will print something like `celery@1234567890ab ready`. At this point you should be able to go to `http://your-host:8000` and log in with the `TA_USER`/`TA_PASSWORD` credentials.
+
+You can bring the application down by running `docker compose down` in the same directory.
+
+Use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tag is for intermediate testing and as the name implies, is **unstable** and not be used on your main installation but in a [testing environment](CONTRIBUTING.md).  
+
+## Installation Details
 
 Tube Archivist depends on three main components split up into separate docker containers:  
 
@@ -72,6 +103,7 @@ The main Python application that displays and serves your video collection, buil
   - And another volume to save application data at **/cache**.
   - The environment variables `ES_URL` and `REDIS_HOST` are needed to tell Tube Archivist where Elasticsearch and Redis respectively are located.
   - The environment variables `HOST_UID` and `HOST_GID` allows Tube Archivist to `chown` the video files to the main host system user instead of the container user. Those two variables are optional, not setting them will disable that functionality. That might be needed if the underlying filesystem doesn't support `chown` like *NFS*. 
+  - Set the environment variable `TA_HOST` to match with the system running Tube Archivist. This can be a domain like *example.com*, a subdomain like *ta.example.com* or an IP address like *192.168.1.20*, add without the protocol and without the port. You can add multiple hostnames separated with a space. Any wrong configurations here will result in a `Bad Request (400)` response.
   - Change the environment variables `TA_USERNAME` and `TA_PASSWORD` to create the initial credentials. 
   - `ELASTIC_PASSWORD` is for the password for Elasticsearch. The environment variable `ELASTIC_USER` is optional, should you want to change the username from the default *elastic*.
   - For the scheduler to know what time it is, set your timezone with the `TZ` environment variable, defaults to *UTC*.
@@ -85,8 +117,25 @@ Should that not be an option, the Tube Archivist container takes these two addit
 
 Changing any of these two environment variables will change the files *nginx.conf* and *uwsgi.ini* at startup using `sed` in your container.
 
+### LDAP Authentication
+You can configure LDAP with the following environment variables:
+
+ - `TA_LDAP` (ex: `true`) Set to anything besides empty string to use LDAP authentication **instead** of local user authentication.
+ - `TA_LDAP_SERVER_URI` (ex: `ldap://ldap-server:389`) Set to the uri of your LDAP server.
+ - `TA_LDAP_DISABLE_CERT_CHECK` (ex: `true`) Set to anything besides empty string to disable certificate checking when connecting over LDAPS.
+ - `TA_LDAP_BIND_DN` (ex: `uid=search-user,ou=users,dc=your-server`) DN of the user that is able to perform searches on your LDAP account.
+ - `TA_LDAP_BIND_PASSWORD` (ex: `yoursecretpassword`) Password for the search user.
+ - `TA_LDAP_USER_ATTR_MAP_USERNAME` (default: `uid`) Bind attribute used to map LDAP user's username
+ - `TA_LDAP_USER_ATTR_MAP_PERSONALNAME` (default: `givenName`) Bind attribute used to match LDAP user's First Name/Personal Name.
+ - `TA_LDAP_USER_ATTR_MAP_SURNAME` (default: `sn`) Bind attribute used to match LDAP user's Last Name/Surname.
+ - `TA_LDAP_USER_ATTR_MAP_EMAIL` (default: `mail`) Bind attribute used to match LDAP user's EMail address
+ - `TA_LDAP_USER_BASE` (ex: `ou=users,dc=your-server`) Search base for user filter.
+ - `TA_LDAP_USER_FILTER` (ex: `(objectClass=user)`) Filter for valid users. Login usernames are matched using the attribute specified in `TA_LDAP_USER_ATTR_MAP_USERNAME` and should not be specified in this filter.
+
+When LDAP authentication is enabled, django passwords (e.g. the password defined in TA_PASSWORD), will not allow you to login, only the LDAP server is used.
+
 ### Elasticsearch
-**Note**: Newest Tube Archivist depends on Elasticsearch version 7.17 to provide an automatic updatepath in the future. 
+**Note**: Tube Archivist depends on Elasticsearch 8. 
 
 Use `bbilly1/tubearchivist-es` to automatically get the recommended version, or use the official image with the version tag in the docker-compose file.
 
@@ -112,12 +161,9 @@ You will see the current version number of **Tube Archivist** in the footer of t
 * There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.  
 * All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version.
 
-### Alternative installation instructions:
-- **arm64**: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, you can use `bbilly1/rejson`, an unofficial rebuild for arm64.
-- **Synology**: There is a [discussion thread](https://github.com/tubearchivist/tubearchivist/discussions/48) with Synology installation instructions.
-- **Unraid**: The three containers needed are all in the Community Applications. First install `TubeArchivist RedisJSON` followed by `TubeArchivist ES`, and finally you can install `TubeArchivist`. If you have unraid specific issues, report those to the [support thread](https://forums.unraid.net/topic/114073-support-crocs-tube-archivist/ "support thread").
-- **Helm Chart**: There is a Helm Chart available at https://github.com/insuusvenerati/helm-charts. Mostly self-explanatory but feel free to ask questions in the discord / subreddit.
+### Helm charts
 
+There is a Helm Chart available at https://github.com/insuusvenerati/helm-charts. Mostly self-explanatory but feel free to ask questions in the discord / subreddit.
 
 ## Potential pitfalls
 ### vm.max_map_count
@@ -134,12 +180,12 @@ To apply the change permanently depends on your host operating system:
 - On any other platform look up in the documentation on how to pass kernel parameters.
 
 ### Permissions for elasticsearch
-If you see a message similar to `AccessDeniedException[/usr/share/elasticsearch/data/nodes]` when initially starting elasticsearch, that means the container is not allowed to write files to the volume.  
-That's most likely the case when you run `docker-compose` as an unprivileged user. To fix that issue, shutdown the container and on your host machine run:
+If you see a message similar to `failed to obtain node locks, tried [/usr/share/elasticsearch/data]` and `maybe these locations are not writable` when initially starting elasticsearch, that probably means the container is not allowed to write files to the volume.  
+To fix that issue, shutdown the container and on your host machine run:
 ```
-chown 1000:0 /path/to/mount/point
+chown 1000:0 -R /path/to/mount/point
 ```
-This will match the permissions with the **UID** and **GID** of elasticsearch within the container and should fix the issue.
+This will match the permissions with the **UID** and **GID** of elasticsearch process within the container and should fix the issue.
 
 ### Disk usage
 The Elasticsearch index will turn to *read only* if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`, [link](https://github.com/tubearchivist/tubearchivist#disk-usage).  
@@ -149,7 +195,7 @@ Similar to that, TubeArchivist will become all sorts of messed up when running o
 ## Getting Started
 1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS and some other browsers a compatible format must be specified. For example:
 ```
-bestvideo[VCODEC=avc1]+bestaudio[ACODEC=mp4a]/mp4
+bestvideo[vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4
 ```
 2. Subscribe to some of your favorite YouTube channels on the **channels** page. 
 3. On the **downloads** page, click on *Rescan subscriptions* to add videos from the subscribed channels to your Download queue or click on *Add to download queue* to manually add Video IDs, links, channels or playlists.
@@ -170,13 +216,13 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex
 - [ ] Show total video downloaded vs total videos available in channel
 - [ ] Add statistics of index
 - [ ] Download speed schedule ([#198](https://github.com/tubearchivist/tubearchivist/issues/198))
-- [ ] Implement complete offline media file import from json file ([#138](https://github.com/tubearchivist/tubearchivist/issues/138))
-- [ ] Filter and query in search form, search by url query ([#134](https://github.com/tubearchivist/tubearchivist/issues/134), [#139](https://github.com/tubearchivist/tubearchivist/issues/139))
 - [ ] Auto ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163))
 - [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144))
 - [ ] Download video comments
 
 Implemented:
+- [X] Implement complete offline media file import from json file [2022-08-20]
+- [X] Filter and query in search form, search by url query [2022-07-23]
 - [X] Make items in grid row configurable to use more of the screen [2022-06-04]
 - [X] Add passing browser cookies to yt-dlp [2022-05-08]
 - [X] Add [SponsorBlock](https://sponsor.ajay.app/) integration [2022-04-16]
@@ -199,13 +245,22 @@ Implemented:
 ## Known limitations
 - Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
 - Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
-- For now this is meant to be run in a trusted network environment. Not everything is properly authenticated.
 - There is currently no flexibility in naming of the media files.
 
 
 ## Donate
 The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.  
 Second best way to support the development is to provide for caffeinated beverages:
+* [GitHub Sponsor](https://github.com/sponsors/bbilly1) become a sponsor here on GitHub
 * [Paypal.me](https://paypal.me/bbilly1) for a one time coffee
 * [Paypal Subscription](https://www.paypal.com/webapps/billing/plans/subscribe?plan_id=P-03770005GR991451KMFGVPMQ) for a monthly coffee
 * [ko-fi.com](https://ko-fi.com/bbilly1) for an alternative platform
+
+
+## Sponsor
+Big thank you to [Digitalocean](https://www.digitalocean.com/) for generously donating credit for the tubearchivist.com VPS and buildserver. 
+<p>
+  <a href="https://www.digitalocean.com/">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/PoweredByDO/DO_Powered_by_Badge_blue.svg" width="201px">
+  </a>
+</p>

deploy.sh

@@ -18,11 +18,7 @@ set -e
 
 function sync_blackhole {
 
-    # docker commands need sudo, only build amd64
     host="blackhole.local"
-
-    read -sp 'Password: ' remote_pw
-    export PASS=$remote_pw
     
     rsync -a --progress --delete-after \
         --exclude ".git" \
@@ -32,8 +28,8 @@ function sync_blackhole {
         --exclude "db.sqlite3" \
         . -e ssh "$host":tubearchivist
 
-    echo "$PASS" | ssh "$host" 'sudo -S docker buildx build --platform linux/amd64 -t bbilly1/tubearchivist:latest tubearchivist --load 2>/dev/null'
-    echo "$PASS" | ssh "$host" 'sudo -S docker compose up -d 2>/dev/null'
+    ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist'
+    ssh "$host" 'docker compose up -d'
 
 }
 
@@ -86,10 +82,12 @@ function validate {
 
     echo "run validate on $check_path"
 
+    # note: this logic is duplicated in the `./github/workflows/lint_python.yml` config
+    # if you update this file, you should update that as well
     echo "running black"
     black --diff --color --check -l 79 "$check_path"
     echo "running codespell"
-    codespell --skip="./.git" "$check_path"
+    codespell --skip="./.git,./package.json,./package-lock.json,./node_modules" "$check_path"
     echo "running flake8"
     flake8 "$check_path" --count --max-complexity=10 --max-line-length=79 \
         --show-source --statistics
@@ -197,7 +195,8 @@ function sync_docker_old {
     # start build
     sudo docker buildx build \
         --platform linux/amd64,linux/arm64 \
-        -t bbilly1/tubearchivist:latest \
+        -t bbilly1/tubearchivist \
+        -t bbilly1/tubearchivist:unstable \
         -t bbilly1/tubearchivist:"$VERSION" --push .
 
     # create release tag
@@ -214,9 +213,9 @@ if [[ $1 == "blackhole" ]]; then
 elif [[ $1 == "test" ]]; then
     sync_test "$2"
 elif [[ $1 == "validate" ]]; then
-    # check package versions in requirements.txt for updates
-    python version_check.py
     validate "$2"
+elif [[ $1 == "versioncheck" ]]; then
+    python version_check.py
 elif [[ $1 == "docker" ]]; then
     sync_docker
 elif [[ $1 == "unstable" ]]; then
@@ -224,7 +223,7 @@ elif [[ $1 == "unstable" ]]; then
 elif [[ $1 == "es" ]]; then
     sync_latest_es
 else
-    echo "valid options are: blackhole | test | validate | docker | unstable | es"
+    echo "valid options are: blackhole | test | validate | versioncheck | docker | unstable | es"
 fi
 
 

docker-compose.yml

@@ -15,6 +15,7 @@ services:
       - REDIS_HOST=archivist-redis          # don't add protocol
       - HOST_UID=1000
       - HOST_GID=1000
+      - TA_HOST=tubearchivist.local         # set your host name
       - TA_USERNAME=tubearchivist           # your initial TA credentials
       - TA_PASSWORD=verysecret              # your initial TA credentials
       - ELASTIC_PASSWORD=verysecret         # set password for Elasticsearch
@@ -33,14 +34,15 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 7.17.4
+    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.4.3
     container_name: archivist-es
     restart: unless-stopped
     environment:
-      - "xpack.security.enabled=true"
       - "ELASTIC_PASSWORD=verysecret"       # matching Elasticsearch password
-      - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "xpack.security.enabled=true"
+      - "discovery.type=single-node"
+      - "path.repo=/usr/share/elasticsearch/data/snapshot"
     ulimits:
       memlock:
         soft: -1

docker_assets/nginx.conf

@@ -3,18 +3,22 @@ server {
     listen 8000;
     
     location /cache/videos/ {
+        auth_request /api/ping/;
         alias /cache/videos/;
     }
     
     location /cache/channels/ {
+        auth_request /api/ping/;
         alias /cache/channels/;
     }
     
     location /cache/playlists/ {
+        auth_request /api/ping/;
         alias /cache/playlists/;
     }
     
     location /media/ {
+        auth_request /api/ping/;
         alias /youtube/;
         types {
             text/vtt vtt;

docker_assets/run.sh

@@ -13,6 +13,7 @@ required="Missing required environment variable"
 [[ -f $lockfile ]] || : "${TA_USERNAME:?$required}"
 : "${TA_PASSWORD:?$required}"
 : "${ELASTIC_PASSWORD:?$required}"
+: "${TA_HOST:?$required}"
 
 # ugly nginx and uwsgi port overwrite with env vars
 if [[ -n "$TA_PORT" ]]; then
@@ -32,6 +33,7 @@ until curl -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL" -fs; do
     if [[ $counter -eq 12 ]]; then
         # fail after 2 min
         echo "failed to connect to elastic search, exiting..."
+        curl -v -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL"?pretty
         exit 1
     fi
     sleep 10

docker_assets/uwsgi.ini

@@ -5,4 +5,7 @@ pidfile = /tmp/project-master.pid
 vacuum = True
 max-requests = 5000
 socket = :8080
-buffer-size = 8192
+buffer-size = 8192
+log-5xx = true
+log-4xx = true
+disable-logging = true

docs/Channels.md

@@ -17,16 +17,20 @@ The **Subscribe to Channels** button <img src="assets/icon-add.png?raw=true" alt
 You can search your indexed channels by clicking on the search icon <img src="assets/icon-search.png?raw=true" alt="search icon" width="20px" style="margin:0 5px;">. This will open a dedicated page.
 
 ## Channel Detail
-Each channel will get a dedicated channel detail page accessible at `/channel/<channel-id>/` of your Tube Archivist. This page shows all the videos you have downloaded from this channel plus additional metadata. 
-- If you are subscribed to the channel, an *Unsubscribe* button will show, else the *Subscribe* button will show.
-- You can *Show* the channel description, that matches with the *About* tab on YouTube.
-- The **Mark as Watched** button will mark all videos of this channel as watched.
-- The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata additionally this will also delete playlists metadata belonging to that channel.
-- The button **Show Playlists** will go to the [playlists](Playlists) page and filter the list to only show playlists from this channel.
+Each channel will get a dedicated channel detail page accessible at `/channel/<channel-id>/` of your Tube Archivist. This page shows all the videos you have downloaded from this channel.
 
-### Channel Customize
-Clicking on the *Configure* button will open a form with options to configure settings on a per channel basis. Any configurations here will overwrite your settings from the [settings](Settings) page.
-- **Download Format**: Overwrite the download qualities for videos from this channel.
+- If you are subscribed to the channel, an *Unsubscribe* button will show, else the *Subscribe* button will show.
+- The **Mark as Watched** button will mark all videos of this channel as watched.
+
+Additionally there is a *Channel Playlist* page, accessible at `/channel/<channel-id>/playlist/` to show all indexed playlists from this channel. 
+
+On the *Channel About* page, accessible at `/channel/<channel-id>/about/`, you can see additional metadata.
+- The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata additionally this will also delete playlists metadata belonging to that channel.
+
+The channel customize form gives options to change settings on a per channel basis. Any configurations here will overwrite your configurations from the [settings](Settings) page.
+- **Download Format**: Overwrite the download quality for videos from this channel.
 - **Auto Delete**: Automatically delete watched videos from this channel after selected days.
 - **Index Playlists**: Automatically add all Playlists with at least a video downloaded to your index. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.
 - **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. Customize per channel: You can *disable* or *enable* SponsorBlock for certain channels only to overwrite the behavior set on the [Settings](settings) page. Selecting *unset* will remove the overwrite and your setting will fall back to the default on the settings page.
+
+If you have any videos pending in the download queue, a *Downloads* link will show, bringing you directly to the [downloads](Downloads) page, filtering the list by the selected channel.

docs/Downloads.md

@@ -29,6 +29,8 @@ The **Add to Download Queue** icon <img src="assets/icon-add.png?raw=true" alt="
 ## The Download Queue
 Below the three buttons you find the download queue. New items will get added at the bottom of the queue, the next video to download once you click on **Start Download** will be the first in the list.
 
+You can filter the download queue with the **filter** dropdown box, the filter will show once you have more than one channel in the download queue. Select the channel to filter by name, the number in parentheses indicates how many videos you have pending from this channel. Reset the filter by selecting *all* from the dropdown. This will generate links for the top 30 channels with pending videos.
+
 Every video in the download queue has two buttons:
 - **Ignore**: This will remove that video from the download queue and this video will not get added again, even when you **Rescan Subscriptions**.
 - **Download now**: This will give priority to this video. If the download process is already running, the prioritized video will get downloaded as soon as the current video is finished. If there is no download process running, this will start downloading this single video and stop after that.  

docs/FAQ.md

@@ -15,7 +15,7 @@ Defining the scope is important for the success of any project:
 - A too narrow scope will make this project uninteresting and will exclude audiences that could also benefit from this project.
 - Not defining a scope will easily lead to misunderstandings and false hopes of where this project tries to go.
 
-Of course this is subject to change, as this project continues to grow and more people contribute.
+Of course this is subject to change: The scope can be expanded as this project continues to grow and more people contribute.
 
 ## 2. Emby/Plex/Jellyfin/Kodi integrations
 Although there are similarities between these excellent projects and Tube Archivist, they have a very different use case. Trying to fit the metadata relations and database structure of a YouTube archival project into these media servers that specialize in Movies and TV shows is always going to be limiting.
@@ -30,5 +30,7 @@ So Docker is the only supported installation method. If you don't have any exper
 ## 4. Finetuning Elasticsearch
 A minimal configuration of Elasticsearch (ES) is provided in the example docker-compose.yml file. ES is highly configurable and very interesting to learn more about. Refer to the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) if you want to get into it.
 
-## 5. Advanced Authentication
-If you like to use things like SSO, LDAP or 2FA to login, consider using something like Authelia as a reverse proxy so this project can focus on the core task. Tube Archivist has a *remember me* checkbox at login to extend your sessions lifetime in your browser. 
+## 5. When I subscribe to a channel it only downloads the most recent 50 videos
+Subscribing to a channel is a different operation from downloading it. You can [add the channel to the download queue](https://github.com/tubearchivist/tubearchivist/wiki/Downloads#add-to-download-queue) to download all past videos.
+
+If you want to download the existing videos and also automatically download new videos, then you should both download the channel and subscribe to it.

docs/Home.md

@@ -9,7 +9,8 @@ Table of contents:
 * [Settings](Settings): All the configuration options
 * [Video](Video): All details of a single video and playlist navigation.
 * [Users](Users): User management admin interface
-* [Installation](Installation): WIP - detailed installation instructions for various platforms.
+* [Search](Search): Search your archive
+* [Installation](Installation): Detailed installation instructions for various platforms.
 
 ## Getting Started
 1. [Subscribe](Channels#channels-overview) to some of your favourite YouTube channels.
@@ -29,3 +30,12 @@ Table of contents:
 An empty checkbox icon <img src="assets/icon-unseen.png?raw=true" alt="unseen icon" width="20px" style="margin:0 5px;"> will show for videos you haven't marked as watched. Click on it and the icon will change to a filled checkbox <img src="assets/icon-seen.png?raw=true" alt="seen icon" width="20px" style="margin:0 5px;"> indicating it as watched - click again to revert.
 
 When available the <img src="assets/icon-gridview.png?raw=true" alt="gridview icon" width="20px" style="margin:0 5px;"> gridview icon will display the list in a grid. A grid row holds 3 items by default, use the <img src="assets/icon-add.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> icon to add more or the <img src="assets/icon-substract.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> icon to remove items per row, depending on your screen size. The <img src="assets/icon-listview.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> listview icon will arrange the items in a list. The sort icon <img src="assets/icon-sort.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> will open additional sort options.
+
+You can control the video player with the following keyboard shortcuts:
+- `?`: Show help
+- `m`: toggle mute
+- `c`: toggle subtitles if available
+- `>`: increase playback speed
+- `<`: decrease playback speed
+- `←` (left arrow): jump back 5 seconds
+- `→` (right arrow): jump forward 5 seconds

docs/Installation.md

@@ -1,5 +1,12 @@
 # Detailed Installation Instructions for Various Platforms
 
+## Table of Contents
+- [Unraid](#unraid)
+- [Truenas Scale](#truenas-scale)
+- [Synology](#synology)
+
+These are beginners guides installation instructions for additional platforms generously provided by users of these platforms. When in doubt, verify the details with the [Readme](https://github.com/tubearchivist/tubearchivist#installing-and-updating). If you see any issues here while using these instructions, please contribute. 
+
 ## Unraid
 
 Tube Archivist, and all if it's dependencies are located in the [community applications](https://forums.unraid.net/topic/38582-plug-in-community-applications/) store. The three containers you will need are as follows:
@@ -67,3 +74,182 @@ However, there is a step by step guide available for it's users here:
 https://heavysetup.info/applications/tube-archivist/dataset/
 
 - Ensure you are navigating the columns under `Tube Archivist` on the left hand side of the screen
+
+<br />
+<br />
+
+## Synology
+
+There are several different methods to install TubeArchivist on Synology platforms. This will focus on the available `docker` package and `docker-compose` implementations.
+
+### Prepare Directories/Folders
+Before we setup TubeArchivist, we need to setup the directories/folders. You are assumed to be logged into the Synology NAS.
+#### 1. Docker Base Folder
+   1. Open the `File Station` utility.
+   2. Click on the **Create🔽** button and choose *Create New Shared Folder*.
+   3. **Name** the folder "Docker".
+   4. Add a **Description**.
+   5. Select the **Volume Location**. 
+      > Note: By default, this will be where all data is stored. Change the folders as best meets your requirements.
+   6. Select the appropriate options from the remaining checkbox configurations.
+![Synology - Create Docker Folder](assets/Synology_0.2.0_Docker-Folder-Create.png)
+   7. Click the **Next** button.
+   8. If you are going to **Encrypt** your folder, check the appropriate box and provide the Encryption Key and its confirmation.
+   9. Click the **Next** button.
+   10. On the **Advanced Settings** page, you can select the *Enable data checksum for advanced data integrity* setting. This may cause a performance impact, but will allow for potential file self-healing. **This cannot be changed later.**
+       > Note: This is not recommended, as we will be hosting databases within this folder.
+   11. If you are enabling a quota for how large the folder can get, you can select the *Enabled shared folder quota* setting and choose the maximum size this folder can grow. This can be changed later.
+   12. Click the **Next** button.
+   13. Confirm the settings, then click the **Apply** button. This will create the folder.
+#### 2. TubeArchivist Base Folder
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Click on the `Create🔽` button and choose *create Folder*.
+   4. **Name** the folder "TubeArchivist".
+#### 3. Redis Data
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Select the "TubeArchivist" folder beneath "Docker".
+   4. Click on the `Create🔽` button and choose *create Folder*.
+   5. **Name** the folder "redis".
+#### 4. Elastic Search Data
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Select the "TubeArchivist" folder beneath "Docker".
+   4. Click on the `Create🔽` button and choose *create Folder*.
+   5. **Name** the folder "es".
+#### 5. TubeArchivist Cache
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Select the "TubeArchivist" folder beneath "Docker".
+   4. Click on the `Create🔽` button and choose *create Folder*.
+   5. **Name** the folder "cache".
+#### 6. TubeArchivist Output
+   1. Open the `File Station` utility.
+   2. Select the "Docker" folder on the left-hand side.
+   3. Select the "TubeArchivist" folder beneath "Docker".
+   4. Click on the `Create🔽` button and choose *create Folder*.
+   5. **Name** the folder "media".
+#### 7. Confirm Folder Structure
+Once all of the folders have been created, it should have a folder structure within Docker\TubeArchivist that includes "cache", "es", "media", and "redis" folders.
+![Synology - Docker Folder Structure](assets/Synology_0.2.0_Docker-Folder-Structure.png)
+
+#### 8. Change Permissions - CLI Required
+> If you do not have SSH access enabled for CLI, [enable it](https://kb.synology.com/en-sg/DSM/tutorial/How_to_login_to_DSM_with_root_permission_via_SSH_Telnet) before continuing.
+   1. Open the SSH connection to the Synology. Login as your primary `Admin` user, or the user that was enabled for SSH access.
+   2. Elevate your access to `root`. Steps are provided [here](https://kb.synology.com/en-sg/DSM/tutorial/How_to_login_to_DSM_with_root_permission_via_SSH_Telnet).
+   3. Change directories to the **Volume** where the "Docker" folder resides.
+      </br>Example: `cd /volume1`
+   4. Change directories to the "Docker" folder.
+      </br>Example: `cd Docker`
+   5. Change directories to the "TubeArchivist" folder.
+      </br>Example: `cd TubeArchivist`
+   6. Change the owner of the "redis" folder. *If correct, this does not have an output.*
+      </br>Example: `chown 999:100 redis`
+   7. Change the owner of the "es" folder. *If correct, this does not have an output.*
+      </br>Example: `chown 1000:1000 es`
+   8. Confirm that the folders have the correct permissions.
+      </br>Example: `ls -hl`
+![Synology - Docker Folder Permissions Command](assets/Synology_0.2.0_Docker-Folder-Permissions-Commands.png)
+   9. Logout from root.
+      </br>Example: `logout`
+   10. Disconnect from the SSH connection.
+      </br>Example: `exit`
+### Docker Setup
+1. Install the `Docker` Synology Package.
+  1. Log in to your Synology NAS.
+  2. Open the `Package Center` utility.
+  3. Search for `Docker`.
+  4. Click `Install`.
+
+![Synology - Install Docker Utility](assets/Synology_0.2.0_Docker-Install.png)
+
+2. After `Docker` is installed, open the `Docker` utility.
+3. Go to the `Registry` tab.
+4. Search for the following `images` and download them. Follow the recommended versions for each of the images.
+  - `redislabs/rejson`
+![Synology - Redis Image Search](assets/Synology_0.2.0_Docker-Redis-Search.png)
+  - `bbilly1/tubearchivist-es`
+![Synology - ElasticSearch Image Search](assets/Synology_0.2.0_Docker-ES-Search.png)
+  - `bbilly1/tubearchivist`
+![Synology - TubeArchivist Image Search](assets/Synology_0.2.0_Docker-TA-Search.png)
+
+
+
+5. Go to the `Image` tab. From here, create an container based on each image with the associated configurations below.
+  - ElasticSearch
+    1. Select the associated image.
+    2. Click the **Launch** button in the top.
+    3. Edit the **Container Name** to be "tubearchivist-es".
+    4. Click on the **Advanced Settings** button.
+    5. In the **Advanced Settings** tab, check the box for `Enable auto-restart`.
+    6. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/es" folder, then type in `/usr/share/elasticsearch/data` for the mount path.
+    7. In the **Network** tab, leave the default `bridge` Network (unless you have a specific Network design that you know how to implement).
+    8. In the **Port Settings** tab, replace the "Auto" entry under **Local Port** with the port that will be used to connect to ElasticSearch (default is 9200).
+    9. In the **Port Settings** tab, select the entryline for port 9300 and **➖ delete** the line. It is not needed for this container.
+    10. The **Links** tab does not require configuration for this container.
+    11. In the **Environment** tab, add in the following ElasticSearch specific environment variables that may apply.
+        - "discovery.type=single-node"
+        - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+        - "UID=1000"
+        - "GID=0"
+        - "xpack.security.enabled=true"
+        - "ELASTIC_PASSWORD=verysecret" 
+          > Do not use the default password as it is very insecure.
+        ![Synology - ElasticSearch Environment Configurations](assets/Synology_0.2.0_Docker-ES-Env-Conf.png)
+    12. Click on the **Apply** button.
+    13. Back on the **Create Container** screen, click the **Next** button.
+    14. Review the settings to confirm, then click the **Apply** button.
+  - Redis
+    1. Select the associated image.
+    2. Click the **Launch** button in the top.
+    3. Edit the **Container Name** to be "tubearchivist-redis".
+    4. Click on the **Advanced Settings** button.
+    5. In the **Advanced Settings** tab, check the box for `Enable auto-restart`.
+    6. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/redis" folder, then type in `/data` for the mount path.
+    7. In the **Network** tab, leave the default `bridge` Network (unless you have a specific Network design that you know how to implement).
+    8. In the **Port Settings** tab, replace the "Auto" entry under **Local Port** with the port that will be used to connect to Redis (default is 6379).
+    9. In the **Links** tab, select the "tubearchivist-es" container from the **Container Name** dropdown and provide it the same alias, "tubearchivist-es".
+    10. In the **Environment** tab, add in any Redis specific environment variables that may apply (none by default).
+    11. Click on the **Apply** button.
+    12. Back on the **Create Container** screen, click the **Next** button.
+    13. Review the settings to confirm, then click the **Apply** button.
+  
+  - TubeArchivist
+    1. Select the associated image.
+    2. Click the **Launch** button in the top.
+    3. Edit the **Container Name** to be "tubearchivist".
+    4. Click on the **Advanced Settings** button.
+    5. In the **Advanced Settings** tab, check the box for `Enable auto-restart`.
+    6. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/cache" folder, then type in `/cache` for the mount path.
+    7. In the **Volume** tab, click the **Add Folder** button and select the "Docker/TubeArchivist/media" folder, then type in `/youtube` for the mount path.
+    8. In the **Network** tab, leave the default `bridge` Network (unless you have a specific Network design that you know how to implement).
+    9. In the **Port Settings** tab, replace the "Auto" entry under **Local Port** with the port that will be used to connect to TubeArchivist (default is 8000).
+    10. In the **Links** tab, select the "tubearchivist-es" container from the **Container Name** dropdown and provide it the same alias, "tubearchivist-es".
+    11. In the **Links** tab, select the "tubearchivist-redis" container from the **Container Name** dropdown and provide it the same alias, "tubearchivist-redis".
+    12. In the **Environment** tab, add in the following TubeArchivist specific environment variables that may apply. **Change the variables as-is appropriate to your use case. Follow the [README section](https://github.com/tubearchivist/tubearchivist#tube-archivist) for details on what to set each variable.**
+        - "TA_HOST=synology.local"
+        - "ES_URL=http://tubearchivist-es:9200"
+        - "REDIS_HOST=tubearchivist-redis"
+        - "HOST_UID=1000"
+        - "HOST_GID=1000"
+        - "TA_USERNAME=tubearchivist"
+        - "TA_PASSWORD=verysecret"
+        - "ELASTIC_PASSWORD=verysecret"
+        - "TZ=America/New_York"
+          > Do not use the default password as it is very insecure.
+          > Ensure that ELASTIC_PASSWORD matches the password used on the tubearchivist-es container.
+        ![Synology - TubeArchivist Environment Configurations](assets/Synology_0.2.0_Docker-TA-Env-Conf.png)
+    13. Click on the **Apply** button.
+    14. Back on the **Create Container** screen, click the **Next** button.
+    15. Review the settings to confirm, then click the **Apply** button.
+6. After the containers have been configured and started, you can go to the **Container** tab and monitor the containers.
+7. To review the logs to ensure that the system has started successfully, select the "tubearchivist" container and click on the **Details** button. In the new window, go to the **Log** tab. Monitor the logs until either an error occurs or the message `celery@tubearchivist ready.` is in the logs. This may take a few minutes, especially for a first time setup.
+   > Note: Synology Docker presents the logs in a pagination format. If you are not seeing the logs update, check if there are additional pages.
+8. After it has started, go to the location in the `TA_HOST`. This should give you the standard TubeArchivist login screen.
+<!-- 
+### Docker-Compose Setup -->
+<!-- This section is a Work In Progress -->
+
+### From there, you should be able to start up your containers and you're good to go!
+If you're still having trouble, join us on [discord](https://discord.gg/AFwz8nE7BK) and come to the #synology channel.

docs/Search.md

@@ -0,0 +1,57 @@
+# Search Page
+Accessible at `/search/` of your **Tube Archivist**, search your archive for Videos, Channels and Playlists - or even full text search throughout your indexed subtitles.
+
+- All your queries are case insensitive and are normalized to lowercase.
+- All your queries are analyzed for the english language, this means *singular*, *plural* and word variations like *-ing*, *-ed*, *-able* etc are treated as synonyms.
+- Fuzzy search is activated for all your searches by default. This can catch typos in your queries or in the matching documents with one to two letters difference, depending on the query length. You can configure fuzziness with the secondary keyword `fuzzy:`, e.g:
+  - `fuzzy:0` or `fuzzy:no`: Deactivate fuzzy matching.
+  - `fuzzy:1`: Set fuzziness to one letter difference.
+  - `fuzzy:2`: Set fuzziness to two letters difference.
+- All text searches are ranked, meaning the better a match the higher ranked the result. Unless otherwise stated, queries with multiple words are processed with the `and` operator, meaning all words need to match so each word will narrow down the result.
+- This will return 30 results per query, pagination is not implemented yet.
+
+Just start typing to start a *simple* search or start your query with a primary keyword to search for a specific type and narrow down the result with secondary keywords. Secondary keywords can be in any order. Use *yes* or *no* for boolean values.
+
+## Simple
+Start your query without a keyword to make a simple query. This will search in *video titles*, *channel names* and *playlist titles* and will return matching videos, channels and playlists. Keyword searches will return more results in a particular category due to the fact that more fields are searched for matches.
+
+## Video
+Start your query with the primary keyword `video:` to search for videos only. This will search through the *video titles*, *tags* and *category* fields. Narrow your search down with secondary keywords:
+- `channel:` search for videos matching the channel name.
+- `active:` is a boolean value, to search for videos that are still active on youtube or that are not active any more.
+
+**Example**:
+- `video:learn python channel:corey shafer active:yes`: This will return all videos with the term *Learn Python* from the channel *Corey Shafer* that are still *Active* on YouTube.
+- `video: channel:tom scott active:no`: Note the omitted term after the primary key, this will show all videos from the channel *Tom Scott* that are no longer active on YouTube.
+
+## Channel
+Start with the `channel:` primary keyword to search for channels matching your query. This will search through the *channel name* and *channel description* fields. Narrow your search down with secondary keywords:
+- `subscribed:` is a boolean value, search for channels that you are subscribed to or not.
+- `active:` is a boolean value, to search for channels that are still active on YouTube or that are no longer active.
+
+**Example**:
+- `channel:linux subscribed:yes`: Search for channels with the term *Linux* that you are subscribed to.
+- `channel: active:no`: Note the omitted term after the primary key, this will return all channels that are no longer active on YouTube.
+
+## Playlist
+Start your query with the primary keyword `playlist:` to search for playlists only. This will search through the *playlist title* and *playlist description* fields. Narrow down your search with these secondary keywords:
+- `subscribed`: is a boolean value, search for playlists that you are subscribed to or not.
+- `active:` is a boolean value, to search for playlists that are still active on YouTube or that are no longer active.
+
+**Example**:
+- `playlist:backend engineering subscribed:yes`: Search for playlists about *Backend Engineering* that you are subscribed to.
+- `playlist: active:yes subscribed:yes`: Note the omitted primary search term, this will return all playlists active on YouTube that you are subscribed to.
+- `playlist:html css active:yes`: Search for playlists containing *HTML CSS* that are still active on YouTube.
+
+## Full
+Start a full text search by beginning your query with the primary keyword `full:`. This will search through your indexed Subtitles showing segments with possible matches. This will only show any results if you have activated *subtitle download and index* on the settings page. The operator for full text searches is `or` meaning when searching for multiple words not all words need to match, but additional words will change the ranking of the result, the more words match and the better they match, the higher ranked the result. The matching words will get highlighted in the text preview.
+
+Clicking the play button on the thumbnail will open the inplace player at the timestamp from where the segment starts. Same when clicking the video title, this will open the video page and put the player at the segment timestamp. This will overwrite any previous playback position.
+
+Narrow down your search with these secondary keywords:
+- `lang`: Search for matches only within a language. Use the same two letter ISO country code as you have set on the settings page.
+- `source`: Can either be *auto* to search through auto generated subtitles only or *user* to search through user uploaded subtitles only.
+
+**Example**:
+- `full:contribute to open source lang:en` search for subtitle segments matching with the words *Contribute to Open Source* in the language *en*.
+- `full:flight simulator cockpit source:user` to search for the words *Flight Simulator Cockpit* from *user* uploaded subtitle segments.

docs/Settings.md

@@ -63,17 +63,30 @@ All third party integrations of TubeArchivist will **always** be *opt in*.
 - **API**: Your access token for the Tube Archivist API.
 - **returnyoutubedislike.com**: This will get return dislikes and average ratings for each video by integrating with the API from [returnyoutubedislike.com](https://www.returnyoutubedislike.com/).
 - **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. If a video doesn't have timestamps, or has unlocked timestamps, use the browser addon to contribute to this excellent project. Can also be activated and deactivated as a per [channel overwrite](Settings#channel-customize).
-- **Cast**: Enabling the cast integration in the settings page will load an additional JS library from **Google**.
-    * Requirements
-        - HTTPS
-            * To use the cast integration HTTPS needs to be enabled, which can be done using a reverse proxy. This is a requirement by Google as communication to the cast device is required to be encrypted, but the content itself is not.
-        - Supported Browser
-            * A supported browser is required for this integration such as Google Chrome. Other browsers, especially Chromium-based browsers, may support casting by enabling it in the settings.
-        - Subtitles
-            * Subtitles are supported however they do not work out of the box and require additional configuration. Due to requirements by Google, to use subtitles you need additional headers which will need to be configured in your reverse proxy. See this [page](https://developers.google.com/cast/docs/web_sender/advanced#cors_requirements) for the specific requirements.
-               > You need the following headers: Content-Type, Accept-Encoding, and Range. Note that the last two headers, Accept-Encoding and Range, are additional headers that you may not have needed previously.
-               > Wildcards "*" cannot be used for the Access-Control-Allow-Origin header. If the page has protected media content, it must use a domain instead of a wildcard.
+- **Cast**: Enabling the cast integration in the settings page will load an additional JS library from **Google**.  
+*NOTE*: This feature is currently broken due to an authentication issue, see #331.
+Requirements:
+    - HTTPS: To use the cast integration HTTPS needs to be enabled, which can be done using a reverse proxy. This is a requirement by Google as communication to the cast device is required to be encrypted, but the content itself is not.
+    - Supported Browser:A supported browser is required for this integration such as Google Chrome. Other browsers, especially Chromium-based browsers, may support casting by enabling it in the settings.
+    - Subtitles: Subtitles are supported however they do not work out of the box and require additional configuration. Due to requirements by Google, to use subtitles you need additional headers which will need to be configured in your reverse proxy. See this [page](https://developers.google.com/cast/docs/web_sender/advanced#cors_requirements) for the specific requirements.  
+    You need the following headers: Content-Type, Accept-Encoding, and Range. Note that the last two headers, Accept-Encoding and Range, are additional headers that you may not have needed previously.  
+    Wildcards "*" cannot be used for the Access-Control-Allow-Origin header. If the page has protected media content, it must use a domain instead of a wildcard.
 
+## Snapshots
+System snapshots will automatically make daily snapshots of the Elasticsearch index. The task will start at 12pm your local time. Snapshots are deduplicated, meaning that each snapshot will only have to backup changes since the last snapshot. The initial snapshot may be slow, but subsequent runs will be much faster. There is also a cleanup function implemented, that will remove snapshots older than 30 days.
+
+This will make a snapshot of your metadata index only, no media files or additional configuration variables you have set on the settings page will be backed up.
+
+Due to these improvements compared to the previous backup solution, system snapshots will replace the current backup system in a future version.
+
+Before activating system snapshots, you'll have to add one additional environment variables to the *archivist-es* container:
+```
+path.repo=/usr/share/elasticsearch/data/snapshot
+```
+The variable `path.repo` will set the folder where the snapshots will go inside the Elasticsearch container, you can't change it, but the variable needs to be set. Rebuild the container for changes to take effect, e.g `docker compose up -d`.
+
+- **Create snapshot now**: Will start the snapshot process now, outside of the regular daily schedule.
+- **Restore**: Restore your index to that point in time.
 
 # Scheduler Setup
 Schedule settings expect a cron like format, where the first value is minute, second is hour and third is day of the week. Day 0 is Sunday, day 1 is Monday etc.
@@ -116,25 +129,34 @@ Additional database functionality.
 The button **Delete all queued** will delete all pending videos from the download queue. The button **Delete all ignored** will delete all videos you have previously ignored.
 
 ## Manual Media Files Import
-So far this depends on the video you are trying to import to be still available on YouTube to get the metadata. Add the files you'd like to import to the */cache/import* folder. Then start the process from the settings page *Manual Media Files Import*. Make sure to follow one of the two methods below.
+NOTE: This is inherently error prone, as there are many variables, some outside of the control of this project. Read this carefully and use at your own risk. 
+
+Add the files you'd like to import to the */cache/import* folder. Only add files, don't add subdirectories. All files you are adding, need to have the same *base name* as the media file. Then start the process from the settings page *Manual Media Files Import*.
+
+Valid media extensions are *.mp4*, *.mkv* or *.webm*. If you have other file extensions or incompatible codecs, convert them first to mp4. **Tube Archivist** can identify the videos with one of the following methods.
 
 ### Method 1:
-Add a matching *.json* file with the media file. Both files need to have the same base name, for example:
-- For the media file: \<base-name>.mp4
-- For the JSON file: \<base-name>.info.json
-- Alternate JSON file: \<base-name>.json
+Add a matching *.info.json* file with the media file. Both files need to have the same base name, for example:
+- For the media file: `<base-name>.mp4`
+- For the JSON file: `<base-name>.info.json`
 
-**Tube Archivist** then looks for the 'id' key within the JSON file to identify the video.
+The import process then looks for the 'id' key within the JSON file to identify the video.
 
 ### Method 2:
 Detect the YouTube ID from filename, this accepts the default yt-dlp naming convention for file names like:
-- \<base-name>[\<youtube-id>].mp4
+- `<base-name>[<youtube-id>].mp4`
 - The YouTube ID in square brackets at the end of the filename is the crucial part.
 
+### Offline import:
+If the video you are trying to import is not available on YouTube any more, **Tube Archivist** can import the required metadata:
+- The file `<base-name>.info.json` is required to extract the required information.
+- Add the thumbnail as `<base-name>.<ext>`, where valid file extensions are *.jpg*, *.png* or *.webp*. If there is no thumbnail file, **Tube Archivist** will try to extract the embedded cover from the media file or will fallback to a default thumbnail.
+- Add subtitles as `<base-name>.<lang>.vtt` where *lang* is the two letter ISO country code. This will archive all subtitle files you add to the import folder, independent from your configurations. Subtitles can be archived and used in the player, but they can't be indexed or made searchable due to the fact, that they have a very different structure than the subtitles as **Tube Archivist** needs them.
+- For videos, where the whole channel is not available any more, you can add the `<channel-id>.info.json` file as generated by *youtube-dl/yt-dlp* to get the full metadata. Alternatively **Tube Archivist** will extract as much info as possible from the video info.json file. 
+
 ### Some notes:
 - This will **consume** the files you put into the import folder: Files will get converted to mp4 if needed (this might take a long time...) and moved to the archive, *.json* files will get deleted upon completion to avoid having duplicates on the next run.
-- For best file transcoding quality, convert your media files with desired settings first before importing (#138).
-- There should be no subdirectories added to */cache/import*, only video files. If your existing video library has video files inside subdirectories, you can get all the files into one directory by running `find ./ -mindepth 2 -type f -exec mv '{}' . \;` from the top-level directory of your existing video library. You can also delete any remaining empty subdirectories with `find ./ -mindepth 1 -type d -delete`.
+- For best file transcoding quality, convert your media files with desired settings first before importing.
 - Maybe start with a subset of your files to import to make sure everything goes well...
 - Follow the logs to monitor progress and errors: `docker-compose logs -f tubearchivist`.
 
@@ -142,7 +164,7 @@ Detect the YouTube ID from filename, this accepts the default yt-dlp naming conv
 This will write or overwrite all thumbnails in the media file using the downloaded thumbnail. This is only necessary if you didn't download the files with the option *Embed Thumbnail* enabled or want to make sure all media files get the newest thumbnail. Follow the docker-compose logs to monitor progress.
 
 ## Backup Database
-This will backup your metadata into a zip file. The file will get stored at *cache/backup* and will contain the necessary files to restore the Elasticsearch index formatted **nd-json** files plus a complete export of the index in a set of conventional **json** files.  
+This will backup your metadata into a zip file. The file will get stored at *cache/backup* and will contain the necessary files to restore the Elasticsearch index formatted **nd-json** files.  
 
 BE AWARE: This will **not** backup any media files, just the metadata from the Elasticsearch.
 

package.json

@@ -0,0 +1,17 @@
+{
+  "private": true,
+  "scripts": {
+    "lint": "eslint 'tubearchivist/static/**/*.js'",
+    "format": "prettier --write 'tubearchivist/static/**/*.js'"
+  },
+  "devDependencies": {
+    "eslint": "^8.26.0",
+    "prettier": "^2.7.1",
+    "eslint-config-prettier": "^8.5.0"
+  },
+  "prettier": {
+    "singleQuote": true,
+    "arrowParens": "avoid",
+    "printWidth": 100
+  }
+}

tubearchivist/api/README.md

@@ -5,6 +5,41 @@ Note:
 - This is very early stages and will change!
 - Check the commit history to see if a documented feature is already in your release
 
+## Table of contents
+- [Authentication](#authentication)
+- [Pagination](#pagination)
+
+**Video**
+- [Video List](#video-list-view)
+- [Video Single](#video-item-view)
+- [Video Single Progress](#video-progress-view)
+- [Video Single Sponsorblock](#sponsor-block-view) WIP
+
+**Channel**
+- [Channel List](#channel-list-view)
+- [Channel Single](#channel-item-view)
+- [Channel Video List](#channel-videos-view)
+
+**Playlist**
+- [Playlist List](#playlist-list-view)
+- [Playlist Single](#playlist-item-view)
+- [Playlist Videos List](#playlist-videos-view)
+
+**Download queue**
+- [Download Queue List](#download-queue-list-view)
+- [Download Queue Single](#download-queue-item-view)
+
+**Snapshot management**
+- [Snapshot List](#snapshot-list-view)
+- [Snapshot Single](#snapshot-item-view)
+
+**Additional**
+- [Login](#login-view)
+- [Task](#task-view) WIP
+- [Cookie](#cookie-view)
+- [Search](#search-view)
+- [Ping](#ping-view)
+
 ## Authentication
 API token will get automatically created, accessible on the settings page. Token needs to be passed as an authorization header with every request. Additionally session based authentication is enabled too: When you are logged into your TubeArchivist instance, you'll have access to the api in the browser for testing.
 
@@ -25,35 +60,18 @@ response = requests.get(url, headers=headers)
 
 ## Pagination
 The list views return a paginate object with the following keys:
-- page_size: int current page size set in config
-- page_from: int first result idx
-- prev_pages: array of ints of previous pages, if available
-- current_page: int current page from query
-- max_hits: reached: bool if max of 10k results is reached
-- last_page: int of last page link
-- next_pages: array of ints of next pages
-- total_hits: int total results
+- page_size: *int* current page size set in config
+- page_from: *int* first result idx
+- prev_pages: *array of ints* of previous pages, if available
+- current_page: *int* current page from query
+- max_hits: *bool* if max of 10k results is reached
+- params: *str* additional url encoded query parameters
+- last_page: *int* of last page link
+- next_pages: *array of ints* of next pages
+- total_hits: *int* total results
 
 Pass page number as a query parameter: `page=2`. Defaults to *0*, `page=1` is redundant and falls back to *0*. If a page query doesn't return any results, you'll get `HTTP 404 Not Found`.
 
-## Login View
-Return token and user ID for username and password:  
-POST /api/login
-```json
-{
-    "username": "tubearchivist",
-    "password": "verysecret"
-}
-```
-
-after successful login returns 
-```json
-{
-    "token": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
-    "user_id": 1
-}
-```
-
 ## Video List View
 /api/video/
 
@@ -145,7 +163,7 @@ POST /api/channel/
 ## Playlist List View
 /api/playlist/
 
-## Playlists Item View
+## Playlist Item View
 /api/playlist/\<playlist_id>/
 
 ## Playlist Videos View
@@ -156,6 +174,7 @@ GET /api/download/
 
 Parameter:
 - filter: pending, ignore
+- channel: channel-id
 
 ### Add list of videos to download queue
 POST /api/download/
@@ -192,15 +211,63 @@ Add to queue previously ignored video:
 DELETE /api/download/\<video_id>/  
 Forget or delete from download queue
 
-## Ping View
-Validate your connection with the API  
-GET /api/ping
+## Snapshot List View
+GET /api/snapshot/  
+Return snapshot config and a list of available snapshots.
 
-When valid returns message with user id: 
 ```json
 {
-    "response": "pong",
-    "user": 1
+    "next_exec": epoch,
+    "next_exec_str": "date_str",
+    "expire_after": "30d",
+    "snapshots": []
+}
+```
+
+POST /api/snapshot/  
+Create new snapshot now, will return immediately, task will run async in the background, will return snapshot name: 
+```json
+{
+    "snapshot_name": "ta_daily_<random-id>
+}
+```
+
+## Snapshot Item View
+GET /api/snapshot/\<snapshot-id>/  
+Return metadata of a single snapshot
+```json
+{
+    "id": "ta_daily_<random-id>,
+    "state": "SUCCESS",
+    "es_version": "0.0.0",
+    "start_date": "date_str",
+    "end_date": "date_str",
+    "end_stamp": epoch,
+    "duration_s": 0
+}
+```
+
+GET /api/snapshot/\<snapshot-id>/  
+Restore this snapshot
+
+DELETE /api/snapshot/\<snapshot-id>/  
+Remove this snapshot from index
+
+## Login View
+Return token and user ID for username and password:  
+POST /api/login
+```json
+{
+    "username": "tubearchivist",
+    "password": "verysecret"
+}
+```
+
+after successful login returns 
+```json
+{
+    "token": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+    "user_id": 1
 }
 ```
 
@@ -273,3 +340,20 @@ Or returns status code 400 on failure:
     "cookie_validated": false
 }
 ```
+
+## Search View
+GET /api/search/?query=\<query>
+
+Returns search results from your query.
+
+## Ping View
+Validate your connection with the API  
+GET /api/ping
+
+When valid returns message with user id: 
+```json
+{
+    "response": "pong",
+    "user": 1
+}
+```

tubearchivist/api/src/search_processor.py

@@ -22,7 +22,7 @@ class SearchProcess:
         self.processed = False
 
     def process(self):
-        """dedect type and process"""
+        """detect type and process"""
         if "_source" in self.response.keys():
             # single
             self.processed = self._process_result(self.response)
@@ -37,7 +37,7 @@ class SearchProcess:
         return self.processed
 
     def _process_result(self, result):
-        """dedect which type of data to process"""
+        """detect which type of data to process"""
         index = result["_index"]
         processed = False
         if index == "ta_video":
@@ -74,7 +74,7 @@ class SearchProcess:
         media_url = urllib.parse.quote(video_dict["media_url"])
         vid_last_refresh = date_praser(video_dict["vid_last_refresh"])
         published = date_praser(video_dict["published"])
-        vid_thumb_url = ThumbManager().vid_thumb_path(video_id)
+        vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
         channel = self._process_channel(video_dict["channel"])
 
         if "subtitles" in video_dict:
@@ -113,7 +113,7 @@ class SearchProcess:
     def _process_download(self, download_dict):
         """run on single download item"""
         video_id = download_dict["youtube_id"]
-        vid_thumb_url = ThumbManager().vid_thumb_path(video_id)
+        vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
         published = date_praser(download_dict["published"])
 
         download_dict.update(

tubearchivist/api/urls.py

@@ -12,6 +12,9 @@ from api.views import (
     PlaylistApiListView,
     PlaylistApiVideoView,
     PlaylistApiView,
+    SearchView,
+    SnapshotApiListView,
+    SnapshotApiView,
     TaskApiView,
     VideoApiListView,
     VideoApiView,
@@ -88,9 +91,24 @@ urlpatterns = [
         TaskApiView.as_view(),
         name="api-task",
     ),
+    path(
+        "snapshot/",
+        SnapshotApiListView.as_view(),
+        name="api-snapshot-list",
+    ),
+    path(
+        "snapshot/<slug:snapshot_id>/",
+        SnapshotApiView.as_view(),
+        name="api-snapshot",
+    ),
     path(
         "cookie/",
         CookieView.as_view(),
         name="api-cookie",
     ),
+    path(
+        "search/",
+        SearchView.as_view(),
+        name="api-search",
+    ),
 ]

tubearchivist/api/views.py

@@ -5,6 +5,8 @@ from api.src.task_processor import TaskHandler
 from home.src.download.queue import PendingInteract
 from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.connect import ElasticWrap
+from home.src.es.snapshot import ElasticSnapshot
+from home.src.frontend.searching import SearchForm
 from home.src.index.generic import Pagination
 from home.src.index.video import SponsorBlock
 from home.src.ta.config import AppConfig
@@ -52,9 +54,7 @@ class ApiBaseView(APIView):
 
     def initiate_pagination(self, request):
         """set initial pagination values"""
-        user_id = request.user.id
-        page_get = int(request.GET.get("page", 0))
-        self.pagination_handler = Pagination(page_get, user_id)
+        self.pagination_handler = Pagination(request)
         self.data.update(
             {
                 "size": self.pagination_handler.pagination["page_size"],
@@ -367,13 +367,23 @@ class DownloadApiListView(ApiBaseView):
         """get request"""
         query_filter = request.GET.get("filter", False)
         self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
+
+        must_list = []
         if query_filter:
             if query_filter not in self.valid_filter:
                 message = f"invalid url query filder: {query_filter}"
                 print(message)
                 return Response({"message": message}, status=400)
 
-            self.data["query"] = {"term": {"status": {"value": query_filter}}}
+            must_list.append({"term": {"status": {"value": query_filter}}})
+
+        filter_channel = request.GET.get("channel", False)
+        if filter_channel:
+            must_list.append(
+                {"term": {"channel_id": {"value": filter_channel}}}
+            )
+
+        self.data["query"] = {"bool": {"must": must_list}}
 
         self.get_document_list(request)
         return Response(self.response)
@@ -476,6 +486,70 @@ class TaskApiView(ApiBaseView):
         return Response(response)
 
 
+class SnapshotApiListView(ApiBaseView):
+    """resolves to /api/snapshot/
+    GET: returns snashot config plus list of existing snapshots
+    POST: take snapshot now
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        snapshots = ElasticSnapshot().get_snapshot_stats()
+
+        return Response(snapshots)
+
+    @staticmethod
+    def post(request):
+        """take snapshot now with post request"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().take_snapshot_now()
+
+        return Response(response)
+
+
+class SnapshotApiView(ApiBaseView):
+    """resolves to /api/snapshot/<snapshot-id>/
+    GET: return a single snapshot
+    POST: restore snapshot
+    DELETE: delete a snapshot
+    """
+
+    @staticmethod
+    def get(request, snapshot_id):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
+
+        if not snapshot:
+            return Response({"message": "snapshot not found"}, status=404)
+
+        return Response(snapshot)
+
+    @staticmethod
+    def post(request, snapshot_id):
+        """restore snapshot with post request"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().restore_all(snapshot_id)
+        if not response:
+            message = {"message": "failed to restore snapshot"}
+            return Response(message, status=400)
+
+        return Response(response)
+
+    @staticmethod
+    def delete(request, snapshot_id):
+        """delete snapshot from index"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
+        if not response:
+            message = {"message": "failed to delete snapshot"}
+            return Response(message, status=400)
+
+        return Response(response)
+
+
 class CookieView(ApiBaseView):
     """resolves to /api/cookie/
     GET: check if cookie is enabled
@@ -526,3 +600,22 @@ class CookieView(ApiBaseView):
 
         message = {"cookie_import": "done", "cookie_validated": validated}
         return Response(message)
+
+
+class SearchView(ApiBaseView):
+    """resolves to /api/search/
+    GET: run a search with the string in the ?query parameter
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request
+        search through all indexes"""
+        search_query = request.GET.get("query", None)
+        if search_query is None:
+            return Response(
+                {"message": "no search query specified"}, status=400
+            )
+        print("searching for: " + search_query)
+        search_results = SearchForm().multi_search(search_query)
+        return Response(search_results)

tubearchivist/config/settings.py

@@ -14,7 +14,9 @@ import hashlib
 from os import environ, path
 from pathlib import Path
 
+import ldap
 from corsheaders.defaults import default_headers
+from django_auth_ldap.config import LDAPSearch
 from home.src.ta.config import AppConfig
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -30,7 +32,14 @@ SECRET_KEY = PW_HASH.hexdigest()
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = bool(environ.get("DJANGO_DEBUG"))
 
-ALLOWED_HOSTS = ["*"]
+ALLOWED_HOSTS = [i.strip() for i in environ.get("TA_HOST").split()]
+
+CSRF_TRUSTED_ORIGINS = []
+for host in ALLOWED_HOSTS:
+    if host.startswith("http://") or host.startswith("https://"):
+        CSRF_TRUSTED_ORIGINS.append(host)
+    else:
+        CSRF_TRUSTED_ORIGINS.append(f"http://{host}")
 
 
 # Application definition
@@ -83,6 +92,96 @@ TEMPLATES = [
 
 WSGI_APPLICATION = "config.wsgi.application"
 
+if bool(environ.get("TA_LDAP")):
+    # pylint: disable=global-at-module-level
+    global AUTH_LDAP_SERVER_URI
+    AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
+
+    global AUTH_LDAP_BIND_DN
+    AUTH_LDAP_BIND_DN = environ.get("TA_LDAP_BIND_DN")
+
+    global AUTH_LDAP_BIND_PASSWORD
+    AUTH_LDAP_BIND_PASSWORD = environ.get("TA_LDAP_BIND_PASSWORD")
+
+    """
+    Since these are new environment variables, taking the opporunity to use
+    more accurate env names.
+    Given Names are *_technically_* different from Personal names, as people
+    who change their names have different given names and personal names,
+    and they go by personal names. Additionally, "LastName" is actually
+    incorrect for many cultures, such as Korea, where the
+    family name comes first, and the personal name comes last.
+
+    But we all know people are going to try to guess at these, so still want
+    to include names that people will guess, hence using first/last as well.
+    """
+
+    # Attribute mapping options
+
+    global AUTH_LDAP_USER_ATTR_MAP_USERNAME
+    AUTH_LDAP_USER_ATTR_MAP_USERNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_USERNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_UID")
+        or "uid"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME
+    AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_PERSONALNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_FIRSTNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_GIVENNAME")
+        or "givenName"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_SURNAME
+    AUTH_LDAP_USER_ATTR_MAP_SURNAME = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_SURNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_LASTNAME")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_FAMILYNAME")
+        or "sn"
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP_EMAIL
+    AUTH_LDAP_USER_ATTR_MAP_EMAIL = (
+        environ.get("TA_LDAP_USER_ATTR_MAP_EMAIL")
+        or environ.get("TA_LDAP_USER_ATTR_MAP_MAIL")
+        or "mail"
+    )
+
+    global AUTH_LDAP_USER_BASE
+    AUTH_LDAP_USER_BASE = environ.get("TA_LDAP_USER_BASE")
+
+    global AUTH_LDAP_USER_FILTER
+    AUTH_LDAP_USER_FILTER = environ.get("TA_LDAP_USER_FILTER")
+
+    global AUTH_LDAP_USER_SEARCH
+    # pylint: disable=no-member
+    AUTH_LDAP_USER_SEARCH = LDAPSearch(
+        AUTH_LDAP_USER_BASE,
+        ldap.SCOPE_SUBTREE,
+        "(&("
+        + AUTH_LDAP_USER_ATTR_MAP_USERNAME
+        + "=%(user)s)"
+        + AUTH_LDAP_USER_FILTER
+        + ")",
+    )
+
+    global AUTH_LDAP_USER_ATTR_MAP
+    AUTH_LDAP_USER_ATTR_MAP = {
+        "username": AUTH_LDAP_USER_ATTR_MAP_USERNAME,
+        "first_name": AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME,
+        "last_name": AUTH_LDAP_USER_ATTR_MAP_SURNAME,
+        "email": AUTH_LDAP_USER_ATTR_MAP_EMAIL,
+    }
+
+    if bool(environ.get("TA_LDAP_DISABLE_CERT_CHECK")):
+        global AUTH_LDAP_GLOBAL_OPTIONS
+        AUTH_LDAP_GLOBAL_OPTIONS = {
+            ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
+        }
+
+    global AUTHENTICATION_BACKENDS
+    AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
 
 # Database
 # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
@@ -163,4 +262,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.1.7"
+TA_VERSION = "v0.2.4"

tubearchivist/home/apps.py

@@ -6,6 +6,7 @@ import sys
 from django.apps import AppConfig
 from home.src.es.connect import ElasticWrap
 from home.src.es.index_setup import index_check
+from home.src.es.snapshot import ElasticSnapshot
 from home.src.ta.config import AppConfig as ArchivistConfig
 from home.src.ta.ta_redis import RedisArchivist
 
@@ -13,8 +14,8 @@ from home.src.ta.ta_redis import RedisArchivist
 class StartupCheck:
     """checks to run at application startup"""
 
-    MIN_MAJOR, MAX_MAJOR = 7, 7
-    MIN_MINOR = 17
+    MIN_MAJOR, MAX_MAJOR = 8, 8
+    MIN_MINOR = 0
 
     def __init__(self):
         self.config_handler = ArchivistConfig()
@@ -30,6 +31,7 @@ class StartupCheck:
         self.sync_redis_state()
         self.set_redis_conf()
         self.make_folders()
+        self.snapshot_check()
         self.set_has_run()
 
     def get_has_run(self):
@@ -63,10 +65,7 @@ class StartupCheck:
         cache_dir = self.config_handler.config["application"]["cache_dir"]
         for folder in folders:
             folder_path = os.path.join(cache_dir, folder)
-            try:
-                os.makedirs(folder_path)
-            except FileExistsError:
-                continue
+            os.makedirs(folder_path, exist_ok=True)
 
     def release_lock(self):
         """make sure there are no leftover locks set in redis"""
@@ -77,12 +76,21 @@ class StartupCheck:
             "dl_queue",
             "dl_queue_id",
             "rescan",
+            "run_backup",
         ]
         for lock in all_locks:
             response = self.redis_con.del_message(lock)
             if response:
                 print("deleted leftover key from redis: " + lock)
 
+    def snapshot_check(self):
+        """setup snapshot config, create if needed"""
+        app = self.config_handler.config["application"]
+        if not app.get("enable_snapshot"):
+            return
+
+        ElasticSnapshot().setup()
+
     def is_invalid(self, version):
         """return true if es version is invalid, false if ok"""
         major, minor = [int(i) for i in version.split(".")[:2]]

tubearchivist/home/config.json

@@ -36,9 +36,9 @@
         "app_root": "/app",
         "cache_dir": "/cache",
         "videos": "/youtube",
-        "file_template": "%(id)s_%(title)s.mp4",
         "colors": "dark",
-        "enable_cast": false
+        "enable_cast": false,
+        "enable_snapshot": false
     },
     "scheduler": {
         "update_subscribed": false,

tubearchivist/home/src/download/queue.py

@@ -161,10 +161,7 @@ class PendingList(PendingIndex):
             self._parse_channel(entry["url"])
         elif entry["type"] == "playlist":
             self._parse_playlist(entry["url"])
-            new_thumbs = PlaylistSubscription().process_url_str(
-                [entry], subscribed=False
-            )
-            ThumbManager().download_playlist(new_thumbs)
+            PlaylistSubscription().process_url_str([entry], subscribed=False)
         else:
             raise ValueError(f"invalid url_type: {entry}")
 
@@ -198,8 +195,8 @@ class PendingList(PendingIndex):
         self.get_channels()
         bulk_list = []
 
-        thumb_handler = ThumbManager()
         for idx, youtube_id in enumerate(self.missing_videos):
+            print(f"{youtube_id}: add to download queue")
             video_details = self.get_youtube_details(youtube_id)
             if not video_details:
                 continue
@@ -209,8 +206,9 @@ class PendingList(PendingIndex):
             bulk_list.append(json.dumps(action))
             bulk_list.append(json.dumps(video_details))
 
-            thumb_needed = [(youtube_id, video_details["vid_thumb_url"])]
-            thumb_handler.download_vid(thumb_needed)
+            url = video_details["vid_thumb_url"]
+            ThumbManager(youtube_id).download_video_thumb(url)
+
             self._notify_add(idx)
 
         if bulk_list:
@@ -248,7 +246,7 @@ class PendingList(PendingIndex):
             print(f"{youtube_id}: skipping premium video, id not matching")
             return False
         # stop if video is streaming live now
-        if vid["is_live"]:
+        if vid["live_status"] in ["is_upcoming", "is_live"]:
             return False
 
         return self._parse_youtube_details(vid)

tubearchivist/home/src/download/subscriptions.py

@@ -5,6 +5,7 @@ Functionality:
 """
 
 from home.src.download import queue  # partial import
+from home.src.download.thumbnails import ThumbManager
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import IndexPaginate
 from home.src.index.channel import YoutubeChannel
@@ -62,6 +63,7 @@ class ChannelSubscription:
 
         for idx, channel in enumerate(all_channels):
             channel_id = channel["channel_id"]
+            print(f"{channel_id}: find missing videos.")
             last_videos = self.get_last_youtube_videos(channel_id)
 
             if last_videos:
@@ -129,11 +131,9 @@ class PlaylistSubscription:
         all_indexed = IndexPaginate("ta_video", data).get_results()
         all_youtube_ids = [i["youtube_id"] for i in all_indexed]
 
-        new_thumbs = []
         for idx, playlist in enumerate(new_playlists):
-            url_type = playlist["type"]
             playlist_id = playlist["url"]
-            if not url_type == "playlist":
+            if not playlist["type"] == "playlist":
                 print(f"{playlist_id} not a playlist, skipping...")
                 continue
 
@@ -144,8 +144,11 @@ class PlaylistSubscription:
             playlist_h.upload_to_es()
             playlist_h.add_vids_to_playlist()
             self.channel_validate(playlist_h.json_data["playlist_channel_id"])
-            thumb = playlist_h.json_data["playlist_thumbnail"]
-            new_thumbs.append((playlist_id, thumb))
+
+            url = playlist_h.json_data["playlist_thumbnail"]
+            thumb = ThumbManager(playlist_id, item_type="playlist")
+            thumb.download_playlist_thumb(url)
+
             # notify
             message = {
                 "status": "message:subplaylist",
@@ -157,8 +160,6 @@ class PlaylistSubscription:
                 "message:subplaylist", message=message, expire=True
             )
 
-        return new_thumbs
-
     @staticmethod
     def channel_validate(channel_id):
         """make sure channel of playlist is there"""

tubearchivist/home/src/download/thumbnails.py

@@ -6,136 +6,71 @@ functionality:
 
 import base64
 import os
-from collections import Counter
 from io import BytesIO
 from time import sleep
 
 import requests
 from home.src.download import queue  # partial import
-from home.src.download import subscriptions  # partial import
+from home.src.es.connect import IndexPaginate
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import ignore_filelist
-from home.src.ta.ta_redis import RedisArchivist
 from mutagen.mp4 import MP4, MP4Cover
-from PIL import Image, ImageFile, ImageFilter
+from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
 
 ImageFile.LOAD_TRUNCATED_IMAGES = True
 
 
-class ThumbManager:
-    """handle thumbnails related functions"""
+class ThumbManagerBase:
+    """base class for thumbnail management"""
 
     CONFIG = AppConfig().config
-    MEDIA_DIR = CONFIG["application"]["videos"]
     CACHE_DIR = CONFIG["application"]["cache_dir"]
     VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
     CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
     PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
 
-    def get_all_thumbs(self):
-        """get all video artwork already downloaded"""
-        all_thumb_folders = ignore_filelist(os.listdir(self.VIDEO_DIR))
-        all_thumbs = []
-        for folder in all_thumb_folders:
-            folder_path = os.path.join(self.VIDEO_DIR, folder)
-            if os.path.isfile(folder_path):
-                self.update_path(folder)
-                all_thumbs.append(folder_path)
-                continue
-                # raise exemption here in a future version
-                # raise FileExistsError("video cache dir has files inside")
+    def __init__(self, item_id, item_type, fallback=False):
+        self.item_id = item_id
+        self.item_type = item_type
+        self.fallback = fallback
 
-            all_folder_thumbs = ignore_filelist(os.listdir(folder_path))
-            all_thumbs.extend(all_folder_thumbs)
+    def download_raw(self, url):
+        """download thumbnail for video"""
+        if not url:
+            return self.get_fallback()
 
-        return all_thumbs
+        for i in range(3):
+            try:
+                response = requests.get(url, stream=True, timeout=5)
+                if response.ok:
+                    try:
+                        return Image.open(response.raw)
+                    except UnidentifiedImageError:
+                        print(f"failed to open thumbnail: {url}")
+                        return self.get_fallback()
 
-    def update_path(self, file_name):
-        """reorganize thumbnails into folders as update path from v0.0.5"""
-        folder_name = file_name[0].lower()
-        folder_path = os.path.join(self.VIDEO_DIR, folder_name)
-        old_file = os.path.join(self.VIDEO_DIR, file_name)
-        new_file = os.path.join(folder_path, file_name)
-        os.makedirs(folder_path, exist_ok=True)
-        os.rename(old_file, new_file)
+                if response.status_code == 404:
+                    return self.get_fallback()
 
-    def get_needed_thumbs(self, missing_only=False):
-        """get a list of all missing thumbnails"""
-        all_thumbs = self.get_all_thumbs()
+            except requests.exceptions.RequestException:
+                print(f"{self.item_id}: retry thumbnail download {url}")
+                sleep((i + 1) ** i)
 
-        pending = queue.PendingList()
-        pending.get_download()
-        pending.get_indexed()
+        return False
 
-        needed_thumbs = []
-        for video in pending.all_videos:
-            youtube_id = video["youtube_id"]
-            thumb_url = video["vid_thumb_url"]
-            if missing_only:
-                if youtube_id + ".jpg" not in all_thumbs:
-                    needed_thumbs.append((youtube_id, thumb_url))
-            else:
-                needed_thumbs.append((youtube_id, thumb_url))
+    def get_fallback(self):
+        """get fallback thumbnail if not available"""
+        if self.fallback:
+            img_raw = Image.open(self.fallback)
+            return img_raw
 
-        for video in pending.all_pending + pending.all_ignored:
-            youtube_id = video["youtube_id"]
-            thumb_url = video["vid_thumb_url"]
-            if missing_only:
-                if youtube_id + ".jpg" not in all_thumbs:
-                    needed_thumbs.append((youtube_id, thumb_url))
-            else:
-                needed_thumbs.append((youtube_id, thumb_url))
-
-        return needed_thumbs
-
-    def get_missing_channels(self):
-        """get all channel artwork"""
-        all_channel_art = os.listdir(self.CHANNEL_DIR)
-        files = [i[0:24] for i in all_channel_art]
-        cached_channel_ids = [k for (k, v) in Counter(files).items() if v > 1]
-        channel_sub = subscriptions.ChannelSubscription()
-        channels = channel_sub.get_channels(subscribed_only=False)
-
-        missing_channels = []
-        for channel in channels:
-            channel_id = channel["channel_id"]
-            if channel_id not in cached_channel_ids:
-                channel_banner = channel["channel_banner_url"]
-                channel_thumb = channel["channel_thumb_url"]
-                missing_channels.append(
-                    (channel_id, channel_thumb, channel_banner)
-                )
-
-        return missing_channels
-
-    def get_missing_playlists(self):
-        """get all missing playlist artwork"""
-        all_downloaded = ignore_filelist(os.listdir(self.PLAYLIST_DIR))
-        all_ids_downloaded = [i.replace(".jpg", "") for i in all_downloaded]
-        playlist_sub = subscriptions.PlaylistSubscription()
-        playlists = playlist_sub.get_playlists(subscribed_only=False)
-
-        missing_playlists = []
-        for playlist in playlists:
-            playlist_id = playlist["playlist_id"]
-            if playlist_id not in all_ids_downloaded:
-                playlist_thumb = playlist["playlist_thumbnail"]
-                missing_playlists.append((playlist_id, playlist_thumb))
-
-        return missing_playlists
-
-    def get_raw_img(self, img_url, thumb_type):
-        """get raw image from youtube and handle 404"""
-        try:
-            app_root = self.CONFIG["application"]["app_root"]
-        except KeyError:
-            # lazy keyerror fix to not have to deal with a strange startup
-            # racing contition between the threads in HomeConfig.ready()
-            app_root = "/app"
+        app_root = self.CONFIG["application"]["app_root"]
         default_map = {
             "video": os.path.join(
                 app_root, "static/img/default-video-thumb.jpg"
             ),
+            "playlist": os.path.join(
+                app_root, "static/img/default-video-thumb.jpg"
+            ),
             "icon": os.path.join(
                 app_root, "static/img/default-channel-icon.jpg"
             ),
@@ -143,116 +78,138 @@ class ThumbManager:
                 app_root, "static/img/default-channel-banner.jpg"
             ),
         }
-        if img_url:
-            try:
-                response = requests.get(img_url, stream=True)
-            except ConnectionError:
-                sleep(5)
-                response = requests.get(img_url, stream=True)
-            if not response.ok and not response.status_code == 404:
-                print("retry thumbnail download for " + img_url)
-                sleep(5)
-                response = requests.get(img_url, stream=True)
-        else:
-            response = False
-        if not response or response.status_code == 404:
-            # use default
-            img_raw = Image.open(default_map[thumb_type])
-        else:
-            # use response
-            img_obj = response.raw
-            img_raw = Image.open(img_obj)
+
+        img_raw = Image.open(default_map[self.item_type])
 
         return img_raw
 
-    def download_vid(self, missing_thumbs, notify=True):
-        """download all missing thumbnails from list"""
-        print(f"downloading {len(missing_thumbs)} thumbnails")
-        for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs):
-            folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower())
-            thumb_path = os.path.join(
-                self.CACHE_DIR, self.vid_thumb_path(youtube_id)
-            )
 
+class ThumbManager(ThumbManagerBase):
+    """handle thumbnails related functions"""
+
+    def __init__(self, item_id, item_type="video", fallback=False):
+        super().__init__(item_id, item_type, fallback=fallback)
+
+    def download(self, url):
+        """download thumbnail"""
+        print(f"{self.item_id}: download {self.item_type} thumbnail")
+        if self.item_type == "video":
+            self.download_video_thumb(url)
+        elif self.item_type == "channel":
+            self.download_channel_art(url)
+        elif self.item_type == "playlist":
+            self.download_playlist_thumb(url)
+
+    def delete(self):
+        """delete thumbnail file"""
+        print(f"{self.item_id}: delete {self.item_type} thumbnail")
+        if self.item_type == "video":
+            self.delete_video_thumb()
+        elif self.item_type == "channel":
+            self.delete_channel_thumb()
+        elif self.item_type == "playlist":
+            self.delete_playlist_thumb()
+
+    def download_video_thumb(self, url, skip_existing=False):
+        """pass url for video thumbnail"""
+        folder_path = os.path.join(self.VIDEO_DIR, self.item_id[0].lower())
+        thumb_path = self.vid_thumb_path(absolute=True)
+
+        if skip_existing and os.path.exists(thumb_path):
+            return
+
+        os.makedirs(folder_path, exist_ok=True)
+        img_raw = self.download_raw(url)
+        width, height = img_raw.size
+
+        if not width / height == 16 / 9:
+            new_height = width / 16 * 9
+            offset = (height - new_height) / 2
+            img_raw = img_raw.crop((0, offset, width, height - offset))
+
+        img_raw.convert("RGB").save(thumb_path)
+
+    def vid_thumb_path(self, absolute=False, create_folder=False):
+        """build expected path for video thumbnail from youtube_id"""
+        folder_name = self.item_id[0].lower()
+        folder_path = os.path.join("videos", folder_name)
+        thumb_path = os.path.join(folder_path, f"{self.item_id}.jpg")
+        if absolute:
+            thumb_path = os.path.join(self.CACHE_DIR, thumb_path)
+
+        if create_folder:
+            folder_path = os.path.join(self.CACHE_DIR, folder_path)
             os.makedirs(folder_path, exist_ok=True)
-            img_raw = self.get_raw_img(thumb_url, "video")
 
-            width, height = img_raw.size
-            if not width / height == 16 / 9:
-                new_height = width / 16 * 9
-                offset = (height - new_height) / 2
-                img_raw = img_raw.crop((0, offset, width, height - offset))
-            img_raw.convert("RGB").save(thumb_path)
+        return thumb_path
 
-            progress = f"{idx + 1}/{len(missing_thumbs)}"
-            if notify:
-                mess_dict = {
-                    "status": "message:add",
-                    "level": "info",
-                    "title": "Processing Videos",
-                    "message": "Downloading Thumbnails, Progress: " + progress,
-                }
-                if idx + 1 == len(missing_thumbs):
-                    expire = 4
-                else:
-                    expire = True
+    def download_channel_art(self, urls, skip_existing=False):
+        """pass tuple of channel thumbnails"""
+        channel_thumb, channel_banner = urls
+        self._download_channel_thumb(channel_thumb, skip_existing)
+        self._download_channel_banner(channel_banner, skip_existing)
 
-                RedisArchivist().set_message(
-                    "message:add", mess_dict, expire=expire
-                )
+    def _download_channel_thumb(self, channel_thumb, skip_existing):
+        """download channel thumbnail"""
 
-            if idx + 1 % 25 == 0:
-                print("thumbnail progress: " + progress)
+        thumb_path = os.path.join(
+            self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg"
+        )
+        self.item_type = "icon"
 
-    def download_chan(self, missing_channels):
-        """download needed artwork for channels"""
-        print(f"downloading {len(missing_channels)} channel artwork")
-        for channel in missing_channels:
-            channel_id, channel_thumb, channel_banner = channel
+        if skip_existing and os.path.exists(thumb_path):
+            return
 
-            thumb_path = os.path.join(
-                self.CHANNEL_DIR, channel_id + "_thumb.jpg"
-            )
-            img_raw = self.get_raw_img(channel_thumb, "icon")
-            img_raw.convert("RGB").save(thumb_path)
+        img_raw = self.download_raw(channel_thumb)
+        img_raw.convert("RGB").save(thumb_path)
 
-            banner_path = os.path.join(
-                self.CHANNEL_DIR, channel_id + "_banner.jpg"
-            )
-            img_raw = self.get_raw_img(channel_banner, "banner")
-            img_raw.convert("RGB").save(banner_path)
+    def _download_channel_banner(self, channel_banner, skip_existing):
+        """download channel banner"""
 
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Processing Channels",
-                "message": "Downloading Channel Art.",
-            }
-            key = "message:download"
-            RedisArchivist().set_message(key, mess_dict, expire=True)
+        banner_path = os.path.join(
+            self.CHANNEL_DIR, self.item_id + "_banner.jpg"
+        )
+        self.item_type = "banner"
+        if skip_existing and os.path.exists(banner_path):
+            return
 
-    def download_playlist(self, missing_playlists):
-        """download needed artwork for playlists"""
-        print(f"downloading {len(missing_playlists)} playlist artwork")
-        for playlist in missing_playlists:
-            playlist_id, playlist_thumb_url = playlist
-            thumb_path = os.path.join(self.PLAYLIST_DIR, playlist_id + ".jpg")
-            img_raw = self.get_raw_img(playlist_thumb_url, "video")
-            img_raw.convert("RGB").save(thumb_path)
+        img_raw = self.download_raw(channel_banner)
+        img_raw.convert("RGB").save(banner_path)
 
-            mess_dict = {
-                "status": "message:download",
-                "level": "info",
-                "title": "Processing Playlists",
-                "message": "Downloading Playlist Art.",
-            }
-            key = "message:download"
-            RedisArchivist().set_message(key, mess_dict, expire=True)
+    def download_playlist_thumb(self, url, skip_existing=False):
+        """pass thumbnail url"""
+        thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
+        if skip_existing and os.path.exists(thumb_path):
+            return
 
-    def get_base64_blur(self, youtube_id):
+        img_raw = self.download_raw(url)
+        img_raw.convert("RGB").save(thumb_path)
+
+    def delete_video_thumb(self):
+        """delete video thumbnail if exists"""
+        thumb_path = self.vid_thumb_path()
+        to_delete = os.path.join(self.CACHE_DIR, thumb_path)
+        if os.path.exists(to_delete):
+            os.remove(to_delete)
+
+    def delete_channel_thumb(self):
+        """delete all artwork of channel"""
+        thumb = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg")
+        banner = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_banner.jpg")
+        if os.path.exists(thumb):
+            os.remove(thumb)
+        if os.path.exists(banner):
+            os.remove(banner)
+
+    def delete_playlist_thumb(self):
+        """delete playlist thumbnail"""
+        thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
+        if os.path.exists(thumb_path):
+            os.remove(thumb_path)
+
+    def get_vid_base64_blur(self):
         """return base64 encoded placeholder"""
-        img_path = self.vid_thumb_path(youtube_id)
-        file_path = os.path.join(self.CACHE_DIR, img_path)
+        file_path = os.path.join(self.CACHE_DIR, self.vid_thumb_path())
         img_raw = Image.open(file_path)
         img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20))
         img_blur = img_raw.filter(ImageFilter.BLUR)
@@ -264,40 +221,109 @@ class ThumbManager:
 
         return data_url
 
-    @staticmethod
-    def vid_thumb_path(youtube_id):
-        """build expected path for video thumbnail from youtube_id"""
-        folder_name = youtube_id[0].lower()
-        folder_path = os.path.join("videos", folder_name)
-        thumb_path = os.path.join(folder_path, youtube_id + ".jpg")
-        return thumb_path
 
-    def delete_vid_thumb(self, youtube_id):
-        """delete video thumbnail if exists"""
-        thumb_path = self.vid_thumb_path(youtube_id)
-        to_delete = os.path.join(self.CACHE_DIR, thumb_path)
-        if os.path.exists(to_delete):
-            os.remove(to_delete)
+class ValidatorCallback:
+    """handle callback validate thumbnails page by page"""
 
-    def delete_chan_thumb(self, channel_id):
-        """delete all artwork of channel"""
-        thumb = os.path.join(self.CHANNEL_DIR, channel_id + "_thumb.jpg")
-        banner = os.path.join(self.CHANNEL_DIR, channel_id + "_banner.jpg")
-        if os.path.exists(thumb):
-            os.remove(thumb)
-        if os.path.exists(banner):
-            os.remove(banner)
+    def __init__(self, source, index_name):
+        self.source = source
+        self.index_name = index_name
 
-    def cleanup_downloaded(self):
-        """find downloaded thumbnails without video indexed"""
-        all_thumbs = self.get_all_thumbs()
-        all_indexed = self.get_needed_thumbs()
-        all_needed_thumbs = [i[0] + ".jpg" for i in all_indexed]
-        for thumb in all_thumbs:
-            if thumb not in all_needed_thumbs:
-                # cleanup
-                youtube_id = thumb.rstrip(".jpg")
-                self.delete_vid_thumb(youtube_id)
+    def run(self):
+        """run the task for page"""
+        print(f"{self.index_name}: validate artwork")
+        if self.index_name == "ta_video":
+            self._validate_videos()
+        elif self.index_name == "ta_channel":
+            self._validate_channels()
+        elif self.index_name == "ta_playlist":
+            self._validate_playlists()
+
+    def _validate_videos(self):
+        """check if video thumbnails are correct"""
+        for video in self.source:
+            url = video["_source"]["vid_thumb_url"]
+            handler = ThumbManager(video["_source"]["youtube_id"])
+            handler.download_video_thumb(url, skip_existing=True)
+
+    def _validate_channels(self):
+        """check if all channel artwork is there"""
+        for channel in self.source:
+            urls = (
+                channel["_source"]["channel_thumb_url"],
+                channel["_source"]["channel_banner_url"],
+            )
+            handler = ThumbManager(channel["_source"]["channel_id"])
+            handler.download_channel_art(urls, skip_existing=True)
+
+    def _validate_playlists(self):
+        """check if all playlist artwork is there"""
+        for playlist in self.source:
+            url = playlist["_source"]["playlist_thumbnail"]
+            handler = ThumbManager(playlist["_source"]["playlist_id"])
+            handler.download_playlist_thumb(url, skip_existing=True)
+
+
+class ThumbValidator:
+    """validate thumbnails"""
+
+    def download_missing(self):
+        """download all missing artwork"""
+        self.download_missing_videos()
+        self.download_missing_channels()
+        self.download_missing_playlists()
+
+    def download_missing_videos(self):
+        """get all missing video thumbnails"""
+        data = {
+            "query": {"term": {"active": {"value": True}}},
+            "sort": [{"youtube_id": {"order": "asc"}}],
+            "_source": ["vid_thumb_url", "youtube_id"],
+        }
+        paginate = IndexPaginate(
+            "ta_video", data, size=5000, callback=ValidatorCallback
+        )
+        _ = paginate.get_results()
+
+    def download_missing_channels(self):
+        """get all missing channel thumbnails"""
+        data = {
+            "query": {"term": {"channel_active": {"value": True}}},
+            "sort": [{"channel_id": {"order": "asc"}}],
+            "_source": {
+                "excludes": ["channel_description", "channel_overwrites"]
+            },
+        }
+        paginate = IndexPaginate(
+            "ta_channel", data, callback=ValidatorCallback
+        )
+        _ = paginate.get_results()
+
+    def download_missing_playlists(self):
+        """get all missing playlist artwork"""
+        data = {
+            "query": {"term": {"playlist_active": {"value": True}}},
+            "sort": [{"playlist_id": {"order": "asc"}}],
+            "_source": ["playlist_id", "playlist_thumbnail"],
+        }
+        paginate = IndexPaginate(
+            "ta_playlist", data, callback=ValidatorCallback
+        )
+        _ = paginate.get_results()
+
+
+class ThumbFilesystem:
+    """filesystem tasks for thumbnails"""
+
+    CONFIG = AppConfig().config
+    CACHE_DIR = CONFIG["application"]["cache_dir"]
+    MEDIA_DIR = CONFIG["application"]["videos"]
+    VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
+
+    def sync(self):
+        """embed thumbnails to mediafiles"""
+        video_list = self.get_thumb_list()
+        self._embed_thumbs(video_list)
 
     def get_thumb_list(self):
         """get list of mediafiles and matching thumbnails"""
@@ -307,10 +333,10 @@ class ThumbManager:
 
         video_list = []
         for video in pending.all_videos:
-            youtube_id = video["youtube_id"]
+            video_id = video["youtube_id"]
             media_url = os.path.join(self.MEDIA_DIR, video["media_url"])
             thumb_path = os.path.join(
-                self.CACHE_DIR, self.vid_thumb_path(youtube_id)
+                self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path()
             )
             video_list.append(
                 {
@@ -322,7 +348,7 @@ class ThumbManager:
         return video_list
 
     @staticmethod
-    def write_all_thumbs(video_list):
+    def _embed_thumbs(video_list):
         """rewrite the thumbnail into media file"""
 
         counter = 1
@@ -340,15 +366,3 @@ class ThumbManager:
             if counter % 50 == 0:
                 print(f"thumbnail write progress {counter}/{len(video_list)}")
             counter = counter + 1
-
-
-def validate_thumbnails():
-    """check if all thumbnails are there and organized correctly"""
-    handler = ThumbManager()
-    thumbs_to_download = handler.get_needed_thumbs(missing_only=True)
-    handler.download_vid(thumbs_to_download)
-    missing_channels = handler.get_missing_channels()
-    handler.download_chan(missing_channels)
-    missing_playlists = handler.get_missing_playlists()
-    handler.download_playlist(missing_playlists)
-    handler.cleanup_downloaded()

tubearchivist/home/src/download/yt_dlp_base.py

@@ -20,7 +20,8 @@ class YtWrap:
         "default_search": "ytsearch",
         "quiet": True,
         "check_formats": "selected",
-        "socket_timeout": 2,
+        "socket_timeout": 3,
+        "extractor_retries": 3,
     }
 
     def __init__(self, obs_request, config=False):
@@ -83,8 +84,13 @@ class CookieHandler:
         """import cookie from file"""
         cache_path = self.config["application"]["cache_dir"]
         import_path = os.path.join(cache_path, "import", "cookies.google.txt")
-        with open(import_path, encoding="utf-8") as cookie_file:
-            cookie = cookie_file.read()
+
+        try:
+            with open(import_path, encoding="utf-8") as cookie_file:
+                cookie = cookie_file.read()
+        except FileNotFoundError as err:
+            print(f"cookie: {import_path} file not found")
+            raise err
 
         self.set_cookie(cookie)
 

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -175,6 +175,14 @@ class VideoDownloader:
             if not success:
                 continue
 
+            mess_dict = {
+                "status": self.MSG,
+                "level": "info",
+                "title": "Indexing....",
+                "message": "Add video metadata to index.",
+            }
+            RedisArchivist().set_message(self.MSG, mess_dict, expire=60)
+
             vid_dict = index_new_video(
                 youtube_id, video_overwrites=self.video_overwrites
             )
@@ -187,12 +195,17 @@ class VideoDownloader:
             }
             RedisArchivist().set_message(self.MSG, mess_dict)
 
+            if queue.has_item():
+                message = "Continue with next video."
+            else:
+                message = "Download queue is finished."
+
             self.move_to_archive(vid_dict)
             mess_dict = {
                 "status": self.MSG,
                 "level": "info",
                 "title": "Completed",
-                "message": "",
+                "message": message,
             }
             RedisArchivist().set_message(self.MSG, mess_dict, expire=10)
             self._delete_from_pending(youtube_id)
@@ -241,12 +254,8 @@ class VideoDownloader:
 
     def _progress_hook(self, response):
         """process the progress_hooks from yt_dlp"""
-        # title
-        path = os.path.split(response["filename"])[-1][12:]
-        filename = os.path.splitext(os.path.splitext(path)[0])[0]
-        filename_clean = filename.replace("_", " ")
-        title = "Downloading: " + filename_clean
-        # message
+        title = "Downloading: " + response["info_dict"]["title"]
+
         try:
             percent = response["_percent_str"]
             size = response["_total_bytes_str"]
@@ -255,6 +264,7 @@ class VideoDownloader:
             message = f"{percent} of {size} at {speed} - time left: {eta}"
         except KeyError:
             message = "processing"
+
         mess_dict = {
             "status": self.MSG,
             "level": "info",
@@ -274,11 +284,9 @@ class VideoDownloader:
         self.obs = {
             "default_search": "ytsearch",
             "merge_output_format": "mp4",
-            "restrictfilenames": True,
             "outtmpl": (
                 self.config["application"]["cache_dir"]
-                + "/download/"
-                + self.config["application"]["file_template"]
+                + "/download/%(id)s.mp4"
             ),
             "progress_hooks": [self._progress_hook],
             "noprogress": True,
@@ -395,12 +403,12 @@ class VideoDownloader:
         for file_str in all_cached:
             if vid_dict["youtube_id"] in file_str:
                 old_file = file_str
-        old_file_path = os.path.join(cache_dir, "download", old_file)
-        new_file_path = os.path.join(videos, vid_dict["media_url"])
+        old_path = os.path.join(cache_dir, "download", old_file)
+        new_path = os.path.join(videos, vid_dict["media_url"])
         # move media file and fix permission
-        shutil.move(old_file_path, new_file_path)
+        shutil.move(old_path, new_path, copy_function=shutil.copyfile)
         if host_uid and host_gid:
-            os.chown(new_file_path, host_uid, host_gid)
+            os.chown(new_path, host_uid, host_gid)
 
     @staticmethod
     def _delete_from_pending(youtube_id):

tubearchivist/home/src/es/connect.py

@@ -91,16 +91,22 @@ class ElasticWrap:
 
 
 class IndexPaginate:
-    """use search_after to go through whole index"""
+    """use search_after to go through whole index
+    kwargs:
+    - size: int, overwrite DEFAULT_SIZE
+    - keep_source: bool, keep _source key from es resutls
+    - callback: obj, Class with run method collback for every loop
+    """
 
     DEFAULT_SIZE = 500
 
-    def __init__(self, index_name, data, size=False, keep_source=False):
+    def __init__(self, index_name, data, **kwargs):
         self.index_name = index_name
         self.data = data
         self.pit_id = False
-        self.size = size
-        self.keep_source = keep_source
+        self.size = kwargs.get("size")
+        self.keep_source = kwargs.get("keep_source")
+        self.callback = kwargs.get("callback")
 
     def get_results(self):
         """get all results"""
@@ -122,14 +128,13 @@ class IndexPaginate:
             print(self.data)
             raise ValueError("missing sort key in data")
 
-        size = self.size or self.DEFAULT_SIZE
-
-        self.data["size"] = size
+        self.data["size"] = self.size or self.DEFAULT_SIZE
         self.data["pit"] = {"id": self.pit_id, "keep_alive": "10m"}
 
     def run_loop(self):
         """loop through results until last hit"""
         all_results = []
+        counter = 0
         while True:
             response, _ = ElasticWrap("_search").get(data=self.data)
             all_hits = response["hits"]["hits"]
@@ -139,10 +144,18 @@ class IndexPaginate:
                         source = hit
                     else:
                         source = hit["_source"]
-                    search_after = hit["sort"]
-                    all_results.append(source)
+
+                    if not self.callback:
+                        all_results.append(source)
+
+                if self.callback:
+                    self.callback(all_hits, self.index_name).run()
+                    if counter % 10 == 0:
+                        print(f"{self.index_name}: processing page {counter}")
+                    counter = counter + 1
+
                 # update search_after with last hit data
-                self.data["search_after"] = search_after
+                self.data["search_after"] = all_hits[-1]["sort"]
             else:
                 break
 

tubearchivist/home/src/es/index_setup.py

@@ -137,35 +137,24 @@ class ElasticIndex:
         _, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
 
 
-class ElasticBackup:
-    """dump index to nd-json files for later bulk import"""
+class BackupCallback:
+    """handle backup ndjson writer as callback for IndexPaginate"""
 
-    def __init__(self, index_config, reason):
-        self.config = AppConfig().config
-        self.cache_dir = self.config["application"]["cache_dir"]
-        self.index_config = index_config
-        self.reason = reason
+    def __init__(self, source, index_name):
+        self.source = source
+        self.index_name = index_name
         self.timestamp = datetime.now().strftime("%Y%m%d")
-        self.backup_files = []
 
-    @staticmethod
-    def get_all_documents(index_name):
-        """export all documents of a single index"""
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"_doc": {"order": "desc"}}],
-        }
-        paginate = IndexPaginate(f"ta_{index_name}", data, keep_source=True)
-        all_results = paginate.get_results()
+    def run(self):
+        """run the junk task"""
+        file_content = self._build_bulk()
+        self._write_es_json(file_content)
 
-        return all_results
-
-    @staticmethod
-    def build_bulk(all_results):
+    def _build_bulk(self):
         """build bulk query data from all_results"""
         bulk_list = []
 
-        for document in all_results:
+        for document in self.source:
             document_id = document["_id"]
             es_index = document["_index"]
             action = {"index": {"_index": es_index, "_id": document_id}}
@@ -179,40 +168,56 @@ class ElasticBackup:
 
         return file_content
 
-    def write_es_json(self, file_content, index_name):
+    def _write_es_json(self, file_content):
         """write nd-json file for es _bulk API to disk"""
-        file_name = f"es_{index_name}-{self.timestamp}.json"
-        file_path = os.path.join(self.cache_dir, "backup", file_name)
-        with open(file_path, "w", encoding="utf-8") as f:
+        cache_dir = AppConfig().config["application"]["cache_dir"]
+        file_name = f"es_{self.index_name.lstrip('ta_')}-{self.timestamp}.json"
+        file_path = os.path.join(cache_dir, "backup", file_name)
+        with open(file_path, "a+", encoding="utf-8") as f:
             f.write(file_content)
 
-        self.backup_files.append(file_path)
 
-    def write_ta_json(self, all_results, index_name):
-        """write generic json file to disk"""
-        file_name = f"ta_{index_name}-{self.timestamp}.json"
-        file_path = os.path.join(self.cache_dir, "backup", file_name)
-        to_write = [i["_source"] for i in all_results]
-        file_content = json.dumps(to_write)
-        with open(file_path, "w", encoding="utf-8") as f:
-            f.write(file_content)
+class ElasticBackup:
+    """dump index to nd-json files for later bulk import"""
 
-        self.backup_files.append(file_path)
+    def __init__(self, index_config, reason):
+        self.config = AppConfig().config
+        self.cache_dir = self.config["application"]["cache_dir"]
+        self.timestamp = datetime.now().strftime("%Y%m%d")
+        self.index_config = index_config
+        self.reason = reason
+
+    @staticmethod
+    def backup_index(index_name):
+        """export all documents of a single index"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"_doc": {"order": "desc"}}],
+        }
+        paginate = IndexPaginate(
+            f"ta_{index_name}", data, keep_source=True, callback=BackupCallback
+        )
+        _ = paginate.get_results()
 
     def zip_it(self):
         """pack it up into single zip file"""
         file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
-        backup_folder = os.path.join(self.cache_dir, "backup")
-        backup_file = os.path.join(backup_folder, file_name)
+        folder = os.path.join(self.cache_dir, "backup")
 
-        with zipfile.ZipFile(
-            backup_file, "w", compression=zipfile.ZIP_DEFLATED
-        ) as zip_f:
-            for backup_file in self.backup_files:
+        to_backup = []
+        for file in os.listdir(folder):
+            if file.endswith(".json"):
+                to_backup.append(os.path.join(folder, file))
+
+        backup_file = os.path.join(folder, file_name)
+
+        comp = zipfile.ZIP_DEFLATED
+        with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
+            for backup_file in to_backup:
                 zip_f.write(backup_file, os.path.basename(backup_file))
 
         # cleanup
-        for backup_file in self.backup_files:
+        for backup_file in to_backup:
             os.remove(backup_file)
 
     def post_bulk_restore(self, file_name):
@@ -380,10 +385,7 @@ def backup_all_indexes(reason):
         print(f"backup: export in progress for {index_name}")
         if not backup_handler.index_exists(index_name):
             continue
-        all_results = backup_handler.get_all_documents(index_name)
-        file_content = backup_handler.build_bulk(all_results)
-        backup_handler.write_es_json(file_content, index_name)
-        backup_handler.write_ta_json(all_results, index_name)
+        backup_handler.backup_index(index_name)
 
     backup_handler.zip_it()
 

tubearchivist/home/src/es/snapshot.py

@@ -0,0 +1,257 @@
+"""
+functionality:
+- handle snapshots in ES
+"""
+
+from datetime import datetime
+from os import environ
+from zoneinfo import ZoneInfo
+
+from home.src.es.connect import ElasticWrap
+from home.src.es.index_setup import get_mapping
+
+
+class ElasticSnapshot:
+    """interact with snapshots on ES"""
+
+    REPO = "ta_snapshot"
+    REPO_SETTINGS = {
+        "compress": "true",
+        "chunk_size": "1g",
+        "location": "/usr/share/elasticsearch/data/snapshot",
+    }
+    POLICY = "ta_daily"
+
+    def __init__(self):
+        self.all_indices = self._get_all_indices()
+
+    def _get_all_indices(self):
+        """return all indices names managed by TA"""
+        mapping = get_mapping()
+        all_indices = [f"ta_{i['index_name']}" for i in mapping]
+
+        return all_indices
+
+    def setup(self):
+        """setup the snapshot in ES, create or update if needed"""
+        print("snapshot: run setup")
+        repo_exists = self._check_repo_exists()
+        if not repo_exists:
+            self.create_repo()
+
+        policy_exists = self._check_policy_exists()
+        if not policy_exists:
+            self.create_policy()
+
+        is_outdated = self._needs_startup_snapshot()
+        if is_outdated:
+            _ = self.take_snapshot_now()
+
+    def _check_repo_exists(self):
+        """check if expected repo already exists"""
+        path = f"_snapshot/{self.REPO}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 200:
+            print(f"snapshot: repo {self.REPO} already created")
+            matching = response[self.REPO]["settings"] == self.REPO_SETTINGS
+            if not matching:
+                print(f"snapshot: update repo settings {self.REPO_SETTINGS}")
+
+            return matching
+
+        print(f"snapshot: setup repo {self.REPO} config {self.REPO_SETTINGS}")
+        return False
+
+    def create_repo(self):
+        """create filesystem repo"""
+        path = f"_snapshot/{self.REPO}"
+        data = {
+            "type": "fs",
+            "settings": self.REPO_SETTINGS,
+        }
+        response, statuscode = ElasticWrap(path).post(data=data)
+        if statuscode == 200:
+            print(f"snapshot: repo setup correctly: {response}")
+
+    def _check_policy_exists(self):
+        """check if snapshot policy is set correctly"""
+        policy = self._get_policy()
+        expected_policy = self._build_policy_data()
+        if not policy:
+            print(f"snapshot: create policy {self.POLICY} {expected_policy}")
+            return False
+
+        if policy["policy"] != expected_policy:
+            print(f"snapshot: update policy settings {expected_policy}")
+            return False
+
+        print("snapshot: policy is set.")
+        return True
+
+    def _get_policy(self):
+        """get policy from es"""
+        path = f"_slm/policy/{self.POLICY}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode != 200:
+            return False
+
+        return response[self.POLICY]
+
+    def create_policy(self):
+        """create snapshot lifetime policy"""
+        path = f"_slm/policy/{self.POLICY}"
+        data = self._build_policy_data()
+        response, statuscode = ElasticWrap(path).put(data)
+        if statuscode == 200:
+            print(f"snapshot: policy setup correctly: {response}")
+
+    def _build_policy_data(self):
+        """build policy dict from config"""
+        at_12 = datetime.now().replace(hour=12, minute=0, second=0)
+        hour = at_12.astimezone(ZoneInfo("UTC")).hour
+
+        return {
+            "schedule": f"0 0 {hour} * * ?",
+            "name": f"<{self.POLICY}_>",
+            "repository": self.REPO,
+            "config": {
+                "indices": self.all_indices,
+                "include_global_state": True,
+            },
+            "retention": {
+                "expire_after": "30d",
+                "min_count": 5,
+                "max_count": 50,
+            },
+        }
+
+    def _needs_startup_snapshot(self):
+        """check if last snapshot is expired"""
+        snap_dicts = self._get_all_snapshots()
+        if not snap_dicts:
+            print("snapshot: create initial snapshot")
+            return True
+
+        last_stamp = snap_dicts[0]["end_stamp"]
+        now = int(datetime.now().strftime("%s"))
+        outdated = (now - last_stamp) / 60 / 60 > 24
+        if outdated:
+            print("snapshot: is outdated, create new now")
+
+        print("snapshot: last snapshot is up-to-date")
+        return outdated
+
+    def take_snapshot_now(self):
+        """execute daily snapshot now"""
+        path = f"_slm/policy/{self.POLICY}/_execute"
+        response, statuscode = ElasticWrap(path).post()
+        if statuscode == 200:
+            print(f"snapshot: executing now: {response}")
+
+        return response
+
+    def get_snapshot_stats(self):
+        """get snapshot info for frontend"""
+        snapshot_info = self._build_policy_details()
+        if snapshot_info:
+            snapshot_info.update({"snapshots": self._get_all_snapshots()})
+
+        return snapshot_info
+
+    def get_single_snapshot(self, snapshot_id):
+        """get single snapshot metadata"""
+        path = f"_snapshot/{self.REPO}/{snapshot_id}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 404:
+            print(f"snapshots: not found: {snapshot_id}")
+            return False
+
+        snapshot = response["snapshots"][0]
+        return self._parse_single_snapshot(snapshot)
+
+    def _get_all_snapshots(self):
+        """get a list of all registered snapshots"""
+        path = f"_snapshot/{self.REPO}/*?sort=start_time&order=desc"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 404:
+            print("snapshots: not configured")
+            return False
+
+        all_snapshots = response["snapshots"]
+        if not all_snapshots:
+            print("snapshots: no snapshots found")
+            return False
+
+        snap_dicts = []
+        for snapshot in all_snapshots:
+            snap_dict = self._parse_single_snapshot(snapshot)
+            snap_dicts.append(snap_dict)
+
+        return snap_dicts
+
+    def _parse_single_snapshot(self, snapshot):
+        """extract relevant metadata from single snapshot"""
+        snap_dict = {
+            "id": snapshot["snapshot"],
+            "state": snapshot["state"],
+            "es_version": snapshot["version"],
+            "start_date": self._date_converter(snapshot["start_time"]),
+            "end_date": self._date_converter(snapshot["end_time"]),
+            "end_stamp": snapshot["end_time_in_millis"] // 1000,
+            "duration_s": snapshot["duration_in_millis"] // 1000,
+        }
+        return snap_dict
+
+    def _build_policy_details(self):
+        """get additional policy details"""
+        policy = self._get_policy()
+        if not policy:
+            return False
+
+        next_exec = policy["next_execution_millis"] // 1000
+        next_exec_date = datetime.fromtimestamp(next_exec)
+        next_exec_str = next_exec_date.strftime("%Y-%m-%d %H:%M")
+        expire_after = policy["policy"]["retention"]["expire_after"]
+        policy_metadata = {
+            "next_exec": next_exec,
+            "next_exec_str": next_exec_str,
+            "expire_after": expire_after,
+        }
+        return policy_metadata
+
+    @staticmethod
+    def _date_converter(date_utc):
+        """convert datetime string"""
+        expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
+        date = datetime.strptime(date_utc, expected_format)
+        local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
+        converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ")))
+        converted_str = converted.strftime("%Y-%m-%d %H:%M")
+
+        return converted_str
+
+    def restore_all(self, snapshot_name):
+        """restore snapshot by name"""
+        for index in self.all_indices:
+            _, _ = ElasticWrap(index).delete()
+
+        path = f"_snapshot/{self.REPO}/{snapshot_name}/_restore"
+        data = {"indices": "*"}
+        response, statuscode = ElasticWrap(path).post(data=data)
+        if statuscode == 200:
+            print(f"snapshot: executing now: {response}")
+            return response
+
+        print(f"snapshot: failed to restore, {statuscode} {response}")
+        return False
+
+    def delete_single_snapshot(self, snapshot_id):
+        """delete single snapshot from index"""
+        path = f"_snapshot/{self.REPO}/{snapshot_id}"
+        response, statuscode = ElasticWrap(path).delete()
+        if statuscode == 200:
+            print(f"snapshot: deleting {snapshot_id} {response}")
+            return response
+
+        print(f"snapshot: failed to delete, {statuscode} {response}")
+        return False

tubearchivist/home/src/frontend/api_calls.py

@@ -9,7 +9,6 @@ from home.src.download.subscriptions import (
     ChannelSubscription,
     PlaylistSubscription,
 )
-from home.src.frontend.searching import SearchForm
 from home.src.frontend.watched import WatchState
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
@@ -74,7 +73,6 @@ class PostData:
             "db-backup": self._db_backup,
             "db-restore": self._db_restore,
             "fs-rescan": self._fs_rescan,
-            "multi_search": self._multi_search,
             "delete-video": self._delete_video,
             "delete-channel": self._delete_channel,
             "delete-playlist": self._delete_playlist,
@@ -286,13 +284,6 @@ class PostData:
         rescan_filesystem.delay()
         return {"success": True}
 
-    def _multi_search(self):
-        """search through all indexes"""
-        search_query = self.exec_val
-        print("searching for: " + search_query)
-        search_results = SearchForm().multi_search(search_query)
-        return search_results
-
     def _delete_video(self):
         """delete media file, metadata and thumb"""
         youtube_id = self.exec_val

tubearchivist/home/src/frontend/forms.py

@@ -74,6 +74,12 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable Cast"),
     ]
 
+    SNAPSHOT_CHOICES = [
+        ("", "-- change snapshot settings --"),
+        ("0", "disable system snapshots"),
+        ("1", "enable system snapshots"),
+    ]
+
     SUBTITLE_SOURCE_CHOICES = [
         ("", "-- change subtitle source settings"),
         ("user", "only download user created"),
@@ -92,7 +98,9 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable cookie"),
     ]
 
-    subscriptions_channel_size = forms.IntegerField(required=False)
+    subscriptions_channel_size = forms.IntegerField(
+        required=False, min_value=1
+    )
     downloads_limit_count = forms.IntegerField(required=False)
     downloads_limit_speed = forms.IntegerField(required=False)
     downloads_throttledratelimit = forms.IntegerField(required=False)
@@ -124,6 +132,9 @@ class ApplicationSettingsForm(forms.Form):
     application_enable_cast = forms.ChoiceField(
         widget=forms.Select, choices=CAST_CHOICES, required=False
     )
+    application_enable_snapshot = forms.ChoiceField(
+        widget=forms.Select, choices=SNAPSHOT_CHOICES, required=False
+    )
 
 
 class SchedulerSettingsForm(forms.Form):
@@ -159,12 +170,14 @@ class MultiSearchForm(forms.Form):
 class AddToQueueForm(forms.Form):
     """text area form to add to downloads"""
 
+    HELP_TEXT = "Enter at least one video, channel or playlist id/URL here..."
+
     vid_url = forms.CharField(
         label=False,
         widget=forms.Textarea(
             attrs={
                 "rows": 4,
-                "placeholder": "Enter Video Urls or IDs here...",
+                "placeholder": HELP_TEXT,
             }
         ),
     )

tubearchivist/home/src/frontend/searching.py

@@ -119,7 +119,7 @@ class SearchHandler:
 
         if "vid_thumb_url" in hit_keys:
             youtube_id = hit["source"]["youtube_id"]
-            thumb_path = ThumbManager().vid_thumb_path(youtube_id)
+            thumb_path = ThumbManager(youtube_id).vid_thumb_path()
             hit["source"]["vid_thumb_url"] = thumb_path
 
         if "channel_last_refresh" in hit_keys:
@@ -136,6 +136,11 @@ class SearchHandler:
                 date_str = datetime.strftime(date_refresh, "%d %b, %Y")
                 hit["source"]["channel"]["channel_last_refresh"] = date_str
 
+        if "subtitle_fragment_id" in hit_keys:
+            youtube_id = hit["source"]["youtube_id"]
+            thumb_path = ThumbManager(youtube_id).vid_thumb_path()
+            hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
+
         return hit
 
 
@@ -146,38 +151,12 @@ class SearchForm:
 
     def multi_search(self, search_query):
         """searching through index"""
-        path = "ta_video,ta_channel,ta_playlist/_search"
-        data = {
-            "size": 30,
-            "query": {
-                "multi_match": {
-                    "query": search_query,
-                    "type": "bool_prefix",
-                    "operator": "and",
-                    "fuzziness": "auto",
-                    "fields": [
-                        "category",
-                        "channel_description",
-                        "channel_name._2gram",
-                        "channel_name._3gram",
-                        "channel_name.search_as_you_type",
-                        "playlist_description",
-                        "playlist_name._2gram",
-                        "playlist_name._3gram",
-                        "playlist_name.search_as_you_type",
-                        "tags",
-                        "title._2gram",
-                        "title._3gram",
-                        "title.search_as_you_type",
-                    ],
-                }
-            },
-        }
-        look_up = SearchHandler(path, config=self.CONFIG, data=data)
+        path, query, query_type = SearchParser(search_query).run()
+        look_up = SearchHandler(path, config=self.CONFIG, data=query)
         search_results = look_up.get_data()
         all_results = self.build_results(search_results)
 
-        return {"results": all_results}
+        return {"results": all_results, "queryType": query_type}
 
     @staticmethod
     def build_results(search_results):
@@ -185,6 +164,7 @@ class SearchForm:
         video_results = []
         channel_results = []
         playlist_results = []
+        fulltext_results = []
         if search_results:
             for result in search_results:
                 if result["_index"] == "ta_video":
@@ -193,11 +173,346 @@ class SearchForm:
                     channel_results.append(result)
                 elif result["_index"] == "ta_playlist":
                     playlist_results.append(result)
+                elif result["_index"] == "ta_subtitle":
+                    fulltext_results.append(result)
 
         all_results = {
             "video_results": video_results,
             "channel_results": channel_results,
             "playlist_results": playlist_results,
+            "fulltext_results": fulltext_results,
         }
 
         return all_results
+
+
+class SearchParser:
+    """handle structured searches"""
+
+    def __init__(self, search_query):
+        self.query_words = search_query.lower().split()
+        self.query_map = {"term": [], "fuzzy": []}
+        self.append_to = "term"
+
+    def run(self):
+        """collection, return path and query dict for es"""
+        print(f"query words: {self.query_words}")
+        query_type = self._find_map()
+        self._run_words()
+        self._delete_unset()
+        self._match_data_types()
+
+        path, query = QueryBuilder(self.query_map, query_type).run()
+
+        return path, query, query_type
+
+    def _find_map(self):
+        """find query in keyword map"""
+        first_word = self.query_words[0]
+        key_word_map = self._get_map()
+
+        if ":" in first_word:
+            index_match, query_string = first_word.split(":")
+            if index_match in key_word_map:
+                self.query_map.update(key_word_map.get(index_match))
+                self.query_words[0] = query_string
+                return index_match
+
+        self.query_map.update(key_word_map.get("simple"))
+        print(f"query_map: {self.query_map}")
+
+        return "simple"
+
+    @staticmethod
+    def _get_map():
+        """return map to build on"""
+        return {
+            "simple": {
+                "index": "ta_video,ta_channel,ta_playlist",
+            },
+            "video": {
+                "index": "ta_video",
+                "channel": [],
+                "active": [],
+            },
+            "channel": {
+                "index": "ta_channel",
+                "active": [],
+                "subscribed": [],
+            },
+            "playlist": {
+                "index": "ta_playlist",
+                "active": [],
+                "subscribed": [],
+            },
+            "full": {
+                "index": "ta_subtitle",
+                "lang": [],
+                "source": [],
+            },
+        }
+
+    def _run_words(self):
+        """append word by word"""
+        for word in self.query_words:
+            if ":" in word:
+                keyword, search_string = word.split(":")
+                if keyword in self.query_map:
+                    self.append_to = keyword
+                    word = search_string
+
+            if word:
+                self.query_map[self.append_to].append(word)
+
+    def _delete_unset(self):
+        """delete unset keys"""
+        new_query_map = {}
+        for key, value in self.query_map.items():
+            if value:
+                new_query_map.update({key: value})
+        self.query_map = new_query_map
+
+    def _match_data_types(self):
+        """match values with data types"""
+        for key, value in self.query_map.items():
+            if key in ["term", "channel"]:
+                self.query_map[key] = " ".join(self.query_map[key])
+            if key in ["active", "subscribed"]:
+                self.query_map[key] = "yes" in value
+
+
+class QueryBuilder:
+    """build query for ES from form data"""
+
+    def __init__(self, query_map, query_type):
+        self.query_map = query_map
+        self.query_type = query_type
+
+    def run(self):
+        """build query"""
+        path = self._build_path()
+        query = self.build_query()
+        print(f"es path: {path}")
+        print(f"query: {query}")
+
+        return path, query
+
+    def _build_path(self):
+        """build es index search path"""
+        return f"{self.query_map.get('index')}/_search"
+
+    def build_query(self):
+        """build query based on query_type"""
+
+        exec_map = {
+            "simple": self._build_simple,
+            "video": self._build_video,
+            "channel": self._build_channel,
+            "playlist": self._build_playlist,
+            "full": self._build_fulltext,
+        }
+
+        build_must_list = exec_map[self.query_type]
+
+        if self.query_type == "full":
+            query = build_must_list()
+        else:
+            query = {
+                "size": 30,
+                "query": {"bool": {"must": build_must_list()}},
+            }
+
+        return query
+
+    def _get_fuzzy(self):
+        """return fuziness valuee"""
+        fuzzy_value = self.query_map.get("fuzzy", ["auto"])[0]
+        if fuzzy_value == "no":
+            return 0
+
+        if not fuzzy_value.isdigit():
+            return "auto"
+
+        if int(fuzzy_value) > 2:
+            return "2"
+
+        return fuzzy_value
+
+    def _build_simple(self):
+        """build simple cross index query"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "channel_name._2gram",
+                            "channel_name._3gram",
+                            "channel_name.search_as_you_type",
+                            "playlist_name._2gram",
+                            "playlist_name._3gram",
+                            "playlist_name.search_as_you_type",
+                            "title._2gram",
+                            "title._3gram",
+                            "title.search_as_you_type",
+                        ],
+                    }
+                }
+            )
+
+        return must_list
+
+    def _build_video(self):
+        """build video query"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "title._2gram^2",
+                            "title._3gram^2",
+                            "title.search_as_you_type^2",
+                            "tags",
+                            "category",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"active": {"value": active}}})
+
+        if (channel := self.query_map.get("channel")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": channel,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "channel.channel_name._2gram",
+                            "channel.channel_name._3gram",
+                            "channel.channel_name.search_as_you_type",
+                        ],
+                    }
+                }
+            )
+
+        return must_list
+
+    def _build_channel(self):
+        """build query for channel"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "channel_description",
+                            "channel_name._2gram^2",
+                            "channel_name._3gram^2",
+                            "channel_name.search_as_you_type^2",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"channel_active": {"value": active}}})
+
+        if (subscribed := self.query_map.get("subscribed")) is not None:
+            must_list.append(
+                {"term": {"channel_subscribed": {"value": subscribed}}}
+            )
+
+        return must_list
+
+    def _build_playlist(self):
+        """build query for playlist"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "multi_match": {
+                        "query": term,
+                        "type": "bool_prefix",
+                        "fuzziness": self._get_fuzzy(),
+                        "operator": "and",
+                        "fields": [
+                            "playlist_description",
+                            "playlist_name._2gram^2",
+                            "playlist_name._3gram^2",
+                            "playlist_name.search_as_you_type^2",
+                        ],
+                    }
+                }
+            )
+
+        if (active := self.query_map.get("active")) is not None:
+            must_list.append({"term": {"playlist_active": {"value": active}}})
+
+        if (subscribed := self.query_map.get("subscribed")) is not None:
+            must_list.append(
+                {"term": {"playlist_subscribed": {"value": subscribed}}}
+            )
+
+        return must_list
+
+    def _build_fulltext(self):
+        """build query for fulltext search"""
+        must_list = []
+
+        if (term := self.query_map.get("term")) is not None:
+            must_list.append(
+                {
+                    "match": {
+                        "subtitle_line": {
+                            "query": term,
+                            "fuzziness": self._get_fuzzy(),
+                        }
+                    }
+                }
+            )
+
+        if (lang := self.query_map.get("lang")) is not None:
+            must_list.append({"term": {"subtitle_lang": {"value": lang[0]}}})
+
+        if (source := self.query_map.get("source")) is not None:
+            must_list.append(
+                {"term": {"subtitle_source": {"value": source[0]}}}
+            )
+
+        query = {
+            "size": 30,
+            "_source": {"excludes": "subtitle_line"},
+            "query": {"bool": {"must": must_list}},
+            "highlight": {
+                "fields": {
+                    "subtitle_line": {
+                        "number_of_fragments": 0,
+                        "pre_tags": ['<span class="settings-current">'],
+                        "post_tags": ["</span>"],
+                    }
+                }
+            },
+        }
+
+        return query

tubearchivist/home/src/frontend/watched.py

@@ -57,6 +57,14 @@ class WatchState:
             print(response)
             raise ValueError("failed to mark video as watched")
 
+    def _get_source(self):
+        """build source line for update_by_query script"""
+        source = [
+            "ctx._source.player['watched'] = true",
+            f"ctx._source.player['watched_date'] = {self.stamp}",
+        ]
+        return "; ".join(source)
+
     def mark_channel_watched(self):
         """change watched status of every video in channel"""
         path = "ta_video/_update_by_query"
@@ -67,7 +75,7 @@ class WatchState:
         data = {
             "query": {"bool": {"must": must_list}},
             "script": {
-                "source": "ctx._source.player['watched'] = true",
+                "source": self._get_source(),
                 "lang": "painless",
             },
         }
@@ -87,7 +95,7 @@ class WatchState:
         data = {
             "query": {"bool": {"must": must_list}},
             "script": {
-                "source": "ctx._source.player['watched'] = true",
+                "source": self._get_source(),
                 "lang": "painless",
             },
         }

tubearchivist/home/src/index/channel.py

@@ -37,6 +37,9 @@ class ChannelScraper:
         """main method to return channel dict"""
         self.get_soup()
         self._extract_yt_json()
+        if self._is_deactivated():
+            return False
+
         self._parse_channel_main()
         self._parse_channel_meta()
         return self.json_data
@@ -68,6 +71,17 @@ class ChannelScraper:
         json_raw = script_content.rstrip(";</script>")
         self.yt_json = json.loads(json_raw)
 
+    def _is_deactivated(self):
+        """check if channel is deactivated"""
+        alerts = self.yt_json.get("alerts")
+        if not alerts:
+            return False
+
+        for alert in alerts:
+            alert_text = alert["alertRenderer"]["text"]["simpleText"]
+            print(f"{self.channel_id}: failed to extract, {alert_text}")
+            return True
+
     def _parse_channel_main(self):
         """extract maintab values from scraped channel json data"""
         main_tab = self.yt_json["header"]["c4TabbedHeaderRenderer"]
@@ -160,30 +174,72 @@ class YoutubeChannel(YouTubeItem):
         self.es_path = f"{self.index_name}/_doc/{youtube_id}"
         self.all_playlists = False
 
-    def build_json(self, upload=False):
+    def build_json(self, upload=False, fallback=False):
         """get from es or from youtube"""
         self.get_from_es()
         if self.json_data:
             return
 
-        self.get_from_youtube()
+        self.get_from_youtube(fallback)
+
         if upload:
             self.upload_to_es()
         return
 
-    def get_from_youtube(self):
+    def get_from_youtube(self, fallback=False):
         """use bs4 to scrape channel about page"""
         self.json_data = ChannelScraper(self.youtube_id).get_json()
+
+        if not self.json_data and fallback:
+            self._video_fallback(fallback)
+
         self.get_channel_art()
 
+    def _video_fallback(self, fallback):
+        """use video metadata as fallback"""
+        print(f"{self.youtube_id}: fallback to video metadata")
+        self.json_data = {
+            "channel_active": False,
+            "channel_last_refresh": int(datetime.now().strftime("%s")),
+            "channel_subs": fallback.get("channel_follower_count", 0),
+            "channel_name": fallback["uploader"],
+            "channel_banner_url": False,
+            "channel_tvart_url": False,
+            "channel_id": self.youtube_id,
+            "channel_subscribed": False,
+            "channel_description": False,
+            "channel_thumb_url": False,
+            "channel_views": 0,
+        }
+        self._info_json_fallback()
+
+    def _info_json_fallback(self):
+        """read channel info.json for additional metadata"""
+        info_json = os.path.join(
+            self.config["application"]["cache_dir"],
+            "import",
+            f"{self.youtube_id}.info.json",
+        )
+        if os.path.exists(info_json):
+            print(f"{self.youtube_id}: read info.json file")
+            with open(info_json, "r", encoding="utf-8") as f:
+                content = json.loads(f.read())
+
+            self.json_data.update(
+                {
+                    "channel_subs": content.get("channel_follower_count", 0),
+                    "channel_description": content.get("description", False),
+                }
+            )
+            os.remove(info_json)
+
     def get_channel_art(self):
         """download channel art for new channels"""
-        channel_id = self.youtube_id
-        channel_thumb = self.json_data["channel_thumb_url"]
-        channel_banner = self.json_data["channel_banner_url"]
-        ThumbManager().download_chan(
-            [(channel_id, channel_thumb, channel_banner)]
+        urls = (
+            self.json_data["channel_thumb_url"],
+            self.json_data["channel_banner_url"],
         )
+        ThumbManager(self.youtube_id, item_type="channel").download(urls)
 
     def sync_to_videos(self):
         """sync new channel_dict to all videos of channel"""

tubearchivist/home/src/index/filesystem.py

@@ -12,13 +12,17 @@ import shutil
 import subprocess
 
 from home.src.download.queue import PendingList
-from home.src.download.yt_dlp_handler import VideoDownloader
+from home.src.download.thumbnails import ThumbManager
 from home.src.es.connect import ElasticWrap
 from home.src.index.reindex import Reindex
-from home.src.index.video import index_new_video
+from home.src.index.video import YoutubeVideo, index_new_video
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import clean_string, ignore_filelist
 from home.src.ta.ta_redis import RedisArchivist
+from PIL import Image, ImageFile
+from yt_dlp.utils import ISO639Utils
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
 
 
 class FilesystemScanner:
@@ -157,63 +161,152 @@ class FilesystemScanner:
             _, _ = ElasticWrap(path).delete()
 
 
-class ManualImport:
-    """import and indexing existing video files"""
+class ImportFolderScanner:
+    """import and indexing existing video files
+    - identify all media files belonging to a video
+    - identify youtube id
+    - convert if needed
+    """
 
     CONFIG = AppConfig().config
     CACHE_DIR = CONFIG["application"]["cache_dir"]
     IMPORT_DIR = os.path.join(CACHE_DIR, "import")
 
+    EXT_MAP = {
+        "media": [".mp4", ".mkv", ".webm"],
+        "metadata": [".json"],
+        "thumb": [".jpg", ".png", ".webp"],
+        "subtitle": [".vtt"],
+    }
+
     def __init__(self):
-        self.identified = self.import_folder_parser()
+        self.to_import = False
 
-    def import_folder_parser(self):
-        """detect files in import folder"""
-        import_files = os.listdir(self.IMPORT_DIR)
-        to_import = ignore_filelist(import_files)
-        to_import.sort()
-        video_files = [i for i in to_import if not i.endswith(".json")]
+    def scan(self):
+        """scan and match media files"""
+        all_files = self.get_all_files()
+        self.match_files(all_files)
+        self.process_videos()
 
-        identified = []
+        return self.to_import
 
-        for file_path in video_files:
+    def get_all_files(self):
+        """get all files in /import"""
+        rel_paths = ignore_filelist(os.listdir(self.IMPORT_DIR))
+        all_files = [os.path.join(self.IMPORT_DIR, i) for i in rel_paths]
+        all_files.sort()
 
-            file_dict = {"video_file": file_path}
-            file_name, _ = os.path.splitext(file_path)
-
-            matching_json = [
-                i
-                for i in to_import
-                if i.startswith(file_name) and i.endswith(".json")
-            ]
-            if matching_json:
-                json_file = matching_json[0]
-                youtube_id = self.extract_id_from_json(json_file)
-                file_dict.update({"json_file": json_file})
-            else:
-                youtube_id = self.extract_id_from_filename(file_name)
-                file_dict.update({"json_file": False})
-
-            file_dict.update({"youtube_id": youtube_id})
-            identified.append(file_dict)
-
-        return identified
+        return all_files
 
     @staticmethod
-    def extract_id_from_filename(file_name):
+    def _get_template():
+        """base dict for video"""
+        return {
+            "media": False,
+            "video_id": False,
+            "metadata": False,
+            "thumb": False,
+            "subtitle": [],
+        }
+
+    def match_files(self, all_files):
+        """loop through all files, join what matches"""
+        self.to_import = []
+
+        current_video = self._get_template()
+        last_base = False
+
+        for file_path in all_files:
+            base_name, ext = self._detect_base_name(file_path)
+            key, file_path = self._detect_type(file_path, ext)
+            if not key or not file_path:
+                continue
+
+            if base_name != last_base:
+                if last_base:
+                    print(f"manual import: {current_video}")
+                    self.to_import.append(current_video)
+
+                current_video = self._get_template()
+                last_base = base_name
+
+            if key == "subtitle":
+                current_video["subtitle"].append(file_path)
+            else:
+                current_video[key] = file_path
+
+        if current_video.get("media"):
+            print(f"manual import: {current_video}")
+            self.to_import.append(current_video)
+
+    def _detect_base_name(self, file_path):
+        """extract base_name and ext for matching"""
+        base_name_raw, ext = os.path.splitext(file_path)
+        base_name, ext2 = os.path.splitext(base_name_raw)
+
+        if ext2:
+            if ISO639Utils.short2long(ext2.strip(".")) or ext2 == ".info":
+                # valid secondary extension
+                return base_name, ext
+
+        return base_name_raw, ext
+
+    def _detect_type(self, file_path, ext):
+        """detect metadata type for file"""
+
+        for key, value in self.EXT_MAP.items():
+            if ext in value:
+                return key, file_path
+
+        return False, False
+
+    def process_videos(self):
+        """loop through all videos"""
+        for current_video in self.to_import:
+            if not current_video["media"]:
+                print(f"{current_video}: no matching media file found.")
+                raise ValueError
+
+            self._detect_youtube_id(current_video)
+            self._dump_thumb(current_video)
+            self._convert_thumb(current_video)
+            self._get_subtitles(current_video)
+            self._convert_video(current_video)
+            print(f"manual import: {current_video}")
+
+            ManualImport(current_video, self.CONFIG).run()
+
+    def _detect_youtube_id(self, current_video):
+        """find video id from filename or json"""
+        youtube_id = self._extract_id_from_filename(current_video["media"])
+        if youtube_id:
+            current_video["video_id"] = youtube_id
+            return
+
+        youtube_id = self._extract_id_from_json(current_video["metadata"])
+        if youtube_id:
+            current_video["video_id"] = youtube_id
+            return
+
+        raise ValueError("failed to find video id")
+
+    @staticmethod
+    def _extract_id_from_filename(file_name):
         """
         look at the file name for the youtube id
         expects filename ending in [<youtube_id>].<ext>
         """
-        id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", file_name)
+        base_name, _ = os.path.splitext(file_name)
+        id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
         if id_search:
             youtube_id = id_search.group(1)
             return youtube_id
 
-        print("failed to extract youtube id for: " + file_name)
-        raise Exception
+        print(f"id extraction failed from filename: {file_name}")
 
-    def extract_id_from_json(self, json_file):
+        return False
+
+    def _extract_id_from_json(self, json_file):
         """open json file and extract id"""
         json_path = os.path.join(self.CACHE_DIR, "import", json_file)
         with open(json_path, "r", encoding="utf-8") as f:
@@ -223,66 +316,276 @@ class ManualImport:
 
         return youtube_id
 
-    def process_import(self):
-        """go through identified media files"""
+    def _dump_thumb(self, current_video):
+        """extract embedded thumb before converting"""
+        if current_video["thumb"]:
+            return
 
-        all_videos_added = []
+        media_path = current_video["media"]
+        _, ext = os.path.splitext(media_path)
 
-        for media_file in self.identified:
-            json_file = media_file["json_file"]
-            video_file = media_file["video_file"]
-            youtube_id = media_file["youtube_id"]
+        new_path = False
+        if ext == ".mkv":
+            idx, thumb_type = self._get_mkv_thumb_stream(media_path)
+            if idx is not None:
+                new_path = self.dump_mpv_thumb(media_path, idx, thumb_type)
 
-            video_path = os.path.join(self.CACHE_DIR, "import", video_file)
+        elif ext == ".mp4":
+            thumb_type = self.get_mp4_thumb_type(media_path)
+            if thumb_type:
+                new_path = self.dump_mp4_thumb(media_path, thumb_type)
 
-            self.move_to_cache(video_path, youtube_id)
+        if new_path:
+            current_video["thumb"] = new_path
 
-            # identify and archive
-            vid_dict = index_new_video(youtube_id)
-            VideoDownloader([youtube_id]).move_to_archive(vid_dict)
-            youtube_id = vid_dict["youtube_id"]
-            thumb_url = vid_dict["vid_thumb_url"]
-            all_videos_added.append((youtube_id, thumb_url))
+    def _get_mkv_thumb_stream(self, media_path):
+        """get stream idx of thumbnail for mkv files"""
+        streams = self._get_streams(media_path)
+        attachments = [
+            i for i in streams["streams"] if i["codec_type"] == "attachment"
+        ]
 
-            # cleanup
-            if os.path.exists(video_path):
-                os.remove(video_path)
-            if json_file:
-                json_path = os.path.join(self.CACHE_DIR, "import", json_file)
-                os.remove(json_path)
+        for idx, stream in enumerate(attachments):
+            tags = stream["tags"]
+            if "mimetype" in tags and tags["filename"].startswith("cover"):
+                _, ext = os.path.splitext(tags["filename"])
+                return idx, ext
 
-        return all_videos_added
+        return None, None
 
-    def move_to_cache(self, video_path, youtube_id):
-        """move identified video file to cache, convert to mp4"""
-        file_name = os.path.split(video_path)[-1]
-        video_file, ext = os.path.splitext(file_name)
+    @staticmethod
+    def dump_mpv_thumb(media_path, idx, thumb_type):
+        """write cover to disk for mkv"""
+        _, media_ext = os.path.splitext(media_path)
+        new_path = f"{media_path.rstrip(media_ext)}{thumb_type}"
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-v",
+                "quiet",
+                f"-dump_attachment:t:{idx}",
+                new_path,
+                "-i",
+                media_path,
+            ],
+            check=False,
+        )
 
-        # make sure youtube_id is in filename
-        if youtube_id not in video_file:
-            video_file = f"{video_file}_{youtube_id}"
+        return new_path
 
-        # move, convert if needed
+    def get_mp4_thumb_type(self, media_path):
+        """detect filetype of embedded thumbnail"""
+        streams = self._get_streams(media_path)
+
+        for stream in streams["streams"]:
+            if stream["codec_name"] in ["png", "jpg"]:
+                return stream["codec_name"]
+
+        return False
+
+    def _convert_thumb(self, current_video):
+        """convert all thumbnails to jpg"""
+        if not current_video["thumb"]:
+            return
+
+        thumb_path = current_video["thumb"]
+
+        base_path, ext = os.path.splitext(thumb_path)
+        if ext == ".jpg":
+            return
+
+        new_path = f"{base_path}.jpg"
+        img_raw = Image.open(thumb_path)
+        img_raw.convert("RGB").save(new_path)
+
+        os.remove(thumb_path)
+        current_video["thumb"] = new_path
+
+    def _get_subtitles(self, current_video):
+        """find all subtitles in media file"""
+        if current_video["subtitle"]:
+            return
+
+        media_path = current_video["media"]
+        streams = self._get_streams(media_path)
+        base_path, ext = os.path.splitext(media_path)
+
+        if ext == ".webm":
+            print(f"{media_path}: subtitle extract from webm not supported")
+            return
+
+        for idx, stream in enumerate(streams["streams"]):
+            if stream["codec_type"] == "subtitle":
+                lang = ISO639Utils.long2short(stream["tags"]["language"])
+                sub_path = f"{base_path}.{lang}.vtt"
+                self._dump_subtitle(idx, media_path, sub_path)
+                current_video["subtitle"].append(sub_path)
+
+    @staticmethod
+    def _dump_subtitle(idx, media_path, sub_path):
+        """extract subtitle from media file"""
+        subprocess.run(
+            ["ffmpeg", "-i", media_path, "-map", f"0:{idx}", sub_path],
+            check=True,
+        )
+
+    @staticmethod
+    def _get_streams(media_path):
+        """return all streams from media_path"""
+        streams_raw = subprocess.run(
+            [
+                "ffprobe",
+                "-v",
+                "error",
+                "-show_streams",
+                "-print_format",
+                "json",
+                media_path,
+            ],
+            capture_output=True,
+            check=True,
+        )
+        streams = json.loads(streams_raw.stdout.decode())
+
+        return streams
+
+    @staticmethod
+    def dump_mp4_thumb(media_path, thumb_type):
+        """save cover to disk"""
+        _, ext = os.path.splitext(media_path)
+        new_path = f"{media_path.rstrip(ext)}.{thumb_type}"
+
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-i",
+                media_path,
+                "-map",
+                "0:v",
+                "-map",
+                "-0:V",
+                "-c",
+                "copy",
+                new_path,
+            ],
+            check=True,
+        )
+
+        return new_path
+
+    def _convert_video(self, current_video):
+        """convert if needed"""
+        current_path = current_video["media"]
+        base_path, ext = os.path.splitext(current_path)
         if ext == ".mp4":
-            new_file = video_file + ext
-            dest_path = os.path.join(self.CACHE_DIR, "download", new_file)
-            shutil.move(video_path, dest_path)
+            return
+
+        new_path = base_path + ".mp4"
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-i",
+                current_path,
+                new_path,
+                "-loglevel",
+                "warning",
+                "-stats",
+            ],
+            check=True,
+        )
+        current_video["media"] = new_path
+        os.remove(current_path)
+
+
+class ManualImport:
+    """import single identified video"""
+
+    def __init__(self, current_video, config):
+        self.current_video = current_video
+        self.config = config
+
+    def run(self):
+        """run all"""
+        json_data = self.index_metadata()
+        self._move_to_archive(json_data)
+        self._cleanup(json_data)
+
+    def index_metadata(self):
+        """get metadata from yt or json"""
+        video_id = self.current_video["video_id"]
+        video = YoutubeVideo(video_id)
+        video.build_json(
+            youtube_meta_overwrite=self._get_info_json(),
+            media_path=self.current_video["media"],
+        )
+        if not video.json_data:
+            print(f"{video_id}: manual import failed, and no metadata found.")
+            raise ValueError
+
+        video.check_subtitles(subtitle_files=self.current_video["subtitle"])
+        video.upload_to_es()
+
+        if video.offline_import and self.current_video["thumb"]:
+            old_path = self.current_video["thumb"]
+            thumbs = ThumbManager(video_id)
+            new_path = thumbs.vid_thumb_path(absolute=True, create_folder=True)
+            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
         else:
-            print(f"processing with ffmpeg: {video_file}")
-            new_file = video_file + ".mp4"
-            dest_path = os.path.join(self.CACHE_DIR, "download", new_file)
-            subprocess.run(
-                [
-                    "ffmpeg",
-                    "-i",
-                    video_path,
-                    dest_path,
-                    "-loglevel",
-                    "warning",
-                    "-stats",
-                ],
-                check=True,
-            )
+            url = video.json_data["vid_thumb_url"]
+            ThumbManager(video_id).download_video_thumb(url)
+
+        return video.json_data
+
+    def _get_info_json(self):
+        """read info_json from file"""
+        if not self.current_video["metadata"]:
+            return False
+
+        with open(self.current_video["metadata"], "r", encoding="utf-8") as f:
+            info_json = json.loads(f.read())
+
+        return info_json
+
+    def _move_to_archive(self, json_data):
+        """move identified media file to archive"""
+        videos = self.config["application"]["videos"]
+
+        channel, file = os.path.split(json_data["media_url"])
+        channel_folder = os.path.join(videos, channel)
+        if not os.path.exists(channel_folder):
+            os.makedirs(channel_folder)
+
+        old_path = self.current_video["media"]
+        new_path = os.path.join(channel_folder, file)
+        shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+
+        base_name, _ = os.path.splitext(new_path)
+        for old_path in self.current_video["subtitle"]:
+            lang = old_path.split(".")[-2]
+            new_path = f"{base_name}.{lang}.vtt"
+            shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+
+    def _cleanup(self, json_data):
+        """cleanup leftover files"""
+        meta_data = self.current_video["metadata"]
+        if meta_data and os.path.exists(meta_data):
+            os.remove(meta_data)
+
+        thumb = self.current_video["thumb"]
+        if thumb and os.path.exists(thumb):
+            os.remove(thumb)
+
+        for subtitle_file in self.current_video["subtitle"]:
+            if os.path.exists(subtitle_file):
+                os.remove(subtitle_file)
+
+        channel_info = os.path.join(
+            self.config["application"]["cache_dir"],
+            "import",
+            f"{json_data['channel']['channel_id']}.info.json",
+        )
+        if os.path.exists(channel_info):
+            os.remove(channel_info)
 
 
 def scan_filesystem():

tubearchivist/home/src/index/generic.py

@@ -73,16 +73,25 @@ class Pagination:
     figure out the pagination based on page size and total_hits
     """
 
-    def __init__(self, page_get, user_id, search_get=False):
-        self.user_id = user_id
+    def __init__(self, request):
+        self.request = request
+        self.page_get = False
+        self.params = False
+        self.get_params()
         self.page_size = self.get_page_size()
-        self.page_get = page_get
-        self.search_get = search_get
         self.pagination = self.first_guess()
 
+    def get_params(self):
+        """process url query parameters"""
+        query_dict = self.request.GET.copy()
+        self.page_get = int(query_dict.get("page", 0))
+
+        _ = query_dict.pop("page", False)
+        self.params = query_dict.urlencode()
+
     def get_page_size(self):
         """get default or user modified page_size"""
-        key = f"{self.user_id}:page_size"
+        key = f"{self.request.user.id}:page_size"
         page_size = RedisArchivist().get_message(key)["status"]
         if not page_size:
             config = AppConfig().config
@@ -108,9 +117,9 @@ class Pagination:
             "prev_pages": prev_pages,
             "current_page": page_get,
             "max_hits": False,
+            "params": self.params,
         }
-        if self.search_get:
-            pagination.update({"search_get": self.search_get})
+
         return pagination
 
     def validate(self, total_hits):

tubearchivist/home/src/index/playlist.py

@@ -18,7 +18,10 @@ class YoutubePlaylist(YouTubeItem):
 
     es_path = False
     index_name = "ta_playlist"
-    yt_obs = {"extract_flat": True}
+    yt_obs = {
+        "extract_flat": True,
+        "allow_playlist_files": True,
+    }
     yt_base = "https://www.youtube.com/playlist?list="
 
     def __init__(self, youtube_id):
@@ -41,18 +44,23 @@ class YoutubePlaylist(YouTubeItem):
             self.process_youtube_meta()
             self.get_entries()
             self.json_data["playlist_entries"] = self.all_members
-            self.get_playlist_art()
             self.json_data["playlist_subscribed"] = subscribed
 
     def process_youtube_meta(self):
         """extract relevant fields from youtube"""
+        try:
+            playlist_thumbnail = self.youtube_meta["thumbnails"][-1]["url"]
+        except IndexError:
+            print(f"{self.youtube_id}: thumbnail extraction failed")
+            playlist_thumbnail = False
+
         self.json_data = {
             "playlist_id": self.youtube_id,
             "playlist_active": True,
             "playlist_name": self.youtube_meta["title"],
             "playlist_channel": self.youtube_meta["channel"],
             "playlist_channel_id": self.youtube_meta["channel_id"],
-            "playlist_thumbnail": self.youtube_meta["thumbnails"][-1]["url"],
+            "playlist_thumbnail": playlist_thumbnail,
             "playlist_description": self.youtube_meta["description"] or False,
             "playlist_last_refresh": int(datetime.now().strftime("%s")),
         }
@@ -81,12 +89,10 @@ class YoutubePlaylist(YouTubeItem):
 
         self.all_members = all_members
 
-    @staticmethod
-    def get_playlist_art():
+    def get_playlist_art(self):
         """download artwork of playlist"""
-        thumbnails = ThumbManager()
-        missing_playlists = thumbnails.get_missing_playlists()
-        thumbnails.download_playlist(missing_playlists)
+        url = self.json_data["playlist_thumbnail"]
+        ThumbManager(self.youtube_id, item_type="playlist").download(url)
 
     def add_vids_to_playlist(self):
         """sync the playlist id to videos"""
@@ -145,17 +151,15 @@ class YoutubePlaylist(YouTubeItem):
             previous_item = False
         else:
             previous_item = all_entries[current_idx - 1]
-            prev_thumb = ThumbManager().vid_thumb_path(
-                previous_item["youtube_id"]
-            )
-            previous_item["vid_thumb"] = prev_thumb
+            prev_id = previous_item["youtube_id"]
+            previous_item["vid_thumb"] = ThumbManager(prev_id).vid_thumb_path()
 
         if current_idx == len(all_entries) - 1:
             next_item = False
         else:
             next_item = all_entries[current_idx + 1]
-            next_thumb = ThumbManager().vid_thumb_path(next_item["youtube_id"])
-            next_item["vid_thumb"] = next_thumb
+            next_id = next_item["youtube_id"]
+            next_item["vid_thumb"] = ThumbManager(next_id).vid_thumb_path()
 
         self.nav = {
             "playlist_meta": {

tubearchivist/home/src/index/reindex.py

@@ -72,14 +72,11 @@ class Reindex:
 
     def _get_unrated_vids(self):
         """get max 200 videos without rating if ryd integration is enabled"""
-        data = {
-            "size": 200,
-            "query": {
-                "bool": {
-                    "must_not": [{"exists": {"field": "stats.average_rating"}}]
-                }
-            },
-        }
+        must_not_list = [
+            {"exists": {"field": "stats.average_rating"}},
+            {"term": {"active": {"value": False}}},
+        ]
+        data = {"size": 200, "query": {"bool": {"must_not": must_not_list}}}
         response, _ = ElasticWrap("ta_video/_search").get(data=data)
 
         missing_rating = [i["_id"] for i in response["hits"]["hits"]]
@@ -181,10 +178,10 @@ class Reindex:
 
         video.upload_to_es()
 
-        thumb_handler = ThumbManager()
-        thumb_handler.delete_vid_thumb(youtube_id)
-        to_download = (youtube_id, video.json_data["vid_thumb_url"])
-        thumb_handler.download_vid([to_download], notify=False)
+        thumb_handler = ThumbManager(youtube_id)
+        thumb_handler.delete_video_thumb()
+        thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
+
         return
 
     @staticmethod
@@ -195,6 +192,10 @@ class Reindex:
         subscribed = channel.json_data["channel_subscribed"]
         overwrites = channel.json_data.get("channel_overwrites", False)
         channel.get_from_youtube()
+        if not channel.json_data:
+            channel.deactivate()
+            return
+
         channel.json_data["channel_subscribed"] = subscribed
         if overwrites:
             channel.json_data["channel_overwrites"] = overwrites
@@ -301,9 +302,9 @@ class ChannelUrlFixer:
         """fix filepath"""
         print(f"{self.youtube_id}: fixing channel rename.")
         cache_dir = self.config["application"]["cache_dir"]
-        new_file_path = os.path.join(
+        new_path = os.path.join(
             cache_dir, "download", self.youtube_id + ".mp4"
         )
-        shutil.move(video_path_is, new_file_path)
+        shutil.move(video_path_is, new_path, copy_function=shutil.copyfile)
         VideoDownloader().move_to_archive(self.video.json_data)
         self.video.update_media_url()

tubearchivist/home/src/index/subtitle.py

@@ -0,0 +1,321 @@
+"""
+functionality:
+- download subtitles
+- parse subtitles into it's cues
+- index dubtitles
+"""
+
+import json
+import os
+from datetime import datetime
+
+import requests
+from home.src.es.connect import ElasticWrap
+from home.src.ta.helper import requests_headers
+
+
+class YoutubeSubtitle:
+    """handle video subtitle functionality"""
+
+    def __init__(self, video):
+        self.video = video
+        self.languages = False
+
+    def _sub_conf_parse(self):
+        """add additional conf values to self"""
+        languages_raw = self.video.config["downloads"]["subtitle"]
+        if languages_raw:
+            self.languages = [i.strip() for i in languages_raw.split(",")]
+
+    def get_subtitles(self):
+        """check what to do"""
+        self._sub_conf_parse()
+        if not self.languages:
+            # no subtitles
+            return False
+
+        relevant_subtitles = []
+        for lang in self.languages:
+            user_sub = self._get_user_subtitles(lang)
+            if user_sub:
+                relevant_subtitles.append(user_sub)
+                continue
+
+            if self.video.config["downloads"]["subtitle_source"] == "auto":
+                auto_cap = self._get_auto_caption(lang)
+                if auto_cap:
+                    relevant_subtitles.append(auto_cap)
+
+        return relevant_subtitles
+
+    def _get_auto_caption(self, lang):
+        """get auto_caption subtitles"""
+        print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
+        all_subtitles = self.video.youtube_meta.get("automatic_captions")
+
+        if not all_subtitles:
+            return False
+
+        video_media_url = self.video.json_data["media_url"]
+        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
+        all_formats = all_subtitles.get(lang)
+        if not all_formats:
+            return False
+
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
+        subtitle.update(
+            {"lang": lang, "source": "auto", "media_url": media_url}
+        )
+
+        return subtitle
+
+    def _normalize_lang(self):
+        """normalize country specific language keys"""
+        all_subtitles = self.video.youtube_meta.get("subtitles")
+        if not all_subtitles:
+            return False
+
+        all_keys = list(all_subtitles.keys())
+        for key in all_keys:
+            lang = key.split("-")[0]
+            old = all_subtitles.pop(key)
+            if lang == "live_chat":
+                continue
+            all_subtitles[lang] = old
+
+        return all_subtitles
+
+    def _get_user_subtitles(self, lang):
+        """get subtitles uploaded from channel owner"""
+        print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
+        all_subtitles = self._normalize_lang()
+        if not all_subtitles:
+            return False
+
+        video_media_url = self.video.json_data["media_url"]
+        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
+        all_formats = all_subtitles.get(lang)
+        if not all_formats:
+            # no user subtitles found
+            return False
+
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
+        subtitle.update(
+            {"lang": lang, "source": "user", "media_url": media_url}
+        )
+
+        return subtitle
+
+    def download_subtitles(self, relevant_subtitles):
+        """download subtitle files to archive"""
+        videos_base = self.video.config["application"]["videos"]
+        indexed = []
+        for subtitle in relevant_subtitles:
+            dest_path = os.path.join(videos_base, subtitle["media_url"])
+            source = subtitle["source"]
+            lang = subtitle.get("lang")
+            response = requests.get(
+                subtitle["url"], headers=requests_headers()
+            )
+            if not response.ok:
+                print(f"{self.video.youtube_id}: failed to download subtitle")
+                print(response.text)
+                continue
+
+            parser = SubtitleParser(response.text, lang, source)
+            parser.process()
+            if not parser.all_cues:
+                continue
+
+            subtitle_str = parser.get_subtitle_str()
+            self._write_subtitle_file(dest_path, subtitle_str)
+            if self.video.config["downloads"]["subtitle_index"]:
+                query_str = parser.create_bulk_import(self.video, source)
+                self._index_subtitle(query_str)
+
+            indexed.append(subtitle)
+
+        return indexed
+
+    @staticmethod
+    def _write_subtitle_file(dest_path, subtitle_str):
+        """write subtitle file to disk"""
+        # create folder here for first video of channel
+        os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
+        with open(dest_path, "w", encoding="utf-8") as subfile:
+            subfile.write(subtitle_str)
+
+    @staticmethod
+    def _index_subtitle(query_str):
+        """send subtitle to es for indexing"""
+        _, _ = ElasticWrap("_bulk").post(data=query_str, ndjson=True)
+
+    def delete(self, subtitles=False):
+        """delete subtitles from index and filesystem"""
+        youtube_id = self.video.youtube_id
+        videos_base = self.video.config["application"]["videos"]
+        # delete files
+        if subtitles:
+            files = [i["media_url"] for i in subtitles]
+        else:
+            if not self.video.json_data.get("subtitles"):
+                return
+
+            files = [i["media_url"] for i in self.video.json_data["subtitles"]]
+
+        for file_name in files:
+            file_path = os.path.join(videos_base, file_name)
+            try:
+                os.remove(file_path)
+            except FileNotFoundError:
+                print(f"{youtube_id}: {file_path} failed to delete")
+        # delete from index
+        path = "ta_subtitle/_delete_by_query?refresh=true"
+        data = {"query": {"term": {"youtube_id": {"value": youtube_id}}}}
+        _, _ = ElasticWrap(path).post(data=data)
+
+
+class SubtitleParser:
+    """parse subtitle str from youtube"""
+
+    def __init__(self, subtitle_str, lang, source):
+        self.subtitle_raw = json.loads(subtitle_str)
+        self.lang = lang
+        self.source = source
+        self.all_cues = False
+
+    def process(self):
+        """extract relevant que data"""
+        self.all_cues = []
+        all_events = self.subtitle_raw.get("events")
+
+        if not all_events:
+            return
+
+        if self.source == "auto":
+            all_events = self._flat_auto_caption(all_events)
+
+        for idx, event in enumerate(all_events):
+            if "dDurationMs" not in event or "segs" not in event:
+                # some events won't have a duration or segs
+                print(f"skipping subtitle event without content: {event}")
+                continue
+
+            cue = {
+                "start": self._ms_conv(event["tStartMs"]),
+                "end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
+                "text": "".join([i.get("utf8") for i in event["segs"]]),
+                "idx": idx + 1,
+            }
+            self.all_cues.append(cue)
+
+    @staticmethod
+    def _flat_auto_caption(all_events):
+        """flatten autocaption segments"""
+        flatten = []
+        for event in all_events:
+            if "segs" not in event.keys():
+                continue
+            text = "".join([i.get("utf8") for i in event.get("segs")])
+            if not text.strip():
+                continue
+
+            if flatten:
+                # fix overlapping retiming issue
+                last = flatten[-1]
+                if "dDurationMs" not in last or "segs" not in last:
+                    # some events won't have a duration or segs
+                    print(f"skipping subtitle event without content: {event}")
+                    continue
+
+                last_end = last["tStartMs"] + last["dDurationMs"]
+                if event["tStartMs"] < last_end:
+                    joined = last["segs"][0]["utf8"] + "\n" + text
+                    last["segs"][0]["utf8"] = joined
+                    continue
+
+            event.update({"segs": [{"utf8": text}]})
+            flatten.append(event)
+
+        return flatten
+
+    @staticmethod
+    def _ms_conv(ms):
+        """convert ms to timestamp"""
+        hours = str((ms // (1000 * 60 * 60)) % 24).zfill(2)
+        minutes = str((ms // (1000 * 60)) % 60).zfill(2)
+        secs = str((ms // 1000) % 60).zfill(2)
+        millis = str(ms % 1000).zfill(3)
+
+        return f"{hours}:{minutes}:{secs}.{millis}"
+
+    def get_subtitle_str(self):
+        """create vtt text str from cues"""
+        subtitle_str = f"WEBVTT\nKind: captions\nLanguage: {self.lang}"
+
+        for cue in self.all_cues:
+            stamp = f"{cue.get('start')} --> {cue.get('end')}"
+            cue_text = f"\n\n{cue.get('idx')}\n{stamp}\n{cue.get('text')}"
+            subtitle_str = subtitle_str + cue_text
+
+        return subtitle_str
+
+    def create_bulk_import(self, video, source):
+        """subtitle lines for es import"""
+        documents = self._create_documents(video, source)
+        bulk_list = []
+
+        for document in documents:
+            document_id = document.get("subtitle_fragment_id")
+            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(document))
+
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+
+        return query_str
+
+    def _create_documents(self, video, source):
+        """process documents"""
+        documents = self._chunk_list(video.youtube_id)
+        channel = video.json_data.get("channel")
+        meta_dict = {
+            "youtube_id": video.youtube_id,
+            "title": video.json_data.get("title"),
+            "subtitle_channel": channel.get("channel_name"),
+            "subtitle_channel_id": channel.get("channel_id"),
+            "subtitle_last_refresh": int(datetime.now().strftime("%s")),
+            "subtitle_lang": self.lang,
+            "subtitle_source": source,
+        }
+
+        _ = [i.update(meta_dict) for i in documents]
+
+        return documents
+
+    def _chunk_list(self, youtube_id):
+        """join cues for bulk import"""
+        chunk_list = []
+
+        chunk = {}
+        for cue in self.all_cues:
+            if chunk:
+                text = f"{chunk.get('subtitle_line')} {cue.get('text')}\n"
+                chunk["subtitle_line"] = text
+            else:
+                idx = len(chunk_list) + 1
+                chunk = {
+                    "subtitle_index": idx,
+                    "subtitle_line": cue.get("text"),
+                    "subtitle_start": cue.get("start"),
+                }
+
+            chunk["subtitle_fragment_id"] = f"{youtube_id}-{self.lang}-{idx}"
+
+            if cue["idx"] % 5 == 0:
+                chunk["subtitle_end"] = cue.get("end")
+                chunk_list.append(chunk)
+                chunk = {}
+
+        return chunk_list

tubearchivist/home/src/index/video.py

@@ -4,7 +4,6 @@ functionality:
 - index and update in es
 """
 
-import json
 import os
 from datetime import datetime
 
@@ -14,311 +13,12 @@ from home.src.es.connect import ElasticWrap
 from home.src.index import channel as ta_channel
 from home.src.index import playlist as ta_playlist
 from home.src.index.generic import YouTubeItem
-from home.src.ta.helper import (
-    DurationConverter,
-    clean_string,
-    randomizor,
-    requests_headers,
-)
+from home.src.index.subtitle import YoutubeSubtitle
+from home.src.ta.helper import DurationConverter, clean_string, randomizor
 from home.src.ta.ta_redis import RedisArchivist
 from ryd_client import ryd_client
 
 
-class YoutubeSubtitle:
-    """handle video subtitle functionality"""
-
-    def __init__(self, video):
-        self.video = video
-        self.languages = False
-
-    def _sub_conf_parse(self):
-        """add additional conf values to self"""
-        languages_raw = self.video.config["downloads"]["subtitle"]
-        if languages_raw:
-            self.languages = [i.strip() for i in languages_raw.split(",")]
-
-    def get_subtitles(self):
-        """check what to do"""
-        self._sub_conf_parse()
-        if not self.languages:
-            # no subtitles
-            return False
-
-        relevant_subtitles = []
-        for lang in self.languages:
-            user_sub = self._get_user_subtitles(lang)
-            if user_sub:
-                relevant_subtitles.append(user_sub)
-                continue
-
-            if self.video.config["downloads"]["subtitle_source"] == "auto":
-                auto_cap = self._get_auto_caption(lang)
-                if auto_cap:
-                    relevant_subtitles.append(auto_cap)
-
-        return relevant_subtitles
-
-    def _get_auto_caption(self, lang):
-        """get auto_caption subtitles"""
-        print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
-        all_subtitles = self.video.youtube_meta.get("automatic_captions")
-
-        if not all_subtitles:
-            return False
-
-        video_media_url = self.video.json_data["media_url"]
-        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
-        all_formats = all_subtitles.get(lang)
-        if not all_formats:
-            return False
-
-        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
-        subtitle.update(
-            {"lang": lang, "source": "auto", "media_url": media_url}
-        )
-
-        return subtitle
-
-    def _normalize_lang(self):
-        """normalize country specific language keys"""
-        all_subtitles = self.video.youtube_meta.get("subtitles")
-        if not all_subtitles:
-            return False
-
-        all_keys = list(all_subtitles.keys())
-        for key in all_keys:
-            lang = key.split("-")[0]
-            old = all_subtitles.pop(key)
-            if lang == "live_chat":
-                continue
-            all_subtitles[lang] = old
-
-        return all_subtitles
-
-    def _get_user_subtitles(self, lang):
-        """get subtitles uploaded from channel owner"""
-        print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
-        all_subtitles = self._normalize_lang()
-        if not all_subtitles:
-            return False
-
-        video_media_url = self.video.json_data["media_url"]
-        media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
-        all_formats = all_subtitles.get(lang)
-        if not all_formats:
-            # no user subtitles found
-            return False
-
-        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
-        subtitle.update(
-            {"lang": lang, "source": "user", "media_url": media_url}
-        )
-
-        return subtitle
-
-    def download_subtitles(self, relevant_subtitles):
-        """download subtitle files to archive"""
-        videos_base = self.video.config["application"]["videos"]
-        for subtitle in relevant_subtitles:
-            dest_path = os.path.join(videos_base, subtitle["media_url"])
-            source = subtitle["source"]
-            lang = subtitle.get("lang")
-            response = requests.get(
-                subtitle["url"], headers=requests_headers()
-            )
-            if not response.ok:
-                print(f"{self.video.youtube_id}: failed to download subtitle")
-                print(response.text)
-                continue
-
-            parser = SubtitleParser(response.text, lang, source)
-            parser.process()
-            subtitle_str = parser.get_subtitle_str()
-            self._write_subtitle_file(dest_path, subtitle_str)
-            if self.video.config["downloads"]["subtitle_index"]:
-                query_str = parser.create_bulk_import(self.video, source)
-                self._index_subtitle(query_str)
-
-    @staticmethod
-    def _write_subtitle_file(dest_path, subtitle_str):
-        """write subtitle file to disk"""
-        # create folder here for first video of channel
-        os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
-        with open(dest_path, "w", encoding="utf-8") as subfile:
-            subfile.write(subtitle_str)
-
-    @staticmethod
-    def _index_subtitle(query_str):
-        """send subtitle to es for indexing"""
-        _, _ = ElasticWrap("_bulk").post(data=query_str, ndjson=True)
-
-    def delete(self, subtitles=False):
-        """delete subtitles from index and filesystem"""
-        youtube_id = self.video.youtube_id
-        videos_base = self.video.config["application"]["videos"]
-        # delete files
-        if subtitles:
-            files = [i["media_url"] for i in subtitles]
-        else:
-            if not self.video.json_data.get("subtitles"):
-                return
-
-            files = [i["media_url"] for i in self.video.json_data["subtitles"]]
-
-        for file_name in files:
-            file_path = os.path.join(videos_base, file_name)
-            try:
-                os.remove(file_path)
-            except FileNotFoundError:
-                print(f"{youtube_id}: {file_path} failed to delete")
-        # delete from index
-        path = "ta_subtitle/_delete_by_query?refresh=true"
-        data = {"query": {"term": {"youtube_id": {"value": youtube_id}}}}
-        _, _ = ElasticWrap(path).post(data=data)
-
-
-class SubtitleParser:
-    """parse subtitle str from youtube"""
-
-    def __init__(self, subtitle_str, lang, source):
-        self.subtitle_raw = json.loads(subtitle_str)
-        self.lang = lang
-        self.source = source
-        self.all_cues = False
-
-    def process(self):
-        """extract relevant que data"""
-        all_events = self.subtitle_raw.get("events")
-        if self.source == "auto":
-            all_events = self._flat_auto_caption(all_events)
-
-        self.all_cues = []
-        for idx, event in enumerate(all_events):
-            if "dDurationMs" not in event or "segs" not in event:
-                # some events won't have a duration or segs
-                print(f"skipping subtitle event without content: {event}")
-                continue
-
-            cue = {
-                "start": self._ms_conv(event["tStartMs"]),
-                "end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
-                "text": "".join([i.get("utf8") for i in event["segs"]]),
-                "idx": idx + 1,
-            }
-            self.all_cues.append(cue)
-
-    @staticmethod
-    def _flat_auto_caption(all_events):
-        """flatten autocaption segments"""
-        flatten = []
-        for event in all_events:
-            if "segs" not in event.keys():
-                continue
-            text = "".join([i.get("utf8") for i in event.get("segs")])
-            if not text.strip():
-                continue
-
-            if flatten:
-                # fix overlapping retiming issue
-                last = flatten[-1]
-                if "dDurationMs" not in last or "segs" not in last:
-                    # some events won't have a duration or segs
-                    print(f"skipping subtitle event without content: {event}")
-                    continue
-
-                last_end = last["tStartMs"] + last["dDurationMs"]
-                if event["tStartMs"] < last_end:
-                    joined = last["segs"][0]["utf8"] + "\n" + text
-                    last["segs"][0]["utf8"] = joined
-                    continue
-
-            event.update({"segs": [{"utf8": text}]})
-            flatten.append(event)
-
-        return flatten
-
-    @staticmethod
-    def _ms_conv(ms):
-        """convert ms to timestamp"""
-        hours = str((ms // (1000 * 60 * 60)) % 24).zfill(2)
-        minutes = str((ms // (1000 * 60)) % 60).zfill(2)
-        secs = str((ms // 1000) % 60).zfill(2)
-        millis = str(ms % 1000).zfill(3)
-
-        return f"{hours}:{minutes}:{secs}.{millis}"
-
-    def get_subtitle_str(self):
-        """create vtt text str from cues"""
-        subtitle_str = f"WEBVTT\nKind: captions\nLanguage: {self.lang}"
-
-        for cue in self.all_cues:
-            stamp = f"{cue.get('start')} --> {cue.get('end')}"
-            cue_text = f"\n\n{cue.get('idx')}\n{stamp}\n{cue.get('text')}"
-            subtitle_str = subtitle_str + cue_text
-
-        return subtitle_str
-
-    def create_bulk_import(self, video, source):
-        """subtitle lines for es import"""
-        documents = self._create_documents(video, source)
-        bulk_list = []
-
-        for document in documents:
-            document_id = document.get("subtitle_fragment_id")
-            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(document))
-
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-
-        return query_str
-
-    def _create_documents(self, video, source):
-        """process documents"""
-        documents = self._chunk_list(video.youtube_id)
-        channel = video.json_data.get("channel")
-        meta_dict = {
-            "youtube_id": video.youtube_id,
-            "title": video.json_data.get("title"),
-            "subtitle_channel": channel.get("channel_name"),
-            "subtitle_channel_id": channel.get("channel_id"),
-            "subtitle_last_refresh": int(datetime.now().strftime("%s")),
-            "subtitle_lang": self.lang,
-            "subtitle_source": source,
-        }
-
-        _ = [i.update(meta_dict) for i in documents]
-
-        return documents
-
-    def _chunk_list(self, youtube_id):
-        """join cues for bulk import"""
-        chunk_list = []
-
-        chunk = {}
-        for cue in self.all_cues:
-            if chunk:
-                text = f"{chunk.get('subtitle_line')} {cue.get('text')}\n"
-                chunk["subtitle_line"] = text
-            else:
-                idx = len(chunk_list) + 1
-                chunk = {
-                    "subtitle_index": idx,
-                    "subtitle_line": cue.get("text"),
-                    "subtitle_start": cue.get("start"),
-                }
-
-            chunk["subtitle_fragment_id"] = f"{youtube_id}-{self.lang}-{idx}"
-
-            if cue["idx"] % 5 == 0:
-                chunk["subtitle_end"] = cue.get("end")
-                chunk_list.append(chunk)
-                chunk = {}
-
-        return chunk_list
-
-
 class SponsorBlock:
     """handle sponsor block integration"""
 
@@ -348,9 +48,17 @@ class SponsorBlock:
         url = f"{self.API}/skipSegments?videoID={youtube_id}"
         headers = {"User-Agent": self.user_agent}
         print(f"{youtube_id}: get sponsorblock timestamps")
-        response = requests.get(url, headers=headers)
+        try:
+            response = requests.get(url, headers=headers, timeout=10)
+        except requests.ReadTimeout:
+            print(f"{youtube_id}: sponsorblock API timeout")
+            return False
+
         if not response.ok:
-            print(f"{youtube_id}: sponsorblock failed: {response.text}")
+            print(f"{youtube_id}: sponsorblock failed: {response.status_code}")
+            if response.status_code == 503:
+                return False
+
             sponsor_dict = {
                 "last_refresh": self.last_refresh,
                 "is_enabled": True,
@@ -425,18 +133,23 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         self.channel_id = False
         self.video_overwrites = video_overwrites
         self.es_path = f"{self.index_name}/_doc/{youtube_id}"
+        self.offline_import = False
 
-    def build_json(self):
+    def build_json(self, youtube_meta_overwrite=False, media_path=False):
         """build json dict of video"""
         self.get_from_youtube()
-        if not self.youtube_meta:
+        if not self.youtube_meta and not youtube_meta_overwrite:
             return
 
+        if not self.youtube_meta:
+            self.youtube_meta = youtube_meta_overwrite
+            self.offline_import = True
+
         self._process_youtube_meta()
         self._add_channel()
         self._add_stats()
         self.add_file_path()
-        self.add_player()
+        self.add_player(media_path)
         if self.config["downloads"]["integrate_ryd"]:
             self._get_ryd_stats()
 
@@ -487,7 +200,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
     def _add_channel(self):
         """add channel dict to video json_data"""
         channel = ta_channel.YoutubeChannel(self.channel_id)
-        channel.build_json(upload=True)
+        channel.build_json(upload=True, fallback=self.youtube_meta)
         self.json_data.update({"channel": channel.json_data})
 
     def _add_stats(self):
@@ -495,13 +208,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         # likes
         like_count = self.youtube_meta.get("like_count", 0)
         dislike_count = self.youtube_meta.get("dislike_count", 0)
+        average_rating = self.youtube_meta.get("average_rating", 0)
         self.json_data.update(
             {
                 "stats": {
                     "view_count": self.youtube_meta["view_count"],
                     "like_count": like_count,
                     "dislike_count": dislike_count,
-                    "average_rating": self.youtube_meta["average_rating"],
+                    "average_rating": average_rating,
                 }
             }
         )
@@ -518,8 +232,28 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
         raise FileNotFoundError
 
-    def add_player(self):
+    def add_player(self, media_path=False):
         """add player information for new videos"""
+        vid_path = self._get_vid_path(media_path)
+
+        duration_handler = DurationConverter()
+        duration = duration_handler.get_sec(vid_path)
+        duration_str = duration_handler.get_str(duration)
+        self.json_data.update(
+            {
+                "player": {
+                    "watched": False,
+                    "duration": duration,
+                    "duration_str": duration_str,
+                }
+            }
+        )
+
+    def _get_vid_path(self, media_path=False):
+        """get path of media file"""
+        if media_path:
+            return media_path
+
         try:
             # when indexing from download task
             vid_path = self.build_dl_cache_path()
@@ -535,18 +269,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             else:
                 raise FileNotFoundError("could not find video file") from err
 
-        duration_handler = DurationConverter()
-        duration = duration_handler.get_sec(vid_path)
-        duration_str = duration_handler.get_str(duration)
-        self.json_data.update(
-            {
-                "player": {
-                    "watched": False,
-                    "duration": duration,
-                    "duration_str": duration_str,
-                }
-            }
-        )
+        return vid_path
 
     def add_file_path(self):
         """build media_url for where file will be located"""
@@ -605,37 +328,60 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
     def _get_ryd_stats(self):
         """get optional stats from returnyoutubedislikeapi.com"""
+        # pylint: disable=broad-except
         try:
             print(f"{self.youtube_id}: get ryd stats")
             result = ryd_client.get(self.youtube_id)
-        except requests.exceptions.ConnectionError:
-            print(f"{self.youtube_id}: failed to query ryd api, skipping")
-            return False
+        except Exception as err:
+            print(f"{self.youtube_id}: failed to query ryd api {err}")
+            return
 
         if result["status"] == 404:
-            return False
+            return
 
         dislikes = {
-            "dislike_count": result["dislikes"],
-            "average_rating": result["rating"],
+            "dislike_count": result.get("dislikes", 0),
+            "average_rating": result.get("rating", 0),
         }
         self.json_data["stats"].update(dislikes)
 
-        return True
-
     def _get_sponsorblock(self):
         """get optional sponsorblock timestamps from sponsor.ajay.app"""
         sponsorblock = SponsorBlock().get_timestamps(self.youtube_id)
         if sponsorblock:
             self.json_data["sponsorblock"] = sponsorblock
 
-    def check_subtitles(self):
+    def check_subtitles(self, subtitle_files=False):
         """optionally add subtitles"""
+        if self.offline_import and subtitle_files:
+            indexed = self._offline_subtitles(subtitle_files)
+            self.json_data["subtitles"] = indexed
+            return
+
         handler = YoutubeSubtitle(self)
         subtitles = handler.get_subtitles()
         if subtitles:
-            self.json_data["subtitles"] = subtitles
-            handler.download_subtitles(relevant_subtitles=subtitles)
+            indexed = handler.download_subtitles(relevant_subtitles=subtitles)
+            self.json_data["subtitles"] = indexed
+
+    def _offline_subtitles(self, subtitle_files):
+        """import offline subtitles"""
+        base_name, _ = os.path.splitext(self.json_data["media_url"])
+        subtitles = []
+        for subtitle in subtitle_files:
+            lang = subtitle.split(".")[-2]
+            subtitle_media_url = f"{base_name}.{lang}.vtt"
+            to_add = {
+                "ext": "vtt",
+                "url": False,
+                "name": lang,
+                "lang": lang,
+                "source": "file",
+                "media_url": subtitle_media_url,
+            }
+            subtitles.append(to_add)
+
+        return subtitles
 
     def update_media_url(self):
         """update only media_url in es for reindex channel rename"""

tubearchivist/home/src/ta/config.py

@@ -133,6 +133,8 @@ class AppConfig:
 
         # check for customizations
         if not redis_config:
+            config = self.get_config()
+            RedisArchivist().set_message("config", config)
             return
 
         needs_update = False

tubearchivist/home/src/ta/helper.py

@@ -105,6 +105,18 @@ def date_praser(timestamp):
     return datetime.strftime(date_obj, "%d %b, %Y")
 
 
+def time_parser(timestamp):
+    """return seconds from timestamp, false on empty"""
+    if not timestamp:
+        return False
+
+    if timestamp.isnumeric():
+        return int(timestamp)
+
+    hours, minutes, seconds = timestamp.split(":", maxsplit=3)
+    return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
+
+
 class UrlListParser:
     """take a multi line string and detect valid youtube ids"""
 
@@ -146,12 +158,12 @@ class UrlListParser:
         if parsed.query:
             # detect from query string
             query_parsed = parse_qs(parsed.query)
-            if "v" in query_parsed.keys():
+            if "v" in query_parsed:
                 youtube_id = query_parsed["v"][0]
                 _ = self.find_valid_id(youtube_id)
                 return youtube_id, "video"
 
-            if "list" in query_parsed.keys():
+            if "list" in query_parsed:
                 youtube_id = query_parsed["list"][0]
                 return youtube_id, "playlist"
 
@@ -161,13 +173,13 @@ class UrlListParser:
             _ = self.find_valid_id(youtube_id)
             return youtube_id, "channel"
 
-        # dedect channel with yt_dlp
+        # detect channel with yt_dlp
         youtube_id = self.extract_channel_name(parsed.geturl())
         return youtube_id, "channel"
 
     @staticmethod
     def find_valid_id(id_str):
-        """dedect valid id from length of string"""
+        """detect valid id from length of string"""
         str_len = len(id_str)
         if str_len == 11:
             id_type = "video"
@@ -190,13 +202,19 @@ class UrlListParser:
             "playlistend": 0,
         }
         url_info = YtWrap(obs_request).extract(url)
-        try:
-            channel_id = url_info["channel_id"]
-        except KeyError as error:
-            print(f"failed to extract channel id from {url}")
-            raise ValueError from error
+        channel_id = url_info.get("channel_id", False)
+        if channel_id:
+            return channel_id
 
-        return channel_id
+        url = url_info.get("url", False)
+        if url:
+            # handle old channel name redirect with url path split
+            channel_id = urlparse(url).path.strip("/").split("/")[1]
+
+            return channel_id
+
+        print(f"failed to extract channel id from {url}")
+        raise ValueError
 
 
 class DurationConverter:

tubearchivist/home/src/ta/ta_redis.py

@@ -143,3 +143,8 @@ class RedisQueue(RedisBase):
     def trim(self, size):
         """trim the queue based on settings amount"""
         self.conn.execute_command("LTRIM", self.key, 0, size)
+
+    def has_item(self):
+        """check if queue as at least one pending item"""
+        result = self.conn.execute_command("LRANGE", self.key, 0, 0)
+        return bool(result)

tubearchivist/home/tasks.py

@@ -15,12 +15,12 @@ from home.src.download.subscriptions import (
     ChannelSubscription,
     PlaylistSubscription,
 )
-from home.src.download.thumbnails import ThumbManager, validate_thumbnails
+from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator
 from home.src.download.yt_dlp_handler import VideoDownloader
 from home.src.es.index_setup import backup_all_indexes, restore_from_backup
 from home.src.index.channel import YoutubeChannel
 from home.src.index.filesystem import (
-    ManualImport,
+    ImportFolderScanner,
     reindex_old_documents,
     scan_filesystem,
 )
@@ -150,10 +150,7 @@ def run_manual_import():
     try:
         have_lock = my_lock.acquire(blocking=False)
         if have_lock:
-            import_handler = ManualImport()
-            if import_handler.identified:
-                all_videos_added = import_handler.process_import()
-                ThumbManager().download_vid(all_videos_added)
+            ImportFolderScanner().scan()
         else:
             print("Did not acquire lock form import.")
 
@@ -165,8 +162,19 @@ def run_manual_import():
 @shared_task(name="run_backup")
 def run_backup(reason="auto"):
     """called from settings page, dump backup to zip file"""
-    backup_all_indexes(reason)
-    print("backup finished")
+    have_lock = False
+    my_lock = RedisArchivist().get_lock("run_backup")
+
+    try:
+        have_lock = my_lock.acquire(blocking=False)
+        if have_lock:
+            backup_all_indexes(reason)
+        else:
+            print("Did not acquire lock for backup task.")
+    finally:
+        if have_lock:
+            my_lock.release()
+            print("backup finished")
 
 
 @shared_task
@@ -204,21 +212,19 @@ def kill_dl(task_id):
 def rescan_filesystem():
     """check the media folder for mismatches"""
     scan_filesystem()
-    validate_thumbnails()
+    ThumbValidator().download_missing()
 
 
 @shared_task(name="thumbnail_check")
 def thumbnail_check():
     """validate thumbnails"""
-    validate_thumbnails()
+    ThumbValidator().download_missing()
 
 
 @shared_task
 def re_sync_thumbs():
     """sync thumbnails to mediafiles"""
-    handler = ThumbManager()
-    video_list = handler.get_thumb_list()
-    handler.write_all_thumbs(video_list)
+    ThumbFilesystem().sync()
 
 
 @shared_task
@@ -229,9 +235,7 @@ def subscribe_to(url_str):
     for item in to_subscribe_list:
         to_sub_id = item["url"]
         if item["type"] == "playlist":
-            new_thumbs = PlaylistSubscription().process_url_str([item])
-            if new_thumbs:
-                ThumbManager().download_playlist(new_thumbs)
+            PlaylistSubscription().process_url_str([item])
             continue
 
         if item["type"] == "video":

tubearchivist/home/templates/home/base.html

@@ -79,16 +79,16 @@
             <div class="pagination">
                 {% if pagination %}
                     {% if pagination.current_page > 1 %}
-                        {% if pagination.search_get %}
-                        <a class="pagination-item" href="{{ request.path }}?search={{ pagination.search_get }}">First</a>
+                        {% if pagination.params %}
+                        <a class="pagination-item" href="{{ request.path }}?{{ pagination.params }}">First</a>
                         {% else %}
                         <a class="pagination-item" href="{{ request.path }}">First</a>
                         {% endif %}
                     {% endif %}
                     {% if pagination.prev_pages %}
                         {% for page in pagination.prev_pages %}
-                            {% if pagination.search_get %}
-                            <a class="pagination-item" href="?page={{ page }}&search={{ pagination.search_get }}">{{ page }}</a>
+                            {% if pagination.params %}
+                            <a class="pagination-item" href="?page={{ page }}&{{ pagination.params }}">{{ page }}</a>
                             {% else %}
                             <a class="pagination-item" href="?page={{ page }}">{{ page }}</a>
                             {% endif %}
@@ -100,16 +100,16 @@
                     {% if pagination.next_pages %}
                         <span> ></span>
                         {% for page in pagination.next_pages %}
-                            {% if pagination.search_get %}
-                            <a class="pagination-item" href="?page={{ page }}&search={{ pagination.search_get }}">{{ page }}</a>
+                            {% if pagination.params %}
+                            <a class="pagination-item" href="?page={{ page }}&{{ pagination.params }}">{{ page }}</a>
                             {% else %}
                             <a class="pagination-item" href="?page={{ page }}">{{ page }}</a>
                             {% endif %}
                         {% endfor %}
                     {% endif %}
                     {% if pagination.last_page > 0 %}
-                        {% if pagination.search_get %}
-                        <a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">
+                        {% if pagination.params %}
+                        <a class="pagination-item" href="?page={{ pagination.last_page }}&{{ pagination.params }}">
                             {% if pagination.max_hits %}
                                 Max ({{ pagination.last_page }})
                             {% else %}

tubearchivist/home/templates/home/channel.html

@@ -36,7 +36,7 @@
             <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="channel" data-value="list" alt="list view">
         </div>
     </div>
-    <h2>Total matching channels: {{ max_hits }}</h2>
+    <h2>Total channels: {{ max_hits }}</h2>
     <div class="channel-list {{ view_style }}">
         {% if results %}
             {% for channel in results %}
@@ -81,4 +81,4 @@
     </div>
 </div>
 <script type="text/javascript" src="{% static 'progress.js' %}"></script>
-{% endblock content %}
+{% endblock content %}

tubearchivist/home/templates/home/channel_id.html

@@ -6,8 +6,16 @@
     <div class="channel-banner">
         <a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
     </div>
+    <div class="info-box-item channel-nav">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
+        <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
+        <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+        {% if has_pending %}
+            <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
+        {% endif %}
+    </div>
     <div id="notifications" data="channel_id"></div>
-    <div class="info-box info-box-3">
+    <div class="info-box info-box-2">
         <div class="info-box-item">
             <div class="round-img">
                 <a href="{% url 'channel_id' channel_info.channel_id %}">
@@ -30,91 +38,16 @@
         </div>
         <div class="info-box-item">
             <div>
-                <p>Last refreshed: {{ channel_info.channel_last_refresh }}</p>
-                {% if channel_info.channel_active %}
-                    <p>Youtube: <a href="https://www.youtube.com/channel/{{ channel_info.channel_id }}" target="_blank">Active</a></p>
-                {% else %}
-                    <p>Youtube: Deactivated</p>
-                {% endif %}
-                <button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
-                <div class="delete-confirm" id="delete-button">
-                    <span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
-                </div>
-            </div>
-        </div>
-        <div class="info-box-item">
-            <div>
-                {% if channel_info.channel_views >= 1000000 %}
-                    <p>Channel views: {{ channel_info.channel_views|intword }}</p>
-                {% elif channel_info.channel_views > 0 %}
-                    <p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
-                {% endif %}
                 {% if max_hits %}
-                    <p>Total Videos archived: {{ max_hits }}</p>
-                    <p>Watched: <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
+                    <p>Total Videos: {{ max_hits }}</p>
+                    <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button>
                 {% endif %}
-                <button {% if channel_info.channel_overwrites %} class="danger-button"{% endif %} onclick="showOverwrite()" title="Overwrite settings for channel {{ channel_info.channel_name }}">Configure</button>
-                <a href="/playlist/?search={{ channel_info.channel_id }}" title="Show all playlists belonging to {{ channel_info.channel_name }}"><button>Show Playlists</button></a>
             </div>
         </div>
     </div>
-    <div id="overwrite-form" class="info-box{% if not channel_info.channel_overwrites %} hidden-overwrite{% endif %}">
-        <div class="info-box-item">
-            <form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/" method="POST">
-                {% csrf_token %}
-                <div class="overwrite-form-item">
-                    <p>Download format: <span class="settings-current">
-                        {% if channel_info.channel_overwrites.download_format %}
-                            {{ channel_info.channel_overwrites.download_format }}
-                        {% else %}
-                            False
-                        {% endif %}</span></p>
-                    {{ channel_overwrite_form.download_format }}<br>
-                </div>
-                <div class="overwrite-form-item">
-                    <p>Auto delete watched videos after x days: <span class="settings-current">
-                        {% if channel_info.channel_overwrites.autodelete_days %}
-                            {{ channel_info.channel_overwrites.autodelete_days }}
-                        {% else %}
-                            False
-                        {% endif %}</span></p>
-                    {{ channel_overwrite_form.autodelete_days }}<br>
-                </div>
-                <div class="overwrite-form-item">
-                    <p>Index playlists: <span class="settings-current">
-                        {% if channel_info.channel_overwrites.index_playlists %}
-                            {{ channel_info.channel_overwrites.index_playlists }}
-                        {% else %}
-                            False
-                        {% endif %}</span></p>
-                    {{ channel_overwrite_form.index_playlists }}<br>
-                </div>
-                <div class="overwrite-form-item">
-                    <p>Enable <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a>: <span class="settings-current">
-                        {% if channel_info.channel_overwrites.integrate_sponsorblock %}
-                            {{ channel_info.channel_overwrites.integrate_sponsorblock }}
-                        {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %}
-                            Disabled
-                        {% else %}
-                            False
-                        {% endif %}</span></p>
-                    {{ channel_overwrite_form.integrate_sponsorblock }}<br>
-                </div>
-                <button type="submit">Save Channel Overwrites</button>
-            </form>
-        </div>
-    </div>
-    {% if channel_info.channel_description %}
-        <div class="info-box-item description-box">
-            <p>Description: <button onclick="textReveal()" id="text-reveal-button">Show</button></p>
-            <div id="text-reveal" class="description-text">
-                {{ channel_info.channel_description|linebreaks }}
-            </div>
-        </div>
-    {% endif %}
 </div>
 <div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
-    <div class="view-controls">
+    <div class="view-controls three">
         <div class="toggle">
             <span>Hide watched videos:</span>
             <div class="toggleBox">

tubearchivist/home/templates/home/channel_id_about.html

@@ -0,0 +1,115 @@
+{% extends "home/base.html" %}
+{% block content %}
+{% load static %}
+{% load humanize %}
+<div class="boxed-content">
+    <div class="channel-banner">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
+    </div>
+    <div class="info-box-item channel-nav">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
+        <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
+        <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+        {% if has_pending %}
+            <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
+        {% endif %}
+    </div>
+    <div class="info-box info-box-3">
+        <div class="info-box-item">
+            <div class="round-img">
+                <a href="{% url 'channel_id' channel_info.channel_id %}">
+                    <img src="{{ channel_info.channel_thumb_url }}" alt="channel-thumb">
+                </a>
+            </div>
+            <div>
+                <h3><a href="{% url 'channel_id' channel_info.channel_id %}">{{ channel_info.channel_name }}</a></h3>
+                {% if channel_info.channel_subs >= 1000000 %}
+                    <p>Subscribers: {{ channel_info.channel_subs|intword }}</p>
+                {% else %}
+                    <p>Subscribers: {{ channel_info.channel_subs|intcomma }}</p>
+                {% endif %}
+            </div>
+        </div>
+        <div class="info-box-item">
+            <div>
+                <p>Last refreshed: {{ channel_info.channel_last_refresh }}</p>
+                {% if channel_info.channel_active %}
+                    <p>Youtube: <a href="https://www.youtube.com/channel/{{ channel_info.channel_id }}" target="_blank">Active</a></p>
+                {% else %}
+                    <p>Youtube: Deactivated</p>
+                {% endif %}
+            </div>
+        </div>
+        <div class="info-box-item">
+            <div>
+                {% if channel_info.channel_views >= 1000000 %}
+                    <p>Channel views: {{ channel_info.channel_views|intword }}</p>
+                {% elif channel_info.channel_views > 0 %}
+                    <p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
+                {% endif %}
+                <button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
+                <div class="delete-confirm" id="delete-button">
+                    <span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
+                </div>
+            </div>
+        </div>
+    </div>
+    {% if channel_info.channel_description %}
+        <div class="description-box">
+            <p id="text-expand" class="description-text">
+                {{ channel_info.channel_description|linebreaksbr|urlizetrunc:50 }}
+            </p>
+            <button onclick="textExpand()" id="text-expand-button">Show more</button>
+        </div>
+    {% endif %}
+    <div class="description-box">
+        <h2>Customize {{ channel_info.channel_name }}</h2>
+    </div>
+    <div id="overwrite-form" class="info-box">
+        <div class="info-box-item">
+            <form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/about/" method="POST">
+                {% csrf_token %}
+                <div class="overwrite-form-item">
+                    <p>Download format: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.download_format %}
+                            {{ channel_info.channel_overwrites.download_format }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.download_format }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Auto delete watched videos after x days: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.autodelete_days %}
+                            {{ channel_info.channel_overwrites.autodelete_days }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.autodelete_days }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Index playlists: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.index_playlists %}
+                            {{ channel_info.channel_overwrites.index_playlists }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.index_playlists }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Enable <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a>: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.integrate_sponsorblock %}
+                            {{ channel_info.channel_overwrites.integrate_sponsorblock }}
+                        {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %}
+                            Disabled
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.integrate_sponsorblock }}<br>
+                </div>
+                <button type="submit">Save Channel Overwrites</button>
+            </form>
+        </div>
+    </div>
+</div>
+{% endblock content %}

tubearchivist/home/templates/home/channel_id_playlist.html

@@ -0,0 +1,60 @@
+{% extends "home/base.html" %}
+{% block content %}
+{% load static %}
+{% load humanize %}
+<div class="boxed-content">
+    <div class="channel-banner">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
+    </div>
+    <div class="info-box-item channel-nav">
+        <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
+        <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
+        <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+        {% if has_pending %}
+            <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
+        {% endif %}
+    </div>
+    <div class="view-controls">
+        <div class="toggle">
+            <span>Show subscribed only:</span>
+            <div class="toggleBox">
+                <input id="show_subed_only" onclick="toggleCheckbox(this)" type="checkbox" {% if show_subed_only %}checked{% endif %}>
+                {% if not show_subed_only %}
+                    <label for="" class="ofbtn">Off</label>
+                {% else %}
+                    <label for="" class="onbtn">On</label>
+                {% endif %}
+            </div>
+        </div>
+        <div class="view-icons">
+            <img src="{% static 'img/icon-gridview.svg' %}" onclick="changeView(this)" data-origin="playlist" data-value="grid" alt="grid view">
+            <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="playlist" data-value="list" alt="list view">
+        </div>
+    </div>
+    <div class="playlist-list {{ view_style }}">
+        {% if results %}
+            {% for playlist in results %}
+            <div class="playlist-item {{ view_style }}">
+                <div class="playlist-thumbnail">
+                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}">
+                        <img src="/cache/playlists/{{ playlist.source.playlist_id }}.jpg" alt="{{ playlist.source.playlist_id }}-thumbnail">
+                    </a>
+                </div>
+                <div class="playlist-desc {{ view_style }}">
+                    <a href="{% url 'channel_id' playlist.source.playlist_channel_id %}"><h3>{{ playlist.source.playlist_channel }}</h3></a>
+                    <a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
+                    <p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
+                    {% if playlist.source.playlist_subscribed %}
+                        <button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
+                    {% else %}
+                        <button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
+                    {% endif %}
+                </div>
+            </div>
+            {% endfor %}
+        {% else %}
+            <h2>No playlists found...</h2>
+        {% endif %}
+    </div>
+</div>
+{% endblock content %}

tubearchivist/home/templates/home/downloads.html

@@ -3,7 +3,7 @@
 {% block content %}
 <div class="boxed-content">
     <div class="title-bar">
-        <h1>Downloads</h1>
+        <h1>Downloads {% if channel_filter_id %} for {{ channel_filter_name }}{% endif %}</h1>
     </div>
     <div id="notifications" data="download"></div>
     <div id="downloadControl"></div>
@@ -28,7 +28,7 @@
             </div>
         </div>
     </div>
-    <div class="view-controls">
+    <div class="view-controls three">
         <div class="toggle">
             <span>Show only ignored videos:</span>
             <div class="toggleBox">
@@ -41,38 +41,64 @@
             </div>
         </div>
         <div class="view-icons">
+            {% if channel_agg_list|length > 1 %}
+                <select name="channel_filter" id="channel_filter" onchange="channelFilterDownload(this.value)">
+                    <option value="all" {% if not channel_filter_id %}selected{% endif %}>all</option>
+                    {% for channel in channel_agg_list %}
+                        <option {% if channel_filter_id == channel.id %}selected{% endif %} value="{{ channel.id }}">{{ channel.name }} ({{channel.count}})</option>
+                    {% endfor %}
+                </select>
+            {% endif %}
+            {% if view_style == "grid" %}
+                <div class="grid-count">
+                    {% if grid_items < 7 %}
+                        <img src="{% static 'img/icon-add.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"1"}}" alt="grid plus row">
+                    {% endif %}
+                    {% if grid_items > 3 %}
+                        <img src="{% static 'img/icon-substract.svg' %}" onclick="changeGridItems(this)" data-value="{{ grid_items|add:"-1"}}" alt="grid minus row">
+                    {% endif %}
+                </div>
+            {% endif %}
             <img src="{% static 'img/icon-gridview.svg' %}" onclick="changeView(this)" data-origin="downloads" data-value="grid" alt="grid view">
             <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="downloads" data-value="list" alt="list view">
         </div>
     </div>
-    <h3>Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %}</h3>
-    <div class="dl-list {{ view_style }}">
+    <h3>Total videos in queue: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %} {% if channel_filter_id %} - from channel <i>{{ channel_filter_name }}</i>{% endif %}</h3>
+</div>
+<div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
+    <div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
         {% if results %}
             {% for video in results %}
-                <div class="dl-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
-                    <div class="dl-thumb {{ view_style }}">
-                        <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video_thumb">
-                        {% if show_ignored_only %}
-                            <span>ignored</span>
-                        {% else %}
-                            <span>queued</span>
-                        {% endif %}
+                <div class="video-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
+                    <div class="video-thumb-wrap {{ view_style }}">
+                        <div class="video-thumb">
+                            <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video_thumb">
+                            {% if show_ignored_only %}
+                                <span>ignored</span>
+                            {% else %}
+                                <span>queued</span>
+                            {% endif %}
+                        </div>
                     </div>
-                    <div class="dl-desc {{ view_style }}">
-                        <h3>{{ video.source.title }}</h3>
-                        {% if video.source.channel_indexed %}
+                    <div class="video-desc {{ view_style }}">
+                        <div>
+                            {% if video.source.channel_indexed %}
                             <a href="{% url 'channel_id' video.source.channel_id %}">{{ video.source.channel_name }}</a>
-                        {% else %}
-                            <span>{{ video.source.channel_name }}</span>
-                        {% endif %}
+                            {% else %}
+                                <span>{{ video.source.channel_name }}</span>
+                            {% endif %}
+                            <h3>{{ video.source.title }}</h3>
+                        </div>
                         <p>Published: {{ video.source.published }} | Duration: {{ video.source.duration }} | {{ video.source.youtube_id }}</p>
-                        {% if show_ignored_only %}
-                            <button data-id="{{ video.source.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>
-                            <button data-id="{{ video.source.youtube_id }}" onclick="addSingle(this)">Add to queue</button>
-                        {% else %}
-                            <button data-id="{{ video.source.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
-                            <button id="{{ video.source.youtube_id }}" data-id="{{ video.source.youtube_id }}" onclick="downloadNow(this)">Download now</button>
-                        {% endif %}
+                        <div>
+                            {% if show_ignored_only %}
+                                <button data-id="{{ video.source.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>
+                                <button data-id="{{ video.source.youtube_id }}" onclick="addSingle(this)">Add to queue</button>
+                            {% else %}
+                                <button data-id="{{ video.source.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
+                                <button id="{{ video.source.youtube_id }}" data-id="{{ video.source.youtube_id }}" onclick="downloadNow(this)">Download now</button>
+                            {% endif %}
+                        </div>
                     </div>
                 </div>
             {% endfor %}
@@ -80,4 +106,4 @@
     </div>
 </div>
 <script type="text/javascript" src="{% static 'progress.js' %}"></script>
-{% endblock content %}
+{% endblock content %}

tubearchivist/home/templates/home/home.html

@@ -45,7 +45,7 @@
     <div class="title-bar">
         <h1>Recent Videos</h1>
     </div>
-    <div class="view-controls">
+    <div class="view-controls three">
         <div class="toggle">
             <span>Hide watched:</span>
             <div class="toggleBox">

tubearchivist/home/templates/home/playlist_id.html

@@ -56,11 +56,11 @@
         </div>
     </div>
     {% if playlist_info.playlist_description %}
-        <div class="info-box-item description-box">
-            <p>Description: <button onclick="textReveal()" id="text-reveal-button">Show</button></p>
-            <div id="text-reveal" class="description-text">
-                {{ playlist_info.playlist_description|linebreaks }}
-            </div>
+        <div class="description-box">
+            <p id="text-expand" class="description-text">
+                {{ playlist_info.playlist_description|linebreaksbr|urlizetrunc:50 }}
+            </p>
+            <button onclick="textExpand()" id="text-expand-button">Show more</button>
         </div>
     {% endif %}
 </div>

tubearchivist/home/templates/home/search.html

@@ -7,6 +7,7 @@
     </div>
     <div class="multi-search-box">
         {{ search_form }}
+        <p>Start typing or use <span class="settings-current">video:</span>, <span class="settings-current">channel:</span>, <span class="settings-current">playlist:</span> or <span class="settings-current">full:</span> keywords for advanced queries. <a href="https://github.com/tubearchivist/tubearchivist/wiki/Search" target="_blank">Learn more</a>.</p>
     </div>
     <div class="multi-search-result">
         <h2>Video Results</h2>
@@ -26,5 +27,11 @@
             <p>No playlists found.</p>
         </div>
     </div>
+    <div class="multi-search-result">
+        <h2>Fulltext Results</h2>
+        <div id="fulltext-results" class="video-list list">
+            <p>No fulltext results found.</p>
+        </div>
+    </div>
 </div>
 {% endblock content %}

tubearchivist/home/templates/home/settings.html

@@ -34,8 +34,8 @@
         <div class="settings-group">
             <h2 id="subscriptions">Subscriptions</h2>
             <div class="settings-item">
-                <p>Current page size: <span class="settings-current">{{ config.subscriptions.channel_size }}</span></p>
-                <i>Recent videos for channels and playlist to check when running <b>Rescan subscriptions</b>, max recommended 50.</i><br>
+                <p>YouTube page size: <span class="settings-current">{{ config.subscriptions.channel_size }}</span></p>
+                <i>Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
                 {{ app_form.subscriptions_channel_size }}
             </div>
         </div>
@@ -77,7 +77,7 @@
                 <ul>
                     <li><span class="settings-current">bestvideo[height<=720]+bestaudio/best[height<=720]</span>: best audio and max video height of 720p.</li>
                     <li><span class="settings-current">bestvideo[height<=1080]+bestaudio/best[height<=1080]</span>: best audio and max video height of 1080p.</li>
-                    <li><span class="settings-current">bestvideo[height<=1080][VCODEC=avc1]+bestaudio[ACODEC=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
+                    <li><span class="settings-current">bestvideo[height<=1080][VCODEC*=avc1]+bestaudio[ACODEC*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
                     <li><span class="settings-current">0</span>: deactivate and download the best quality possible as decided by yt-dlp.</li>
                 </ul>
                 <i>Make sure your custom format gets merged into a single file. Check out the <a href="https://github.com/yt-dlp/yt-dlp#format-selection" target="_blank">documentation</a> for valid configurations.</i><br>
@@ -153,6 +153,23 @@
                 {{ app_form.application_enable_cast }}
             </div>
         </div>
+        <div class="settings-group">
+            <h2 id="snapshots">Snapshots</h2>
+            <div class="settings-item">
+                <p>Current system snapshot: <span class="settings-current">{{ config.application.enable_snapshot }}</span></p>
+                <i>Automatically create daily deduplicated snapshots of the index, stored in Elasticsearch. Read first before activating: <a target="_blank" href="https://github.com/tubearchivist/tubearchivist/wiki/Settings#snapshots">Wiki</a>.</i><br>
+                {{ app_form.application_enable_snapshot }}
+            </div>
+            <div>
+                {% if snapshots %}
+                    <p>Create next snapshot: <span class="settings-current">{{ snapshots.next_exec_str }}</span>, snapshots expire after <span class="settings-current">{{ snapshots.expire_after }}</span>. <button onclick="createSnapshot()" id="createButton">Create snapshot now</button></p>
+                    <br>
+                    {% for snapshot in snapshots.snapshots %}
+                        <p><button id="{{ snapshot.id }}" onclick="restoreSnapshot(id)">Restore</button> Snapshot created on: <span class="settings-current">{{ snapshot.start_date }}</span>, took <span class="settings-current">{{ snapshot.duration_s }}s</span> to create. State: <i>{{ snapshot.state }}</i></p>
+                    {% endfor %}
+                {% endif %}
+            </div>
+        </div>
         <button type="submit" name="application-settings">Update Application Configurations</button>
     </form>
     <div class="title-bar">

tubearchivist/home/templates/home/video.html

@@ -2,7 +2,9 @@
 {% block content %}
 {% load static %}
 {% load humanize %}
-<div class="video-main"></div>
+<div class="video-main">
+    <div class="video-modal"><span class="video-modal-text"></span></div>
+</div>
 <div class="notifications" id="notifications"></div>
 <div class="sponsorblock" id="sponsorblock">
     {% if video.sponsorblock.is_enabled %}
@@ -67,7 +69,7 @@
                     <p class="thumb-icon"><img class="dislike" src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-down">: {{ video.stats.dislike_count|intcomma }}</p>
                 {% endif %}
                 {% if video.stats.average_rating %}
-                    <p class="rating-stars">Rating: 
+                    <p class="rating-stars">Rating:
                         {% for star in video.stats.average_rating %}
                             <img src="/static/img/icon-star-{{ star }}.svg" alt="{{ star }}">
                         {% endfor %}
@@ -77,11 +79,11 @@
         </div>
     </div>
     {% if video.description %}
-        <div class="info-box-item description-box">
-            <p>Description: <button onclick="textReveal()" id="text-reveal-button">Show</button></p>
-            <div id="text-reveal" class="description-text">
-                {{ video.description|linebreaks }}
-            </div>
+        <div class="description-box">
+            <p id="text-expand" class="description-text">
+                {{ video.description|linebreaksbr|urlizetrunc:50 }}
+            </p>
+            <button onclick="textExpand()" id="text-expand-button">Show more</button>
         </div>
     {% endif %}
     {% if playlist_nav %}
@@ -90,7 +92,7 @@
                 <a href="{% url 'playlist_id' playlist_item.playlist_meta.playlist_id %}">
                     <h3>Playlist [{{ playlist_item.playlist_meta.current_idx|add:"1" }}]: {{ playlist_item.playlist_meta.playlist_name }}</h3>
                 </a>
-                <div class="playlist-nav">    
+                <div class="playlist-nav">
                     <div class="playlist-nav-item">
                         {% if playlist_item.playlist_previous %}
                             <a href="{% url 'video' playlist_item.playlist_previous.youtube_id %}">
@@ -125,7 +127,11 @@
 <script>
     var videoData = getVideoData('{{ video.youtube_id }}');
     sponsorBlock = videoData.data.sponsorblock;
-    var videoProgress = getVideoProgress('{{ video.youtube_id }}').position;
+    {% if position %}
+        var videoProgress = {{ position }}
+    {% else %}
+        var videoProgress = getVideoProgress('{{ video.youtube_id }}').position;
+    {% endif %}
     window.onload = insertVideoTag(videoData, videoProgress);
 </script>
 {% endblock content %}

tubearchivist/home/urls.py

@@ -6,6 +6,8 @@ from django.contrib.auth.views import LogoutView
 from django.urls import path
 from home.views import (
     AboutView,
+    ChannelIdAboutView,
+    ChannelIdPlaylistView,
     ChannelIdView,
     ChannelView,
     DownloadView,
@@ -42,6 +44,16 @@ urlpatterns = [
         login_required(ChannelIdView.as_view()),
         name="channel_id",
     ),
+    path(
+        "channel/<slug:channel_id>/about/",
+        login_required(ChannelIdAboutView.as_view()),
+        name="channel_id_about",
+    ),
+    path(
+        "channel/<slug:channel_id>/playlist/",
+        login_required(ChannelIdPlaylistView.as_view()),
+        name="channel_id_playlist",
+    ),
     path(
         "video/<slug:video_id>/",
         login_required(VideoView.as_view()),

tubearchivist/home/views.py

@@ -8,6 +8,7 @@ import json
 import urllib.parse
 from time import sleep
 
+from api.src.search_processor import SearchProcess
 from django.conf import settings
 from django.contrib.auth import login
 from django.contrib.auth.forms import AuthenticationForm
@@ -15,7 +16,9 @@ from django.http import JsonResponse
 from django.shortcuts import redirect, render
 from django.views import View
 from home.src.download.yt_dlp_base import CookieHandler
+from home.src.es.connect import ElasticWrap
 from home.src.es.index_setup import get_available_backups
+from home.src.es.snapshot import ElasticSnapshot
 from home.src.frontend.api_calls import PostData
 from home.src.frontend.forms import (
     AddToQueueForm,
@@ -29,11 +32,11 @@ from home.src.frontend.forms import (
     UserSettingsForm,
 )
 from home.src.frontend.searching import SearchHandler
-from home.src.index.channel import channel_overwrites
+from home.src.index.channel import YoutubeChannel, channel_overwrites
 from home.src.index.generic import Pagination
 from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.config import AppConfig, ScheduleBuilder
-from home.src.ta.helper import UrlListParser
+from home.src.ta.helper import UrlListParser, time_parser
 from home.src.ta.ta_redis import RedisArchivist
 from home.tasks import extrac_dl, index_channel_playlists, subscribe_to
 from rest_framework.authtoken.models import Token
@@ -235,14 +238,10 @@ class ArchivistResultsView(ArchivistViewConfig):
 
     def initiate_vars(self, request):
         """search in es for vidoe hits"""
-        page_get = int(request.GET.get("page", 0))
         self.user_id = request.user.id
         self.config_builder(self.user_id)
         self.search_get = request.GET.get("search", False)
-        search_encoded = self._url_encode(self.search_get)
-        self.pagination_handler = Pagination(
-            page_get=page_get, user_id=self.user_id, search_get=search_encoded
-        )
+        self.pagination_handler = Pagination(request)
         self.sort_by = self._sort_by_overwrite()
         self._initial_data()
 
@@ -360,29 +359,80 @@ class DownloadView(ArchivistResultsView):
     def get(self, request):
         """handle get request"""
         self.initiate_vars(request)
-        self._update_view_data()
+        self._update_view_data(request)
         self.find_results()
         self.context.update(
             {
                 "title": "Downloads",
                 "add_form": AddToQueueForm(),
+                "channel_agg_list": self._get_channel_agg(),
             }
         )
         return render(request, "home/downloads.html", self.context)
 
-    def _update_view_data(self):
+    def _update_view_data(self, request):
         """update downloads view specific data dict"""
         if self.context["show_ignored_only"]:
             filter_view = "ignore"
         else:
             filter_view = "pending"
+
+        must_list = [{"term": {"status": {"value": filter_view}}}]
+
+        channel_filter = request.GET.get("channel", False)
+        if channel_filter:
+            must_list.append(
+                {"term": {"channel_id": {"value": channel_filter}}}
+            )
+
+            channel = YoutubeChannel(channel_filter)
+            channel.get_from_es()
+            self.context.update(
+                {
+                    "channel_filter_id": channel_filter,
+                    "channel_filter_name": channel.json_data["channel_name"],
+                }
+            )
+
         self.data.update(
             {
-                "query": {"term": {"status": {"value": filter_view}}},
+                "query": {"bool": {"must": must_list}},
                 "sort": [{"timestamp": {"order": "asc"}}],
             }
         )
 
+    def _get_channel_agg(self):
+        """get pending channel with count"""
+        data = {
+            "size": 0,
+            "query": {"term": {"status": {"value": "pending"}}},
+            "aggs": {
+                "channel_downloads": {
+                    "multi_terms": {
+                        "size": 30,
+                        "terms": [
+                            {"field": "channel_name.keyword"},
+                            {"field": "channel_id"},
+                        ],
+                        "order": {"_count": "desc"},
+                    }
+                }
+            },
+        }
+        response, _ = ElasticWrap(self.es_search).get(data=data)
+        buckets = response["aggregations"]["channel_downloads"]["buckets"]
+
+        buckets_sorted = []
+        for i in buckets:
+            bucket = {
+                "name": i["key"][0],
+                "id": i["key"][1],
+                "count": i["doc_count"],
+            }
+            buckets_sorted.append(bucket)
+
+        return buckets_sorted
+
     @staticmethod
     def post(request):
         """handle post requests"""
@@ -412,7 +462,37 @@ class DownloadView(ArchivistResultsView):
         return redirect("downloads", permanent=True)
 
 
-class ChannelIdView(ArchivistResultsView):
+class ChannelIdBaseView(ArchivistResultsView):
+    """base class for all channel-id views"""
+
+    def get_channel_meta(self, channel_id):
+        """get metadata for channel"""
+        path = f"ta_channel/_doc/{channel_id}"
+        response, _ = ElasticWrap(path).get()
+        channel_info = SearchProcess(response).process()
+
+        return channel_info
+
+    def channel_has_pending(self, channel_id):
+        """check if channel has pending videos in queue"""
+        path = "ta_download/_search"
+        data = {
+            "size": 1,
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {"status": {"value": "pending"}}},
+                        {"term": {"channel_id": {"value": channel_id}}},
+                    ]
+                }
+            },
+        }
+        response, _ = ElasticWrap(path).get(data=data)
+
+        self.context.update({"has_pending": bool(response["hits"]["hits"])})
+
+
+class ChannelIdView(ChannelIdBaseView):
     """resolves to /channel/<channel-id>/
     display single channel page from channel_id
     """
@@ -426,6 +506,7 @@ class ChannelIdView(ArchivistResultsView):
         self._update_view_data(channel_id)
         self.find_results()
         self.match_progress()
+        self.channel_has_pending(channel_id)
 
         if self.context["results"]:
             channel_info = self.context["results"][0]["source"]["channel"]
@@ -440,7 +521,6 @@ class ChannelIdView(ArchivistResultsView):
             {
                 "title": "Channel: " + channel_name,
                 "channel_info": channel_info,
-                "channel_overwrite_form": ChannelOverwriteForm,
             }
         )
 
@@ -477,6 +557,87 @@ class ChannelIdView(ArchivistResultsView):
         return redirect("channel_id", channel_id, permanent=True)
 
 
+class ChannelIdAboutView(ChannelIdBaseView):
+    """resolves to /channel/<channel-id>/about/
+    show metadata, handle per channel conf
+    """
+
+    view_origin = "channel"
+
+    def get(self, request, channel_id):
+        """handle get request"""
+        self.initiate_vars(request)
+        self.channel_has_pending(channel_id)
+
+        path = f"ta_channel/_doc/{channel_id}"
+        response, _ = ElasticWrap(path).get()
+
+        channel_info = SearchProcess(response).process()
+        channel_name = channel_info["channel_name"]
+
+        self.context.update(
+            {
+                "title": "Channel: About " + channel_name,
+                "channel_info": channel_info,
+                "channel_overwrite_form": ChannelOverwriteForm,
+            }
+        )
+
+        return render(request, "home/channel_id_about.html", self.context)
+
+    @staticmethod
+    def post(request, channel_id):
+        """handle post request"""
+        print(f"handle post from {channel_id}")
+        channel_overwrite_form = ChannelOverwriteForm(request.POST)
+        if channel_overwrite_form.is_valid():
+            overwrites = channel_overwrite_form.cleaned_data
+            print(f"{channel_id}: set overwrites {overwrites}")
+            channel_overwrites(channel_id, overwrites=overwrites)
+            if overwrites.get("index_playlists") == "1":
+                index_channel_playlists.delay(channel_id)
+
+        sleep(1)
+        return redirect("channel_id_about", channel_id, permanent=True)
+
+
+class ChannelIdPlaylistView(ChannelIdBaseView):
+    """resolves to /channel/<channel-id>/playlist/
+    show all playlists of channel
+    """
+
+    view_origin = "playlist"
+    es_search = "ta_playlist/_search"
+
+    def get(self, request, channel_id):
+        """handle get request"""
+        self.initiate_vars(request)
+        self._update_view_data(channel_id)
+        self.find_results()
+        self.channel_has_pending(channel_id)
+
+        channel_info = self.get_channel_meta(channel_id)
+        channel_name = channel_info["channel_name"]
+        self.context.update(
+            {
+                "title": "Channel: Playlists " + channel_name,
+                "channel_info": channel_info,
+            }
+        )
+
+        return render(request, "home/channel_id_playlist.html", self.context)
+
+    def _update_view_data(self, channel_id):
+        """update view specific data dict"""
+        self.data["sort"] = [{"playlist_name.keyword": {"order": "asc"}}]
+        must_list = [{"match": {"playlist_channel_id": channel_id}}]
+
+        if self.context["show_subed_only"]:
+            must_list.append({"match": {"playlist_subscribed": True}})
+
+        self.data["query"] = {"bool": {"must": must_list}}
+
+
 class ChannelView(ArchivistResultsView):
     """resolves to /channel/
     handle functionality for channel overview page, subscribe to channel,
@@ -683,6 +844,7 @@ class VideoView(View):
     def get(self, request, video_id):
         """get single video"""
         config_handler = AppConfig(request.user.id)
+        position = time_parser(request.GET.get("t"))
         path = f"ta_video/_doc/{video_id}"
         look_up = SearchHandler(path, config=False)
         video_hit = look_up.get_data()
@@ -708,6 +870,7 @@ class VideoView(View):
             "cast": config_handler.config["application"]["enable_cast"],
             "version": settings.TA_VERSION,
             "config": config_handler.config,
+            "position": position,
         }
         return render(request, "home/video.html", context)
 
@@ -780,6 +943,7 @@ class SettingsView(View):
         user_form = UserSettingsForm()
         app_form = ApplicationSettingsForm()
         scheduler_form = SchedulerSettingsForm()
+        snapshots = ElasticSnapshot().get_snapshot_stats()
         token = self.get_token(request)
 
         context = {
@@ -791,6 +955,7 @@ class SettingsView(View):
             "user_form": user_form,
             "app_form": app_form,
             "scheduler_form": scheduler_form,
+            "snapshots": snapshots,
             "version": settings.TA_VERSION,
         }
 
@@ -838,27 +1003,40 @@ class SettingsView(View):
         for config_value, updated_value in updated:
             if config_value == "cookie_import":
                 self.process_cookie(config, updated_value)
+            if config_value == "enable_snapshot":
+                ElasticSnapshot().setup()
 
-    @staticmethod
-    def process_cookie(config, updated_value):
+    def process_cookie(self, config, updated_value):
         """import and validate cookie"""
         handler = CookieHandler(config)
         if updated_value:
-            handler.import_cookie()
+            try:
+                handler.import_cookie()
+            except FileNotFoundError:
+                print("cookie: import failed, file not found")
+                handler.revoke()
+                self._fail_message("Cookie file not found.")
+                return
+
             valid = handler.validate()
             if not valid:
                 handler.revoke()
-                key = "message:setting"
-                message = {
-                    "status": key,
-                    "level": "error",
-                    "title": "Cookie import failed",
-                    "message": "",
-                }
-                RedisArchivist().set_message(key, message=message, expire=True)
+                self._fail_message("Failed to validate cookie file.")
         else:
             handler.revoke()
 
+    @staticmethod
+    def _fail_message(message_line):
+        """notify our failure"""
+        key = "message:setting"
+        message = {
+            "status": key,
+            "level": "error",
+            "title": "Cookie import failed",
+            "message": message_line,
+        }
+        RedisArchivist().set_message(key, message=message, expire=True)
+
 
 def progress(request):
     # pylint: disable=unused-argument

tubearchivist/requirements.txt

@@ -1,12 +1,13 @@
 beautifulsoup4==4.11.1
 celery==5.2.7
-Django==4.0.5
+Django==4.1.3
+django-auth-ldap==4.1.0
 django-cors-headers==3.13.0
-djangorestframework==3.13.1
-Pillow==9.2.0
+djangorestframework==3.14.0
+Pillow==9.3.0
 redis==4.3.4
 requests==2.28.1
-ryd-client==0.0.3
-uWSGI==2.0.20
+ryd-client==0.0.6
+uWSGI==2.0.21
 whitenoise==6.2.0
-yt_dlp==2022.6.29
+yt_dlp==2022.10.4

tubearchivist/static/cast-videos.js

@@ -1,148 +1,157 @@
+'use strict';
+
+/* global cast chrome getVideoPlayerVideoId postVideoProgress setProgressBar getVideoPlayer getVideoPlayerWatchStatus watchedThreshold isWatched getVideoData getURL getVideoPlayerCurrentTime */
+
 function initializeCastApi() {
-    cast.framework.CastContext.getInstance().setOptions({
-        receiverApplicationId: chrome.cast.media.DEFAULT_MEDIA_RECEIVER_APP_ID, // Use built in reciver app on cast device, see https://developers.google.com/cast/docs/styled_receiver if you want to be able to add a theme, splash screen or watermark. Has a $5 one time fee.
-        autoJoinPolicy: chrome.cast.AutoJoinPolicy.ORIGIN_SCOPED
-    });
+  cast.framework.CastContext.getInstance().setOptions({
+    receiverApplicationId: chrome.cast.media.DEFAULT_MEDIA_RECEIVER_APP_ID, // Use built in receiver app on cast device, see https://developers.google.com/cast/docs/styled_receiver if you want to be able to add a theme, splash screen or watermark. Has a $5 one time fee.
+    autoJoinPolicy: chrome.cast.AutoJoinPolicy.ORIGIN_SCOPED,
+  });
 
-    var player = new cast.framework.RemotePlayer();
-    var playerController = new cast.framework.RemotePlayerController(player);
+  let player = new cast.framework.RemotePlayer();
+  let playerController = new cast.framework.RemotePlayerController(player);
 
-    // Add event listerner to check if a connection to a cast device is initiated
-    playerController.addEventListener(
-        cast.framework.RemotePlayerEventType.IS_CONNECTED_CHANGED, function() { 
-            castConnectionChange(player) 
-        }
-    );
-    playerController.addEventListener(
-        cast.framework.RemotePlayerEventType.CURRENT_TIME_CHANGED, function() { 
-            castVideoProgress(player) 
-        }
-    );
-    playerController.addEventListener(
-        cast.framework.RemotePlayerEventType.IS_PAUSED_CHANGED, function() { 
-            castVideoPaused(player) 
-        }
-    );
+  // Add event listerner to check if a connection to a cast device is initiated
+  playerController.addEventListener(
+    cast.framework.RemotePlayerEventType.IS_CONNECTED_CHANGED,
+    function () {
+      castConnectionChange(player);
+    }
+  );
+  playerController.addEventListener(
+    cast.framework.RemotePlayerEventType.CURRENT_TIME_CHANGED,
+    function () {
+      castVideoProgress(player);
+    }
+  );
+  playerController.addEventListener(
+    cast.framework.RemotePlayerEventType.IS_PAUSED_CHANGED,
+    function () {
+      castVideoPaused(player);
+    }
+  );
 }
 
-
 function castConnectionChange(player) {
-    // If cast connection is initialized start cast
-    if (player.isConnected) {
-        // console.log("Cast Connected.");
-        castStart();
-    } else if (!player.isConnected) {
-        // console.log("Cast Disconnected.");
-    }
+  // If cast connection is initialized start cast
+  if (player.isConnected) {
+    // console.log("Cast Connected.");
+    castStart();
+  } else if (!player.isConnected) {
+    // console.log("Cast Disconnected.");
+  }
 }
 
 function castVideoProgress(player) {
-    var videoId = getVideoPlayerVideoId();
-    if (player.mediaInfo.contentId.includes(videoId)) {
-        var currentTime = player.currentTime;
-        var duration = player.duration;
-        if ((currentTime % 10) <= 1.0 && currentTime != 0 && duration != 0) { // Check progress every 10 seconds or else progress is checked a few times a second
-            postVideoProgress(videoId, currentTime);
-            setProgressBar(videoId, currentTime, duration);
-            if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
-                if (watchedThreshold(currentTime, duration)) {
-                    isWatched(videoId);
-                }
-            }
+  let videoId = getVideoPlayerVideoId();
+  if (player.mediaInfo.contentId.includes(videoId)) {
+    let currentTime = player.currentTime;
+    let duration = player.duration;
+    if (currentTime % 10 <= 1.0 && currentTime !== 0 && duration !== 0) {
+      // Check progress every 10 seconds or else progress is checked a few times a second
+      postVideoProgress(videoId, currentTime);
+      setProgressBar(videoId, currentTime, duration);
+      if (!getVideoPlayerWatchStatus()) {
+        // Check if video is already marked as watched
+        if (watchedThreshold(currentTime, duration)) {
+          isWatched(videoId);
         }
+      }
     }
+  }
 }
 
 function castVideoPaused(player) {
-    var videoId = getVideoPlayerVideoId();
-    var currentTime = player.currentTime;
-    var duration = player.duration;
-    if (player.mediaInfo != null) {
-        if (player.mediaInfo.contentId.includes(videoId)) {
-            if (currentTime != 0 && duration != 0) {
-                postVideoProgress(videoId, currentTime);
-            }
-        }
+  let videoId = getVideoPlayerVideoId();
+  let currentTime = player.currentTime;
+  let duration = player.duration;
+  if (player.mediaInfo != null) {
+    if (player.mediaInfo.contentId.includes(videoId)) {
+      if (currentTime !== 0 && duration !== 0) {
+        postVideoProgress(videoId, currentTime);
+      }
     }
+  }
 }
 
 function castStart() {
-    var castSession = cast.framework.CastContext.getInstance().getCurrentSession();
-    // Check if there is already media playing on the cast target to prevent recasting on page reload or switching to another video page
-    if (!castSession.getMediaSession()) { 
-        var videoId = getVideoPlayerVideoId();
-        var videoData = getVideoData(videoId);
-        var contentId = getURL() + videoData.data.media_url;
-        var contentTitle = videoData.data.title;
-        var contentImage = getURL() + videoData.data.vid_thumb_url;
+  let castSession = cast.framework.CastContext.getInstance().getCurrentSession();
+  // Check if there is already media playing on the cast target to prevent recasting on page reload or switching to another video page
+  if (!castSession.getMediaSession()) {
+    let videoId = getVideoPlayerVideoId();
+    let videoData = getVideoData(videoId);
+    let contentId = getURL() + videoData.data.media_url;
+    let contentTitle = videoData.data.title;
+    let contentImage = getURL() + videoData.data.vid_thumb_url;
 
-        contentType = 'video/mp4'; // Set content type, only videos right now so it is hard coded
-        contentCurrentTime = getVideoPlayerCurrentTime(); // Get video's current position
-        contentActiveSubtitle = [];
-        // Check if a subtitle is turned on.
-        for (var i = 0; i < getVideoPlayer().textTracks.length; i++) {
-            if (getVideoPlayer().textTracks[i].mode == "showing") {
-                contentActiveSubtitle =[i + 1];
-            }
-        }
-        contentSubtitles = [];
-        var videoSubtitles = videoData.data.subtitles; // Array of subtitles
-        if (typeof(videoSubtitles) != 'undefined' && videoData.config.downloads.subtitle) {
-            for (var i = 0; i < videoSubtitles.length; i++) {
-                subtitle = new chrome.cast.media.Track(i, chrome.cast.media.TrackType.TEXT);
-                subtitle.trackContentId = videoSubtitles[i].media_url;
-                subtitle.trackContentType = 'text/vtt';
-                subtitle.subtype = chrome.cast.media.TextTrackType.SUBTITLES;
-                subtitle.name = videoSubtitles[i].name;
-                subtitle.language = videoSubtitles[i].lang;
-                subtitle.customData = null;
-                contentSubtitles.push(subtitle);
-            }
-        }
+    let contentType = 'video/mp4'; // Set content type, only videos right now so it is hard coded
+    let contentCurrentTime = getVideoPlayerCurrentTime(); // Get video's current position
+    let contentActiveSubtitle = [];
+    // Check if a subtitle is turned on.
+    for (let i = 0; i < getVideoPlayer().textTracks.length; i++) {
+      if (getVideoPlayer().textTracks[i].mode === 'showing') {
+        contentActiveSubtitle = [i + 1];
+      }
+    }
+    let contentSubtitles = [];
+    let videoSubtitles = videoData.data.subtitles; // Array of subtitles
+    if (typeof videoSubtitles !== 'undefined' && videoData.config.downloads.subtitle) {
+      for (let i = 0; i < videoSubtitles.length; i++) {
+        let subtitle = new chrome.cast.media.Track(i, chrome.cast.media.TrackType.TEXT);
+        subtitle.trackContentId = videoSubtitles[i].media_url;
+        subtitle.trackContentType = 'text/vtt';
+        subtitle.subtype = chrome.cast.media.TextTrackType.SUBTITLES;
+        subtitle.name = videoSubtitles[i].name;
+        subtitle.language = videoSubtitles[i].lang;
+        subtitle.customData = null;
+        contentSubtitles.push(subtitle);
+      }
+    }
 
-        mediaInfo = new chrome.cast.media.MediaInfo(contentId, contentType); // Create MediaInfo var that contains url and content type
-        // mediaInfo.streamType = chrome.cast.media.StreamType.BUFFERED; // Set type of stream, BUFFERED, LIVE, OTHER
-        mediaInfo.metadata = new chrome.cast.media.GenericMediaMetadata(); // Create metadata var and add it to MediaInfo
-        mediaInfo.metadata.title = contentTitle.replace("&amp;", "&"); // Set the video title
-        mediaInfo.metadata.images = [new chrome.cast.Image(contentImage)]; // Set the video thumbnail
-        // mediaInfo.textTrackStyle = new chrome.cast.media.TextTrackStyle();
-        mediaInfo.tracks = contentSubtitles;
+    let mediaInfo = new chrome.cast.media.MediaInfo(contentId, contentType); // Create MediaInfo var that contains url and content type
+    // mediaInfo.streamType = chrome.cast.media.StreamType.BUFFERED; // Set type of stream, BUFFERED, LIVE, OTHER
+    mediaInfo.metadata = new chrome.cast.media.GenericMediaMetadata(); // Create metadata var and add it to MediaInfo
+    mediaInfo.metadata.title = contentTitle.replace('&amp;', '&'); // Set the video title
+    mediaInfo.metadata.images = [new chrome.cast.Image(contentImage)]; // Set the video thumbnail
+    // mediaInfo.textTrackStyle = new chrome.cast.media.TextTrackStyle();
+    mediaInfo.tracks = contentSubtitles;
 
-        var request = new chrome.cast.media.LoadRequest(mediaInfo); // Create request with the previously set MediaInfo. 
-        // request.queueData = new chrome.cast.media.QueueData(); // See https://developers.google.com/cast/docs/reference/web_sender/chrome.cast.media.QueueData for playlist support.
-        request.currentTime = shiftCurrentTime(contentCurrentTime); // Set video start position based on the browser video position
-        request.activeTrackIds = contentActiveSubtitle; // Set active subtitle based on video player
-        // request.autoplay = false; // Set content to auto play, true by default
-        castSession.loadMedia(request).then(
-            function() { 
-                castSuccessful();
-            }, 
-            function() { 
-                castFailed(errorCode); 
-            }
-        ); // Send request to cast device
-    }   
+    let request = new chrome.cast.media.LoadRequest(mediaInfo); // Create request with the previously set MediaInfo.
+    // request.queueData = new chrome.cast.media.QueueData(); // See https://developers.google.com/cast/docs/reference/web_sender/chrome.cast.media.QueueData for playlist support.
+    request.currentTime = shiftCurrentTime(contentCurrentTime); // Set video start position based on the browser video position
+    request.activeTrackIds = contentActiveSubtitle; // Set active subtitle based on video player
+    // request.autoplay = false; // Set content to auto play, true by default
+    castSession.loadMedia(request).then(
+      function () {
+        castSuccessful();
+      },
+      function (error) {
+        castFailed(error.code);
+      }
+    ); // Send request to cast device
+  }
 }
 
-function shiftCurrentTime(contentCurrentTime) { // Shift media back 3 seconds to prevent missing some of the content
-    if (contentCurrentTime > 5) {
-        return(contentCurrentTime - 3);
-    } else {
-        return(0);
-    }
+function shiftCurrentTime(contentCurrentTime) {
+  // Shift media back 3 seconds to prevent missing some of the content
+  if (contentCurrentTime > 5) {
+    return contentCurrentTime - 3;
+  } else {
+    return 0;
+  }
 }
 
 function castSuccessful() {
-    // console.log('Cast Successful.');
-    getVideoPlayer().pause(); // Pause browser video on successful cast
+  // console.log('Cast Successful.');
+  getVideoPlayer().pause(); // Pause browser video on successful cast
 }
 
 function castFailed(errorCode) {
-    console.log('Error code: ' + errorCode);
+  console.log('Error code: ' + errorCode);
 }
 
-window['__onGCastApiAvailable'] = function(isAvailable) {
-    if (isAvailable) {
-        initializeCastApi();
-    }
-}
+window['__onGCastApiAvailable'] = function (isAvailable) {
+  if (isAvailable) {
+    initializeCastApi();
+  }
+};

tubearchivist/static/css/style.css

@@ -344,6 +344,7 @@ button:hover {
 .grid-count {
     display: flex;
     justify-content: end;
+    align-items: center;
 }
 
 .view-icons img {
@@ -365,10 +366,17 @@ button:hover {
 }
 
 #text-reveal {
-    height: 0px;
+    height: 0;
     overflow: hidden;
 }
 
+#text-expand {
+    overflow: hidden;
+    display: -webkit-inline-box;
+    -webkit-box-orient: vertical;
+    -webkit-line-clamp: 4;
+}
+
 
 /* video player */
 .player-wrapper {
@@ -380,6 +388,7 @@ button:hover {
     display: grid;
     align-content: space-evenly;
     height: 100vh;
+    position: relative; /* needed for modal */
 }
 
 .notifications {
@@ -408,6 +417,12 @@ button:hover {
     margin: 10px 10px 10px 0;
 }
 
+/* fix for safari full screen not scaling full */
+video:-webkit-full-screen {
+    max-height: unset !important;
+    max-width: unset !important;
+}
+
 /* video list */
 .video-list {
     display: grid;
@@ -443,9 +458,15 @@ button:hover {
     overflow: hidden;
 }
 
+.video-item:hover .video-thumb span {
+    opacity: 1;
+}
+
 .video-item.list {
     display: grid;
-    grid-template-columns: 25% auto;
+    grid-template-columns: 26% auto;
+    background-color: var(--highlight-bg);
+    align-items: center;
 }
 
 .video-progress-bar {
@@ -461,6 +482,16 @@ button:hover {
     position: relative;
 }
 
+.video-thumb span {
+    position: absolute;
+    top: 5px;
+    left: 5px;
+    background-color: var(--accent-font-light);
+    padding: 5px;
+    opacity: 0;
+    transition: 300ms ease-in-out;
+}
+
 .video-play img {
     width: 40px;
     filter: var(--img-filter);
@@ -495,10 +526,9 @@ button:hover {
 
 .video-desc.list {
     padding: 10px;
-    height: unset;
-    background-color: var(--highlight-bg);
+    height: 100%;
     display: flex;
-    flex-wrap: wrap-reverse;
+    flex-wrap: wrap;
     align-content: center;
 }
 
@@ -591,6 +621,8 @@ button:hover {
 
 .description-box {
     margin-top: 1rem;
+    padding: 15px;
+    background-color: var(--highlight-bg);
 }
 
 .info-box-3 {
@@ -713,6 +745,22 @@ button:hover {
 /* video page */
 .video-main {
     margin: 1rem 0;
+    position: relative; /* needed for modal */
+}
+
+.video-modal {
+    position: absolute;
+    z-index: 1;
+    top: 20%;
+    width: 100%;
+    text-align: center;
+}
+
+.video-modal-text {
+    background: rgba(0,0,0,.5);
+    color: #eeeeee;
+    font-size: 1.3em;
+    display: none;
 }
 
 .video-main video {
@@ -772,6 +820,10 @@ button:hover {
 }
 
 /* multi search page */
+.multi-search-box {
+    padding-right: 20px;
+}
+
 .multi-search-box input {
     width: 100%;
 }
@@ -817,6 +869,18 @@ button:hover {
     transform: translateX(-30%);
 }
 
+.info-box-item.channel-nav {
+    justify-content: center;
+}
+
+.info-box-item.channel-nav a {
+    padding: 0 1rem;
+}
+
+.info-box-item.channel-nav a:hover {
+    text-decoration: underline;
+}
+
 /* playlist overview page */
 .playlist-list.list {
     display: grid;
@@ -875,73 +939,6 @@ button:hover {
     cursor: pointer;
 }
 
-.dl-list.list {
-    display: block;
-}
-
-.dl-list.grid {
-    display: grid;
-    grid-template-columns: 1fr 1fr 1fr;
-    grid-gap: 1rem;
-}
-
-.dl-item.list {
-    display: flex;
-    margin: 15px 0;
-    align-items: center;
-    background-color: var(--highlight-bg);
-}
-
-.dl-item.grid {
-    display: flex;
-    flex-wrap: wrap;
-    background-color: var(--highlight-bg);
-}
-
-.dl-check {
-    width: 30px;
-}
-
-.dl-item:hover > .dl-thumb span {
-    opacity: 1;
-}
-
-.dl-thumb {
-    position: relative;
-}
-
-.dl-thumb span {
-    position: absolute;
-    top: 5px;
-    left: 5px;
-    background-color: var(--accent-font-light);
-    padding: 5px;
-    opacity: 0;
-    transition: 300ms ease-in-out;
-}
-
-.dl-thumb.list {
-    width: 25%;
-}
-
-.dl-thumb.grid {
-    width: 100%;
-}
-
-.dl-item img {
-    width: 100%;
-}
-
-.dl-desc.list {
-    padding: 0 15px;
-    width: 75%;
-}
-
-.dl-desc.grid {
-    padding: 15px;
-    width: 100%;
-}
-
 .dl-control-icons {
     display: flex;
     justify-content: center;
@@ -1099,14 +1096,10 @@ button:hover {
     .video-list.grid.grid-5,
     .video-list.grid.grid-6,
     .video-list.grid.grid-7,
-    .dl-list.grid,
     .channel-list.grid,
     .playlist-list.grid {
         grid-template-columns: 1fr 1fr;
     }
-    .dl-thumb.list {
-        width: 35%;
-    }
     .video-item.list,
     .playlist-item.list {
         display: grid;
@@ -1150,7 +1143,6 @@ button:hover {
     .video-list.grid.grid-5,
     .video-list.grid.grid-6,
     .video-list.grid.grid-7,
-    .dl-list.grid,
     .channel-list.grid,
     .video-item.list,
     .playlist-list.list,
@@ -1190,6 +1182,10 @@ button:hover {
         margin: 15px;
         text-align: center;
     }
+    .view-controls.three {
+        grid-template-columns: unset;
+        justify-content: center;
+    }
     .sort {
         display: block;
     }
@@ -1199,15 +1195,6 @@ button:hover {
     .description-box {
         display: block;
     }
-    .dl-item {
-        display: block;
-    }
-    .dl-thumb {
-        width: 100%;
-    }
-    .dl-desc {
-        padding: 15px;
-    }
     .backup-grid-row {
         display: flex;
         flex-wrap: wrap;

tubearchivist/static/progress.js

@@ -1,106 +1,110 @@
 /**
  * Handle multi channel notifications
- * 
+ *
  */
 
-checkMessages()
+'use strict';
+
+checkMessages();
 
 // page map to notification status
 const messageTypes = {
-    "download": ["message:download", "message:add", "message:rescan", "message:playlistscan"],
-    "channel": ["message:subchannel"],
-    "channel_id": ["message:playlistscan"],
-    "playlist": ["message:subplaylist"],
-    "setting": ["message:setting"]
-}
+  download: ['message:download', 'message:add', 'message:rescan', 'message:playlistscan'],
+  channel: ['message:subchannel'],
+  channel_id: ['message:playlistscan'],
+  playlist: ['message:subplaylist'],
+  setting: ['message:setting'],
+};
 
 // start to look for messages
 function checkMessages() {
-    var notifications = document.getElementById("notifications");
-    if (notifications) {
-        var dataOrigin = notifications.getAttribute("data");
-        getMessages(dataOrigin);
-    }
+  let notifications = document.getElementById('notifications');
+  if (notifications) {
+    let dataOrigin = notifications.getAttribute('data');
+    getMessages(dataOrigin);
+  }
 }
 
 // get messages for page on timer
 function getMessages(dataOrigin) {
-    fetch('/progress/').then(response => {
-        return response.json();
-    }).then(responseData => {
-        var messages = buildMessage(responseData, dataOrigin);
-        if (messages.length > 0) {
-            // restart itself
-            setTimeout(function() {
-                getMessages(dataOrigin);
-            }, 3000);
-        };
+  fetch('/progress/')
+    .then(response => {
+      return response.json();
+    })
+    .then(responseData => {
+      let messages = buildMessage(responseData, dataOrigin);
+      if (messages.length > 0) {
+        // restart itself
+        setTimeout(function () {
+          getMessages(dataOrigin);
+        }, 3000);
+      }
     });
 }
 
 // make div for all messages, return relevant
 function buildMessage(responseData, dataOrigin) {
-    // filter relevan messages
-    var allMessages = responseData["messages"];
-    var messages = allMessages.filter(function(value) {
-        return messageTypes[dataOrigin].includes(value["status"])
-    }, dataOrigin);
-    // build divs
-    var notificationDiv = document.getElementById("notifications");
-    var nots = notificationDiv.childElementCount;
-    notificationDiv.innerHTML = "";
-    for (let i = 0; i < messages.length; i++) {
-        var messageData = messages[i];
-        var messageStatus = messageData["status"];
-        var messageBox = document.createElement("div");
-        var title = document.createElement("h3");
-        title.innerHTML = messageData["title"];
-        var message = document.createElement("p");
-        message.innerHTML = messageData["message"];
-        messageBox.appendChild(title);
-        messageBox.appendChild(message);
-        messageBox.classList.add(messageData["level"], "notification");
-        notificationDiv.appendChild(messageBox);
-        if (messageStatus === "message:download") {
-            checkDownloadIcons();
-        };
-    };
-    // reload page when no more notifications
-    if (nots > 0 && messages.length === 0) {
-        location.reload();
-    };
-    return messages
+  // filter relevan messages
+  let allMessages = responseData['messages'];
+  let messages = allMessages.filter(function (value) {
+    return messageTypes[dataOrigin].includes(value['status']);
+  }, dataOrigin);
+  // build divs
+  let notificationDiv = document.getElementById('notifications');
+  let nots = notificationDiv.childElementCount;
+  notificationDiv.innerHTML = '';
+  for (let i = 0; i < messages.length; i++) {
+    let messageData = messages[i];
+    let messageStatus = messageData['status'];
+    let messageBox = document.createElement('div');
+    let title = document.createElement('h3');
+    title.innerHTML = messageData['title'];
+    let message = document.createElement('p');
+    message.innerHTML = messageData['message'];
+    messageBox.appendChild(title);
+    messageBox.appendChild(message);
+    messageBox.classList.add(messageData['level'], 'notification');
+    notificationDiv.appendChild(messageBox);
+    if (messageStatus === 'message:download') {
+      checkDownloadIcons();
+    }
+  }
+  // reload page when no more notifications
+  if (nots > 0 && messages.length === 0) {
+    location.reload();
+  }
+  return messages;
 }
 
 // check if download icons are needed
 function checkDownloadIcons() {
-    var iconBox = document.getElementById("downloadControl");
-    if (iconBox.childElementCount === 0) {
-        var downloadIcons = buildDownloadIcons();
-        iconBox.appendChild(downloadIcons);
-    };
+  let iconBox = document.getElementById('downloadControl');
+  if (iconBox.childElementCount === 0) {
+    let downloadIcons = buildDownloadIcons();
+    iconBox.appendChild(downloadIcons);
+  }
 }
 
 // add dl control icons
 function buildDownloadIcons() {
-    var downloadIcons = document.createElement('div');
-    downloadIcons.classList = 'dl-control-icons';
-    // stop icon
-    var stopIcon = document.createElement('img');
-    stopIcon.setAttribute('id', "stop-icon");
-    stopIcon.setAttribute('title', "Stop Download Queue");
-    stopIcon.setAttribute('src', "/static/img/icon-stop.svg");
-    stopIcon.setAttribute('alt', "stop icon");
-    stopIcon.setAttribute('onclick', 'stopQueue()');
-    // kill icon
-    var killIcon = document.createElement('img');
-    killIcon.setAttribute('id', "kill-icon");
-    killIcon.setAttribute('title', "Kill Download Queue");
-    killIcon.setAttribute('src', "/static/img/icon-close.svg");
-    killIcon.setAttribute('alt', "kill icon");
-    killIcon.setAttribute('onclick', 'killQueue()');
-    // stich together
-    downloadIcons.appendChild(stopIcon);
-    downloadIcons.appendChild(killIcon);
-    return downloadIcons
+  let downloadIcons = document.createElement('div');
+  downloadIcons.classList = 'dl-control-icons';
+  // stop icon
+  let stopIcon = document.createElement('img');
+  stopIcon.setAttribute('id', 'stop-icon');
+  stopIcon.setAttribute('title', 'Stop Download Queue');
+  stopIcon.setAttribute('src', '/static/img/icon-stop.svg');
+  stopIcon.setAttribute('alt', 'stop icon');
+  stopIcon.setAttribute('onclick', 'stopQueue()');
+  // kill icon
+  let killIcon = document.createElement('img');
+  killIcon.setAttribute('id', 'kill-icon');
+  killIcon.setAttribute('title', 'Kill Download Queue');
+  killIcon.setAttribute('src', '/static/img/icon-close.svg');
+  killIcon.setAttribute('alt', 'kill icon');
+  killIcon.setAttribute('onclick', 'killQueue()');
+  // stich together
+  downloadIcons.appendChild(stopIcon);
+  downloadIcons.appendChild(killIcon);
+  return downloadIcons;
 }