update TA_VERSION

Changed:
- Changed split settings page
- Changed reset autostart on queue stop
- Added stats dashbord
- Fixed for wrong date epoch indexing

.github/workflows/lint_python.yml

@@ -8,20 +8,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-      # note: this logic is duplicated in the `validate` function in ./deploy.sh
-      # if you update this file, you should update that as well
       - run: pip install --upgrade pip wheel
       - run: pip install bandit black codespell flake8 flake8-bugbear
                          flake8-comprehensions isort requests
       - run: ./deploy.sh validate
-      # - run: black --check --diff --line-length 79 .
-      # - run: codespell --skip="./.git,./package.json,./package-lock.json,./node_modules"
-      # - run: flake8 . --count --max-complexity=10 --max-line-length=79
-      #                 --show-source --statistics
-      # - run: isort --check-only --line-length 79 --profile black .
-      # - run: pip install -r tubearchivist/requirements.txt
-      # - run: mkdir --parents --verbose .mypy_cache
-      # - run: mypy --ignore-missing-imports --install-types --non-interactive .
-      # - run: python3 tubearchivist/manage.py test || true
-      # - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true
-      # - run: safety check

.gitignore

@@ -1,5 +1,6 @@
 # python testing cache
 __pycache__
+.venv
 
 # django testing db
 db.sqlite3

CONTRIBUTING.md

@@ -8,6 +8,8 @@ Welcome, and thanks for showing interest in improving Tube Archivist!
   - [Feature Request](#feature-request)
   - [Installation Help](#installation-help)
 - [How to make a Pull Request](#how-to-make-a-pull-request)
+- [Contributions beyond the scope](#contributions-beyond-the-scope)
+- [User Scripts](#user-scripts)
 - [Improve to the Documentation](#improve-to-the-documentation)
 - [Development Environment](#development-environment)
 ---
@@ -87,6 +89,33 @@ to validate your changes. If you omit the path, all the project files will get c
 
 ---
 
+## Contributions beyond the scope
+
+As you have read the [FAQ](https://docs.tubearchivist.com/faq/) and the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and have gotten an idea what this project tries to do, there will be some obvious shortcomings that stand out, that have been explicitly excluded from the scope of this project, at least for the time being.
+
+Extending the scope of this project will only be feasible with more [regular contributors](https://github.com/tubearchivist/tubearchivist/graphs/contributors) that are willing to help improve this project in the long run. Contributors that have an overall improvement of the project in mind and not just about implementing this *one* thing.  
+
+Small minor additions, or making a PR for a documented feature request or bug, even if that was and will be your only contribution to this project, are always welcome and is *not* what this is about.
+
+Beyond that, general rules to consider:
+
+- Maintainability is key: It's not just about implementing something and being done with it, it's about maintaining it, fixing bugs as they occur, improving on it and supporting it in the long run.
+- Others can do it better: Some problems have been solved by very talented developers. These things don't need to be reinvented again here in this project.
+- Develop for the 80%: New features and additions *should* be beneficial for 80% of the users. If you are trying to solve your own problem that only applies to you, maybe that would be better to do in your own fork or if possible by a standalone implementation using the API.
+- If all of that sounds too strict for you, as stated above, start becoming a regular contributor to this project.
+
+---
+
+## User Scripts
+Some of you might have created useful scripts or API integrations around this project. Sharing is caring! Please add a link to your script to the Readme [here](https://github.com/tubearchivist/tubearchivist#user-scripts).
+- Your repo should have a `LICENSE` file with one of the common open source licenses. People are expected to fork, adapt and build upon your great work.
+- Your script should not modify the *official* files of Tube Archivist. E.g. your symlink script should build links *outside* of your `/youtube` folder. Or your fancy script that creates a beautiful artwork gallery should do that *outside* of the `/cache` folder. Modifying the *official* files and folders of TA are probably not supported.
+- On the top of the repo you should have a mention and a link back to the Tube Archivist repo. Clearly state to **not** to open any issues on the main TA repo regarding your script.
+- Example template:
+  - `[<user>/<repo>](https://linktoyourrepo.com)`: A short one line description.
+
+---
+
 ## Improve to the Documentation
 
 The documentation available at [docs.tubearchivist.com](https://docs.tubearchivist.com/) and is build from a separate repo [tubearchivist/docs](https://github.com/tubearchivist/docs). The Readme has additional instructions on how to make changes.
@@ -136,14 +165,15 @@ bin/elasticsearch-service-tokens create elastic/kibana kibana
 
 Example docker compose, use same version as for Elasticsearch:
 ```yml
-kibana:
-  image: docker.elastic.co/kibana/kibana:0.0.0
-  container_name: kibana
-  environment:
-	- "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
-	- "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
-  ports:
-	- "5601:5601"
+services:
+  kibana:
+    image: docker.elastic.co/kibana/kibana:0.0.0
+    container_name: kibana
+    environment:
+    - "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
+    - "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
+    ports:
+    - "5601:5601"
 ```
 
 If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.

Dockerfile

@@ -3,7 +3,7 @@
 
 
 # First stage to build python wheel
-FROM python:3.10.9-slim-bullseye AS builder
+FROM python:3.11.3-slim-bullseye AS builder
 ARG TARGETPLATFORM
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -14,7 +14,7 @@ COPY ./tubearchivist/requirements.txt /requirements.txt
 RUN pip install --user -r requirements.txt
 
 # build final image
-FROM python:3.10.9-slim-bullseye as tubearchivist
+FROM python:3.11.3-slim-bullseye as tubearchivist
 
 ARG TARGETPLATFORM
 ARG INSTALL_DEBUG

README.md

@@ -1,6 +1,6 @@
-![Tube Archivist](assets/tube-archivist-banner.jpg?raw=true "Tube Archivist Banner")  
+![Tube Archivist](assets/tube-archivist-front.jpg?raw=true "Tube Archivist Banner")  
+[*more screenshots and video*](SHOWCASE.MD)
 
-<h1 align="center">Your self hosted YouTube media server</h1>
 <div align="center">
 <a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-docker.png" alt="tubearchivist-docker" title="Tube Archivist Docker Pulls" height="50" width="190"/></a>
 <a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-star.png" alt="tubearchivist-github-star" title="Tube Archivist GitHub Stars" height="50" width="190"/></a>
@@ -8,9 +8,6 @@
 <a href="https://www.tubearchivist.com/discord" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-discord.png" alt="tubearchivist-discord" title="TA Discord Server Members" height="50" width="190"/></a>
 </div>
 
-![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")
-[*more screenshots and video*](SHOWCASE.MD)
-
 ## Table of contents:
 * [Docs](https://docs.tubearchivist.com/) with [FAQ](https://docs.tubearchivist.com/faq/), and API documentation
 * [Core functionality](#core-functionality)
@@ -37,6 +34,7 @@ Once your YouTube video collection grows, it becomes hard to search and find a s
 - [Discord](https://www.tubearchivist.com/discord): Connect with us on our Discord server.
 - [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit.
 - [Browser Extension](https://github.com/tubearchivist/browser-extension) Tube Archivist Companion, for [Firefox](https://addons.mozilla.org/addon/tubearchivist-companion/) and [Chrome](https://chrome.google.com/webstore/detail/tubearchivist-companion/jjnkmicfnfojkkgobdfeieblocadmcie)
+- [Jellyfin Integration](https://github.com/tubearchivist/tubearchivist-jf): Add your videos to Jellyfin.
 - [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.  
 
 ## Installing
@@ -135,7 +133,7 @@ Similar to that, TubeArchivist will become all sorts of messed up when running o
 ## Known limitations
 - Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection. 
 - Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
-- There is currently no flexibility in naming of the media files.
+- There is no flexibility in naming of the media files.
 
 ## Roadmap
 We have come far, nonetheless we are not short of ideas on how to improve and extend this project. Issues waiting for you to be tackled in no particular order:
@@ -143,7 +141,6 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex
 - [ ] User roles
 - [ ] Podcast mode to serve channel as mp3
 - [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage
-- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/tubearchivist/tubearchivist/issues/97))
 - [ ] User created playlists, random and repeat controls ([#108](https://github.com/tubearchivist/tubearchivist/issues/108), [#220](https://github.com/tubearchivist/tubearchivist/issues/220))
 - [ ] Auto play or play next link ([#226](https://github.com/tubearchivist/tubearchivist/issues/226))
 - [ ] Multi language support
@@ -154,6 +151,7 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex
 - [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144))
 
 Implemented:
+- [X] Implement [Apprise](https://github.com/caronc/apprise) for notifications [2023-08-05]
 - [X] Download video comments [2022-11-30]
 - [X] Show similar videos on video page [2022-11-30]
 - [X] Implement complete offline media file import from json file [2022-08-20]
@@ -177,6 +175,11 @@ Implemented:
 - [X] Backup and restore [2021-09-22]
 - [X] Scan your file system to index already downloaded videos [2021-09-14]
 
+## User Scripts
+This is a list of useful user scripts, generously created from folks like you to extend this project and its functionality. This is your time to shine, [read this](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#user-scripts) then open a PR to add your script here.
+
+- Example 1
+
 ## Donate
 The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.  
 Second best way to support the development is to provide for caffeinated beverages:
@@ -187,6 +190,8 @@ Second best way to support the development is to provide for caffeinated beverag
 
 ## Notable mentions
 This is a selection of places where this project has been featured on reddit, in the news, blogs or any other online media, newest on top.
+* **ycombinator**: Tube Archivist on Hackernews front page, [2023-07-16][[link](https://news.ycombinator.com/item?id=36744395)]
+* **linux-community.de**: Tube Archivist bringt Ordnung in die Youtube-Sammlung, [German][2023-05-01][[link](https://www.linux-community.de/ausgaben/linuxuser/2023/05/tube-archivist-bringt-ordnung-in-die-youtube-sammlung/)]
 * **noted.lol**: Dev Debrief, An Interview With the Developer of Tube Archivist, [2023-03-30] [[link](https://noted.lol/dev-debrief-tube-archivist/)]
 * **console.substack.com**: Interview With Simon of Tube Archivist, [2023-01-29] [[link](https://console.substack.com/p/console-142#%C2%A7interview-with-simon-of-tube-archivist)]
 * **reddit.com**: Tube Archivist v0.3.0 - Now Archiving Comments, [2022-12-02] [[link](https://www.reddit.com/r/selfhosted/comments/zaonzp/tube_archivist_v030_now_archiving_comments/)]

SHOWCASE.MD

@@ -3,17 +3,23 @@
 Video featuring Tube Archivist generously created by [IBRACORP](https://www.youtube.com/@IBRACORP).
 
 ## Screenshots
-![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")  
+![login screenshot](assets/tube-archivist-login.png?raw=true "Tube Archivist Login")  
+*Login Page*: Secure way to access your media collection.
+
+![home screenshot](assets/tube-archivist-home.png?raw=true "Tube Archivist Home")  
 *Home Page*: Your recent videos, continue watching incomplete videos.
 
-![channels screenshot](assets/tube-archivist-screenshot-channels.png?raw=true "Tube Archivist Channels")  
+![channels screenshot](assets/tube-archivist-channels.png?raw=true "Tube Archivist Channels")  
 *All Channels*: A list of all your indexed channels, filtered by subscribed only.
 
-![single channel screenshot](assets/tube-archivist-screenshot-single-channel.png?raw=true "Tube Archivist Single Channel")  
+![single channel screenshot](assets/tube-archivist-single-channel.png?raw=true "Tube Archivist Single Channel")  
 *Single Channel*: Single channel page with additional metadata and sub pages.
 
-![video page screenshot](assets/tube-archivist-screenshot-video.png?raw=true "Tube Archivist Video Page")  
+![video page screenshot](assets/tube-archivist-video.png?raw=true "Tube Archivist Video Page")  
 *Video Page*: Stream your video directly from the interface.
 
-![video page screenshot](assets/tube-archivist-screenshot-download.png?raw=true "Tube Archivist Video Page")  
-*Downloads Page*: Add, control and monitor your download queue.
+![video page screenshot](assets/tube-archivist-download.png?raw=true "Tube Archivist Video Page")  
+*Downloads Page*: Add, control, and monitor your download queue.
+
+![search page screenshot](assets/tube-archivist-search.png?raw=true "Tube Archivist Search Page")  
+*Search Page*. Use expressions to quickly search through your collection.

deploy.sh

@@ -25,6 +25,7 @@ function sync_blackhole {
         --exclude ".gitignore" \
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
+        --exclude ".venv" \
         --exclude "db.sqlite3" \
         --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
@@ -49,6 +50,7 @@ function sync_test {
         --exclude ".gitignore" \
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
+        --exclude ".venv" \
         --exclude "db.sqlite3" \
         --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
@@ -87,14 +89,14 @@ function validate {
     # note: this logic is duplicated in the `./github/workflows/lint_python.yml` config
     # if you update this file, you should update that as well
     echo "running black"
-    black --exclude "migrations/*" --diff --color --check -l 79 "$check_path"
+    black --force-exclude "migrations/*" --diff --color --check -l 79 "$check_path"
     echo "running codespell"
-    codespell --skip="./.git,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
+    codespell --skip="./.git,./.venv,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
     echo "running flake8"
-    flake8 "$check_path" --exclude "migrations" --count --max-complexity=10 \
+    flake8 "$check_path" --exclude "migrations,.venv" --count --max-complexity=10 \
         --max-line-length=79 --show-source --statistics
     echo "running isort"
-    isort --skip "migrations" --check-only --diff --profile black -l 79 "$check_path"
+    isort --skip "migrations" --skip ".venv" --check-only --diff --profile black -l 79 "$check_path"
     printf "    \n> all validations passed\n"
 
 }

docker-compose.yml

@@ -34,7 +34,7 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.7.0
+    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.9.0
     container_name: archivist-es
     restart: unless-stopped
     environment:

docker_assets/run.sh

@@ -14,6 +14,7 @@ fi
 python manage.py ta_envcheck
 python manage.py ta_connection
 python manage.py ta_startup
+python manage.py ta_migpath
 
 # start all tasks
 nginx &

tubearchivist/api/src/aggs.py

@@ -0,0 +1,242 @@
+"""aggregations"""
+
+from home.src.es.connect import ElasticWrap
+from home.src.index.video_streams import DurationConverter
+
+
+class AggBase:
+    """base class for aggregation calls"""
+
+    path: str = ""
+    data: dict = {}
+    name: str = ""
+
+    def get(self):
+        """make get call"""
+        response, _ = ElasticWrap(self.path).get(self.data)
+        print(f"[agg][{self.name}] took {response.get('took')} ms to process")
+
+        return response.get("aggregations")
+
+    def process(self):
+        """implement in subclassess"""
+        raise NotImplementedError
+
+
+class Primary(AggBase):
+    """primary aggregation for total documents indexed"""
+
+    name = "primary"
+    path = "ta_video,ta_channel,ta_playlist,ta_subtitle,ta_download/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "video_type": {
+                "filter": {"exists": {"field": "active"}},
+                "aggs": {"filtered": {"terms": {"field": "vid_type"}}},
+            },
+            "channel_total": {"value_count": {"field": "channel_active"}},
+            "channel_sub": {"terms": {"field": "channel_subscribed"}},
+            "playlist_total": {"value_count": {"field": "playlist_active"}},
+            "playlist_sub": {"terms": {"field": "playlist_subscribed"}},
+            "download": {"terms": {"field": "status"}},
+        },
+    }
+
+    def process(self):
+        """make the call"""
+        aggregations = self.get()
+
+        videos = {"total": aggregations["video_type"].get("doc_count")}
+        videos.update(
+            {
+                i.get("key"): i.get("doc_count")
+                for i in aggregations["video_type"]["filtered"]["buckets"]
+            }
+        )
+        channels = {"total": aggregations["channel_total"].get("value")}
+        channels.update(
+            {
+                "sub_" + i.get("key_as_string"): i.get("doc_count")
+                for i in aggregations["channel_sub"]["buckets"]
+            }
+        )
+        playlists = {"total": aggregations["playlist_total"].get("value")}
+        playlists.update(
+            {
+                "sub_" + i.get("key_as_string"): i.get("doc_count")
+                for i in aggregations["playlist_sub"]["buckets"]
+            }
+        )
+        downloads = {
+            i.get("key"): i.get("doc_count")
+            for i in aggregations["download"]["buckets"]
+        }
+
+        response = {
+            "videos": videos,
+            "channels": channels,
+            "playlists": playlists,
+            "downloads": downloads,
+        }
+
+        return response
+
+
+class WatchProgress(AggBase):
+    """get watch progress"""
+
+    name = "watch_progress"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            name: {
+                "terms": {"field": "player.watched"},
+                "aggs": {
+                    "watch_docs": {
+                        "filter": {"terms": {"player.watched": [True, False]}},
+                        "aggs": {
+                            "true_count": {"value_count": {"field": "_index"}},
+                            "duration": {"sum": {"field": "player.duration"}},
+                        },
+                    },
+                },
+            },
+            "total_duration": {"sum": {"field": "player.duration"}},
+            "total_vids": {"value_count": {"field": "_index"}},
+        },
+    }
+
+    def process(self):
+        """make the call"""
+        aggregations = self.get()
+        buckets = aggregations[self.name]["buckets"]
+
+        response = {}
+        all_duration = int(aggregations["total_duration"].get("value"))
+        response.update(
+            {
+                "all": {
+                    "duration": all_duration,
+                    "duration_str": DurationConverter().get_str(all_duration),
+                    "items": aggregations["total_vids"].get("value"),
+                }
+            }
+        )
+
+        for bucket in buckets:
+            response.update(self._build_bucket(bucket, all_duration))
+
+        return response
+
+    @staticmethod
+    def _build_bucket(bucket, all_duration):
+        """parse bucket"""
+
+        duration = int(bucket["watch_docs"]["duration"]["value"])
+        duration_str = DurationConverter().get_str(duration)
+        items = bucket["watch_docs"]["true_count"]["value"]
+        if bucket["key_as_string"] == "false":
+            key = "unwatched"
+        else:
+            key = "watched"
+
+        bucket_parsed = {
+            key: {
+                "duration": duration,
+                "duration_str": duration_str,
+                "progress": duration / all_duration if all_duration else 0,
+                "items": items,
+            }
+        }
+
+        return bucket_parsed
+
+
+class DownloadHist(AggBase):
+    """get downloads histogram last week"""
+
+    name = "videos_last_week"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            name: {
+                "date_histogram": {
+                    "field": "date_downloaded",
+                    "calendar_interval": "day",
+                    "format": "yyyy-MM-dd",
+                    "order": {"_key": "desc"},
+                },
+                "aggs": {
+                    "total_videos": {"value_count": {"field": "youtube_id"}}
+                },
+            }
+        },
+        "query": {"range": {"date_downloaded": {"gte": "now-7d/d"}}},
+    }
+
+    def process(self):
+        """process query"""
+        aggregations = self.get()
+        buckets = aggregations[self.name]["buckets"]
+
+        response = [
+            {
+                "date": i.get("key_as_string"),
+                "count": i.get("doc_count"),
+            }
+            for i in buckets
+        ]
+
+        return response
+
+
+class BiggestChannel(AggBase):
+    """get channel aggregations"""
+
+    name = "channel_stats"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            name: {
+                "multi_terms": {
+                    "terms": [
+                        {"field": "channel.channel_name.keyword"},
+                        {"field": "channel.channel_id"},
+                    ],
+                    "order": {"doc_count": "desc"},
+                },
+                "aggs": {
+                    "doc_count": {"value_count": {"field": "_index"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                    "media_size": {"sum": {"field": "media_size"}},
+                },
+            },
+        },
+    }
+    order_choices = ["doc_count", "duration", "media_size"]
+
+    def process(self):
+        """process aggregation, order_by validated in the view"""
+
+        aggregations = self.get()
+        buckets = aggregations[self.name]["buckets"]
+
+        response = [
+            {
+                "id": i["key"][1],
+                "name": i["key"][0].title(),
+                "doc_count": i["doc_count"]["value"],
+                "duration": i["duration"]["value"],
+                "duration_str": DurationConverter().get_str(
+                    i["duration"]["value"]
+                ),
+                "media_size": i["media_size"]["value"],
+            }
+            for i in buckets
+        ]
+
+        return response

tubearchivist/api/urls.py

@@ -41,6 +41,11 @@ urlpatterns = [
         views.ChannelApiListView.as_view(),
         name="api-channel-list",
     ),
+    path(
+        "channel/search/",
+        views.ChannelApiSearchView.as_view(),
+        name="api-channel-search",
+    ),
     path(
         "channel/<slug:channel_id>/",
         views.ChannelApiView.as_view(),
@@ -131,4 +136,24 @@ urlpatterns = [
         views.NotificationView.as_view(),
         name="api-notification",
     ),
+    path(
+        "stats/primary/",
+        views.StatPrimaryView.as_view(),
+        name="api-stats-primary",
+    ),
+    path(
+        "stats/watch/",
+        views.StatWatchProgress.as_view(),
+        name="api-stats-watch",
+    ),
+    path(
+        "stats/downloadhist/",
+        views.StatDownloadHist.as_view(),
+        name="api-stats-downloadhist",
+    ),
+    path(
+        "stats/biggestchannels/",
+        views.StatBiggestChannel.as_view(),
+        name="api-stats-biggestchannels",
+    ),
 ]

tubearchivist/api/views.py

@@ -1,7 +1,12 @@
 """all API views"""
 
+from api.src.aggs import BiggestChannel, DownloadHist, Primary, WatchProgress
 from api.src.search_processor import SearchProcess
 from home.src.download.queue import PendingInteract
+from home.src.download.subscriptions import (
+    ChannelSubscription,
+    PlaylistSubscription,
+)
 from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.connect import ElasticWrap
 from home.src.es.snapshot import ElasticSnapshot
@@ -9,9 +14,10 @@ from home.src.frontend.searching import SearchForm
 from home.src.frontend.watched import WatchState
 from home.src.index.channel import YoutubeChannel
 from home.src.index.generic import Pagination
+from home.src.index.playlist import YoutubePlaylist
 from home.src.index.reindex import ReindexProgress
 from home.src.index.video import SponsorBlock, YoutubeVideo
-from home.src.ta.config import AppConfig
+from home.src.ta.config import AppConfig, ReleaseVersion
 from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.task_manager import TaskCommand, TaskManager
 from home.src.ta.urlparser import Parser
@@ -134,7 +140,7 @@ class VideoApiListView(ApiBaseView):
 
 
 class VideoProgressView(ApiBaseView):
-    """resolves to /api/video/<video_id>/
+    """resolves to /api/video/<video_id>/progress/
     handle progress status for video
     """
 
@@ -189,7 +195,7 @@ class VideoCommentView(ApiBaseView):
 
 class VideoSimilarView(ApiBaseView):
     """resolves to /api/video/<video-id>/similar/
-    GET: return max 3 videos similar to this
+    GET: return max 6 videos similar to this
     """
 
     search_base = "ta_video/_search/"
@@ -223,6 +229,10 @@ class VideoSponsorView(ApiBaseView):
         # pylint: disable=unused-argument
 
         self.get_document(video_id)
+        if not self.response.get("data"):
+            message = {"message": "video not found"}
+            return Response(message, status=404)
+
         sponsorblock = self.response["data"].get("sponsorblock")
 
         return Response(sponsorblock)
@@ -318,9 +328,8 @@ class ChannelApiListView(ApiBaseView):
 
         return Response(self.response)
 
-    @staticmethod
-    def post(request):
-        """subscribe to list of channels"""
+    def post(self, request):
+        """subscribe/unsubscribe to list of channels"""
         data = request.data
         try:
             to_add = data["data"]
@@ -329,12 +338,58 @@ class ChannelApiListView(ApiBaseView):
             print(message)
             return Response({"message": message}, status=400)
 
-        pending = [i["channel_id"] for i in to_add if i["channel_subscribed"]]
-        url_str = " ".join(pending)
-        subscribe_to.delay(url_str)
+        pending = []
+        for channel_item in to_add:
+            channel_id = channel_item["channel_id"]
+            if channel_item["channel_subscribed"]:
+                pending.append(channel_id)
+            else:
+                self._unsubscribe(channel_id)
+
+        if pending:
+            url_str = " ".join(pending)
+            subscribe_to.delay(url_str, expected_type="channel")
 
         return Response(data)
 
+    @staticmethod
+    def _unsubscribe(channel_id: str):
+        """unsubscribe"""
+        print(f"[{channel_id}] unsubscribe from channel")
+        ChannelSubscription().change_subscribe(
+            channel_id, channel_subscribed=False
+        )
+
+
+class ChannelApiSearchView(ApiBaseView):
+    """resolves to /api/channel/search/
+    search for channel
+    """
+
+    search_base = "ta_channel/_doc/"
+
+    def get(self, request):
+        """handle get request, search with s parameter"""
+
+        query = request.GET.get("q")
+        if not query:
+            message = "missing expected q parameter"
+            return Response({"message": message, "data": False}, status=400)
+
+        try:
+            parsed = Parser(query).parse()[0]
+        except (ValueError, IndexError, AttributeError):
+            message = f"channel not found: {query}"
+            return Response({"message": message, "data": False}, status=404)
+
+        if not parsed["type"] == "channel":
+            message = "expected type channel"
+            return Response({"message": message, "data": False}, status=400)
+
+        self.get_document(parsed["url"])
+
+        return Response(self.response, status=self.status_code)
+
 
 class ChannelApiVideoView(ApiBaseView):
     """resolves to /api/channel/<channel-id>/video
@@ -373,6 +428,38 @@ class PlaylistApiListView(ApiBaseView):
         self.get_document_list(request)
         return Response(self.response)
 
+    def post(self, request):
+        """subscribe/unsubscribe to list of playlists"""
+        data = request.data
+        try:
+            to_add = data["data"]
+        except KeyError:
+            message = "missing expected data key"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        pending = []
+        for playlist_item in to_add:
+            playlist_id = playlist_item["playlist_id"]
+            if playlist_item["playlist_subscribed"]:
+                pending.append(playlist_id)
+            else:
+                self._unsubscribe(playlist_id)
+
+        if pending:
+            url_str = " ".join(pending)
+            subscribe_to.delay(url_str, expected_type="playlist")
+
+        return Response(data)
+
+    @staticmethod
+    def _unsubscribe(playlist_id: str):
+        """unsubscribe"""
+        print(f"[{playlist_id}] unsubscribe from playlist")
+        PlaylistSubscription().change_subscribe(
+            playlist_id, subscribe_status=False
+        )
+
 
 class PlaylistApiView(ApiBaseView):
     """resolves to /api/playlist/<playlist_id>/
@@ -387,6 +474,17 @@ class PlaylistApiView(ApiBaseView):
         self.get_document(playlist_id)
         return Response(self.response, status=self.status_code)
 
+    def delete(self, request, playlist_id):
+        """delete playlist"""
+        print(f"{playlist_id}: delete playlist")
+        delete_videos = request.GET.get("delete-videos", False)
+        if delete_videos:
+            YoutubePlaylist(playlist_id).delete_videos_playlist()
+        else:
+            YoutubePlaylist(playlist_id).delete_metadata()
+
+        return Response({"success": True})
+
 
 class PlaylistApiVideoView(ApiBaseView):
     """resolves to /api/playlist/<playlist_id>/video
@@ -535,7 +633,11 @@ class PingView(ApiBaseView):
     @staticmethod
     def get(request):
         """get pong"""
-        data = {"response": "pong", "user": request.user.id}
+        data = {
+            "response": "pong",
+            "user": request.user.id,
+            "version": ReleaseVersion().get_local_version(),
+        }
         return Response(data)
 
 
@@ -874,3 +976,58 @@ class NotificationView(ApiBaseView):
             query = f"{query}:{filter_by}"
 
         return Response(RedisArchivist().list_items(query))
+
+
+class StatPrimaryView(ApiBaseView):
+    """resolves to /api/stats/primary/
+    GET: return document count
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Primary().process())
+
+
+class StatWatchProgress(ApiBaseView):
+    """resolves to /api/stats/watchprogress/
+    GET: return watch/unwatch progress stats
+    """
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+
+        return Response(WatchProgress().process())
+
+
+class StatDownloadHist(ApiBaseView):
+    """resolves to /api/stats/downloadhist/
+    GET: return download video count histogram for last days
+    """
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+
+        return Response(DownloadHist().process())
+
+
+class StatBiggestChannel(ApiBaseView):
+    """resolves to /api/stats/biggestchannels/
+    GET: return biggest channels
+    param: order
+    """
+
+    order_choices = ["doc_count", "duration", "media_size"]
+
+    def get(self, request):
+        """handle get request"""
+
+        order = request.GET.get("order", False)
+        if order and order not in self.order_choices:
+            message = {"message": f"invalid order parameter {order}"}
+            return Response(message, status=400)
+
+        return Response(BiggestChannel().process())

tubearchivist/config/management/commands/ta_connection.py

@@ -81,11 +81,16 @@ class Command(BaseCommand):
                 _, status_code = ElasticWrap("/").get(
                     timeout=1, print_error=False
                 )
-            except requests.exceptions.ConnectionError:
+            except (
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout,
+            ):
                 sleep(5)
                 continue
 
             if status_code and status_code == 200:
+                path = "_cluster/health?wait_for_status=yellow&timeout=60s"
+                _, _ = ElasticWrap(path).get(timeout=60)
                 self.stdout.write(
                     self.style.SUCCESS("    ✓ ES connection established")
                 )
@@ -116,7 +121,7 @@ class Command(BaseCommand):
             return
 
         message = (
-            "    🗙 ES connection failed. "
+            "    🗙 ES version check failed. "
             + f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}"
         )
         self.stdout.write(self.style.ERROR(f"{message}"))

tubearchivist/config/management/commands/ta_migpath.py

@@ -0,0 +1,182 @@
+"""filepath migration from v0.3.6 to v0.3.7"""
+
+import json
+import os
+import shutil
+
+from django.core.management.base import BaseCommand
+from home.src.es.connect import ElasticWrap, IndexPaginate
+from home.src.ta.config import AppConfig
+from home.src.ta.helper import ignore_filelist
+
+TOPIC = """
+
+########################
+# Filesystem Migration #
+########################
+
+"""
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+
+    def handle(self, *args, **options):
+        """run commands"""
+        self.stdout.write(TOPIC)
+
+        handler = FolderMigration()
+        to_migrate = handler.get_to_migrate()
+        if not to_migrate:
+            self.stdout.write(
+                self.style.SUCCESS("    no channel migration needed\n")
+            )
+            return
+
+        self.stdout.write(self.style.SUCCESS("    migrating channels"))
+        total_channels = handler.create_folders(to_migrate)
+        self.stdout.write(
+            self.style.SUCCESS(f"    created {total_channels} channels")
+        )
+
+        self.stdout.write(
+            self.style.SUCCESS(f"    migrating {len(to_migrate)} videos")
+        )
+        handler.migrate_videos(to_migrate)
+        self.stdout.write(self.style.SUCCESS("    update videos in index"))
+        handler.send_bulk()
+
+        self.stdout.write(self.style.SUCCESS("    cleanup old folders"))
+        handler.delete_old()
+
+        self.stdout.write(self.style.SUCCESS("    ✓ migration completed\n"))
+
+
+class FolderMigration:
+    """migrate video archive folder"""
+
+    def __init__(self):
+        self.config = AppConfig().config
+        self.videos = self.config["application"]["videos"]
+        self.bulk_list = []
+
+    def get_to_migrate(self):
+        """get videos to migrate"""
+        script = (
+            "doc['media_url'].value == "
+            + "doc['channel.channel_id'].value + '/'"
+            + " + doc['youtube_id'].value + '.mp4'"
+        )
+        data = {
+            "query": {"bool": {"must_not": [{"script": {"script": script}}]}},
+            "_source": [
+                "youtube_id",
+                "media_url",
+                "channel.channel_id",
+                "subtitles",
+            ],
+        }
+        response = IndexPaginate("ta_video", data).get_results()
+
+        return response
+
+    def create_folders(self, to_migrate):
+        """create required channel folders"""
+        host_uid = self.config["application"]["HOST_UID"]
+        host_gid = self.config["application"]["HOST_GID"]
+        all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate}
+
+        for channel_id in all_channel_ids:
+            new_folder = os.path.join(self.videos, channel_id)
+            os.makedirs(new_folder, exist_ok=True)
+            if host_uid and host_gid:
+                os.chown(new_folder, host_uid, host_gid)
+
+        return len(all_channel_ids)
+
+    def migrate_videos(self, to_migrate):
+        """migrate all videos of channel"""
+        total = len(to_migrate)
+        for idx, video in enumerate(to_migrate):
+            new_media_url = self._move_video_file(video)
+            if not new_media_url:
+                continue
+
+            all_subtitles = self._move_subtitles(video)
+            action = {
+                "update": {"_id": video["youtube_id"], "_index": "ta_video"}
+            }
+            source = {"doc": {"media_url": new_media_url}}
+            if all_subtitles:
+                source["doc"].update({"subtitles": all_subtitles})
+
+            self.bulk_list.append(json.dumps(action))
+            self.bulk_list.append(json.dumps(source))
+            if idx % 1000 == 0:
+                print(f"processing migration [{idx}/{total}]")
+                self.send_bulk()
+
+    def _move_video_file(self, video):
+        """move video file to new location"""
+        old_path = os.path.join(self.videos, video["media_url"])
+        if not os.path.exists(old_path):
+            print(f"did not find expected video at {old_path}")
+            return False
+
+        new_media_url = os.path.join(
+            video["channel"]["channel_id"], video["youtube_id"] + ".mp4"
+        )
+        new_path = os.path.join(self.videos, new_media_url)
+        os.rename(old_path, new_path)
+
+        return new_media_url
+
+    def _move_subtitles(self, video):
+        """move subtitle files to new location"""
+        all_subtitles = video.get("subtitles")
+        if not all_subtitles:
+            return False
+
+        for subtitle in all_subtitles:
+            old_path = os.path.join(self.videos, subtitle["media_url"])
+            if not os.path.exists(old_path):
+                print(f"did not find expected subtitle at {old_path}")
+                continue
+
+            new_media_url = os.path.join(
+                video["channel"]["channel_id"],
+                f"{video.get('youtube_id')}.{subtitle.get('lang')}.vtt",
+            )
+            new_path = os.path.join(self.videos, new_media_url)
+            os.rename(old_path, new_path)
+            subtitle["media_url"] = new_media_url
+
+        return all_subtitles
+
+    def send_bulk(self):
+        """send bulk request to update index with new urls"""
+        if not self.bulk_list:
+            print("nothing to update")
+            return
+
+        self.bulk_list.append("\n")
+        path = "_bulk?refresh=true"
+        data = "\n".join(self.bulk_list)
+        response, status = ElasticWrap(path).post(data=data, ndjson=True)
+        if not status == 200:
+            print(response)
+
+        self.bulk_list = []
+
+    def delete_old(self):
+        """delete old empty folders"""
+        all_folders = ignore_filelist(os.listdir(self.videos))
+        for folder in all_folders:
+            folder_path = os.path.join(self.videos, folder)
+            if not os.path.isdir(folder_path):
+                continue
+
+            if not ignore_filelist(os.listdir(folder_path)):
+                shutil.rmtree(folder_path)

tubearchivist/config/management/commands/ta_startup.py

@@ -11,7 +11,6 @@ from django.core.management.base import BaseCommand, CommandError
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.es.index_setup import ElasitIndexWrap
 from home.src.es.snapshot import ElasticSnapshot
-from home.src.index.filesystem import Filesystem
 from home.src.index.video_streams import MediaStreamExtractor
 from home.src.ta.config import AppConfig, ReleaseVersion
 from home.src.ta.helper import clear_dl_cache
@@ -163,8 +162,6 @@ class Command(BaseCommand):
             self.stdout.write("    no videos need updating")
             return
 
-        self.stdout.write("    start filesystem rescan")
-        Filesystem().process()
         total = len(all_missing)
         for idx, missing in enumerate(all_missing):
             media_url = missing["media_url"]
@@ -172,6 +169,7 @@ class Command(BaseCommand):
             media_path = os.path.join(videos, media_url)
             if not os.path.exists(media_path):
                 self.stdout.write(f"    file not found: {media_path}")
+                self.stdout.write("    run file system rescan to fix")
                 continue
 
             media = MediaStreamExtractor(media_path)

tubearchivist/config/settings.py

@@ -256,4 +256,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.3.6"
+TA_VERSION = "v0.4.1"

tubearchivist/home/config.json

@@ -47,8 +47,11 @@
     },
     "scheduler": {
         "update_subscribed": false,
+        "update_subscribed_notify": false,
         "download_pending": false,
+        "download_pending_notify": false,
         "check_reindex":  {"minute": "0", "hour": "12", "day_of_week": "*"},
+        "check_reindex_notify": false,
         "check_reindex_days": 90,
         "thumbnail_check": {"minute": "0", "hour": "17", "day_of_week": "*"},
         "run_backup": false,

tubearchivist/home/src/download/queue.py

@@ -114,7 +114,13 @@ class PendingInteract:
     def update_status(self):
         """update status of pending item"""
         if self.status == "priority":
-            data = {"doc": {"status": "pending", "auto_start": True}}
+            data = {
+                "doc": {
+                    "status": "pending",
+                    "auto_start": True,
+                    "message": None,
+                }
+            }
         else:
             data = {"doc": {"status": self.status}}
 

tubearchivist/home/src/download/subscriptions.py

@@ -12,7 +12,6 @@ from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
-from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.urlparser import Parser
 
 
@@ -197,16 +196,13 @@ class PlaylistSubscription:
             thumb = ThumbManager(playlist_id, item_type="playlist")
             thumb.download_playlist_thumb(url)
 
-            # notify
-            message = {
-                "status": "message:subplaylist",
-                "level": "info",
-                "title": "Subscribing to Playlists",
-                "message": f"Processing {idx + 1} of {len(new_playlists)}",
-            }
-            RedisArchivist().set_message(
-                "message:subplaylist", message=message, expire=True
-            )
+            if self.task:
+                self.task.send_progress(
+                    message_lines=[
+                        f"Processing {idx + 1} of {len(new_playlists)}"
+                    ],
+                    progress=(idx + 1) / len(new_playlists),
+                )
 
     @staticmethod
     def channel_validate(channel_id):
@@ -332,7 +328,7 @@ class SubscriptionHandler:
         self.task = task
         self.to_subscribe = False
 
-    def subscribe(self):
+    def subscribe(self, expected_type=False):
         """subscribe to url_str items"""
         if self.task:
             self.task.send_progress(["Processing form content."])
@@ -343,11 +339,16 @@ class SubscriptionHandler:
             if self.task:
                 self._notify(idx, item, total)
 
-            self.subscribe_type(item)
+            self.subscribe_type(item, expected_type=expected_type)
 
-    def subscribe_type(self, item):
+    def subscribe_type(self, item, expected_type):
         """process single item"""
         if item["type"] == "playlist":
+            if expected_type and expected_type != "playlist":
+                raise TypeError(
+                    f"expected {expected_type} url but got {item.get('type')}"
+                )
+
             PlaylistSubscription().process_url_str([item])
             return
 
@@ -360,6 +361,11 @@ class SubscriptionHandler:
         else:
             raise ValueError("failed to subscribe to: " + item["url"])
 
+        if expected_type and expected_type != "channel":
+            raise TypeError(
+                f"expected {expected_type} url but got {item.get('type')}"
+            )
+
         self._subscribe(channel_id)
 
     def _subscribe(self, channel_id):

tubearchivist/home/src/download/thumbnails.py

@@ -54,11 +54,14 @@ class ThumbManagerBase:
                 if response.status_code == 404:
                     return self.get_fallback()
 
-            except requests.exceptions.RequestException:
+            except (
+                requests.exceptions.RequestException,
+                requests.exceptions.ReadTimeout,
+            ):
                 print(f"{self.item_id}: retry thumbnail download {url}")
                 sleep((i + 1) ** i)
 
-        return False
+        return self.get_fallback()
 
     def get_fallback(self):
         """get fallback thumbnail if not available"""
@@ -270,7 +273,7 @@ class ValidatorCallback:
             urls = (
                 channel["_source"]["channel_thumb_url"],
                 channel["_source"]["channel_banner_url"],
-                channel["_source"]["channel_tvart_url"],
+                channel["_source"].get("channel_tvart_url", False),
             )
             handler = ThumbManager(channel["_source"]["channel_id"])
             handler.download_channel_art(urls, skip_existing=True)

tubearchivist/home/src/download/yt_dlp_base.py

@@ -48,11 +48,14 @@ class YtWrap:
         with yt_dlp.YoutubeDL(self.obs) as ydl:
             try:
                 ydl.download([url])
-            except yt_dlp.utils.DownloadError:
-                print(f"{url}: failed to download.")
-                return False
+            except yt_dlp.utils.DownloadError as err:
+                print(f"{url}: failed to download with message {err}")
+                if "Temporary failure in name resolution" in str(err):
+                    raise ConnectionError("lost the internet, abort!") from err
 
-        return True
+                return False, str(err)
+
+        return True, True
 
     def extract(self, url):
         """make extract request"""
@@ -61,8 +64,17 @@ class YtWrap:
         except cookiejar.LoadError:
             print("cookie file is invalid")
             return False
-        except (yt_dlp.utils.ExtractorError, yt_dlp.utils.DownloadError):
-            print(f"{url}: failed to get info from youtube")
+        except yt_dlp.utils.ExtractorError as err:
+            print(f"{url}: failed to extract with message: {err}, continue...")
+            return False
+        except yt_dlp.utils.DownloadError as err:
+            if "This channel does not have a" in str(err):
+                return False
+
+            print(f"{url}: failed to get info from youtube with message {err}")
+            if "Temporary failure in name resolution" in str(err):
+                raise ConnectionError("lost the internet, abort!") from err
+
             return False
 
         return response
@@ -100,9 +112,9 @@ class CookieHandler:
 
     def set_cookie(self, cookie):
         """set cookie str and activate in cofig"""
-        RedisArchivist().set_message("cookie", cookie)
+        RedisArchivist().set_message("cookie", cookie, save=True)
         path = ".downloads.cookie_import"
-        RedisArchivist().set_message("config", True, path=path)
+        RedisArchivist().set_message("config", True, path=path, save=True)
         self.config["downloads"]["cookie_import"] = True
         print("cookie: activated and stored in Redis")
 

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -20,7 +20,7 @@ from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo, index_new_video
 from home.src.index.video_constants import VideoTypeEnum
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import clean_string, ignore_filelist
+from home.src.ta.helper import ignore_filelist
 
 
 class DownloadPostProcess:
@@ -163,6 +163,7 @@ class VideoDownloader:
         while True:
             video_data = self._get_next(auto_only)
             if self.task.is_stopped() or not video_data:
+                self._reset_auto()
                 break
 
             youtube_id = video_data.get("youtube_id")
@@ -191,6 +192,8 @@ class VideoDownloader:
         self._add_subscribed_channels()
         DownloadPostProcess(self).run()
 
+        return self.videos
+
     def _notify(self, video_data, message):
         """send progress notification to task"""
         if not self.task:
@@ -203,12 +206,13 @@ class VideoDownloader:
     def _get_next(self, auto_only):
         """get next item in queue"""
         must_list = [{"term": {"status": {"value": "pending"}}}]
+        must_not_list = [{"exists": {"field": "message"}}]
         if auto_only:
             must_list.append({"term": {"auto_start": {"value": True}}})
 
         data = {
             "size": 1,
-            "query": {"bool": {"must": must_list}},
+            "query": {"bool": {"must": must_list, "must_not": must_not_list}},
             "sort": [
                 {"auto_start": {"order": "desc"}},
                 {"timestamp": {"order": "asc"}},
@@ -344,7 +348,9 @@ class VideoDownloader:
             if youtube_id in file_name:
                 obs["outtmpl"] = os.path.join(dl_cache, file_name)
 
-        success = YtWrap(obs, self.config).download(youtube_id)
+        success, message = YtWrap(obs, self.config).download(youtube_id)
+        if not success:
+            self._handle_error(youtube_id, message)
 
         if self.obs["writethumbnail"]:
             # webp files don't get cleaned up automatically
@@ -356,28 +362,27 @@ class VideoDownloader:
 
         return success
 
+    @staticmethod
+    def _handle_error(youtube_id, message):
+        """store error message"""
+        data = {"doc": {"message": message}}
+        _, _ = ElasticWrap(f"ta_download/_update/{youtube_id}").post(data=data)
+
     def move_to_archive(self, vid_dict):
         """move downloaded video from cache to archive"""
         videos = self.config["application"]["videos"]
         host_uid = self.config["application"]["HOST_UID"]
         host_gid = self.config["application"]["HOST_GID"]
-        channel_name = clean_string(vid_dict["channel"]["channel_name"])
-        if len(channel_name) <= 3:
-            # fall back to channel id
-            channel_name = vid_dict["channel"]["channel_id"]
-        # make archive folder with correct permissions
-        new_folder = os.path.join(videos, channel_name)
-        if not os.path.exists(new_folder):
-            os.makedirs(new_folder)
-        if host_uid and host_gid:
-            os.chown(new_folder, host_uid, host_gid)
-        # find real filename
+        # make folder
+        folder = os.path.join(videos, vid_dict["channel"]["channel_id"])
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+            if host_uid and host_gid:
+                os.chown(folder, host_uid, host_gid)
+        # move media file
+        media_file = vid_dict["youtube_id"] + ".mp4"
         cache_dir = self.config["application"]["cache_dir"]
-        all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))
-        for file_str in all_cached:
-            if vid_dict["youtube_id"] in file_str:
-                old_file = file_str
-        old_path = os.path.join(cache_dir, "download", old_file)
+        old_path = os.path.join(cache_dir, "download", media_file)
         new_path = os.path.join(videos, vid_dict["media_url"])
         # move media file and fix permission
         shutil.move(old_path, new_path, copy_function=shutil.copyfile)
@@ -401,3 +406,18 @@ class VideoDownloader:
             self.channels.add(channel_id)
 
         return
+
+    def _reset_auto(self):
+        """reset autostart to defaults after queue stop"""
+        path = "ta_download/_update_by_query"
+        data = {
+            "query": {"term": {"auto_start": {"value": True}}},
+            "script": {
+                "source": "ctx._source.auto_start = false",
+                "lang": "painless",
+            },
+        }
+        response, _ = ElasticWrap(path, config=self.config).post(data=data)
+        updated = response.get("updated")
+        if updated:
+            print(f"[download] reset auto start on {updated} videos.")

tubearchivist/home/src/es/connect.py

@@ -127,6 +127,12 @@ class IndexPaginate:
 
     def validate_data(self):
         """add pit and size to data"""
+        if not self.data:
+            self.data = {}
+
+        if "query" not in self.data.keys():
+            self.data.update({"query": {"match_all": {}}})
+
         if "sort" not in self.data.keys():
             self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
 

tubearchivist/home/src/es/index_mapping.json

@@ -37,7 +37,8 @@
                     "type": "text"
                 },
                 "channel_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "channel_tags": {
                     "type": "text",
@@ -90,7 +91,8 @@
                     "index": false
                 },
                 "date_downloaded": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "channel": {
                     "properties": {
@@ -129,7 +131,8 @@
                             "type": "text"
                         },
                         "channel_last_refresh": {
-                            "type": "date"
+                            "type": "date",
+                            "format": "epoch_second"
                         },
                         "channel_tags": {
                             "type": "text",
@@ -196,7 +199,8 @@
                     }
                 },
                 "vid_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "youtube_id": {
                     "type": "keyword"
@@ -289,7 +293,8 @@
                 "sponsorblock": {
                     "properties": {
                         "last_refresh": {
-                            "type": "date"
+                            "type": "date",
+                            "format": "epoch_second"
                         },
                         "has_unlocked": {
                             "type": "boolean"
@@ -341,7 +346,8 @@
             "index_name": "download",
             "expected_map": {
                 "timestamp": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "channel_id": {
                     "type": "keyword"
@@ -380,6 +386,9 @@
                 },
                 "auto_start": {
                     "type": "boolean"
+                },
+                "message": {
+                    "type": "text"
                 }
             },
             "expected_set": {
@@ -436,7 +445,8 @@
                     "type": "keyword"
                 },
                 "playlist_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 }
             },
             "expected_set": {
@@ -490,7 +500,8 @@
                     "type": "text"
                 },
                 "subtitle_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "subtitle_index": {
                     "type" : "long"
@@ -525,7 +536,8 @@
                     "type": "keyword"
                 },
                 "comment_last_refresh": {
-                    "type": "date"
+                    "type": "date",
+                    "format": "epoch_second"
                 },
                 "comment_channel_id": {
                     "type": "keyword"
@@ -539,7 +551,8 @@
                             "type" : "text"
                         },
                         "comment_timestamp": {
-                            "type": "date"
+                            "type": "date",
+                            "format": "epoch_second"
                         },
                         "comment_time_text": {
                             "type" : "text"

tubearchivist/home/src/frontend/api_calls.py

@@ -4,14 +4,8 @@ Functionality:
 - called via user input
 """
 
-from home.src.download.subscriptions import (
-    ChannelSubscription,
-    PlaylistSubscription,
-)
-from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.ta_redis import RedisArchivist
-from home.src.ta.urlparser import Parser
-from home.tasks import run_restore_backup, subscribe_to
+from home.tasks import run_restore_backup
 
 
 class PostData:
@@ -36,14 +30,11 @@ class PostData:
         exec_map = {
             "change_view": self._change_view,
             "change_grid": self._change_grid,
-            "unsubscribe": self._unsubscribe,
-            "subscribe": self._subscribe,
             "sort_order": self._sort_order,
             "hide_watched": self._hide_watched,
             "show_subed_only": self._show_subed_only,
             "show_ignored_only": self._show_ignored_only,
             "db-restore": self._db_restore,
-            "delete-playlist": self._delete_playlist,
         }
 
         return exec_map[self.to_exec]
@@ -67,34 +58,6 @@ class PostData:
         RedisArchivist().set_message(key, {"status": grid_items})
         return {"success": True}
 
-    def _unsubscribe(self):
-        """unsubscribe from channels or playlists"""
-        id_unsub = self.exec_val
-        print(f"{id_unsub}: unsubscribe")
-        to_unsub_list = Parser(id_unsub).parse()
-        for to_unsub in to_unsub_list:
-            unsub_type = to_unsub["type"]
-            unsub_id = to_unsub["url"]
-            if unsub_type == "playlist":
-                PlaylistSubscription().change_subscribe(
-                    unsub_id, subscribe_status=False
-                )
-            elif unsub_type == "channel":
-                ChannelSubscription().change_subscribe(
-                    unsub_id, channel_subscribed=False
-                )
-            else:
-                raise ValueError("failed to process " + id_unsub)
-
-        return {"success": True}
-
-    def _subscribe(self):
-        """subscribe to channel or playlist, called from js buttons"""
-        id_sub = self.exec_val
-        print(f"{id_sub}: subscribe")
-        subscribe_to.delay(id_sub)
-        return {"success": True}
-
     def _sort_order(self):
         """change the sort between published to downloaded"""
         sort_order = {"status": self.exec_val}
@@ -139,16 +102,3 @@ class PostData:
         filename = self.exec_val
         run_restore_backup.delay(filename)
         return {"success": True}
-
-    def _delete_playlist(self):
-        """delete playlist, only metadata or incl all videos"""
-        playlist_dict = self.exec_val
-        playlist_id = playlist_dict["playlist-id"]
-        playlist_action = playlist_dict["playlist-action"]
-        print(f"{playlist_id}: delete playlist {playlist_action}")
-        if playlist_action == "metadata":
-            YoutubePlaylist(playlist_id).delete_metadata()
-        elif playlist_action == "all":
-            YoutubePlaylist(playlist_id).delete_videos_playlist()
-
-        return {"success": True}

tubearchivist/home/src/frontend/forms.py

@@ -157,9 +157,41 @@ class ApplicationSettingsForm(forms.Form):
 class SchedulerSettingsForm(forms.Form):
     """handle scheduler settings"""
 
+    HELP_TEXT = "Add Apprise notification URLs, one per line"
+
     update_subscribed = forms.CharField(required=False)
+    update_subscribed_notify = forms.CharField(
+        label=False,
+        widget=forms.Textarea(
+            attrs={
+                "rows": 2,
+                "placeholder": HELP_TEXT,
+            }
+        ),
+        required=False,
+    )
     download_pending = forms.CharField(required=False)
+    download_pending_notify = forms.CharField(
+        label=False,
+        widget=forms.Textarea(
+            attrs={
+                "rows": 2,
+                "placeholder": HELP_TEXT,
+            }
+        ),
+        required=False,
+    )
     check_reindex = forms.CharField(required=False)
+    check_reindex_notify = forms.CharField(
+        label=False,
+        widget=forms.Textarea(
+            attrs={
+                "rows": 2,
+                "placeholder": HELP_TEXT,
+            }
+        ),
+        required=False,
+    )
     check_reindex_days = forms.IntegerField(required=False)
     thumbnail_check = forms.CharField(required=False)
     run_backup = forms.CharField(required=False)

tubearchivist/home/src/index/channel.py

@@ -14,7 +14,6 @@ from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.generic import YouTubeItem
 from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.helper import clean_string
 
 
 class YoutubeChannel(YouTubeItem):
@@ -93,8 +92,9 @@ class YoutubeChannel(YouTubeItem):
     def _get_tv_art(self):
         """extract tv artwork"""
         for i in self.youtube_meta["thumbnails"]:
-            if i.get("id") == "avatar_uncropped":
+            if i.get("id") == "banner_uncropped":
                 return i["url"]
+        for i in self.youtube_meta["thumbnails"]:
             if not i.get("width"):
                 continue
             if i["width"] // i["height"] < 2 and not i["width"] == i["height"]:
@@ -177,12 +177,10 @@ class YoutubeChannel(YouTubeItem):
 
     def get_folder_path(self):
         """get folder where media files get stored"""
-        channel_name = self.json_data["channel_name"]
-        folder_name = clean_string(channel_name)
-        if len(folder_name) <= 3:
-            # fall back to channel id
-            folder_name = self.json_data["channel_id"]
-        folder_path = os.path.join(self.app_conf["videos"], folder_name)
+        folder_path = os.path.join(
+            self.app_conf["videos"],
+            self.json_data["channel_id"],
+        )
         return folder_path
 
     def delete_es_videos(self):

tubearchivist/home/src/index/comments.py

@@ -63,6 +63,7 @@ class Comments:
             "check_formats": None,
             "skip_download": True,
             "getcomments": True,
+            "ignoreerrors": True,
             "extractor_args": {
                 "youtube": {
                     "max_comments": max_comments_list,
@@ -119,12 +120,14 @@ class Comments:
             "comment_text": comment["text"].replace("\xa0", ""),
             "comment_timestamp": comment["timestamp"],
             "comment_time_text": time_text,
-            "comment_likecount": comment["like_count"],
-            "comment_is_favorited": comment["is_favorited"],
+            "comment_likecount": comment.get("like_count", None),
+            "comment_is_favorited": comment.get("is_favorited", False),
             "comment_author": comment["author"],
             "comment_author_id": comment["author_id"],
             "comment_author_thumbnail": comment["author_thumbnail"],
-            "comment_author_is_uploader": comment["author_is_uploader"],
+            "comment_author_is_uploader": comment.get(
+                "comment_author_is_uploader", False
+            ),
             "comment_parent": comment["parent"],
         }
 

tubearchivist/home/src/index/filesystem.py

@@ -1,198 +1,85 @@
 """
 Functionality:
-- reindexing old documents
-- syncing updated values between indexes
 - scan the filesystem to delete or index
 """
 
-import json
 import os
 
-from home.src.download.queue import PendingList
-from home.src.es.connect import ElasticWrap
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.comments import CommentList
-from home.src.index.video import index_new_video
+from home.src.index.video import YoutubeVideo, index_new_video
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import clean_string, ignore_filelist
-from PIL import ImageFile
-
-ImageFile.LOAD_TRUNCATED_IMAGES = True
+from home.src.ta.helper import ignore_filelist
 
 
-class ScannerBase:
-    """scan the filesystem base class"""
+class Scanner:
+    """scan index and filesystem"""
 
-    CONFIG = AppConfig().config
-    VIDEOS = CONFIG["application"]["videos"]
+    VIDEOS: str = AppConfig().config["application"]["videos"]
 
-    def __init__(self):
-        self.to_index = False
-        self.to_delete = False
-        self.mismatch = False
-        self.to_rename = False
-
-    def scan(self):
-        """entry point, scan and compare"""
-        all_downloaded = self._get_all_downloaded()
-        all_indexed = self._get_all_indexed()
-        self.list_comarison(all_downloaded, all_indexed)
-
-    def _get_all_downloaded(self):
-        """get a list of all video files downloaded"""
-        channels = os.listdir(self.VIDEOS)
-        all_channels = ignore_filelist(channels)
-        all_channels.sort()
-        all_downloaded = []
-        for channel_name in all_channels:
-            channel_path = os.path.join(self.VIDEOS, channel_name)
-            channel_files = os.listdir(channel_path)
-            channel_files_clean = ignore_filelist(channel_files)
-            all_videos = [i for i in channel_files_clean if i.endswith(".mp4")]
-            for video in all_videos:
-                youtube_id = video[9:20]
-                all_downloaded.append((channel_name, video, youtube_id))
-
-        return all_downloaded
-
-    @staticmethod
-    def _get_all_indexed():
-        """get a list of all indexed videos"""
-        index_handler = PendingList()
-        index_handler.get_download()
-        index_handler.get_indexed()
-
-        all_indexed = []
-        for video in index_handler.all_videos:
-            youtube_id = video["youtube_id"]
-            media_url = video["media_url"]
-            published = video["published"]
-            title = video["title"]
-            all_indexed.append((youtube_id, media_url, published, title))
-        return all_indexed
-
-    def list_comarison(self, all_downloaded, all_indexed):
-        """compare the lists to figure out what to do"""
-        self._find_unindexed(all_downloaded, all_indexed)
-        self._find_missing(all_downloaded, all_indexed)
-        self._find_bad_media_url(all_downloaded, all_indexed)
-
-    def _find_unindexed(self, all_downloaded, all_indexed):
-        """find video files without a matching document indexed"""
-        all_indexed_ids = [i[0] for i in all_indexed]
-        self.to_index = []
-        for downloaded in all_downloaded:
-            if downloaded[2] not in all_indexed_ids:
-                self.to_index.append(downloaded)
-
-    def _find_missing(self, all_downloaded, all_indexed):
-        """find indexed videos without matching media file"""
-        all_downloaded_ids = [i[2] for i in all_downloaded]
-        self.to_delete = []
-        for video in all_indexed:
-            youtube_id = video[0]
-            if youtube_id not in all_downloaded_ids:
-                self.to_delete.append(video)
-
-    def _find_bad_media_url(self, all_downloaded, all_indexed):
-        """rename media files not matching the indexed title"""
-        self.mismatch = []
-        self.to_rename = []
-
-        for downloaded in all_downloaded:
-            channel, filename, downloaded_id = downloaded
-            # find in indexed
-            for indexed in all_indexed:
-                indexed_id, media_url, published, title = indexed
-                if indexed_id == downloaded_id:
-                    # found it
-                    pub = published.replace("-", "")
-                    expected = f"{pub}_{indexed_id}_{clean_string(title)}.mp4"
-                    new_url = os.path.join(channel, expected)
-                    if expected != filename:
-                        # file to rename
-                        self.to_rename.append((channel, filename, expected))
-                    if media_url != new_url:
-                        # media_url to update in es
-                        self.mismatch.append((indexed_id, new_url))
-
-                    break
-
-
-class Filesystem(ScannerBase):
-    """handle scanning and fixing from filesystem"""
-
-    def __init__(self, task=False):
-        super().__init__()
+    def __init__(self, task=False) -> None:
         self.task = task
+        self.to_delete: set[str] = set()
+        self.to_index: set[str] = set()
 
-    def process(self):
-        """entry point"""
+    def scan(self) -> None:
+        """scan the filesystem"""
+        downloaded: set[str] = self._get_downloaded()
+        indexed: set[str] = self._get_indexed()
+        self.to_index = downloaded - indexed
+        self.to_delete = indexed - downloaded
+
+    def _get_downloaded(self) -> set[str]:
+        """get downloaded ids"""
         if self.task:
-            self.task.send_progress(["Scanning your archive and index."])
-        self.scan()
-        self.rename_files()
-        self.send_mismatch_bulk()
-        self.delete_from_index()
-        self.add_missing()
+            self.task.send_progress(["Scan your filesystem for videos."])
 
-    def rename_files(self):
-        """rename media files as identified by find_bad_media_url"""
-        if not self.to_rename:
-            return
+        downloaded: set = set()
+        channels = ignore_filelist(os.listdir(self.VIDEOS))
+        for channel in channels:
+            folder = os.path.join(self.VIDEOS, channel)
+            files = ignore_filelist(os.listdir(folder))
+            downloaded.update({i.split(".")[0] for i in files})
 
-        total = len(self.to_rename)
+        return downloaded
+
+    def _get_indexed(self) -> set:
+        """get all indexed ids"""
         if self.task:
-            self.task.send_progress([f"Rename {total} media files."])
-        for bad_filename in self.to_rename:
-            channel, filename, expected_filename = bad_filename
-            print(f"renaming [{filename}] to [{expected_filename}]")
-            old_path = os.path.join(self.VIDEOS, channel, filename)
-            new_path = os.path.join(self.VIDEOS, channel, expected_filename)
-            os.rename(old_path, new_path)
+            self.task.send_progress(["Get all videos indexed."])
 
-    def send_mismatch_bulk(self):
-        """build bulk update"""
-        if not self.mismatch:
-            return
+        data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
+        response = IndexPaginate("ta_video", data).get_results()
+        return {i["youtube_id"] for i in response}
 
-        total = len(self.mismatch)
-        if self.task:
-            self.task.send_progress([f"Fix media urls for {total} files"])
-        bulk_list = []
-        for video_mismatch in self.mismatch:
-            youtube_id, media_url = video_mismatch
-            print(f"{youtube_id}: fixing media url {media_url}")
-            action = {"update": {"_id": youtube_id, "_index": "ta_video"}}
-            source = {"doc": {"media_url": media_url}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(source))
-        # add last newline
-        bulk_list.append("\n")
-        data = "\n".join(bulk_list)
-        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
+    def apply(self) -> None:
+        """apply all changes"""
+        self.delete()
+        self.index()
+        self.url_fix()
 
-    def delete_from_index(self):
-        """find indexed but deleted mediafile"""
+    def delete(self) -> None:
+        """delete videos from index"""
         if not self.to_delete:
+            print("nothing to delete")
             return
 
-        total = len(self.to_delete)
         if self.task:
-            self.task.send_progress([f"Clean up {total} items from index."])
-        for indexed in self.to_delete:
-            youtube_id = indexed[0]
-            print(f"deleting {youtube_id} from index")
-            path = f"ta_video/_doc/{youtube_id}"
-            _, _ = ElasticWrap(path).delete()
+            self.task.send_progress(
+                [f"Remove {len(self.to_delete)} videos from index."]
+            )
 
-    def add_missing(self):
-        """add missing videos to index"""
-        video_ids = [i[2] for i in self.to_index]
-        if not video_ids:
+        for youtube_id in self.to_delete:
+            YoutubeVideo(youtube_id).delete_media_file()
+
+    def index(self) -> None:
+        """index new"""
+        if not self.to_index:
+            print("nothing to index")
             return
 
-        total = len(video_ids)
-        for idx, youtube_id in enumerate(video_ids):
+        total = len(self.to_index)
+        for idx, youtube_id in enumerate(self.to_index):
             if self.task:
                 self.task.send_progress(
                     message_lines=[
@@ -202,4 +89,36 @@ class Filesystem(ScannerBase):
                 )
             index_new_video(youtube_id)
 
-        CommentList(video_ids, task=self.task).index()
+        CommentList(self.to_index, task=self.task).index()
+
+    def url_fix(self) -> None:
+        """
+        update path v0.3.6 to v0.3.7
+        fix url not matching channel-videoid pattern
+        """
+        bool_must = (
+            "doc['media_url'].value == "
+            + "(doc['channel.channel_id'].value + '/' + "
+            + "doc['youtube_id'].value) + '.mp4'"
+        )
+        to_update = (
+            "ctx._source['media_url'] = "
+            + "ctx._source.channel['channel_id'] + '/' + "
+            + "ctx._source['youtube_id'] + '.mp4'"
+        )
+        data = {
+            "query": {
+                "bool": {
+                    "must_not": [{"script": {"script": {"source": bool_must}}}]
+                }
+            },
+            "script": {"source": to_update},
+        }
+        response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data)
+        updated = response.get("updates")
+        if updated:
+            print(f"updated {updated} bad media_url")
+            if self.task:
+                self.task.send_progress(
+                    [f"Updated {updated} wrong media urls."]
+                )

tubearchivist/home/src/index/manual.py

@@ -31,6 +31,9 @@ class ImportFolderScanner:
     CACHE_DIR = CONFIG["application"]["cache_dir"]
     IMPORT_DIR = os.path.join(CACHE_DIR, "import")
 
+    """All extensions should be in lowercase until better handling is in place.
+    Described in Issue #502.
+    """
     EXT_MAP = {
         "media": [".mp4", ".mkv", ".webm"],
         "metadata": [".json"],
@@ -118,7 +121,7 @@ class ImportFolderScanner:
         """detect metadata type for file"""
 
         for key, value in self.EXT_MAP.items():
-            if ext in value:
+            if ext.lower() in value:
                 return key, file_path
 
         return False, False

tubearchivist/home/src/index/reindex.py

@@ -6,7 +6,6 @@ functionality:
 
 import json
 import os
-import shutil
 from datetime import datetime
 from time import sleep
 
@@ -14,7 +13,6 @@ from home.src.download.queue import PendingList
 from home.src.download.subscriptions import ChannelSubscription
 from home.src.download.thumbnails import ThumbManager
 from home.src.download.yt_dlp_base import CookieHandler
-from home.src.download.yt_dlp_handler import VideoDownloader
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
 from home.src.index.comments import Comments
@@ -54,6 +52,7 @@ class ReindexBase:
     def __init__(self):
         self.config = AppConfig().config
         self.now = int(datetime.now().timestamp())
+        self.total = None
 
     def populate(self, all_ids, reindex_config):
         """add all to reindex ids to redis queue"""
@@ -61,6 +60,7 @@ class ReindexBase:
             return
 
         RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
+        self.total = None
 
 
 class ReindexPopulate(ReindexBase):
@@ -227,6 +227,11 @@ class Reindex(ReindexBase):
         super().__init__()
         self.task = task
         self.all_indexed_ids = False
+        self.processed = {
+            "videos": 0,
+            "channels": 0,
+            "playlists": 0,
+        }
 
     def reindex_all(self):
         """reindex all in queue"""
@@ -238,18 +243,19 @@ class Reindex(ReindexBase):
             if not RedisQueue(index_config["queue_name"]).has_item():
                 continue
 
-            total = RedisQueue(index_config["queue_name"]).length()
+            self.total = RedisQueue(index_config["queue_name"]).length()
             while True:
-                has_next = self.reindex_index(name, index_config, total)
+                has_next = self.reindex_index(name, index_config)
                 if not has_next:
                     break
 
-    def reindex_index(self, name, index_config, total):
+    def reindex_index(self, name, index_config):
         """reindex all of a single index"""
         reindex = self.get_reindex_map(index_config["index_name"])
         youtube_id = RedisQueue(index_config["queue_name"]).get_next()
         if youtube_id:
-            self._notify(name, index_config, total)
+            if self.task:
+                self._notify(name, index_config)
             reindex(youtube_id)
             sleep_interval = self.config["downloads"].get("sleep_interval", 0)
             sleep(sleep_interval)
@@ -266,23 +272,18 @@ class Reindex(ReindexBase):
 
         return def_map.get(index_name)
 
-    def _notify(self, name, index_config, total):
+    def _notify(self, name, index_config):
         """send notification back to task"""
+        if self.total is None:
+            self.total = RedisQueue(index_config["queue_name"]).length()
+
         remaining = RedisQueue(index_config["queue_name"]).length()
-        idx = total - remaining
-        message = [f"Reindexing {name.title()}s {idx}/{total}"]
-        progress = idx / total
+        idx = self.total - remaining
+        message = [f"Reindexing {name.title()}s {idx}/{self.total}"]
+        progress = idx / self.total
         self.task.send_progress(message, progress=progress)
 
     def _reindex_single_video(self, youtube_id):
-        """wrapper to handle channel name changes"""
-        try:
-            self._reindex_single_video_call(youtube_id)
-        except FileNotFoundError:
-            ChannelUrlFixer(youtube_id, self.config).run()
-            self._reindex_single_video_call(youtube_id)
-
-    def _reindex_single_video_call(self, youtube_id):
         """refresh data for single video"""
         video = YoutubeVideo(youtube_id)
 
@@ -291,7 +292,10 @@ class Reindex(ReindexBase):
         es_meta = video.json_data.copy()
 
         # get new
-        video.build_json()
+        media_url = os.path.join(
+            self.config["application"]["videos"], es_meta["media_url"]
+        )
+        video.build_json(media_path=media_url)
         if not video.youtube_meta:
             video.deactivate()
             return
@@ -317,6 +321,7 @@ class Reindex(ReindexBase):
         thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
 
         Comments(youtube_id, config=self.config).reindex_comments()
+        self.processed["videos"] += 1
 
         return
 
@@ -328,8 +333,7 @@ class Reindex(ReindexBase):
         new_path = os.path.join(videos, media_url_should)
         os.rename(old_path, new_path)
 
-    @staticmethod
-    def _reindex_single_channel(channel_id):
+    def _reindex_single_channel(self, channel_id):
         """refresh channel data and sync to videos"""
         # read current state
         channel = YoutubeChannel(channel_id)
@@ -355,6 +359,7 @@ class Reindex(ReindexBase):
 
         channel.upload_to_es()
         ChannelFullScan(channel_id).scan()
+        self.processed["channels"] += 1
 
     def _reindex_single_playlist(self, playlist_id):
         """refresh playlist data"""
@@ -370,6 +375,7 @@ class Reindex(ReindexBase):
 
         playlist.json_data["playlist_subscribed"] = subscribed
         playlist.upload_to_es()
+        self.processed["playlists"] += 1
         return
 
     def _get_all_videos(self):
@@ -391,6 +397,18 @@ class Reindex(ReindexBase):
         valid = CookieHandler(self.config).validate()
         return valid
 
+    def build_message(self):
+        """build progress message"""
+        message = ""
+        for key, value in self.processed.items():
+            if value:
+                message = message + f"{value} {key}, "
+
+        if message:
+            message = f"reindexed {message.rstrip(', ')}"
+
+        return message
+
 
 class ReindexProgress(ReindexBase):
     """
@@ -466,65 +484,6 @@ class ReindexProgress(ReindexBase):
         return state_dict
 
 
-class ChannelUrlFixer:
-    """fix not matching channel names in reindex"""
-
-    def __init__(self, youtube_id, config):
-        self.youtube_id = youtube_id
-        self.config = config
-        self.video = False
-
-    def run(self):
-        """check and run if needed"""
-        print(f"{self.youtube_id}: failed to build channel path, try to fix.")
-        video_path_is, video_folder_is = self.get_as_is()
-        if not os.path.exists(video_path_is):
-            print(f"giving up reindex, video in video: {self.video.json_data}")
-            raise ValueError
-
-        _, video_folder_should = self.get_as_should()
-
-        if video_folder_is != video_folder_should:
-            self.process(video_path_is)
-        else:
-            print(f"{self.youtube_id}: skip channel url fixer")
-
-    def get_as_is(self):
-        """get video object as is"""
-        self.video = YoutubeVideo(self.youtube_id)
-        self.video.get_from_es()
-        video_path_is = os.path.join(
-            self.config["application"]["videos"],
-            self.video.json_data["media_url"],
-        )
-        video_folder_is = os.path.split(video_path_is)[0]
-
-        return video_path_is, video_folder_is
-
-    def get_as_should(self):
-        """add fresh metadata from remote"""
-        self.video.get_from_youtube()
-        self.video.add_file_path()
-
-        video_path_should = os.path.join(
-            self.config["application"]["videos"],
-            self.video.json_data["media_url"],
-        )
-        video_folder_should = os.path.split(video_path_should)[0]
-        return video_path_should, video_folder_should
-
-    def process(self, video_path_is):
-        """fix filepath"""
-        print(f"{self.youtube_id}: fixing channel rename.")
-        cache_dir = self.config["application"]["cache_dir"]
-        new_path = os.path.join(
-            cache_dir, "download", self.youtube_id + ".mp4"
-        )
-        shutil.move(video_path_is, new_path, copy_function=shutil.copyfile)
-        VideoDownloader().move_to_archive(self.video.json_data)
-        self.video.update_media_url()
-
-
 class ChannelFullScan:
     """
     update from v0.3.0 to v0.3.1

tubearchivist/home/src/index/subtitle.py

@@ -62,7 +62,12 @@ class YoutubeSubtitle:
         if not all_formats:
             return False
 
-        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
+        subtitle_json3 = [i for i in all_formats if i["ext"] == "json3"]
+        if not subtitle_json3:
+            print(f"{self.video.youtube_id}-{lang}: json3 not processed")
+            return False
+
+        subtitle = subtitle_json3[0]
         subtitle.update(
             {"lang": lang, "source": "auto", "media_url": media_url}
         )

tubearchivist/home/src/index/video.py

@@ -20,7 +20,7 @@ from home.src.index.video_streams import (
     DurationConverter,
     MediaStreamExtractor,
 )
-from home.src.ta.helper import clean_string, randomizor
+from home.src.ta.helper import randomizor
 from home.src.ta.ta_redis import RedisArchivist
 from ryd_client import ryd_client
 
@@ -231,18 +231,24 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
     def build_dl_cache_path(self):
         """find video path in dl cache"""
         cache_dir = self.app_conf["cache_dir"]
-        cache_path = f"{cache_dir}/download/"
-        all_cached = os.listdir(cache_path)
-        for file_cached in all_cached:
-            if self.youtube_id in file_cached:
-                vid_path = os.path.join(cache_path, file_cached)
-                return vid_path
+        video_id = self.json_data["youtube_id"]
+        cache_path = f"{cache_dir}/download/{video_id}.mp4"
+        if os.path.exists(cache_path):
+            return cache_path
+
+        channel_path = os.path.join(
+            self.app_conf["videos"],
+            self.json_data["channel"]["channel_id"],
+            f"{video_id}.mp4",
+        )
+        if os.path.exists(channel_path):
+            return channel_path
 
         raise FileNotFoundError
 
     def add_player(self, media_path=False):
         """add player information for new videos"""
-        vid_path = self._get_vid_path(media_path)
+        vid_path = media_path or self.build_dl_cache_path()
 
         duration_handler = DurationConverter()
         duration = duration_handler.get_sec(vid_path)
@@ -259,7 +265,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
     def add_streams(self, media_path=False):
         """add stream metadata"""
-        vid_path = self._get_vid_path(media_path)
+        vid_path = media_path or self.build_dl_cache_path()
         media = MediaStreamExtractor(vid_path)
         self.json_data.update(
             {
@@ -268,43 +274,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             }
         )
 
-    def _get_vid_path(self, media_path=False):
-        """get path of media file"""
-        if media_path:
-            return media_path
-
-        try:
-            # when indexing from download task
-            vid_path = self.build_dl_cache_path()
-        except FileNotFoundError as err:
-            # when reindexing needs to handle title rename
-            channel = os.path.split(self.json_data["media_url"])[0]
-            channel_dir = os.path.join(self.app_conf["videos"], channel)
-            all_files = os.listdir(channel_dir)
-            for file in all_files:
-                if self.youtube_id in file and file.endswith(".mp4"):
-                    vid_path = os.path.join(channel_dir, file)
-                    break
-            else:
-                raise FileNotFoundError("could not find video file") from err
-
-        return vid_path
-
     def add_file_path(self):
         """build media_url for where file will be located"""
-        channel_name = self.json_data["channel"]["channel_name"]
-        clean_channel_name = clean_string(channel_name)
-        if len(clean_channel_name) <= 3:
-            # fall back to channel id
-            clean_channel_name = self.json_data["channel"]["channel_id"]
-
-        timestamp = self.json_data["published"].replace("-", "")
-        youtube_id = self.json_data["youtube_id"]
-        title = self.json_data["title"]
-        clean_title = clean_string(title)
-        filename = f"{timestamp}_{youtube_id}_{clean_title}.mp4"
-        media_url = os.path.join(clean_channel_name, filename)
-        self.json_data["media_url"] = media_url
+        self.json_data["media_url"] = os.path.join(
+            self.json_data["channel"]["channel_id"],
+            self.json_data["youtube_id"] + ".mp4",
+        )
 
     def delete_media_file(self):
         """delete video file, meta data"""

tubearchivist/home/src/index/video_streams.py

@@ -35,24 +35,27 @@ class DurationConverter:
         return duration_sec
 
     @staticmethod
-    def get_str(duration_sec):
+    def get_str(seconds):
         """takes duration in sec and returns clean string"""
-        if not duration_sec:
+        if not seconds:
             # failed to extract
             return "NA"
 
-        hours = int(duration_sec // 3600)
-        minutes = int((duration_sec - (hours * 3600)) // 60)
-        secs = int(duration_sec - (hours * 3600) - (minutes * 60))
+        days = int(seconds // (24 * 3600))
+        hours = int((seconds % (24 * 3600)) // 3600)
+        minutes = int((seconds % 3600) // 60)
+        seconds = int(seconds % 60)
 
         duration_str = str()
+        if days:
+            duration_str = f"{days}d "
         if hours:
-            duration_str = str(hours).zfill(2) + ":"
+            duration_str = duration_str + str(hours).zfill(2) + ":"
         if minutes:
             duration_str = duration_str + str(minutes).zfill(2) + ":"
         else:
             duration_str = duration_str + "00:"
-        duration_str = duration_str + str(secs).zfill(2)
+        duration_str = duration_str + str(seconds).zfill(2)
         return duration_str
 
 
@@ -122,8 +125,8 @@ class MediaStreamExtractor:
             {
                 "type": "audio",
                 "index": stream["index"],
-                "codec": stream["codec_name"],
-                "bitrate": int(stream["bit_rate"]),
+                "codec": stream.get("codec_name", "undefined"),
+                "bitrate": int(stream.get("bit_rate", 0)),
             }
         )
 

tubearchivist/home/src/ta/config.py

@@ -8,6 +8,7 @@ import json
 import os
 import re
 from random import randint
+from time import sleep
 
 import requests
 from celery.schedules import crontab
@@ -67,11 +68,19 @@ class AppConfig:
     @staticmethod
     def get_config_redis():
         """read config json set from redis to overwrite defaults"""
-        config = RedisArchivist().get_message("config")
-        if not list(config.values())[0]:
-            return False
+        for i in range(10):
+            try:
+                config = RedisArchivist().get_message("config")
+                if not list(config.values())[0]:
+                    return False
 
-        return config
+                return config
+
+            except Exception:  # pylint: disable=broad-except
+                print(f"... Redis connection failed, retry [{i}/10]")
+                sleep(3)
+
+        raise ConnectionError("failed to connect to redis")
 
     def update_config(self, form_post):
         """update config values from settings form"""
@@ -91,7 +100,7 @@ class AppConfig:
             self.config[config_dict][config_value] = to_write
             updated.append((config_value, to_write))
 
-        RedisArchivist().set_message("config", self.config)
+        RedisArchivist().set_message("config", self.config, save=True)
         return updated
 
     @staticmethod
@@ -103,7 +112,7 @@ class AppConfig:
 
             message = {"status": value}
             redis_key = f"{user_id}:{key}"
-            RedisArchivist().set_message(redis_key, message)
+            RedisArchivist().set_message(redis_key, message, save=True)
 
     def get_colors(self):
         """overwrite config if user has set custom values"""
@@ -175,6 +184,11 @@ class ScheduleBuilder:
         "version_check": "0 11 *",
     }
     CONFIG = ["check_reindex_days", "run_backup_rotate"]
+    NOTIFY = [
+        "update_subscribed_notify",
+        "download_pending_notify",
+        "check_reindex_notify",
+    ]
     MSG = "message:setting"
 
     def __init__(self):
@@ -191,10 +205,11 @@ class ScheduleBuilder:
                 except ValueError:
                     print(f"failed: {key} {value}")
                     mess_dict = {
-                        "status": self.MSG,
+                        "group": "setting:schedule",
                         "level": "error",
                         "title": "Scheduler update failed.",
-                        "message": "Invalid schedule input",
+                        "messages": ["Invalid schedule input"],
+                        "id": "0000",
                     }
                     RedisArchivist().set_message(
                         self.MSG, mess_dict, expire=True
@@ -204,12 +219,20 @@ class ScheduleBuilder:
                 redis_config["scheduler"][key] = to_write
             if key in self.CONFIG and value:
                 redis_config["scheduler"][key] = int(value)
-        RedisArchivist().set_message("config", redis_config)
+            if key in self.NOTIFY and value:
+                if value == "0":
+                    to_write = False
+                else:
+                    to_write = value
+                redis_config["scheduler"][key] = to_write
+
+        RedisArchivist().set_message("config", redis_config, save=True)
         mess_dict = {
-            "status": self.MSG,
+            "group": "setting:schedule",
             "level": "info",
             "title": "Scheduler changed.",
-            "message": "Please restart container for changes to take effect",
+            "messages": ["Restart container for changes to take effect"],
+            "id": "0000",
         }
         RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
 
@@ -317,8 +340,13 @@ class ReleaseVersion:
             RedisArchivist().set_message(self.NEW_KEY, message)
             print(f"[{self.local_version}]: found new version {new_version}")
 
+    def get_local_version(self):
+        """read version from local"""
+        return self.local_version
+
     def get_remote_version(self):
         """read version from remote"""
+        sleep(randint(0, 60))
         self.response = requests.get(self.REMOTE_URL, timeout=20).json()
         remote_version_str = self.response["release_version"]
         self.remote_version = self._parse_version(remote_version_str)

tubearchivist/home/src/ta/helper.py

@@ -6,28 +6,21 @@ Loose collection of helper functions
 import json
 import os
 import random
-import re
 import string
-import unicodedata
 from datetime import datetime
 from urllib.parse import urlparse
 
 import requests
 
 
-def clean_string(file_name: str) -> str:
-    """clean string to only asci characters"""
-    whitelist = "-_.() " + string.ascii_letters + string.digits
-    normalized = unicodedata.normalize("NFKD", file_name)
-    ascii_only = normalized.encode("ASCII", "ignore").decode().strip()
-    white_listed: str = "".join(c for c in ascii_only if c in whitelist)
-    cleaned: str = re.sub(r"[ ]{2,}", " ", white_listed)
-    return cleaned
-
-
 def ignore_filelist(filelist: list[str]) -> list[str]:
     """ignore temp files for os.listdir sanitizer"""
-    to_ignore = ["Icon\r\r", "Temporary Items", "Network Trash Folder"]
+    to_ignore = [
+        "@eaDir",
+        "Icon\r\r",
+        "Network Trash Folder",
+        "Temporary Items",
+    ]
     cleaned: list[str] = []
     for file_name in filelist:
         if file_name.startswith(".") or file_name in to_ignore:
@@ -103,7 +96,7 @@ def date_praser(timestamp: int | str) -> str:
     elif isinstance(timestamp, str):
         date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
 
-    return datetime.strftime(date_obj, "%d %b, %Y")
+    return date_obj.date().isoformat()
 
 
 def time_parser(timestamp: str) -> float:
@@ -122,7 +115,7 @@ def clear_dl_cache(config: dict) -> int:
     """clear leftover files from dl cache"""
     print("clear download cache")
     cache_dir = os.path.join(config["application"]["cache_dir"], "download")
-    leftover_files = os.listdir(cache_dir)
+    leftover_files = ignore_filelist(os.listdir(cache_dir))
     for cached in leftover_files:
         to_delete = os.path.join(cache_dir, cached)
         os.remove(to_delete)
@@ -150,8 +143,14 @@ def is_shorts(youtube_id: str) -> bool:
 
 def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
     """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
-    allowed_hosts: list[str] = []
-    csrf_trusted_origins: list[str] = []
+    allowed_hosts: list[str] = [
+        "localhost",
+        "tubearchivist",
+    ]
+    csrf_trusted_origins: list[str] = [
+        "http://localhost",
+        "http://tubearchivist",
+    ]
     for host in ta_host.split():
         host_clean = host.strip()
         if not host_clean.startswith("http"):

tubearchivist/home/src/ta/notify.py

@@ -0,0 +1,55 @@
+"""send notifications using apprise"""
+
+import apprise
+from home.src.ta.config import AppConfig
+from home.src.ta.task_manager import TaskManager
+
+
+class Notifications:
+    """notification handler"""
+
+    def __init__(self, name: str, task_id: str, task_title: str):
+        self.name: str = name
+        self.task_id: str = task_id
+        self.task_title: str = task_title
+
+    def send(self) -> None:
+        """send notifications"""
+        apobj = apprise.Apprise()
+        hooks: str | None = self.get_url()
+        if not hooks:
+            return
+
+        hook_list: list[str] = self.parse_hooks(hooks=hooks)
+        title, body = self.build_message()
+
+        if not body:
+            return
+
+        for hook in hook_list:
+            apobj.add(hook)
+
+        apobj.notify(body=body, title=title)
+
+    def get_url(self) -> str | None:
+        """get apprise urls for task"""
+        config = AppConfig().config
+        hooks: str = config["scheduler"].get(f"{self.name}_notify")
+
+        return hooks
+
+    def parse_hooks(self, hooks: str) -> list[str]:
+        """create list of hooks"""
+
+        hook_list: list[str] = [i.strip() for i in hooks.split()]
+
+        return hook_list
+
+    def build_message(self) -> tuple[str, str | None]:
+        """build message to send notification"""
+        task = TaskManager().get_task(self.task_id)
+        status = task.get("status")
+        title: str = f"[TA] {self.task_title} process ended with {status}"
+        body: str | None = task.get("result")
+
+        return title, body

tubearchivist/home/src/ta/ta_redis.py

@@ -41,6 +41,7 @@ class RedisArchivist(RedisBase):
         message: dict,
         path: str = ".",
         expire: bool | int = False,
+        save: bool = False,
     ) -> None:
         """write new message to redis"""
         self.conn.execute_command(
@@ -54,6 +55,16 @@ class RedisArchivist(RedisBase):
                 secs = expire
             self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
 
+        if save:
+            self.bg_save()
+
+    def bg_save(self) -> None:
+        """save to aof"""
+        try:
+            self.conn.bgsave()
+        except redis.exceptions.ResponseError:
+            pass
+
     def get_message(self, key: str) -> dict:
         """get message dict from redis"""
         reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)

tubearchivist/home/tasks.py

@@ -19,10 +19,11 @@ from home.src.download.yt_dlp_handler import VideoDownloader
 from home.src.es.backup import ElasticBackup
 from home.src.es.index_setup import ElasitIndexWrap
 from home.src.index.channel import YoutubeChannel
-from home.src.index.filesystem import Filesystem
+from home.src.index.filesystem import Scanner
 from home.src.index.manual import ImportFolderScanner
 from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
 from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
+from home.src.ta.notify import Notifications
 from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.task_manager import TaskManager
 from home.src.ta.urlparser import Parser
@@ -113,7 +114,7 @@ class BaseTask(Task):
         """callback for task failure"""
         print(f"{task_id} Failed callback")
         message, key = self._build_message(level="error")
-        message.update({"messages": ["Task failed"]})
+        message.update({"messages": [f"Task failed: {exc}"]})
         RedisArchivist().set_message(key, message, expire=20)
 
     def on_success(self, retval, task_id, args, kwargs):
@@ -130,6 +131,12 @@ class BaseTask(Task):
         message.update({"messages": ["New task received."]})
         RedisArchivist().set_message(key, message)
 
+    def after_return(self, status, retval, task_id, args, kwargs, einfo):
+        """callback after task returns"""
+        print(f"{task_id} return callback")
+        task_title = self.TASK_CONFIG.get(self.name).get("title")
+        Notifications(self.name, task_id, task_title).send()
+
     def send_progress(self, message_lines, progress=False, title=False):
         """send progress message"""
         message, key = self._build_message()
@@ -169,7 +176,7 @@ def update_subscribed(self):
     if manager.is_pending(self):
         print(f"[task][{self.name}] rescan already running")
         self.send_progress("Rescan already in progress.")
-        return
+        return None
 
     manager.init(self)
     handler = SubscriptionScanner(task=self)
@@ -178,6 +185,10 @@ def update_subscribed(self):
     if missing_videos:
         print(missing_videos)
         extrac_dl.delay(missing_videos, auto_start=auto_start)
+        message = f"Found {len(missing_videos)} videos to add to the queue."
+        return message
+
+    return None
 
 
 @shared_task(name="download_pending", bind=True, base=BaseTask)
@@ -187,10 +198,16 @@ def download_pending(self, auto_only=False):
     if manager.is_pending(self):
         print(f"[task][{self.name}] download queue already running")
         self.send_progress("Download Queue is already running.")
-        return
+        return None
 
     manager.init(self)
-    VideoDownloader(task=self).run_queue(auto_only=auto_only)
+    downloader = VideoDownloader(task=self)
+    videos_downloaded = downloader.run_queue(auto_only=auto_only)
+
+    if videos_downloaded:
+        return f"downloaded {len(videos_downloaded)} videos."
+
+    return None
 
 
 @shared_task(name="extract_download", bind=True, base=BaseTask)
@@ -235,7 +252,10 @@ def check_reindex(self, data=False, extract_videos=False):
         self.send_progress("Add outdated documents to the reindex Queue.")
         populate.add_outdated()
 
-    Reindex(task=self).reindex_all()
+    handler = Reindex(task=self)
+    handler.reindex_all()
+
+    return handler.build_message()
 
 
 @shared_task(bind=True, name="manual_import", base=BaseTask)
@@ -290,7 +310,9 @@ def rescan_filesystem(self):
         return
 
     manager.init(self)
-    Filesystem(task=self).process()
+    handler = Scanner(task=self)
+    handler.scan()
+    handler.apply()
     ThumbValidator(task=self).validate()
 
 
@@ -321,9 +343,12 @@ def re_sync_thumbs(self):
 
 
 @shared_task(bind=True, name="subscribe_to", base=BaseTask)
-def subscribe_to(self, url_str):
-    """take a list of urls to subscribe to"""
-    SubscriptionHandler(url_str, task=self).subscribe()
+def subscribe_to(self, url_str: str, expected_type: str | bool = False):
+    """
+    take a list of urls to subscribe to
+    optionally validate expected_type channel / playlist
+    """
+    SubscriptionHandler(url_str, task=self).subscribe(expected_type)
 
 
 @shared_task(bind=True, name="index_playlists", base=BaseTask)

tubearchivist/home/templates/home/base_settings.html

@@ -0,0 +1,17 @@
+{# Base file for all of the settings pages to ensure a common menu #}
+{% extends "home/base.html" %}
+{% load static %}
+{% block content %}
+<div class="boxed-content">
+    <div class="info-box-item child-page-nav">
+        <a href="{% url 'settings' %}"><h3>Dashboard</h3></a>
+        <a href="{% url 'settings_user' %}"><h3>User</h3></a>
+        <a href="{% url 'settings_application' %}"><h3>Application</h3></a>
+        <a href="{% url 'settings_scheduling' %}"><h3>Scheduling</h3></a>
+        <a href="{% url 'settings_actions' %}"><h3>Actions</h3></a>
+    </div>
+    <div id="notifications" data=""></div>
+    {% block settings_content %}{% endblock %}
+</div>
+<script type="text/javascript" src="{% static 'progress.js' %}"></script>
+{% endblock content %}

tubearchivist/home/templates/home/channel.html

@@ -66,9 +66,9 @@
                             <div>
                                 <p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
                                 {% if channel.source.channel_subscribed %}
-                                    <button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
+                                    <button class="unsubscribe" type="button" data-type="channel" data-subscribe="" data-id="{{ channel.source.channel_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
                                 {% else %}
-                                    <button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
+                                    <button type="button" data-type="channel" data-subscribe="true" data-id="{{ channel.source.channel_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
                                 {% endif %}
                             </div>
                         </div>

tubearchivist/home/templates/home/channel_id.html

@@ -6,7 +6,7 @@
     <div class="channel-banner">
         <a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
     </div>
-    <div class="info-box-item channel-nav">
+    <div class="info-box-item child-page-nav">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
         {% if has_streams %}
             <a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
@@ -38,9 +38,9 @@
                     <p>Subscribers: {{ channel_info.channel_subs|intcomma }}</p>
                 {% endif %}
                 {% if channel_info.channel_subscribed %}
-                    <button class="unsubscribe" type="button" id="{{ channel_info.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel_info.channel_name }}">Unsubscribe</button>
+                    <button class="unsubscribe" type="button" data-type="channel" data-subscribe="" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ channel_info.channel_name }}">Unsubscribe</button>
                 {% else %}
-                    <button type="button" id="{{ channel_info.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel_info.channel_name }}">Subscribe</button>
+                    <button type="button" data-type="channel" data-subscribe="true" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ channel_info.channel_name }}">Subscribe</button>
                 {% endif %}
             </div>
         </div>

tubearchivist/home/templates/home/channel_id_about.html

@@ -6,7 +6,7 @@
     <div class="channel-banner">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
     </div>
-    <div class="info-box-item channel-nav">
+    <div class="info-box-item child-page-nav">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
         {% if has_streams %}
             <a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>

tubearchivist/home/templates/home/channel_id_playlist.html

@@ -6,7 +6,7 @@
     <div class="channel-banner">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
     </div>
-    <div class="info-box-item channel-nav">
+    <div class="info-box-item child-page-nav">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
         {% if has_streams %}
             <a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
@@ -54,9 +54,9 @@
                     <a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
                     <p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
                     {% if playlist.source.playlist_subscribed %}
-                        <button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
+                        <button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist.source.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
                     {% else %}
-                        <button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
+                        <button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist.source.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
                     {% endif %}
                 </div>
             </div>

tubearchivist/home/templates/home/downloads.html

@@ -97,6 +97,9 @@
                             <a href="https://www.youtube.com/watch?v={{ video.source.youtube_id }}" target="_blank"><h3>{{ video.source.title }}</h3></a>
                         </div>
                         <p>Published: {{ video.source.published }} | Duration: {{ video.source.duration }} | {{ video.source.youtube_id }}</p>
+                        {% if video.source.message %}
+                            <p class="danger-zone">{{ video.source.message }}</p>
+                        {% endif %}
                         <div>
                             {% if show_ignored_only %}
                                 <button data-id="{{ video.source.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>
@@ -105,6 +108,9 @@
                                 <button data-id="{{ video.source.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
                                 <button id="{{ video.source.youtube_id }}" data-id="{{ video.source.youtube_id }}" onclick="downloadNow(this)">Download now</button>
                             {% endif %}
+                            {% if video.source.message %}
+                                <button class="danger-button" data-id="{{ video.source.youtube_id }}" onclick="forgetIgnore(this)">Delete</button>
+                            {% endif %}
                         </div>
                     </div>
                 </div>

tubearchivist/home/templates/home/playlist.html

@@ -49,9 +49,9 @@
                     <a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
                     <p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
                     {% if playlist.source.playlist_subscribed %}
-                        <button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
+                        <button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist.source.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
                     {% else %}
-                        <button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
+                        <button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist.source.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
                     {% endif %}
                 </div>
             </div>

tubearchivist/home/templates/home/playlist_id.html

@@ -27,9 +27,9 @@
                 <p>Last refreshed: {{ playlist_info.playlist_last_refresh }}</p>
                 <p>Playlist:
                     {% if playlist_info.playlist_subscribed %}
-                        <button class="unsubscribe" type="button" id="{{ playlist_info.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist_info.playlist_name }}">Unsubscribe</button>
+                        <button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist_info.playlist_name }}">Unsubscribe</button>
                     {% else %}
-                        <button type="button" id="{{ playlist_info.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist_info.playlist_name }}">Subscribe</button>
+                        <button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist_info.playlist_name }}">Subscribe</button>
                     {% endif %}
                 </p>
                 {% if playlist_info.playlist_active %}
@@ -40,8 +40,8 @@
                 <button onclick="deleteConfirm()" id="delete-item">Delete Playlist</button>
                 <div class="delete-confirm" id="delete-button">
                     <span>Delete {{ playlist_info.playlist_name }}?</span>
-                    <button onclick="deletePlaylist(this)" data-action="metadata" data-id="{{ playlist_info.playlist_id }}">Delete metadata</button>
-                    <button onclick="deletePlaylist(this)" data-action="all" class="danger-button" data-id="{{ playlist_info.playlist_id }}">Delete all</button><br>
+                    <button onclick="deletePlaylist(this)" data-action="" data-id="{{ playlist_info.playlist_id }}">Delete metadata</button>
+                    <button onclick="deletePlaylist(this)" data-action="delete-videos" class="danger-button" data-id="{{ playlist_info.playlist_id }}">Delete all</button><br>
                     <button onclick="cancelDelete()">Cancel</button>
                 </div>
             </div>

tubearchivist/home/templates/home/settings.html

@@ -1,405 +1,42 @@
-{% extends "home/base.html" %}
+{% extends "home/base_settings.html" %}
 {% load static %}
-{% block content %}
-<div class="boxed-content">
-    <div id="notifications" data="setting reindex"></div>
-    <div class="title-bar">
-        <h1>User Configurations</h1>
-    </div>
-    <form action="/settings/" method="POST" name="user-update">
-        {% csrf_token %}
-        <div class="settings-group">
-            <h2>Color scheme</h2>
-            <div class="settings-item">
-                <p>Current color scheme: <span class="settings-current">{{ config.application.colors }}</span></p>
-                <i>Select your preferred color scheme between dark and light mode.</i><br>
-                {{ user_form.colors }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>Archive View</h2>
-            <div class="settings-item">
-                <p>Current page size: <span class="settings-current">{{ config.archive.page_size }}</span></p>
-                <i>Result of videos showing in archive page</i><br>
-                {{ user_form.page_size }}
-            </div>
-        </div>
-        <button type="submit" name="user-settings">Update User Configurations</button>
-    </form>
-    <div class="title-bar">
-        <h1>Application Configurations</h1>
-    </div>
-    <form action="/settings/" method="POST" name="application-update">
-        {% csrf_token %}
-        <div class="settings-group">
-            <h2 id="subscriptions">Subscriptions</h2>
-            <p>Disable shorts or streams by setting their page size to 0 (zero).</p>
-            <div class="settings-item">
-                <p>YouTube page size: <span class="settings-current">{{ config.subscriptions.channel_size }}</span></p>
-                <i>Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
-                {{ app_form.subscriptions_channel_size }}
-            </div>
-            <div class="settings-item">
-                <p>YouTube Live page size: <span class="settings-current">{{ config.subscriptions.live_channel_size }}</span></p>
-                <i>Live Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
-                {{ app_form.subscriptions_live_channel_size }}
-            </div>
-            <div class="settings-item">
-                <p>YouTube Shorts page size: <span class="settings-current">{{ config.subscriptions.shorts_channel_size }}</span></p>
-                <i>Shorts Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
-                {{ app_form.subscriptions_shorts_channel_size }}
-            </div>
-            <div class="settings-item">
-                <p>Auto start download from your subscriptions: <span class="settings-current">{{ config.subscriptions.auto_start}}</span></p>
-                <i>Enable this will automatically start and prioritize videos from your subscriptions.</i><br>
-                {{ app_form.subscriptions_auto_start }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="downloads">Downloads</h2>
-            <div class="settings-item">
-                <p>Current download speed limit in KB/s: <span class="settings-current">{{ config.downloads.limit_speed }}</span></p>
-                <i>Limit download speed. 0 (zero) to deactivate, e.g. 1000 (1MB/s). Speeds are in KB/s. Setting takes effect on new download jobs or application restart.</i><br>
-                {{ app_form.downloads_limit_speed }}
-            </div>
-            <div class="settings-item">
-                <p>Current throttled rate limit in KB/s: <span class="settings-current">{{ config.downloads.throttledratelimit }}</span></p>
-                <i>Download will restart if speeds drop below specified amount. 0 (zero) to deactivate, e.g. 100. Speeds are in KB/s.</i><br>
-                {{ app_form.downloads_throttledratelimit }}
-            </div>
-            <div class="settings-item">
-                <p>Current scraping sleep interval: <span class="settings-current">{{ config.downloads.sleep_interval }}</p>
-                <i>Seconds to sleep between calls to YouTube. Might be necessary to avoid throttling. Recommended 3.</i><br>
-                {{ app_form.downloads_sleep_interval }}
-            </div>
-            <div class="settings-item">
-                <p><span class="danger-zone">Danger Zone</span>: Current auto delete watched videos: <span class="settings-current">{{ config.downloads.autodelete_days }}</span></p>
-                <i>Auto delete watched videos after x days, 0 (zero) to deactivate:</i><br>
-                {{ app_form.downloads_autodelete_days }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="format">Download Format</h2>
-            <div class="settings-item">
-                <p>Limit video and audio quality format for yt-dlp.<br>
-                    Currently: <span class="settings-current">{{ config.downloads.format }}</span>
-                </p>
-                <p>Example configurations:</p>
-                <ul>
-                    <li><span class="settings-current">bestvideo[height<=720]+bestaudio/best[height<=720]</span>: best audio and max video height of 720p.</li>
-                    <li><span class="settings-current">bestvideo[height<=1080]+bestaudio/best[height<=1080]</span>: best audio and max video height of 1080p.</li>
-                    <li><span class="settings-current">bestvideo[height<=1080][vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
-                    <li><span class="settings-current">0</span>: deactivate and download the best quality possible as decided by yt-dlp.</li>
-                </ul>
-                <i>Make sure your custom format gets merged into a single file. Check out the <a href="https://github.com/yt-dlp/yt-dlp#format-selection" target="_blank">documentation</a> for valid configurations.</i><br>
-                {{ app_form.downloads_format }}
-                <br>
-            </div>
-            <div class="settings-item">
-                <p>Force sort order to have precedence over all yt-dlp fields.<br>
-                    Currently: <span class="settings-current">{{ config.downloads.format_sort }}</span>
-                </p>
-                <p>Example configurations:</p>
-                <ul>
-                    <li><span class="settings-current">res,codec:av1</span>: prefer AV1 over all other video codecs.</li>
-                    <li><span class="settings-current">0</span>: deactivate and keep the default as decided by yt-dlp.</li>
-                </ul>
-                <i>Not all codecs are supported by all browsers. The default value ensures best compatibility. Check out the <a href="https://github.com/yt-dlp/yt-dlp#sorting-formats" target="_blank">documentation</a> for valid configurations.</i><br>
-                {{ app_form.downloads_format_sort }}
-                <br>
-            </div>
-            <div class="settings-item">
-                <p>Prefer translated metadata language: <span class="settings-current">{{ config.downloads.extractor_lang }}</span></p>
-                <i>This will change the language this video gets indexed as. That will only be available if the uploader provides translations. Add as two letter ISO language code, check the <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">documentation</a> which languages are available.</i><br>
-                {{ app_form.downloads_extractor_lang}}
-            </div>
-            <div class="settings-item">
-                <p>Current metadata embed setting: <span class="settings-current">{{ config.downloads.add_metadata }}</span></p>
-                <i>Metadata is not embedded into the downloaded files by default.</i><br>
-                {{ app_form.downloads_add_metadata }}
-            </div>
-            <div class="settings-item">
-                <p>Current thumbnail embed setting: <span class="settings-current">{{ config.downloads.add_thumbnail }}</span></p>
-                <i>Embed thumbnail into the mediafile.</i><br>
-                {{ app_form.downloads_add_thumbnail }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="format">Subtitles</h2>
-            <div class="settings-item">
-                <p>Subtitles download setting: <span class="settings-current">{{ config.downloads.subtitle }}</span><br>
-                <i>Choose which subtitles to download, add comma separated two letter language ISO code,<br>
-                    e.g. <span class="settings-current">en, de</span></i><br>
-                {{ app_form.downloads_subtitle }}</p>
-            </div>
-            <div class="settings-item">
-                <p>Subtitle source settings: <span class="settings-current">{{ config.downloads.subtitle_source }}</span></p>
-                <i>Download only user generated, or also less accurate auto generated subtitles.</i><br>
-                {{ app_form.downloads_subtitle_source }}
-            </div>
-            <div class="settings-item">
-                <p>Index and make subtitles searchable: <span class="settings-current">{{ config.downloads.subtitle_index }}</span></p>
-                <i>Store subtitle lines in Elasticsearch. Not recommended for low-end hardware.</i><br>
-                {{ app_form.downloads_subtitle_index }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="comments">Comments</h2>
-            <div class="settings-item">
-                <p>Download and index comments: <span class="settings-current">{{ config.downloads.comment_max }}</span><br>
-                <i>Follow the yt-dlp max_comments documentation, <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">max-comments,max-parents,max-replies,max-replies-per-thread</a>:</i><br>
-                <p>Example configurations:</p>
-                <ul>
-                    <li><span class="settings-current">all,100,all,30</span>: Get 100 max-parents and 30 max-replies-per-thread.</li>
-                    <li><span class="settings-current">1000,all,all,50</span>: Get a total of 1000 comments over all, 50 replies per thread.</li>
-                </ul>
-                {{ app_form.downloads_comment_max }}</p>
-            </div>
-            <div class="settings-item">
-                <p>Selected comment sort method: <span class="settings-current">{{ config.downloads.comment_sort }}</span><br>
-                <i>Select how many comments and threads to download:</i><br>
-                {{ app_form.downloads_comment_sort }}</p>
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="format">Cookie</h2>
-            <div class="settings-item">
-                <p>Import YouTube cookie: <span class="settings-current">{{ config.downloads.cookie_import }}</span><br></p>
-                <p>For automatic cookie import use <b>Tube Archivist Companion</b> <a href="https://github.com/tubearchivist/browser-extension" target="_blank">browser extension</a>.</p> 
-                <i>For manual cookie import, place your cookie file named <span class="settings-current">cookies.google.txt</span> in <span class="settings-current">cache/import</span> before enabling. Instructions in the <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki.</a></i><br>
-                {{ app_form.downloads_cookie_import }}<br>
-                {% if config.downloads.cookie_import %}
-                    <div id="cookieMessage">
-                        <button onclick="handleCookieValidate()" type="button" id="cookieButton">Validate Cookie File</button>
-                    </div>
-                {% endif %}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="integrations">Integrations</h2>
-            <div class="settings-item">
-                <p>API token: <button type="button" onclick="textReveal()" id="text-reveal-button">Show</button></p>
-                <div id="text-reveal" class="description-text">
-                    <p>{{ api_token }}</p>
-                    <button class="danger-button" type="button" onclick="resetToken()">Revoke</button>
-                </div>
-            </div>
-            <div class="settings-item">
-                <p>Integrate with <a href="https://returnyoutubedislike.com/" target="_blank">returnyoutubedislike.com</a> to get dislikes and average ratings back: <span class="settings-current">{{ config.downloads.integrate_ryd }}</span></p>
-                <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
-                {{ app_form.downloads_integrate_ryd }}
-            </div>
-            <div class="settings-item">
-                <p>Integrate with <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a> to get sponsored timestamps: <span class="settings-current">{{ config.downloads.integrate_sponsorblock }}</span></p>
-                <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
-                {{ app_form.downloads_integrate_sponsorblock }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2 id="snapshots">Snapshots</h2>
-            <div class="settings-item">
-                <p>Current system snapshot: <span class="settings-current">{{ config.application.enable_snapshot }}</span></p>
-                <i>Automatically create daily deduplicated snapshots of the index, stored in Elasticsearch. Read first before activating: <a target="_blank" href="https://docs.tubearchivist.com/settings/#snapshots">Wiki</a>.</i><br>
-                {{ app_form.application_enable_snapshot }}
-            </div>
-            <div>
-                {% if snapshots %}
-                    <p>Create next snapshot: <span class="settings-current">{{ snapshots.next_exec_str }}</span>, snapshots expire after <span class="settings-current">{{ snapshots.expire_after }}</span>. <button onclick="createSnapshot()" id="createButton">Create snapshot now</button></p>
-                    <br>
-                    {% for snapshot in snapshots.snapshots %}
-                        <p><button id="{{ snapshot.id }}" onclick="restoreSnapshot(id)">Restore</button> Snapshot created on: <span class="settings-current">{{ snapshot.start_date }}</span>, took <span class="settings-current">{{ snapshot.duration_s }}s</span> to create. State: <i>{{ snapshot.state }}</i></p>
-                    {% endfor %}
-                {% endif %}
-            </div>
-        </div>
-        <button type="submit" name="application-settings">Update Application Configurations</button>
-    </form>
-    <div class="title-bar">
-        <h1>Scheduler Setup</h1>
-        <div class="settings-group">
-            <p>Schedule settings expect a cron like format, where the first value is minute, second is hour and third is day of the week.</p>
-            <p>Examples:</p>
-            <ul>
-                <li><span class="settings-current">0 15 *</span>: Run task every day at 15:00 in the afternoon.</li>
-                <li><span class="settings-current">30 8 */2</span>: Run task every second day of the week (Sun, Tue, Thu, Sat) at 08:30 in the morning.</li>
-                <li><span class="settings-current">auto</span>: Sensible default.</li>
-                <li><span class="settings-current">0</span>: (zero), deactivate that task.</li>
-            </ul>
-            <p>Note:</p>
-            <ul>
-                <li>Changes in the scheduler settings require a container restart to take effect.</li>
-                <li>Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason, the scheduler doesn't support schedules that trigger more than once per hour.</li>
-            </ul>
-        </div>
-    </div>
-    <form action="/settings/" method="POST" name="scheduler-update">
-        {% csrf_token %}
-        <div class="settings-group">
-            <h2>Rescan Subscriptions</h2>
-            <div class="settings-item">
-                <p>Current rescan schedule: <span class="settings-current">
-                    {% if config.scheduler.update_subscribed %}
-                        {% for key, value in config.scheduler.update_subscribed.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Become a sponsor and join <a href="https://members.tubearchivist.com/" target="_blank">members.tubearchivist.com</a> to get access to <span class="settings-current">real time</span> notifications for new videos uploaded by your favorite channels.</p>
-                <p>Periodically rescan your subscriptions:</p>
-                {{ scheduler_form.update_subscribed }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>Start download</h2>
-            <div class="settings-item">
-                <p>Current Download schedule: <span class="settings-current">
-                    {% if config.scheduler.download_pending %}
-                        {% for key, value in config.scheduler.download_pending.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Automatic video download schedule:</p>
-                {{ scheduler_form.download_pending }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>Refresh Metadata</h2>
-            <div class="settings-item">
-                <p>Current Metadata refresh schedule: <span class="settings-current">
-                    {% if config.scheduler.check_reindex %}
-                        {% for key, value in config.scheduler.check_reindex.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Daily schedule to refresh metadata from YouTube:</p>
-                {{ scheduler_form.check_reindex }}
-            </div>
-            <div class="settings-item">
-                <p>Current refresh for metadata older than x days: <span class="settings-current">{{ config.scheduler.check_reindex_days }}</span></p>
-                <p>Refresh older than x days, recommended 90:</p>
-                {{ scheduler_form.check_reindex_days }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>Thumbnail check</h2>
-            <div class="settings-item">
-                <p>Current thumbnail check schedule: <span class="settings-current">
-                    {% if config.scheduler.thumbnail_check %}
-                        {% for key, value in config.scheduler.thumbnail_check.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Periodically check and cleanup thumbnails:</p>
-                {{ scheduler_form.thumbnail_check }}
-            </div>
-        </div>
-        <div class="settings-group">
-            <h2>ZIP file index backup</h2>
-            <div class="settings-item">
-                <p><i>Zip file backups are very slow for large archives and consistency is not guaranteed, use snapshots instead. Make sure no other tasks are running when creating a Zip file backup.</i></p>
-                <p>Current index backup schedule: <span class="settings-current">
-                    {% if config.scheduler.run_backup %}
-                        {% for key, value in config.scheduler.run_backup.items %}
-                            {{ value }}
-                        {% endfor %}
-                    {% else %}
-                        False
-                    {% endif %}
-                </span></p>
-                <p>Automatically backup metadata to a zip file:</p>
-                {{ scheduler_form.run_backup }}
-            </div>
-            <div class="settings-item">
-                <p>Current backup files to keep: <span class="settings-current">{{ config.scheduler.run_backup_rotate }}</span></p>
-                <p>Max auto backups to keep:</p>
-                {{ scheduler_form.run_backup_rotate }}
-            </div>
-        </div>
-        <button type="submit" name="scheduler-settings">Update Scheduler Settings</button>
-    </form>
-    <div class="title-bar">
-        <h1>Actions</h1>
-    </div>
-    <div class="settings-group">
-        <h2>Delete download queue</h2>
-        <p>Delete your pending or previously ignored videos from your download queue.<p>
-        <button onclick="deleteQueue(this)" id="ignore-button" data-id="ignore" title="Delete all previously ignored videos from the queue">Delete all ignored</button>
-        <button onclick="deleteQueue(this)" id="pending-button" data-id="pending" title="Delete all pending videos from the queue">Delete all queued</button>
-    </div>
-    <div class="settings-group">
-        <h2>Manual media files import.</h2>
-        <p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki</a>.</p>
-        <div id="manual-import">
-            <button onclick="manualImport()">Start import</button>
-        </div>
-    </div>
-    <div class="settings-group">
-        <h2>Embed thumbnails into media file.</h2>
-        <p>Set extracted youtube thumbnail as cover art of the media file.</p>
-        <div id="re-embed">
-            <button onclick="reEmbed()">Start process</button>
-        </div>
-    </div>
-    <div class="settings-group">
-        <h2>ZIP file index backup</h2>
-        <p>Export your database to a zip file stored at <span class="settings-current">cache/backup</span>.</p>
-        <p><i>Zip file backups are very slow for large archives and consistency is not guaranteed, use snapshots instead. Make sure no other tasks are running when creating a Zip file backup.</i></p>
-        <div id="db-backup">
-            <button onclick="dbBackup()">Start backup</button>
-        </div>
-    </div>
-    <div class="settings-group">
-        <h2>Restore from backup</h2>
-        <p><span class="danger-zone">Danger Zone</span>: This will replace your existing index with the backup.</p>
-        <p>Restore from available backup files from <span class="settings-current">cache/backup</span>.</p>
-        {% if available_backups %}
-            <div class="backup-grid-row">
-                <span></span>
-                <span>Timestamp</span>
-                <span>Source</span>
-                <span>Filename</span>
-            </div>
-            {% for backup in available_backups %}
-                <div class="backup-grid-row" id="{{ backup.filename }}">
-                    <button onclick="dbRestore(this)" data-id="{{ backup.filename }}">Restore</button>
-                    <span>{{ backup.timestamp }}</span>
-                    <span>{{ backup.reason }}</span>
-                    <span>{{ backup.filename }}</span>
-                </div>
-            {% endfor %}
-        {% else %}
-            <p>No backups found.</p>
-        {% endif %}
-    </div>
-    <div class="settings-group">
-        <h2>Rescan filesystem</h2>
-        <p><span class="danger-zone">Danger Zone</span>: This will delete the metadata of deleted videos from the filesystem.</p>
-        <p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki</a>.</p>
-        <div id="fs-rescan">
-            <button onclick="fsRescan()">Rescan filesystem</button>
-        </div>
-    </div>
-    {% if request.user.is_superuser %}
-        <div class="title-bar">
-            <h1>Users</h1>
-        </div>
-        <div class="settings-group">
-            <h2>User Management</h2>
-            <p>Access the admin interface for basic user management functionality like adding and deleting users, changing passwords and more.</p>
-            <a href="/admin/"><button>Admin Interface</button></a>
-        </div>
-    {% endif %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>Your Archive</h1>
 </div>
-<script type="text/javascript" src="{% static 'progress.js' %}"></script>
-{% endblock content %}
+<div class="settings-item">
+    <h2>Main overview</h2>
+    <div id="primaryBox" class="info-box info-box-4">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Watch Progress</h2>
+    <div id="watchBox" class="info-box info-box-3">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Download History</h2>
+    <div id="downHistBox" class="info-box info-box-4">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Biggest Channels</h2>
+    <div class="info-box description-box">
+        <table>
+            <thead>
+                <tr>
+                    <th class="agg-channel-name">Name</th>
+                    <th>Videos</th>
+                    <th>Duration</th>
+                    <th>Media Size</th>
+                </tr>
+            </thead>
+            <tbody id="biggestChannelTable"></tbody>
+        </table>
+    </div>
+</div>
+<script type="text/javascript" src="{% static 'stats.js' %}"></script>
+{% endblock settings_content %}

tubearchivist/home/templates/home/settings_actions.html

@@ -0,0 +1,66 @@
+{% extends "home/base_settings.html" %}
+{% load static %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>Actions</h1>
+</div>
+<div class="settings-group">
+    <h2>Delete download queue</h2>
+    <p>Delete your pending or previously ignored videos from your download queue.<p>
+    <button onclick="deleteQueue(this)" id="ignore-button" data-id="ignore" title="Delete all previously ignored videos from the queue">Delete all ignored</button>
+    <button onclick="deleteQueue(this)" id="pending-button" data-id="pending" title="Delete all pending videos from the queue">Delete all queued</button>
+</div>
+<div class="settings-group">
+    <h2>Manual media files import.</h2>
+    <p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki</a>.</p>
+    <div id="manual-import">
+        <button onclick="manualImport()">Start import</button>
+    </div>
+</div>
+<div class="settings-group">
+    <h2>Embed thumbnails into media file.</h2>
+    <p>Set extracted youtube thumbnail as cover art of the media file.</p>
+    <div id="re-embed">
+        <button onclick="reEmbed()">Start process</button>
+    </div>
+</div>
+<div class="settings-group">
+    <h2>ZIP file index backup</h2>
+    <p>Export your database to a zip file stored at <span class="settings-current">cache/backup</span>.</p>
+    <p><i>Zip file backups are very slow for large archives and consistency is not guaranteed, use snapshots instead. Make sure no other tasks are running when creating a Zip file backup.</i></p>
+    <div id="db-backup">
+        <button onclick="dbBackup()">Start backup</button>
+    </div>
+</div>
+<div class="settings-group">
+    <h2>Restore from backup</h2>
+    <p><span class="danger-zone">Danger Zone</span>: This will replace your existing index with the backup.</p>
+    <p>Restore from available backup files from <span class="settings-current">cache/backup</span>.</p>
+    {% if available_backups %}
+        <div class="backup-grid-row">
+            <span></span>
+            <span>Timestamp</span>
+            <span>Source</span>
+            <span>Filename</span>
+        </div>
+        {% for backup in available_backups %}
+            <div class="backup-grid-row" id="{{ backup.filename }}">
+                <button onclick="dbRestore(this)" data-id="{{ backup.filename }}">Restore</button>
+                <span>{{ backup.timestamp }}</span>
+                <span>{{ backup.reason }}</span>
+                <span>{{ backup.filename }}</span>
+            </div>
+        {% endfor %}
+    {% else %}
+        <p>No backups found.</p>
+    {% endif %}
+</div>
+<div class="settings-group">
+    <h2>Rescan filesystem</h2>
+    <p><span class="danger-zone">Danger Zone</span>: This will delete the metadata of deleted videos from the filesystem.</p>
+    <p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki</a>.</p>
+    <div id="fs-rescan">
+        <button onclick="fsRescan()">Rescan filesystem</button>
+    </div>
+</div>
+{% endblock settings_content %}

tubearchivist/home/templates/home/settings_application.html

@@ -0,0 +1,192 @@
+{% extends "home/base_settings.html" %}
+{% load static %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>Application Configurations</h1>
+</div>
+<form action="{% url 'settings_application' %}" method="POST" name="application-update">
+    {% csrf_token %}
+    <div class="settings-group">
+        <h2 id="subscriptions">Subscriptions</h2>
+        <p>Disable shorts or streams by setting their page size to 0 (zero).</p>
+        <div class="settings-item">
+            <p>YouTube page size: <span class="settings-current">{{ config.subscriptions.channel_size }}</span></p>
+            <i>Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
+            {{ app_form.subscriptions_channel_size }}
+        </div>
+        <div class="settings-item">
+            <p>YouTube Live page size: <span class="settings-current">{{ config.subscriptions.live_channel_size }}</span></p>
+            <i>Live Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
+            {{ app_form.subscriptions_live_channel_size }}
+        </div>
+        <div class="settings-item">
+            <p>YouTube Shorts page size: <span class="settings-current">{{ config.subscriptions.shorts_channel_size }}</span></p>
+            <i>Shorts Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
+            {{ app_form.subscriptions_shorts_channel_size }}
+        </div>
+        <div class="settings-item">
+            <p>Auto start download from your subscriptions: <span class="settings-current">{{ config.subscriptions.auto_start}}</span></p>
+            <i>Enable this will automatically start and prioritize videos from your subscriptions.</i><br>
+            {{ app_form.subscriptions_auto_start }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="downloads">Downloads</h2>
+        <div class="settings-item">
+            <p>Current download speed limit in KB/s: <span class="settings-current">{{ config.downloads.limit_speed }}</span></p>
+            <i>Limit download speed. 0 (zero) to deactivate, e.g. 1000 (1MB/s). Speeds are in KB/s. Setting takes effect on new download jobs or application restart.</i><br>
+            {{ app_form.downloads_limit_speed }}
+        </div>
+        <div class="settings-item">
+            <p>Current throttled rate limit in KB/s: <span class="settings-current">{{ config.downloads.throttledratelimit }}</span></p>
+            <i>Download will restart if speeds drop below specified amount. 0 (zero) to deactivate, e.g. 100. Speeds are in KB/s.</i><br>
+            {{ app_form.downloads_throttledratelimit }}
+        </div>
+        <div class="settings-item">
+            <p>Current scraping sleep interval: <span class="settings-current">{{ config.downloads.sleep_interval }}</p>
+            <i>Seconds to sleep between calls to YouTube. Might be necessary to avoid throttling. Recommended 3.</i><br>
+            {{ app_form.downloads_sleep_interval }}
+        </div>
+        <div class="settings-item">
+            <p><span class="danger-zone">Danger Zone</span>: Current auto delete watched videos: <span class="settings-current">{{ config.downloads.autodelete_days }}</span></p>
+            <i>Auto delete watched videos after x days, 0 (zero) to deactivate:</i><br>
+            {{ app_form.downloads_autodelete_days }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="format">Download Format</h2>
+        <div class="settings-item">
+            <p>Limit video and audio quality format for yt-dlp.<br>
+                Currently: <span class="settings-current">{{ config.downloads.format }}</span>
+            </p>
+            <p>Example configurations:</p>
+            <ul>
+                <li><span class="settings-current">bestvideo[height<=720]+bestaudio/best[height<=720]</span>: best audio and max video height of 720p.</li>
+                <li><span class="settings-current">bestvideo[height<=1080]+bestaudio/best[height<=1080]</span>: best audio and max video height of 1080p.</li>
+                <li><span class="settings-current">bestvideo[height<=1080][vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
+                <li><span class="settings-current">0</span>: deactivate and download the best quality possible as decided by yt-dlp.</li>
+            </ul>
+            <i>Make sure your custom format gets merged into a single file. Check out the <a href="https://github.com/yt-dlp/yt-dlp#format-selection" target="_blank">documentation</a> for valid configurations.</i><br>
+            {{ app_form.downloads_format }}
+            <br>
+        </div>
+        <div class="settings-item">
+            <p>Force sort order to have precedence over all yt-dlp fields.<br>
+                Currently: <span class="settings-current">{{ config.downloads.format_sort }}</span>
+            </p>
+            <p>Example configurations:</p>
+            <ul>
+                <li><span class="settings-current">res,codec:av1</span>: prefer AV1 over all other video codecs.</li>
+                <li><span class="settings-current">0</span>: deactivate and keep the default as decided by yt-dlp.</li>
+            </ul>
+            <i>Not all codecs are supported by all browsers. The default value ensures best compatibility. Check out the <a href="https://github.com/yt-dlp/yt-dlp#sorting-formats" target="_blank">documentation</a> for valid configurations.</i><br>
+            {{ app_form.downloads_format_sort }}
+            <br>
+        </div>
+        <div class="settings-item">
+            <p>Prefer translated metadata language: <span class="settings-current">{{ config.downloads.extractor_lang }}</span></p>
+            <i>This will change the language this video gets indexed as. That will only be available if the uploader provides translations. Add as two letter ISO language code, check the <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">documentation</a> which languages are available.</i><br>
+            {{ app_form.downloads_extractor_lang}}
+        </div>
+        <div class="settings-item">
+            <p>Current metadata embed setting: <span class="settings-current">{{ config.downloads.add_metadata }}</span></p>
+            <i>Metadata is not embedded into the downloaded files by default.</i><br>
+            {{ app_form.downloads_add_metadata }}
+        </div>
+        <div class="settings-item">
+            <p>Current thumbnail embed setting: <span class="settings-current">{{ config.downloads.add_thumbnail }}</span></p>
+            <i>Embed thumbnail into the mediafile.</i><br>
+            {{ app_form.downloads_add_thumbnail }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="format">Subtitles</h2>
+        <div class="settings-item">
+            <p>Subtitles download setting: <span class="settings-current">{{ config.downloads.subtitle }}</span><br>
+            <i>Choose which subtitles to download, add comma separated language codes,<br>
+                e.g. <span class="settings-current">en, de, zh-Hans</span></i><br>
+            {{ app_form.downloads_subtitle }}</p>
+        </div>
+        <div class="settings-item">
+            <p>Subtitle source settings: <span class="settings-current">{{ config.downloads.subtitle_source }}</span></p>
+            <i>Download only user generated, or also less accurate auto generated subtitles.</i><br>
+            {{ app_form.downloads_subtitle_source }}
+        </div>
+        <div class="settings-item">
+            <p>Index and make subtitles searchable: <span class="settings-current">{{ config.downloads.subtitle_index }}</span></p>
+            <i>Store subtitle lines in Elasticsearch. Not recommended for low-end hardware.</i><br>
+            {{ app_form.downloads_subtitle_index }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="comments">Comments</h2>
+        <div class="settings-item">
+            <p>Download and index comments: <span class="settings-current">{{ config.downloads.comment_max }}</span><br>
+            <i>Follow the yt-dlp max_comments documentation, <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">max-comments,max-parents,max-replies,max-replies-per-thread</a>:</i><br>
+            <p>Example configurations:</p>
+            <ul>
+                <li><span class="settings-current">all,100,all,30</span>: Get 100 max-parents and 30 max-replies-per-thread.</li>
+                <li><span class="settings-current">1000,all,all,50</span>: Get a total of 1000 comments over all, 50 replies per thread.</li>
+            </ul>
+            {{ app_form.downloads_comment_max }}</p>
+        </div>
+        <div class="settings-item">
+            <p>Selected comment sort method: <span class="settings-current">{{ config.downloads.comment_sort }}</span><br>
+            <i>Select how many comments and threads to download:</i><br>
+            {{ app_form.downloads_comment_sort }}</p>
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="format">Cookie</h2>
+        <div class="settings-item">
+            <p>Import YouTube cookie: <span class="settings-current">{{ config.downloads.cookie_import }}</span><br></p>
+            <p>For automatic cookie import use <b>Tube Archivist Companion</b> <a href="https://github.com/tubearchivist/browser-extension" target="_blank">browser extension</a>.</p> 
+            <i>For manual cookie import, place your cookie file named <span class="settings-current">cookies.google.txt</span> in <span class="settings-current">cache/import</span> before enabling. Instructions in the <a href="https://docs.tubearchivist.com/settings/" target="_blank">Wiki.</a></i><br>
+            {{ app_form.downloads_cookie_import }}<br>
+            {% if config.downloads.cookie_import %}
+                <div id="cookieMessage">
+                    <button onclick="handleCookieValidate()" type="button" id="cookieButton">Validate Cookie File</button>
+                </div>
+            {% endif %}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="integrations">Integrations</h2>
+        <div class="settings-item">
+            <p>API token: <button type="button" onclick="textReveal(this)" id="text-reveal-button">Show</button></p>
+            <div id="text-reveal" class="description-text">
+                <p>{{ api_token }}</p>
+                <button class="danger-button" type="button" onclick="resetToken()">Revoke</button>
+            </div>
+        </div>
+        <div class="settings-item">
+            <p>Integrate with <a href="https://returnyoutubedislike.com/" target="_blank">returnyoutubedislike.com</a> to get dislikes and average ratings back: <span class="settings-current">{{ config.downloads.integrate_ryd }}</span></p>
+            <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
+            {{ app_form.downloads_integrate_ryd }}
+        </div>
+        <div class="settings-item">
+            <p>Integrate with <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a> to get sponsored timestamps: <span class="settings-current">{{ config.downloads.integrate_sponsorblock }}</span></p>
+            <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
+            {{ app_form.downloads_integrate_sponsorblock }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2 id="snapshots">Snapshots</h2>
+        <div class="settings-item">
+            <p>Current system snapshot: <span class="settings-current">{{ config.application.enable_snapshot }}</span></p>
+            <i>Automatically create daily deduplicated snapshots of the index, stored in Elasticsearch. Read first before activating: <a target="_blank" href="https://docs.tubearchivist.com/settings/#snapshots">Wiki</a>.</i><br>
+            {{ app_form.application_enable_snapshot }}
+        </div>
+        <div>
+            {% if snapshots %}
+                <p>Create next snapshot: <span class="settings-current">{{ snapshots.next_exec_str }}</span>, snapshots expire after <span class="settings-current">{{ snapshots.expire_after }}</span>. <button onclick="createSnapshot()" id="createButton">Create snapshot now</button></p>
+                <br>
+                {% for snapshot in snapshots.snapshots %}
+                    <p><button id="{{ snapshot.id }}" onclick="restoreSnapshot(id)">Restore</button> Snapshot created on: <span class="settings-current">{{ snapshot.start_date }}</span>, took <span class="settings-current">{{ snapshot.duration_s }}s</span> to create. State: <i>{{ snapshot.state }}</i></p>
+                {% endfor %}
+            {% endif %}
+        </div>
+    </div>
+    <button type="submit" name="application-settings">Update Application Configurations</button>
+</form>
+{% endblock settings_content %}

tubearchivist/home/templates/home/settings_scheduling.html

@@ -0,0 +1,154 @@
+{% extends "home/base_settings.html" %}
+{% load static %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>Scheduler Setup</h1>
+    <div class="settings-group">
+        <p>Schedule settings expect a cron like format, where the first value is minute, second is hour and third is day of the week.</p>
+        <p>Examples:</p>
+        <ul>
+            <li><span class="settings-current">0 15 *</span>: Run task every day at 15:00 in the afternoon.</li>
+            <li><span class="settings-current">30 8 */2</span>: Run task every second day of the week (Sun, Tue, Thu, Sat) at 08:30 in the morning.</li>
+            <li><span class="settings-current">auto</span>: Sensible default.</li>
+            <li><span class="settings-current">0</span>: (zero), deactivate that task.</li>
+        </ul>
+        <p>Note:</p>
+        <ul>
+            <li>Changes in the scheduler settings require a container restart to take effect.</li>
+            <li>Avoid an unnecessary frequent schedule to not get blocked by YouTube. For that reason, the scheduler doesn't support schedules that trigger more than once per hour.</li>
+        </ul>
+    </div>
+</div>
+<form action="{% url 'settings_scheduling' %}" method="POST" name="scheduler-update">
+    {% csrf_token %}
+    <div class="settings-group">
+        <h2>Rescan Subscriptions</h2>
+        <div class="settings-item">
+            <p>Current rescan schedule: <span class="settings-current">
+                {% if config.scheduler.update_subscribed %}
+                    {% for key, value in config.scheduler.update_subscribed.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Become a sponsor and join <a href="https://members.tubearchivist.com/" target="_blank">members.tubearchivist.com</a> to get access to <span class="settings-current">real time</span> notifications for new videos uploaded by your favorite channels.</p>
+            <p>Periodically rescan your subscriptions:</p>
+            {{ scheduler_form.update_subscribed }}
+        </div>
+        <div class="settings-item">
+            <p>Send notification on task completed:</p>
+            {% if config.scheduler.update_subscribed_notify %}
+                <p><button type="button" onclick="textReveal(this)" id="text-reveal-button">Show</button> stored notification links</p>
+                <div id="text-reveal" class="description-text">
+                    <p>{{ config.scheduler.update_subscribed_notify|linebreaks }}</p>
+                </div>
+            {% else %}
+                <p>Current notification urls: <span class="settings-current">{{ config.scheduler.update_subscribed_notify }}</span></p>
+            {% endif %}
+            {{ scheduler_form.update_subscribed_notify }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>Start download</h2>
+        <div class="settings-item">
+            <p>Current Download schedule: <span class="settings-current">
+                {% if config.scheduler.download_pending %}
+                    {% for key, value in config.scheduler.download_pending.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Automatic video download schedule:</p>
+            {{ scheduler_form.download_pending }}
+        </div>
+        <div class="settings-item">
+            <p>Send notification on task completed:</p>
+            {% if config.scheduler.download_pending_notify %}
+                <p><button type="button" onclick="textReveal(this)" id="text-reveal-button">Show</button> stored notification links</p>
+                <div id="text-reveal" class="description-text">
+                    <p>{{ config.scheduler.download_pending_notify|linebreaks }}</p>
+                </div>
+            {% else %}
+                <p>Current notification urls: <span class="settings-current">{{ config.scheduler.download_pending_notify }}</span></p>
+            {% endif %}
+            {{ scheduler_form.download_pending_notify }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>Refresh Metadata</h2>
+        <div class="settings-item">
+            <p>Current Metadata refresh schedule: <span class="settings-current">
+                {% if config.scheduler.check_reindex %}
+                    {% for key, value in config.scheduler.check_reindex.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Daily schedule to refresh metadata from YouTube:</p>
+            {{ scheduler_form.check_reindex }}
+        </div>
+        <div class="settings-item">
+            <p>Current refresh for metadata older than x days: <span class="settings-current">{{ config.scheduler.check_reindex_days }}</span></p>
+            <p>Refresh older than x days, recommended 90:</p>
+            {{ scheduler_form.check_reindex_days }}
+        </div>
+        <div class="settings-item">
+            <p>Send notification on task completed:</p>
+            {% if config.scheduler.check_reindex_notify %}
+                <p><button type="button" onclick="textReveal(this)" id="text-reveal-button">Show</button> stored notification links</p>
+                <div id="text-reveal" class="description-text">
+                    <p>{{ config.scheduler.check_reindex_notify|linebreaks }}</p>
+                </div>
+            {% else %}
+                <p>Current notification urls: <span class="settings-current">{{ config.scheduler.check_reindex_notify }}</span></p>
+            {% endif %}
+            {{ scheduler_form.check_reindex_notify }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>Thumbnail check</h2>
+        <div class="settings-item">
+            <p>Current thumbnail check schedule: <span class="settings-current">
+                {% if config.scheduler.thumbnail_check %}
+                    {% for key, value in config.scheduler.thumbnail_check.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Periodically check and cleanup thumbnails:</p>
+            {{ scheduler_form.thumbnail_check }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>ZIP file index backup</h2>
+        <div class="settings-item">
+            <p><i>Zip file backups are very slow for large archives and consistency is not guaranteed, use snapshots instead. Make sure no other tasks are running when creating a Zip file backup.</i></p>
+            <p>Current index backup schedule: <span class="settings-current">
+                {% if config.scheduler.run_backup %}
+                    {% for key, value in config.scheduler.run_backup.items %}
+                        {{ value }}
+                    {% endfor %}
+                {% else %}
+                    False
+                {% endif %}
+            </span></p>
+            <p>Automatically backup metadata to a zip file:</p>
+            {{ scheduler_form.run_backup }}
+        </div>
+        <div class="settings-item">
+            <p>Current backup files to keep: <span class="settings-current">{{ config.scheduler.run_backup_rotate }}</span></p>
+            <p>Max auto backups to keep:</p>
+            {{ scheduler_form.run_backup_rotate }}
+        </div>
+    </div>
+    <button type="submit" name="scheduler-settings">Update Scheduler Settings</button>
+</form>
+{% endblock settings_content %}

tubearchivist/home/templates/home/settings_user.html

@@ -0,0 +1,37 @@
+{% extends "home/base_settings.html" %}
+{% load static %}
+{% block settings_content %}
+<div class="title-bar">
+    <h1>User Configurations</h1>
+</div>
+<form action="{% url 'settings_user' %}" method="POST" name="user-update">
+    {% csrf_token %}
+    <div class="settings-group">
+        <h2>Color scheme</h2>
+        <div class="settings-item">
+            <p>Current color scheme: <span class="settings-current">{{ config.application.colors }}</span></p>
+            <i>Select your preferred color scheme between dark and light mode.</i><br>
+            {{ user_form.colors }}
+        </div>
+    </div>
+    <div class="settings-group">
+        <h2>Archive View</h2>
+        <div class="settings-item">
+            <p>Current page size: <span class="settings-current">{{ config.archive.page_size }}</span></p>
+            <i>Result of videos showing in archive page</i><br>
+            {{ user_form.page_size }}
+        </div>
+    </div>
+    <button type="submit" name="user-settings">Update User Configurations</button>
+</form>
+{% if request.user.is_superuser %}
+    <div class="title-bar">
+        <h1>Users</h1>
+    </div>
+    <div class="settings-group">
+        <h2>User Management</h2>
+        <p>Access the admin interface for basic user management functionality like adding and deleting users, changing passwords and more.</p>
+        <a href="/admin/"><button>Admin Interface</button></a>
+    </div>
+{% endif %}
+{% endblock settings_content %}

tubearchivist/home/urls.py

@@ -26,6 +26,26 @@ urlpatterns = [
         login_required(views.SettingsView.as_view()),
         name="settings",
     ),
+    path(
+        "settings/user/",
+        login_required(views.SettingsUserView.as_view()),
+        name="settings_user",
+    ),
+    path(
+        "settings/application/",
+        login_required(views.SettingsApplicationView.as_view()),
+        name="settings_application",
+    ),
+    path(
+        "settings/scheduling/",
+        login_required(views.SettingsSchedulingView.as_view()),
+        name="settings_scheduling",
+    ),
+    path(
+        "settings/actions/",
+        login_required(views.SettingsActionsView.as_view()),
+        name="settings_actions",
+    ),
     path("process/", login_required(views.process), name="process"),
     path(
         "channel/",

tubearchivist/home/views.py

@@ -736,7 +736,7 @@ class ChannelView(ArchivistResultsView):
         if subscribe_form.is_valid():
             url_str = request.POST.get("subscribe")
             print(url_str)
-            subscribe_to.delay(url_str)
+            subscribe_to.delay(url_str, expected_type="channel")
 
         sleep(1)
         return redirect("channel", permanent=True)
@@ -879,7 +879,7 @@ class PlaylistView(ArchivistResultsView):
         if subscribe_form.is_valid():
             url_str = request.POST.get("subscribe")
             print(url_str)
-            subscribe_to.delay(url_str)
+            subscribe_to.delay(url_str, expected_type="playlist")
 
         sleep(1)
         return redirect("playlist")
@@ -981,7 +981,21 @@ class SearchView(ArchivistResultsView):
 
 class SettingsView(MinView):
     """resolves to /settings/
-    handle the settings page, display current settings,
+    handle the settings dashboard
+    """
+
+    def get(self, request):
+        """read and display the dashboard"""
+        context = self.get_min_context(request)
+        context.update({"title": "Settings Dashboard"})
+
+        return render(request, "home/settings.html", context)
+
+
+class SettingsUserView(MinView):
+    """resolves to /settings/user/
+    handle the settings sub-page for user settings,
+    display current settings,
     take post request from the form to update settings
     """
 
@@ -990,18 +1004,47 @@ class SettingsView(MinView):
         context = self.get_min_context(request)
         context.update(
             {
-                "title": "Settings",
+                "title": "User Settings",
+                "config": AppConfig(request.user.id).config,
+                "user_form": UserSettingsForm(),
+            }
+        )
+
+        return render(request, "home/settings_user.html", context)
+
+    def post(self, request):
+        """handle form post to update settings"""
+        user_form = UserSettingsForm(request.POST)
+        if user_form.is_valid():
+            user_form_post = user_form.cleaned_data
+            if any(user_form_post.values()):
+                AppConfig().set_user_config(user_form_post, request.user.id)
+
+        sleep(1)
+        return redirect("settings_user", permanent=True)
+
+
+class SettingsApplicationView(MinView):
+    """resolves to /settings/application/
+    handle the settings sub-page for application configuration,
+    display current settings,
+    take post request from the form to update settings
+    """
+
+    def get(self, request):
+        """read and display current application settings"""
+        context = self.get_min_context(request)
+        context.update(
+            {
+                "title": "Application Settings",
                 "config": AppConfig(request.user.id).config,
                 "api_token": self.get_token(request),
-                "available_backups": ElasticBackup().get_all_backup_files(),
-                "user_form": UserSettingsForm(),
                 "app_form": ApplicationSettingsForm(),
-                "scheduler_form": SchedulerSettingsForm(),
                 "snapshots": ElasticSnapshot().get_snapshot_stats(),
             }
         )
 
-        return render(request, "home/settings.html", context)
+        return render(request, "home/settings_application.html", context)
 
     @staticmethod
     def get_token(request):
@@ -1012,12 +1055,7 @@ class SettingsView(MinView):
 
     def post(self, request):
         """handle form post to update settings"""
-        user_form = UserSettingsForm(request.POST)
         config_handler = AppConfig()
-        if user_form.is_valid():
-            user_form_post = user_form.cleaned_data
-            if any(user_form_post.values()):
-                config_handler.set_user_config(user_form_post, request.user.id)
 
         app_form = ApplicationSettingsForm(request.POST)
         if app_form.is_valid():
@@ -1027,15 +1065,8 @@ class SettingsView(MinView):
                 updated = config_handler.update_config(app_form_post)
                 self.post_process_updated(updated, config_handler.config)
 
-        scheduler_form = SchedulerSettingsForm(request.POST)
-        if scheduler_form.is_valid():
-            scheduler_form_post = scheduler_form.cleaned_data
-            if any(scheduler_form_post.values()):
-                print(scheduler_form_post)
-                ScheduleBuilder().update_schedule_conf(scheduler_form_post)
-
         sleep(1)
-        return redirect("settings", permanent=True)
+        return redirect("settings_application", permanent=True)
 
     def post_process_updated(self, updated, config):
         """apply changes for config"""
@@ -1073,13 +1104,66 @@ class SettingsView(MinView):
         key = "message:setting"
         message = {
             "status": key,
+            "group": "setting:application",
             "level": "error",
             "title": "Cookie import failed",
-            "message": message_line,
+            "messages": [message_line],
+            "id": "0000",
         }
         RedisArchivist().set_message(key, message=message, expire=True)
 
 
+class SettingsSchedulingView(MinView):
+    """resolves to /settings/scheduling/
+    handle the settings sub-page for scheduling settings,
+    display current settings,
+    take post request from the form to update settings
+    """
+
+    def get(self, request):
+        """read and display current settings"""
+        context = self.get_min_context(request)
+        context.update(
+            {
+                "title": "Scheduling Settings",
+                "config": AppConfig(request.user.id).config,
+                "scheduler_form": SchedulerSettingsForm(),
+            }
+        )
+
+        return render(request, "home/settings_scheduling.html", context)
+
+    def post(self, request):
+        """handle form post to update settings"""
+        scheduler_form = SchedulerSettingsForm(request.POST)
+        if scheduler_form.is_valid():
+            scheduler_form_post = scheduler_form.cleaned_data
+            if any(scheduler_form_post.values()):
+                print(scheduler_form_post)
+                ScheduleBuilder().update_schedule_conf(scheduler_form_post)
+
+        sleep(1)
+        return redirect("settings_scheduling", permanent=True)
+
+
+class SettingsActionsView(MinView):
+    """resolves to /settings/actions/
+    handle the settings actions sub-page
+    """
+
+    def get(self, request):
+        """read and display current settings"""
+        context = self.get_min_context(request)
+        context.update(
+            {
+                "title": "Actions",
+                "available_backups": ElasticBackup().get_all_backup_files(),
+            }
+        )
+
+        return render(request, "home/settings_actions.html", context)
+
+
 def process(request):
     """handle all the buttons calls via POST ajax"""
     if request.method == "POST":

tubearchivist/requirements.txt

@@ -1,12 +1,13 @@
-celery==5.2.7
-Django==4.2.1
-django-auth-ldap==4.3.0
-django-cors-headers==3.14.0
+apprise==1.4.5
+celery==5.3.1
+Django==4.2.4
+django-auth-ldap==4.5.0
+django-cors-headers==4.2.0
 djangorestframework==3.14.0
-Pillow==9.5.0
-redis==4.5.4
-requests==2.30.0
+Pillow==10.0.0
+redis==5.0.0
+requests==2.31.0
 ryd-client==0.0.6
-uWSGI==2.0.21
-whitenoise==6.4.0
-yt_dlp==2023.3.4
+uWSGI==2.0.22
+whitenoise==6.5.0
+yt_dlp==2023.7.6

tubearchivist/static/css/style.css

@@ -82,6 +82,7 @@ ul {
 td, th, span, label {
     font-family: Sen-Regular, sans-serif;
     color: var(--main-font);
+    text-align: left;
 }
 
 select, input {
@@ -643,6 +644,10 @@ video:-webkit-full-screen {
     background-color: var(--highlight-bg);
 }
 
+.info-box-4 {
+    grid-template-columns: 1fr 1fr 1fr 1fr;
+}
+
 .info-box-3 {
     grid-template-columns: 1fr 1fr 1fr;
 }
@@ -651,6 +656,10 @@ video:-webkit-full-screen {
     grid-template-columns: 1fr 1fr;
 }
 
+.info-box-1 {
+    grid-template-columns: 1fr;
+}
+
 .info-box img {
     width: 80px;
     margin: 0 15px;
@@ -960,15 +969,15 @@ video:-webkit-full-screen {
     transform: translateX(-30%);
 }
 
-.info-box-item.channel-nav {
+.info-box-item.child-page-nav {
     justify-content: center;
 }
 
-.info-box-item.channel-nav a {
+.info-box-item.child-page-nav a {
     padding: 0 1rem;
 }
 
-.info-box-item.channel-nav a:hover {
+.info-box-item.child-page-nav a:hover {
     text-decoration: underline;
 }
 
@@ -1240,6 +1249,7 @@ video:-webkit-full-screen {
     .playlist-list.grid,
     .info-box-2,
     .info-box-3,
+    .info-box-4,
     .overwrite-form {
         grid-template-columns: 1fr;
     }
@@ -1306,4 +1316,12 @@ video:-webkit-full-screen {
     .playlist-nav-item img {
         width: 100%;
     }
+    .agg-channel-name {
+        min-width: 50px;
+        width: 100px;
+        max-width: 200px;
+    }
+    .td, th, span, label {
+        text-align: unset;
+    }
 }

tubearchivist/static/progress.js

@@ -30,9 +30,15 @@ function getMessages(dataOrigin) {
 
 function buildMessage(responseData, dataOrigin) {
   // filter relevant messages
-  let messages = responseData.filter(function (value) {
-    return dataOrigin.split(' ').includes(value.group.split(':')[0]);
-  }, dataOrigin);
+  let messages;
+  if (dataOrigin) {
+    messages = responseData.filter(function (value) {
+      return dataOrigin.split(' ').includes(value.group.split(':')[0]);
+    }, dataOrigin);
+  } else {
+    messages = responseData;
+  }
+
   let notifications = document.getElementById('notifications');
   let currentNotifications = notifications.childElementCount;
 

tubearchivist/static/script.js

@@ -71,20 +71,27 @@ function isWatchedButton(button) {
   }, 1000);
 }
 
-function unsubscribe(id_unsub) {
-  let payload = JSON.stringify({ unsubscribe: id_unsub });
-  sendPost(payload);
+function subscribeStatus(subscribeButton) {
+  let id = subscribeButton.getAttribute('data-id');
+  let type = subscribeButton.getAttribute('data-type');
+  let subscribe = Boolean(subscribeButton.getAttribute('data-subscribe'));
+  let apiEndpoint;
+  let data;
+  if (type === 'channel') {
+    apiEndpoint = '/api/channel/';
+    data = { data: [{ channel_id: id, channel_subscribed: subscribe }] };
+  } else if (type === 'playlist') {
+    apiEndpoint = '/api/playlist/';
+    data = { data: [{ playlist_id: id, playlist_subscribed: subscribe }] };
+  }
+  apiRequest(apiEndpoint, 'POST', data);
   let message = document.createElement('span');
-  message.innerText = 'You are unsubscribed.';
-  document.getElementById(id_unsub).replaceWith(message);
-}
-
-function subscribe(id_sub) {
-  let payload = JSON.stringify({ subscribe: id_sub });
-  sendPost(payload);
-  let message = document.createElement('span');
-  message.innerText = 'You are subscribed.';
-  document.getElementById(id_sub).replaceWith(message);
+  if (subscribe) {
+    message.innerText = 'You are subscribed.';
+  } else {
+    message.innerText = 'You are unsubscribed.';
+  }
+  subscribeButton.replaceWith(message);
 }
 
 function changeView(image) {
@@ -160,12 +167,12 @@ function dlPending() {
   }, 500);
 }
 
-function addToQueue(autostart=false) {
+function addToQueue(autostart = false) {
   let textArea = document.getElementById('id_vid_url');
   if (textArea.value === '') {
-    return
+    return;
   }
-  let toPost = {data: [{youtube_id: textArea.value, status: 'pending'}]};
+  let toPost = { data: [{ youtube_id: textArea.value, status: 'pending' }] };
   let apiEndpoint = '/api/download/';
   if (autostart) {
     apiEndpoint = `${apiEndpoint}?autostart=true`;
@@ -374,13 +381,11 @@ function deleteChannel(button) {
 function deletePlaylist(button) {
   let playlist_id = button.getAttribute('data-id');
   let playlist_action = button.getAttribute('data-action');
-  let payload = JSON.stringify({
-    'delete-playlist': {
-      'playlist-id': playlist_id,
-      'playlist-action': playlist_action,
-    },
-  });
-  sendPost(payload);
+  let apiEndpoint = `/api/playlist/${playlist_id}/`;
+  if (playlist_action === 'delete-videos') {
+    apiEndpoint += '?delete-videos=true';
+  }
+  apiRequest(apiEndpoint, 'DELETE');
   setTimeout(function () {
     window.location.replace('/playlist/');
   }, 1000);
@@ -1057,9 +1062,9 @@ function createChannel(channel, viewStyle) {
   const channelLastRefresh = channel.channel_last_refresh;
   let button;
   if (channel.channel_subscribed) {
-    button = `<button class="unsubscribe" type="button" id="${channelId}" onclick="unsubscribe(this.id)" title="Unsubscribe from ${channelName}">Unsubscribe</button>`;
+    button = `<button class="unsubscribe" type="button" data-id="${channelId}" data-subscribe="" data-type="channel" onclick="subscribeStatus(this)" title="Unsubscribe from ${channelName}">Unsubscribe</button>`;
   } else {
-    button = `<button type="button" id="${channelId}" onclick="subscribe(this.id)" title="Subscribe to ${channelName}">Subscribe</button>`;
+    button = `<button type="button" data-id="${channelId}" data-subscribe="true" data-type="channel" onclick="subscribeStatus(this)" title="Subscribe to ${channelName}">Subscribe</button>`;
   }
   // build markup
   const markup = `
@@ -1103,9 +1108,9 @@ function createPlaylist(playlist, viewStyle) {
   const playlistLastRefresh = playlist.playlist_last_refresh;
   let button;
   if (playlist.playlist_subscribed) {
-    button = `<button class="unsubscribe" type="button" id="${playlistId}" onclick="unsubscribe(this.id)" title="Unsubscribe from ${playlistName}">Unsubscribe</button>`;
+    button = `<button class="unsubscribe" type="button" data-id="${playlistId}" data-subscribe="" data-type="playlist" onclick="subscribeStatus(this)" title="Unsubscribe from ${playlistName}">Unsubscribe</button>`;
   } else {
-    button = `<button type="button" id="${playlistId}" onclick="subscribe(this.id)" title="Subscribe to ${playlistName}">Subscribe</button>`;
+    button = `<button type="button" data-id="${playlistId}" data-subscribe="true" data-type="playlist" onclick="subscribeStatus(this)" title="Subscribe to ${playlistName}">Subscribe</button>`;
   }
   const markup = `
     <div class="playlist-thumbnail">
@@ -1201,7 +1206,7 @@ function writeComments(allComments) {
 
 function createReplyButton(replyId, totalReplies) {
   let replyButton = document.createElement('button');
-  replyButton.innerHTML = `<span id="toggle-icon">+</span> ${totalReplies} replies`;
+  replyButton.innerHTML = `<span id="toggle-icon">▼</span> ${totalReplies} replies`;
   replyButton.setAttribute('data-id', replyId);
   replyButton.setAttribute('onclick', 'toggleCommentReplies(this)');
   return replyButton;
@@ -1213,10 +1218,10 @@ function toggleCommentReplies(button) {
 
   if (state === 'none' || state === '') {
     document.getElementById(commentReplyId).style.display = 'block';
-    button.querySelector('#toggle-icon').innerHTML = '-';
+    button.querySelector('#toggle-icon').innerHTML = '▲';
   } else {
     document.getElementById(commentReplyId).style.display = 'none';
-    button.querySelector('#toggle-icon').innerHTML = '+';
+    button.querySelector('#toggle-icon').innerHTML = '▼';
   }
 }
 
@@ -1318,9 +1323,9 @@ function getCookie(c_name) {
 
 // animations
 
-function textReveal() {
-  let textBox = document.getElementById('text-reveal');
-  let button = document.getElementById('text-reveal-button');
+function textReveal(button) {
+  let revealBox = button.parentElement.parentElement;
+  let textBox = revealBox.querySelector('#text-reveal');
   let textBoxHeight = textBox.style.height;
   if (textBoxHeight === 'unset') {
     textBox.style.height = '0px';
@@ -1478,6 +1483,21 @@ function doShortcut(e) {
       player.muted = !player.muted;
       break;
     }
+    case 'f': {
+      e.preventDefault();
+      if (document.fullscreenElement === null) {
+        player.requestFullscreen().catch(e => {
+          console.error(e);
+          showModal('Unable to enter fullscreen', 3000);
+        });
+      } else {
+        document.exitFullscreen().catch(e => {
+          console.error(e);
+          showModal('Unable to exit fullscreen', 3000);
+        });
+      }
+      break;
+    }
     case 'ArrowLeft': {
       e.preventDefault();
       showModal('- 5 seconds', 500);
@@ -1522,6 +1542,7 @@ function doShortcut(e) {
                 <table style="margin: auto; background: rgba(0,0,0,.5)"><tbody>
                 <tr><td>Show help</td><td>?</td>
                 <tr><td>Toggle mute</td><td>m</td>
+                <tr><td>Toggle fullscreen</td><td>f</td>
                 <tr><td>Toggle subtitles (if available)</td><td>c</td>
                 <tr><td>Increase speed</td><td>&gt;</td>
                 <tr><td>Decrease speed</td><td>&lt;</td>

tubearchivist/static/stats.js

@@ -0,0 +1,169 @@
+// build stats for settings page
+
+'use strict';
+
+/* globals apiRequest */
+
+function primaryStats() {
+  let apiEndpoint = '/api/stats/primary/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let primaryBox = document.getElementById('primaryBox');
+  clearLoading(primaryBox);
+  let videoTile = buildVideoTile(responseData);
+  primaryBox.appendChild(videoTile);
+  let channelTile = buildChannelTile(responseData);
+  primaryBox.appendChild(channelTile);
+  let playlistTile = buildPlaylistTile(responseData);
+  primaryBox.appendChild(playlistTile);
+  let downloadTile = buildDownloadTile(responseData);
+  primaryBox.appendChild(downloadTile);
+}
+
+function clearLoading(dashBox) {
+  dashBox.querySelector('#loading').remove();
+}
+
+function buildTile(titleText) {
+  let tile = document.createElement('div');
+  tile.classList.add('info-box-item');
+  let title = document.createElement('h3');
+  title.innerText = titleText;
+  tile.appendChild(title);
+  return tile;
+}
+
+function buildVideoTile(responseData) {
+  let tile = buildTile(`Total Videos: ${responseData.videos.total || 0}`);
+  let message = document.createElement('p');
+  message.innerHTML = `
+        videos: ${responseData.videos.videos || 0}<br>
+        shorts: ${responseData.videos.shorts || 0}<br>
+        streams: ${responseData.videos.streams || 0}<br>
+    `;
+  tile.appendChild(message);
+
+  return tile;
+}
+
+function buildChannelTile(responseData) {
+  let tile = buildTile(`Total Channels: ${responseData.channels.total || 0}`);
+  let message = document.createElement('p');
+  message.innerHTML = `subscribed: ${responseData.channels.sub_true || 0}`;
+  tile.appendChild(message);
+
+  return tile;
+}
+
+function buildPlaylistTile(responseData) {
+  let tile = buildTile(`Total Playlists: ${responseData.playlists.total || 0}`);
+  let message = document.createElement('p');
+  message.innerHTML = `subscribed: ${responseData.playlists.sub_true || 0}`;
+  tile.appendChild(message);
+
+  return tile;
+}
+
+function buildDownloadTile(responseData) {
+  let tile = buildTile('Downloads');
+  let message = document.createElement('p');
+  message.innerHTML = `
+        pending: ${responseData.downloads.pending || 0}<br>
+        ignored: ${responseData.downloads.ignore || 0}<br>
+    `;
+  tile.appendChild(message);
+
+  return tile;
+}
+
+function watchStats() {
+  let apiEndpoint = '/api/stats/watch/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let watchBox = document.getElementById('watchBox');
+  clearLoading(watchBox);
+
+  for (const property in responseData) {
+    let tile = buildWatchTile(property, responseData[property]);
+    watchBox.appendChild(tile);
+  }
+}
+
+function buildWatchTile(title, watchDetail) {
+  let tile = buildTile(`Total ${title}`);
+  let message = document.createElement('p');
+  message.innerHTML = `
+    ${watchDetail.items} Videos<br>
+    ${watchDetail.duration} Seconds<br>
+    ${watchDetail.duration_str} Playback
+  `;
+  if (watchDetail.progress) {
+    message.innerHTML += `<br>${Number(watchDetail.progress * 100).toFixed(2)}%`;
+  }
+  tile.appendChild(message);
+  return tile;
+}
+
+function downloadHist() {
+  let apiEndpoint = '/api/stats/downloadhist/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let histBox = document.getElementById('downHistBox');
+  clearLoading(histBox);
+  if (responseData.length === 0) {
+    let tile = buildTile('No recent downloads');
+    histBox.appendChild(tile);
+    return;
+  }
+
+  for (let i = 0; i < responseData.length; i++) {
+    const dailyStat = responseData[i];
+    let tile = buildDailyStat(dailyStat);
+    histBox.appendChild(tile);
+  }
+}
+
+function buildDailyStat(dailyStat) {
+  let tile = buildTile(dailyStat.date);
+  let message = document.createElement('p');
+  message.innerText = `new videos: ${dailyStat.count}`;
+  tile.appendChild(message);
+  return tile;
+}
+
+function biggestChannel() {
+  let apiEndpoint = '/api/stats/biggestchannels/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let tBody = document.getElementById('biggestChannelTable');
+  for (let i = 0; i < responseData.length; i++) {
+    const channelData = responseData[i];
+    let tableRow = buildChannelRow(channelData);
+    tBody.appendChild(tableRow);
+  }
+}
+
+function buildChannelRow(channelData) {
+  let tableRow = document.createElement('tr');
+  tableRow.innerHTML = `
+    <td class="agg-channel-name"><a href="/channel/${channelData.id}/">${channelData.name}</a></td>
+    <td>${channelData.doc_count}</td>
+    <td>${channelData.duration_str}</td>
+    <td>${humanFileSize(channelData.media_size)}</td>
+  `;
+  return tableRow;
+}
+
+function humanFileSize(size) {
+  let i = size === 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024));
+  return (size / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
+}
+
+async function buildStats() {
+  primaryStats();
+  watchStats();
+  downloadHist();
+  biggestChannel();
+}
+
+document.addEventListener('DOMContentLoaded', () => {
+  window.requestAnimationFrame(() => {
+    buildStats();
+  });
+});