increase version to v0.1.3

Changed:
- layout of the channel configure form
- hide if no overwrites set
- validate min es version 7.17.x
- improved build to incl debug tools

.github/workflows/lint_python.yml

@@ -11,7 +11,7 @@ jobs:
                          flake8-comprehensions isort
       - run: black --check --diff --line-length 79 .
       - run: codespell
-      - run: flake8 . --count --max-complexity=12 --max-line-length=79
+      - run: flake8 . --count --max-complexity=10 --max-line-length=79
                       --show-source --statistics
       - run: isort --check-only --line-length 79 --profile black .
       # - run: pip install -r tubearchivist/requirements.txt

CONTRIBUTING.md

@@ -25,7 +25,7 @@ This is my setup I have landed on, YMMV:
 ```bash
 ./deploy.sh test
 ```
-- The command above will also copy the file `tubarchivist/testing.sh` into the working folder of the container. Running this script will install additional debugging tools I regularly use in testing. 
+- The command above will call the docker build command with `--build-arg INSTALL_DEBUG=1` to install additional useful debug tools.
 - The `test` argument takes another optional argument to build for a specific architecture valid options are: `amd64`, `arm64` and `multi`, default is `amd64`.
 - This `deploy.sh` file is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking. 
 

Dockerfile

@@ -1,7 +1,8 @@
 # build the tube archivist image from default python slim image
 
-FROM python:3.10.2-slim-bullseye
+FROM python:3.10.3-slim-bullseye
 ARG TARGETPLATFORM
+ARG INSTALL_DEBUG
 
 ENV PYTHONUNBUFFERED 1
 
@@ -16,7 +17,7 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
     curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
         | grep browser_download_url \
-        | grep linux64-gpl-4.4.tar.xz \
+        | grep ".*master.*linux64.*tar.xz" \
         | cut -d '"' -f 4 \
         | xargs curl -L --output ffmpeg.tar.xz && \
         tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
@@ -26,6 +27,13 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
         apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
     ; fi
 
+# install debug tools for testing environment
+RUN if [ "$INSTALL_DEBUG" ] ; then \
+        apt-get -y update && apt-get -y install --no-install-recommends \
+        vim htop bmon net-tools iputils-ping procps \
+        && pip install --no-cache-dir ipython --src /usr/local/src \
+    ; fi
+
 # make folders
 RUN mkdir /cache
 RUN mkdir /youtube

deploy.sh

@@ -43,7 +43,10 @@ function sync_test {
     # pass argument to build for specific platform
 
     host="tubearchivist.local"
+    # make base folder
+    ssh "$host" "mkdir -p docker"
 
+    # copy project files to build image
     rsync -a --progress --delete-after \
         --exclude ".git" \
         --exclude ".gitignore" \
@@ -52,8 +55,8 @@ function sync_test {
         --exclude "db.sqlite3" \
         . -e ssh "$host":tubearchivist
 
-    # uncomment or copy your own docker-compose file
+    # copy default docker-compose file if not exist
-    # rsync -r --progress --delete docker-compose.yml -e ssh "$host":docker
+    rsync --progress --ignore-existing docker-compose.yml -e ssh "$host":docker
 
     if [[ $1 = "amd64" ]]; then
         platform="linux/amd64"
@@ -65,12 +68,9 @@ function sync_test {
         platform="linux/amd64"
     fi
 
-    ssh "$host" "docker buildx build --platform $platform -t bbilly1/tubearchivist:latest tubearchivist --load"
+    ssh "$host" "docker buildx build --build-arg INSTALL_DEBUG=1 --platform $platform -t bbilly1/tubearchivist:latest tubearchivist --load"
     ssh "$host" 'docker-compose -f docker/docker-compose.yml up -d'
 
-    ssh "$host" 'docker cp tubearchivist/tubearchivist/testing.sh tubearchivist:/app/testing.sh'
-    ssh "$host" 'docker exec tubearchivist chmod +x /app/testing.sh'
-
 }
 
 
@@ -91,7 +91,7 @@ function validate {
     echo "running codespell"
     codespell --skip="./.git" "$check_path"
     echo "running flake8"
-    flake8 "$check_path" --count --max-complexity=12 --max-line-length=79 \
+    flake8 "$check_path" --count --max-complexity=10 --max-line-length=79 \
         --show-source --statistics
     echo "running isort"
     isort --check-only --diff --profile black -l 79 "$check_path"

docker-compose.yml

@@ -23,7 +23,7 @@ services:
       - archivist-es
       - archivist-redis
   archivist-redis:
-    image: redislabs/rejson:latest
+    image: redislabs/rejson:latest      # For arm64 just update this line with bbilly1/rejson:latest
     container_name: archivist-redis
     restart: always
     expose:
@@ -33,7 +33,7 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
     container_name: archivist-es
     restart: always
     environment:

docs/Channels.md

@@ -22,5 +22,10 @@ Each channel will get a dedicated channel detail page accessible at `/channel/<c
 - You can *Show* the channel description, that matches with the *About* tab on YouTube.
 - The **Mark as Watched** button will mark all videos of this channel as watched.
 - The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata additionally this will also delete playlists metadata belonging to that channel.
-- The button **Find Playlists** will go out to youtube and look for playlists belonging to this channel and index all with at least one video downloaded. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.
 - The button **Show Playlists** will go to the [playlists](Playlists) page and filter the list to only show playlists from this channel.
+
+### Channel Customize
+Clicking on the *Configure* button will open a form with options to configure settings on a per channel basis. Any configurations here will overwrite your settings from the [settings](Settings) page.
+- **Download Format**: Overwrite the download qualities for videos from this channel.
+- **Auto Delete**: Automatically delete watched videos from this channel after selected days.
+- **Index Playlists**: Automatically add all Playlists with at least a video downloaded to your index. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.

docs/Settings.md

@@ -36,7 +36,17 @@ Additional settings passed to yt-dlp.
 All third party integrations of TubeArchivist will **always** be *opt in*.
 - **API**: Your access token for the Tube Archivist API. 
 - **returnyoutubedislike.com**: This will get return dislikes and average ratings for each video by integrating with the API from [returnyoutubedislike.com](https://www.returnyoutubedislike.com/).
-- **Cast**: Enable Google Cast for videos. Requires a valid SSL certificate and works only in Google Chrome.
+- **Cast**: Enabling the cast integration in the settings page will load an additional JS library from **Google**.
+    * Requirements
+        - HTTPS
+            * To use the cast integration HTTPS needs to be enabled, which can be done using a reverse proxy. This is a requirement by Google as communication to the cast device is required to be encrypted, but the content itself is not.
+        - Supported Browser
+            * A supported browser is required for this integration such as Google Chrome. Other browsers, especially Chromium-based browsers, may support casting by enabling it in the settings.
+        - Subtitles
+            * Subtitles are supported however they do not work out of the box and require additional configuration. Due to requirements by Google, to use subtitles you need additional headers which will need to be configured in your reverse proxy. See this [page](https://developers.google.com/cast/docs/web_sender/advanced#cors_requirements) for the specific requirements.
+               > You need the following headers: Content-Type, Accept-Encoding, and Range. Note that the last two headers, Accept-Encoding and Range, are additional headers that you may not have needed previously.
+               > Wildcards "*" cannot be used for the Access-Control-Allow-Origin header. If the page has protected media content, it must use a domain instead of a wildcard.
+
 
 # Scheduler Setup
 Schedule settings expect a cron like format, where the first value is minute, second is hour and third is day of the week. Day 0 is Sunday, day 1 is Monday etc.

tubearchivist/api/README.md

@@ -23,6 +23,32 @@ response = requests.get(url, headers=headers)
 ## Video Item View
 /api/video/\<video_id>/
 
+## Video Progress View
+/api/video/\<video_id>/progress  
+
+Progress is stored for each user.
+
+### Get last player position of a video
+GET /api/video/\<video_id>/progress
+```json
+{
+    "youtube_id": "<video_id>",
+    "user_id": 1,
+    "position": 100
+}
+```
+
+### Post player position of video
+POST /api/video/\<video_id>/progress
+```json
+{
+    "position": 100
+}
+```
+
+### Delete player position of video
+DELETE /api/video/\<video_id>/progress  
+
 ## Channel List View
 /api/channel/
 

tubearchivist/api/urls.py

@@ -7,6 +7,7 @@ from api.views import (
     DownloadApiView,
     PlaylistApiView,
     VideoApiView,
+    VideoProgressView,
 )
 from django.urls import path
 
@@ -16,6 +17,11 @@ urlpatterns = [
         VideoApiView.as_view(),
         name="api-video",
     ),
+    path(
+        "video/<slug:video_id>/progress/",
+        VideoProgressView.as_view(),
+        name="api-video-progress",
+    ),
     path(
         "channel/",
         ChannelApiListView.as_view(),

tubearchivist/api/views.py

@@ -1,9 +1,10 @@
 """all API views"""
 
-import requests
 from home.src.download.thumbnails import ThumbManager
+from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import UrlListParser
+from home.src.ta.ta_redis import RedisArchivist
 from home.tasks import extrac_dl, subscribe_to
 from rest_framework.authentication import (
     SessionAuthentication,
@@ -23,30 +24,21 @@ class ApiBaseView(APIView):
 
     def __init__(self):
         super().__init__()
-        self.response = {"data": False}
+        self.response = {"data": False, "config": AppConfig().config}
         self.status_code = False
         self.context = False
-        self.default_conf = AppConfig().config
-
-    def config_builder(self):
-        """build confic context"""
-        self.context = {
-            "es_url": self.default_conf["application"]["es_url"],
-            "es_auth": self.default_conf["application"]["es_auth"],
-        }
 
     def get_document(self, document_id):
         """get single document from es"""
-        es_url = self.context["es_url"]
+        path = f"{self.search_base}{document_id}"
-        url = f"{es_url}{self.search_base}{document_id}"
+        print(path)
-        print(url)
+        response, status_code = ElasticWrap(path).get()
-        response = requests.get(url, auth=self.context["es_auth"])
         try:
-            self.response["data"] = response.json()["_source"]
+            self.response["data"] = response["_source"]
         except KeyError:
             print(f"item not found: {document_id}")
             self.response["data"] = False
-        self.status_code = response.status_code
+        self.status_code = status_code
 
     def process_keys(self):
         """process keys for frontend"""
@@ -57,7 +49,7 @@ class ApiBaseView(APIView):
         if "vid_thumb_url" in all_keys:
             youtube_id = self.response["data"]["youtube_id"]
             vid_thumb_url = ThumbManager().vid_thumb_path(youtube_id)
-            cache_dir = self.default_conf["application"]["cache_dir"]
+            cache_dir = self.response["config"]["application"]["cache_dir"]
             new_thumb = f"{cache_dir}/{vid_thumb_url}"
             self.response["data"]["vid_thumb_url"] = new_thumb
         if "subtitles" in all_keys:
@@ -73,13 +65,11 @@ class ApiBaseView(APIView):
 
     def get_document_list(self, data):
         """get a list of results"""
-        es_url = self.context["es_url"]
+        print(self.search_base)
-        url = f"{es_url}{self.search_base}"
+        response, status_code = ElasticWrap(self.search_base).get(data=data)
-        print(url)
+        all_hits = response["hits"]["hits"]
-        response = requests.get(url, json=data, auth=self.context["es_auth"])
-        all_hits = response.json()["hits"]["hits"]
         self.response["data"] = [i["_source"] for i in all_hits]
-        self.status_code = response.status_code
+        self.status_code = status_code
 
 
 class VideoApiView(ApiBaseView):
@@ -87,28 +77,64 @@ class VideoApiView(ApiBaseView):
     GET: returns metadata dict of video
     """
 
-    search_base = "/ta_video/_doc/"
+    search_base = "ta_video/_doc/"
 
     def get(self, request, video_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(video_id)
         self.process_keys()
         return Response(self.response, status=self.status_code)
 
 
+class VideoProgressView(ApiBaseView):
+    """resolves to /api/video/<video_id>/
+    handle progress status for video
+    """
+
+    def get(self, request, video_id):
+        """get progress for a single video"""
+        user_id = request.user.id
+        key = f"{user_id}:progress:{video_id}"
+        video_progress = RedisArchivist().get_message(key)
+        position = video_progress.get("position", 0)
+
+        self.response = {
+            "youtube_id": video_id,
+            "user_id": user_id,
+            "position": position,
+        }
+        return Response(self.response)
+
+    def post(self, request, video_id):
+        """set progress position in redis"""
+        position = request.data.get("position", 0)
+        key = f"{request.user.id}:progress:{video_id}"
+        message = {"position": position, "youtube_id": video_id}
+        RedisArchivist().set_message(key, message, expire=False)
+        self.response = request.data
+
+        return Response(self.response)
+
+    def delete(self, request, video_id):
+        """delete progress position"""
+        key = f"{request.user.id}:progress:{video_id}"
+        RedisArchivist().del_message(key)
+        self.response = {"progress-reset": video_id}
+
+        return Response(self.response)
+
+
 class ChannelApiView(ApiBaseView):
     """resolves to /api/channel/<channel_id>/
     GET: returns metadata dict of channel
     """
 
-    search_base = "/ta_channel/_doc/"
+    search_base = "ta_channel/_doc/"
 
     def get(self, request, channel_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(channel_id)
         return Response(self.response, status=self.status_code)
 
@@ -119,13 +145,12 @@ class ChannelApiListView(ApiBaseView):
     POST: edit a list of channels
     """
 
-    search_base = "/ta_channel/_search/"
+    search_base = "ta_channel/_search/"
 
     def get(self, request):
         # pylint: disable=unused-argument
         """get request"""
         data = {"query": {"match_all": {}}}
-        self.config_builder()
         self.get_document_list(data)
         self.get_paginate()
 
@@ -154,12 +179,11 @@ class PlaylistApiView(ApiBaseView):
     GET: returns metadata dict of playlist
     """
 
-    search_base = "/ta_playlist/_doc/"
+    search_base = "ta_playlist/_doc/"
 
     def get(self, request, playlist_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(playlist_id)
         return Response(self.response, status=self.status_code)
 
@@ -169,12 +193,11 @@ class DownloadApiView(ApiBaseView):
     GET: returns metadata dict of an item in the download queue
     """
 
-    search_base = "/ta_download/_doc/"
+    search_base = "ta_download/_doc/"
 
     def get(self, request, video_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(video_id)
         return Response(self.response, status=self.status_code)
 
@@ -185,13 +208,12 @@ class DownloadApiListView(ApiBaseView):
     POST: add a list of videos to download queue
     """
 
-    search_base = "/ta_download/_search/"
+    search_base = "ta_download/_search/"
 
     def get(self, request):
         # pylint: disable=unused-argument
         """get request"""
         data = {"query": {"match_all": {}}}
-        self.config_builder()
         self.get_document_list(data)
         self.get_paginate()
         return Response(self.response)

tubearchivist/home/apps.py

@@ -1,53 +1,109 @@
 """handle custom startup functions"""
 
 import os
+import sys
 
 from django.apps import AppConfig
+from home.src.es.connect import ElasticWrap
 from home.src.es.index_setup import index_check
 from home.src.ta.config import AppConfig as ArchivistConfig
 from home.src.ta.ta_redis import RedisArchivist
 
 
-def sync_redis_state():
+class StartupCheck:
-    """make sure redis gets new config.json values"""
+    """checks to run at application startup"""
-    print("sync redis")
-    config_handler = ArchivistConfig()
-    config_handler.load_new_defaults()
 
+    MIN_MAJOR, MAX_MAJOR = 7, 7
+    MIN_MINOR = 17
 
-def make_folders():
+    def __init__(self):
-    """make needed cache folders here so docker doesn't mess it up"""
+        self.config_handler = ArchivistConfig()
-    folders = [
+        self.redis_con = RedisArchivist()
-        "download",
+        self.has_run = self.get_has_run()
-        "channels",
-        "videos",
-        "playlists",
-        "import",
-        "backup",
-    ]
-    config = ArchivistConfig().config
-    cache_dir = config["application"]["cache_dir"]
-    for folder in folders:
-        folder_path = os.path.join(cache_dir, folder)
-        try:
-            os.makedirs(folder_path)
-        except FileExistsError:
-            continue
 
+    def run(self):
+        """run all startup checks"""
+        print("run startup checks")
+        self.es_version_check()
+        self.release_lock()
+        index_check()
+        self.sync_redis_state()
+        self.make_folders()
+        self.set_has_run()
 
-def release_lock():
+    def get_has_run(self):
-    """make sure there are no leftover locks set in redis on container start"""
+        """validate if check has already executed"""
-    all_locks = [
+        return self.redis_con.get_message("startup_check")
-        "manual_import",
+
-        "downloading",
+    def set_has_run(self):
-        "dl_queue",
+        """startup checks run"""
-        "dl_queue_id",
+        message = {"status": True}
-        "rescan",
+        self.redis_con.set_message("startup_check", message, expire=120)
-    ]
+
-    for lock in all_locks:
+    def sync_redis_state(self):
-        response = RedisArchivist().del_message(lock)
+        """make sure redis gets new config.json values"""
-        if response:
+        print("sync redis")
-            print("deleted leftover key from redis: " + lock)
+        self.config_handler.load_new_defaults()
+
+    def make_folders(self):
+        """make needed cache folders here so docker doesn't mess it up"""
+        folders = [
+            "download",
+            "channels",
+            "videos",
+            "playlists",
+            "import",
+            "backup",
+        ]
+        cache_dir = self.config_handler.config["application"]["cache_dir"]
+        for folder in folders:
+            folder_path = os.path.join(cache_dir, folder)
+            try:
+                os.makedirs(folder_path)
+            except FileExistsError:
+                continue
+
+    def release_lock(self):
+        """make sure there are no leftover locks set in redis"""
+        all_locks = [
+            "startup_check",
+            "manual_import",
+            "downloading",
+            "dl_queue",
+            "dl_queue_id",
+            "rescan",
+        ]
+        for lock in all_locks:
+            response = self.redis_con.del_message(lock)
+            if response:
+                print("deleted leftover key from redis: " + lock)
+
+    def is_invalid(self, version):
+        """return true if es version is invalid, false if ok"""
+        major, minor = [int(i) for i in version.split(".")[:2]]
+        if not self.MIN_MAJOR <= major <= self.MAX_MAJOR:
+            return True
+
+        if minor >= self.MIN_MINOR:
+            return False
+
+        return True
+
+    def es_version_check(self):
+        """check for minimal elasticsearch version"""
+        response, _ = ElasticWrap("/").get()
+        version = response["version"]["number"]
+        invalid = self.is_invalid(version)
+
+        if invalid:
+            print(
+                "minial required elasticsearch version: "
+                + f"{self.MIN_MAJOR}.{self.MIN_MINOR}, "
+                + "please update to recommended version."
+            )
+            sys.exit(1)
+
+        print("elasticsearch version check passed")
 
 
 class HomeConfig(AppConfig):
@@ -57,7 +113,9 @@ class HomeConfig(AppConfig):
     name = "home"
 
     def ready(self):
-        release_lock()
+        startup = StartupCheck()
-        index_check()
+        if startup.has_run["status"]:
-        sync_redis_state()
+            print("startup checks run in other thread")
-        make_folders()
+            return
+
+        startup.run()

tubearchivist/home/src/download/queue.py

@@ -5,36 +5,132 @@ Functionality:
 """
 
 import json
-import os
 from datetime import datetime
 
-import requests
 import yt_dlp
-from home.src.download.subscriptions import ChannelSubscription
+from home.src.download.subscriptions import (
-from home.src.es.connect import IndexPaginate
+    ChannelSubscription,
+    PlaylistSubscription,
+)
+from home.src.download.thumbnails import ThumbManager
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.config import AppConfig
+from home.src.ta.helper import DurationConverter
-from home.src.ta.helper import DurationConverter, ignore_filelist
 from home.src.ta.ta_redis import RedisArchivist
 
 
-class PendingList:
+class PendingIndex:
-    """manage the pending videos list"""
+    """base class holding all export methods"""
-
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
-    VIDEOS = CONFIG["application"]["videos"]
 
     def __init__(self):
-        self.all_channel_ids = False
+        self.all_pending = False
-        self.all_downloaded = False
+        self.all_ignored = False
-        self.missing_from_playlists = []
+        self.all_videos = False
+        self.all_channels = False
+        self.channel_overwrites = False
+        self.video_overwrites = False
+        self.to_skip = False
 
-    def parse_url_list(self, youtube_ids):
+    def get_download(self):
+        """get a list of all pending videos in ta_download"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"timestamp": {"order": "asc"}}],
+        }
+        all_results = IndexPaginate("ta_download", data).get_results()
+
+        self.all_pending = []
+        self.all_ignored = []
+        self.to_skip = []
+
+        for result in all_results:
+            self.to_skip.append(result["youtube_id"])
+            if result["status"] == "pending":
+                self.all_pending.append(result)
+            elif result["status"] == "ignore":
+                self.all_ignored.append(result)
+
+    def get_indexed(self):
+        """get a list of all videos indexed"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        self.all_videos = IndexPaginate("ta_video", data).get_results()
+        for video in self.all_videos:
+            self.to_skip.append(video["youtube_id"])
+
+    def get_channels(self):
+        """get a list of all channels indexed"""
+        self.all_channels = []
+        self.channel_overwrites = {}
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"channel_id": {"order": "asc"}}],
+        }
+        channels = IndexPaginate("ta_channel", data).get_results()
+
+        for channel in channels:
+            channel_id = channel["channel_id"]
+            self.all_channels.append(channel_id)
+            if channel.get("channel_overwrites"):
+                self.channel_overwrites.update(
+                    {channel_id: channel.get("channel_overwrites")}
+                )
+
+        self._map_overwrites()
+
+    def _map_overwrites(self):
+        """map video ids to channel ids overwrites"""
+        self.video_overwrites = {}
+        for video in self.all_pending:
+            video_id = video["youtube_id"]
+            channel_id = video["channel_id"]
+            overwrites = self.channel_overwrites.get(channel_id, False)
+            if overwrites:
+                self.video_overwrites.update({video_id: overwrites})
+
+
+class PendingInteract:
+    """interact with items in download queue"""
+
+    def __init__(self, video_id=False, status=False):
+        self.video_id = video_id
+        self.status = status
+
+    def delete_item(self):
+        """delete single item from pending"""
+        path = f"ta_download/_doc/{self.video_id}"
+        _, _ = ElasticWrap(path).delete()
+
+    def delete_by_status(self):
+        """delete all matching item by status"""
+        data = {"query": {"term": {"status": {"value": self.status}}}}
+        path = "ta_download/_delete_by_query"
+        _, _ = ElasticWrap(path).post(data=data)
+
+    def update_status(self):
+        """update status field of pending item"""
+        data = {"doc": {"status": self.status}}
+        path = f"ta_download/_update/{self.video_id}"
+        _, _ = ElasticWrap(path).post(data=data)
+
+
+class PendingList(PendingIndex):
+    """manage the pending videos list"""
+
+    def __init__(self, youtube_ids=False):
+        super().__init__()
+        self.youtube_ids = youtube_ids
+        self.to_skip = False
+        self.missing_videos = False
+
+    def parse_url_list(self):
         """extract youtube ids from list"""
-        missing_videos = []
+        self.missing_videos = []
-        for entry in youtube_ids:
+        self.get_download()
+        self.get_indexed()
+        for entry in self.youtube_ids:
             # notify
             mess_dict = {
                 "status": "message:add",
@@ -43,97 +139,89 @@ class PendingList:
                 "message": "Extracting lists",
             }
             RedisArchivist().set_message("message:add", mess_dict)
-            # extract
+            self._process_entry(entry)
-            url = entry["url"]
-            url_type = entry["type"]
-            if url_type == "video":
-                missing_videos.append(url)
-            elif url_type == "channel":
-                video_results = ChannelSubscription().get_last_youtube_videos(
-                    url, limit=False
-                )
-                youtube_ids = [i[0] for i in video_results]
-                missing_videos = missing_videos + youtube_ids
-            elif url_type == "playlist":
-                self.missing_from_playlists.append(entry)
-                playlist = YoutubePlaylist(url)
-                playlist.build_json()
-                video_results = playlist.json_data.get("playlist_entries")
-                youtube_ids = [i["youtube_id"] for i in video_results]
-                missing_videos = missing_videos + youtube_ids
 
-        return missing_videos
+    def _process_entry(self, entry):
+        """process single entry from url list"""
+        if entry["type"] == "video":
+            self._add_video(entry["url"])
+        elif entry["type"] == "channel":
+            self._parse_channel(entry["url"])
+        elif entry["type"] == "playlist":
+            self._parse_playlist(entry["url"])
+            new_thumbs = PlaylistSubscription().process_url_str(
+                [entry], subscribed=False
+            )
+            ThumbManager().download_playlist(new_thumbs)
+        else:
+            raise ValueError(f"invalid url_type: {entry}")
 
-    def add_to_pending(self, missing_videos, ignore=False):
+    def _add_video(self, url):
-        """build the bulk json data from pending"""
+        """add video to list"""
-        # check if channel is indexed
+        if url not in self.missing_videos and url not in self.to_skip:
-        channel_handler = ChannelSubscription()
+            self.missing_videos.append(url)
-        all_indexed = channel_handler.get_channels(subscribed_only=False)
+
-        self.all_channel_ids = [i["channel_id"] for i in all_indexed]
+    def _parse_channel(self, url):
-        # check if already there
+        """add all videos of channel to list"""
-        self.all_downloaded = self.get_all_downloaded()
+        video_results = ChannelSubscription().get_last_youtube_videos(
+            url, limit=False
+        )
+        youtube_ids = [i[0] for i in video_results]
+        for video_id in youtube_ids:
+            self._add_video(video_id)
+
+    def _parse_playlist(self, url):
+        """add all videos of playlist to list"""
+        playlist = YoutubePlaylist(url)
+        playlist.build_json()
+        video_results = playlist.json_data.get("playlist_entries")
+        youtube_ids = [i["youtube_id"] for i in video_results]
+        for video_id in youtube_ids:
+            self._add_video(video_id)
+
+    def add_to_pending(self, status="pending"):
+        """add missing videos to pending list"""
+        self.get_channels()
+        bulk_list = []
+
+        thumb_handler = ThumbManager()
+        for idx, youtube_id in enumerate(self.missing_videos):
+            video_details = self.get_youtube_details(youtube_id)
+            if not video_details:
+                continue
+
+            video_details["status"] = status
+            action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(video_details))
+
+            thumb_needed = [(youtube_id, video_details["vid_thumb_url"])]
+            thumb_handler.download_vid(thumb_needed)
+            self._notify_add(idx)
 
-        bulk_list, all_videos_added = self.build_bulk(missing_videos, ignore)
         # add last newline
         bulk_list.append("\n")
         query_str = "\n".join(bulk_list)
-        headers = {"Content-type": "application/x-ndjson"}
+        _, _ = ElasticWrap("_bulk").post(query_str, ndjson=True)
-        url = self.ES_URL + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request)
-            raise ValueError("failed to add video to download queue")
 
-        return all_videos_added
+    def _notify_add(self, idx):
+        """send notification for adding videos to download queue"""
+        progress = f"{idx + 1}/{len(self.missing_videos)}"
+        mess_dict = {
+            "status": "message:add",
+            "level": "info",
+            "title": "Adding new videos to download queue.",
+            "message": "Progress: " + progress,
+        }
+        if idx + 1 == len(self.missing_videos):
+            RedisArchivist().set_message("message:add", mess_dict, expire=4)
+        else:
+            RedisArchivist().set_message("message:add", mess_dict)
 
-    def build_bulk(self, missing_videos, ignore=False):
+        if idx + 1 % 25 == 0:
-        """build the bulk lists"""
+            print("adding to queue progress: " + progress)
-        bulk_list = []
-        all_videos_added = []
 
-        for idx, youtube_id in enumerate(missing_videos):
+    def get_youtube_details(self, youtube_id):
-            # check if already downloaded
-            if youtube_id in self.all_downloaded:
-                continue
-
-            video = self.get_youtube_details(youtube_id)
-            # skip on download error
-            if not video:
-                continue
-
-            channel_indexed = video["channel_id"] in self.all_channel_ids
-            video["channel_indexed"] = channel_indexed
-            if ignore:
-                video["status"] = "ignore"
-            else:
-                video["status"] = "pending"
-            action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(video))
-            all_videos_added.append((youtube_id, video["vid_thumb_url"]))
-            # notify
-            progress = f"{idx + 1}/{len(missing_videos)}"
-            mess_dict = {
-                "status": "message:add",
-                "level": "info",
-                "title": "Adding new videos to download queue.",
-                "message": "Progress: " + progress,
-            }
-            if idx + 1 == len(missing_videos):
-                RedisArchivist().set_message(
-                    "message:add", mess_dict, expire=4
-                )
-            else:
-                RedisArchivist().set_message("message:add", mess_dict)
-            if idx + 1 % 25 == 0:
-                print("adding to queue progress: " + progress)
-
-        return bulk_list, all_videos_added
-
-    @staticmethod
-    def get_youtube_details(youtube_id):
         """get details from youtubedl for single pending video"""
         obs = {
             "default_search": "ytsearch",
@@ -151,113 +239,29 @@ class PendingList:
         # stop if video is streaming live now
         if vid["is_live"]:
             return False
-        # parse response
+
-        seconds = vid["duration"]
+        return self._parse_youtube_details(vid)
-        duration_str = DurationConverter.get_str(seconds)
+
+    def _parse_youtube_details(self, vid):
+        """parse response"""
+        vid_id = vid.get("id")
+        duration_str = DurationConverter.get_str(vid["duration"])
         if duration_str == "NA":
-            print(f"skip extracting duration for: {youtube_id}")
+            print(f"skip extracting duration for: {vid_id}")
-        upload_date = vid["upload_date"]
+        published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
-        upload_dt = datetime.strptime(upload_date, "%Y%m%d")
+            "%Y-%m-%d"
-        published = upload_dt.strftime("%Y-%m-%d")
+        )
+
         # build dict
         youtube_details = {
-            "youtube_id": youtube_id,
+            "youtube_id": vid_id,
             "channel_name": vid["channel"],
             "vid_thumb_url": vid["thumbnail"],
             "title": vid["title"],
             "channel_id": vid["channel_id"],
+            "channel_indexed": vid["channel_id"] in self.all_channels,
             "duration": duration_str,
             "published": published,
             "timestamp": int(datetime.now().strftime("%s")),
         }
         return youtube_details
-
-    @staticmethod
-    def get_all_pending():
-        """get a list of all pending videos in ta_download"""
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"timestamp": {"order": "asc"}}],
-        }
-        all_results = IndexPaginate("ta_download", data).get_results()
-
-        all_pending = []
-        all_ignore = []
-
-        for result in all_results:
-            if result["status"] == "pending":
-                all_pending.append(result)
-            elif result["status"] == "ignore":
-                all_ignore.append(result)
-
-        return all_pending, all_ignore
-
-    @staticmethod
-    def get_all_indexed():
-        """get a list of all videos indexed"""
-
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"published": {"order": "desc"}}],
-        }
-        all_indexed = IndexPaginate("ta_video", data).get_results()
-
-        return all_indexed
-
-    def get_all_downloaded(self):
-        """get a list of all videos in archive"""
-        channel_folders = os.listdir(self.VIDEOS)
-        all_channel_folders = ignore_filelist(channel_folders)
-        all_downloaded = []
-        for channel_folder in all_channel_folders:
-            channel_path = os.path.join(self.VIDEOS, channel_folder)
-            videos = os.listdir(channel_path)
-            all_videos = ignore_filelist(videos)
-            youtube_vids = [i[9:20] for i in all_videos]
-            for youtube_id in youtube_vids:
-                all_downloaded.append(youtube_id)
-        return all_downloaded
-
-    def delete_from_pending(self, youtube_id):
-        """delete the youtube_id from ta_download"""
-        url = f"{self.ES_URL}/ta_download/_doc/{youtube_id}"
-        response = requests.delete(url, auth=self.ES_AUTH)
-        if not response.ok:
-            print(response.text)
-
-    def delete_pending(self, status):
-        """delete download queue based on status value"""
-        data = {"query": {"term": {"status": {"value": status}}}}
-        payload = json.dumps(data)
-        url = self.ES_URL + "/ta_download/_delete_by_query"
-        headers = {"Content-type": "application/json"}
-        response = requests.post(
-            url, data=payload, headers=headers, auth=self.ES_AUTH
-        )
-        if not response.ok:
-            print(response.text)
-
-    def ignore_from_pending(self, ignore_list):
-        """build the bulk query string"""
-
-        stamp = int(datetime.now().strftime("%s"))
-        bulk_list = []
-
-        for youtube_id in ignore_list:
-            action = {"update": {"_id": youtube_id, "_index": "ta_download"}}
-            source = {"doc": {"status": "ignore", "timestamp": stamp}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(source))
-
-        # add last newline
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-
-        headers = {"Content-type": "application/x-ndjson"}
-        url = self.ES_URL + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request)
-            raise ValueError("failed to set video to ignore")

tubearchivist/home/src/download/subscriptions.py

@@ -48,27 +48,33 @@ class ChannelSubscription:
         }
         if limit:
             obs["playlistend"] = self.channel_size
-        chan = yt_dlp.YoutubeDL(obs).extract_info(url, download=False)
+
+        try:
+            chan = yt_dlp.YoutubeDL(obs).extract_info(url, download=False)
+        except yt_dlp.utils.DownloadError:
+            print(f"{channel_id}: failed to extract videos, skipping.")
+            return False
+
         last_videos = [(i["id"], i["title"]) for i in chan["entries"]]
         return last_videos
 
     def find_missing(self):
         """add missing videos from subscribed channels to pending"""
         all_channels = self.get_channels()
-        pending_handler = queue.PendingList()
+        pending = queue.PendingList()
-        all_pending, all_ignore = pending_handler.get_all_pending()
+        pending.get_download()
-        all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
+        pending.get_indexed()
-        all_downloaded = pending_handler.get_all_downloaded()
-        to_ignore = all_ids + all_downloaded
 
         missing_videos = []
 
         for idx, channel in enumerate(all_channels):
             channel_id = channel["channel_id"]
             last_videos = self.get_last_youtube_videos(channel_id)
-            for video in last_videos:
+
-                if video[0] not in to_ignore:
+            if last_videos:
-                    missing_videos.append(video[0])
+                for video in last_videos:
+                    if video[0] not in pending.to_skip:
+                        missing_videos.append(video[0])
             # notify
             message = {
                 "status": "message:rescan",
@@ -121,7 +127,11 @@ class PlaylistSubscription:
 
     def process_url_str(self, new_playlists, subscribed=True):
         """process playlist subscribe form url_str"""
-        all_indexed = queue.PendingList().get_all_indexed()
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        all_indexed = IndexPaginate("ta_video", data).get_results()
         all_youtube_ids = [i["youtube_id"] for i in all_indexed]
 
         new_thumbs = []
@@ -171,12 +181,11 @@ class PlaylistSubscription:
     @staticmethod
     def get_to_ignore():
         """get all youtube_ids already downloaded or ignored"""
-        pending_handler = queue.PendingList()
+        pending = queue.PendingList()
-        all_pending, all_ignore = pending_handler.get_all_pending()
+        pending.get_download()
-        all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
+        pending.get_indexed()
-        all_downloaded = pending_handler.get_all_downloaded()
+
-        to_ignore = all_ids + all_downloaded
+        return pending.to_skip
-        return to_ignore
 
     def find_missing(self):
         """find videos in subscribed playlists not downloaded yet"""

tubearchivist/home/src/download/thumbnails.py

@@ -58,11 +58,13 @@ class ThumbManager:
     def get_needed_thumbs(self, missing_only=False):
         """get a list of all missing thumbnails"""
         all_thumbs = self.get_all_thumbs()
-        all_indexed = queue.PendingList().get_all_indexed()
+
-        all_in_queue, all_ignored = queue.PendingList().get_all_pending()
+        pending = queue.PendingList()
+        pending.get_download()
+        pending.get_indexed()
 
         needed_thumbs = []
-        for video in all_indexed:
+        for video in pending.all_videos:
             youtube_id = video["youtube_id"]
             thumb_url = video["vid_thumb_url"]
             if missing_only:
@@ -71,7 +73,7 @@ class ThumbManager:
             else:
                 needed_thumbs.append((youtube_id, thumb_url))
 
-        for video in all_in_queue + all_ignored:
+        for video in pending.all_pending + pending.all_ignored:
             youtube_id = video["youtube_id"]
             thumb_url = video["vid_thumb_url"]
             if missing_only:
@@ -276,9 +278,11 @@ class ThumbManager:
 
     def get_thumb_list(self):
         """get list of mediafiles and matching thumbnails"""
-        all_indexed = queue.PendingList().get_all_indexed()
+        pending = queue.PendingList()
+        pending.get_indexed()
+
         video_list = []
-        for video in all_indexed:
+        for video in pending.all_videos:
             youtube_id = video["youtube_id"]
             media_url = os.path.join(self.MEDIA_DIR, video["media_url"])
             thumb_path = os.path.join(

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -11,11 +11,10 @@ import shutil
 from datetime import datetime
 from time import sleep
 
-import requests
 import yt_dlp
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import PlaylistSubscription
-from home.src.es.connect import IndexPaginate
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo, index_new_video
@@ -24,6 +23,123 @@ from home.src.ta.helper import clean_string, ignore_filelist
 from home.src.ta.ta_redis import RedisArchivist, RedisQueue
 
 
+class DownloadPostProcess:
+    """handle task to run after download queue finishes"""
+
+    def __init__(self, download):
+        self.download = download
+        self.now = int(datetime.now().strftime("%s"))
+        self.pending = False
+
+    def run(self):
+        """run all functions"""
+        self.pending = PendingList()
+        self.pending.get_download()
+        self.pending.get_channels()
+        self.pending.get_indexed()
+        self.auto_delete_all()
+        self.auto_delete_overwrites()
+        self.validate_playlists()
+
+    def auto_delete_all(self):
+        """handle auto delete"""
+        autodelete_days = self.download.config["downloads"]["autodelete_days"]
+        if not autodelete_days:
+            return
+
+        print(f"auto delete older than {autodelete_days} days")
+        now_lte = self.now - autodelete_days * 24 * 60 * 60
+        data = {
+            "query": {"range": {"player.watched_date": {"lte": now_lte}}},
+            "sort": [{"player.watched_date": {"order": "asc"}}],
+        }
+        self._auto_delete_watched(data)
+
+    def auto_delete_overwrites(self):
+        """handle per channel auto delete from overwrites"""
+        for channel_id, value in self.pending.channel_overwrites.items():
+            if "autodelete_days" in value:
+                autodelete_days = value.get("autodelete_days")
+                print(f"{channel_id}: delete older than {autodelete_days}d")
+                now_lte = self.now - autodelete_days * 24 * 60 * 60
+                must_list = [
+                    {"range": {"player.watched_date": {"lte": now_lte}}},
+                    {"term": {"channel.channel_id": {"value": channel_id}}},
+                ]
+                data = {
+                    "query": {"bool": {"must": must_list}},
+                    "sort": [{"player.watched_date": {"order": "desc"}}],
+                }
+                self._auto_delete_watched(data)
+
+    @staticmethod
+    def _auto_delete_watched(data):
+        """delete watched videos after x days"""
+        to_delete = IndexPaginate("ta_video", data).get_results()
+        if not to_delete:
+            return
+
+        for video in to_delete:
+            youtube_id = video["youtube_id"]
+            print(f"{youtube_id}: auto delete video")
+            YoutubeVideo(youtube_id).delete_media_file()
+
+        print("add deleted to ignore list")
+        vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
+        pending = PendingList(youtube_ids=vids)
+        pending.parse_url_list()
+        pending.add_to_pending(status="ignore")
+
+    def validate_playlists(self):
+        """look for playlist needing to update"""
+        for id_c, channel_id in enumerate(self.download.channels):
+            channel = YoutubeChannel(channel_id)
+            overwrites = self.pending.channel_overwrites.get(channel_id, False)
+            if overwrites and overwrites.get("index_playlists"):
+                # validate from remote
+                channel.index_channel_playlists()
+                continue
+
+            # validate from local
+            playlists = channel.get_indexed_playlists()
+            all_channel_playlist = [i["playlist_id"] for i in playlists]
+            self._validate_channel_playlist(all_channel_playlist, id_c)
+
+    def _validate_channel_playlist(self, all_channel_playlist, id_c):
+        """scan channel for playlist needing update"""
+        all_youtube_ids = [i["youtube_id"] for i in self.pending.all_videos]
+        for id_p, playlist_id in enumerate(all_channel_playlist):
+            playlist = YoutubePlaylist(playlist_id)
+            playlist.all_youtube_ids = all_youtube_ids
+            playlist.build_json(scrape=True)
+            if not playlist.json_data:
+                playlist.deactivate()
+
+            playlist.add_vids_to_playlist()
+            playlist.upload_to_es()
+            self._notify_playlist_progress(all_channel_playlist, id_c, id_p)
+
+    def _notify_playlist_progress(self, all_channel_playlist, id_c, id_p):
+        """notify to UI"""
+        title = (
+            "Processing playlists for channels: "
+            + f"{id_c + 1}/{len(self.download.channels)}"
+        )
+        message = f"Progress: {id_p + 1}/{len(all_channel_playlist)}"
+        mess_dict = {
+            "status": "message:download",
+            "level": "info",
+            "title": title,
+            "message": message,
+        }
+        if id_p + 1 == len(all_channel_playlist):
+            RedisArchivist().set_message(
+                "message:download", mess_dict, expire=4
+            )
+        else:
+            RedisArchivist().set_message("message:download", mess_dict)
+
+
 class VideoDownloader:
     """
     handle the video download functionality
@@ -32,6 +148,7 @@ class VideoDownloader:
 
     def __init__(self, youtube_id_list=False):
         self.obs = False
+        self.video_overwrites = False
         self.youtube_id_list = youtube_id_list
         self.config = AppConfig().config
         self._build_obs()
@@ -39,7 +156,12 @@ class VideoDownloader:
 
     def run_queue(self):
         """setup download queue in redis loop until no more items"""
-        queue = RedisQueue("dl_queue")
+        pending = PendingList()
+        pending.get_download()
+        pending.get_channels()
+        self.video_overwrites = pending.video_overwrites
+
+        queue = RedisQueue()
 
         limit_queue = self.config["downloads"]["limit_count"]
         if limit_queue:
@@ -60,10 +182,9 @@ class VideoDownloader:
             self.move_to_archive(vid_dict)
             self._delete_from_pending(youtube_id)
 
-        autodelete_days = self.config["downloads"]["autodelete_days"]
+        # post processing
-        if autodelete_days:
+        self._add_subscribed_channels()
-            print(f"auto delete older than {autodelete_days} days")
+        DownloadPostProcess(self).run()
-            self.auto_delete_watched(autodelete_days)
 
     @staticmethod
     def add_pending():
@@ -75,8 +196,9 @@ class VideoDownloader:
             "message": "Scanning your download queue.",
         }
         RedisArchivist().set_message("message:download", mess_dict)
-        all_pending, _ = PendingList().get_all_pending()
+        pending = PendingList()
-        to_add = [i["youtube_id"] for i in all_pending]
+        pending.get_download()
+        to_add = [i["youtube_id"] for i in pending.all_pending]
         if not to_add:
             # there is nothing pending
             print("download queue is empty")
@@ -89,8 +211,7 @@ class VideoDownloader:
             RedisArchivist().set_message("message:download", mess_dict)
             return
 
-        queue = RedisQueue("dl_queue")
+        RedisQueue().add_list(to_add)
-        queue.add_list(to_add)
 
     @staticmethod
     def _progress_hook(response):
@@ -181,16 +302,30 @@ class VideoDownloader:
 
         self.obs["postprocessors"] = postprocessors
 
+    def get_format_overwrites(self, youtube_id):
+        """get overwrites from single video"""
+        overwrites = self.video_overwrites.get(youtube_id, False)
+        if overwrites:
+            return overwrites.get("download_format", False)
+
+        return False
+
     def _dl_single_vid(self, youtube_id):
         """download single video"""
+        obs = self.obs.copy()
+        format_overwrite = self.get_format_overwrites(youtube_id)
+        if format_overwrite:
+            obs["format"] = format_overwrite
+
         dl_cache = self.config["application"]["cache_dir"] + "/download/"
 
         # check if already in cache to continue from there
         all_cached = ignore_filelist(os.listdir(dl_cache))
         for file_name in all_cached:
             if youtube_id in file_name:
-                self.obs["outtmpl"] = os.path.join(dl_cache, file_name)
+                obs["outtmpl"] = os.path.join(dl_cache, file_name)
-        with yt_dlp.YoutubeDL(self.obs) as ydl:
+
+        with yt_dlp.YoutubeDL(obs) as ydl:
             try:
                 ydl.download([youtube_id])
             except yt_dlp.utils.DownloadError:
@@ -234,14 +369,11 @@ class VideoDownloader:
         if host_uid and host_gid:
             os.chown(new_file_path, host_uid, host_gid)
 
-    def _delete_from_pending(self, youtube_id):
+    @staticmethod
+    def _delete_from_pending(youtube_id):
         """delete downloaded video from pending index if its there"""
-        es_url = self.config["application"]["es_url"]
+        path = f"ta_download/_doc/{youtube_id}"
-        es_auth = self.config["application"]["es_auth"]
+        _, _ = ElasticWrap(path).delete()
-        url = f"{es_url}/ta_download/_doc/{youtube_id}"
-        response = requests.delete(url, auth=es_auth)
-        if not response.ok and not response.status_code == 404:
-            print(response.text)
 
     def _add_subscribed_channels(self):
         """add all channels subscribed to refresh"""
@@ -254,63 +386,3 @@ class VideoDownloader:
             self.channels.add(channel_id)
 
         return
-
-    def validate_playlists(self):
-        """look for playlist needing to update"""
-        print("sync playlists")
-        self._add_subscribed_channels()
-        all_indexed = PendingList().get_all_indexed()
-        all_youtube_ids = [i["youtube_id"] for i in all_indexed]
-        for id_c, channel_id in enumerate(self.channels):
-            playlists = YoutubeChannel(channel_id).get_indexed_playlists()
-            all_playlist_ids = [i["playlist_id"] for i in playlists]
-            for id_p, playlist_id in enumerate(all_playlist_ids):
-                playlist = YoutubePlaylist(playlist_id)
-                playlist.all_youtube_ids = all_youtube_ids
-                playlist.build_json(scrape=True)
-                if not playlist.json_data:
-                    playlist.deactivate()
-
-                playlist.add_vids_to_playlist()
-                playlist.upload_to_es()
-
-                # notify
-                title = (
-                    "Processing playlists for channels: "
-                    + f"{id_c + 1}/{len(self.channels)}"
-                )
-                message = f"Progress: {id_p + 1}/{len(all_playlist_ids)}"
-                mess_dict = {
-                    "status": "message:download",
-                    "level": "info",
-                    "title": title,
-                    "message": message,
-                }
-                if id_p + 1 == len(all_playlist_ids):
-                    RedisArchivist().set_message(
-                        "message:download", mess_dict, expire=4
-                    )
-                else:
-                    RedisArchivist().set_message("message:download", mess_dict)
-
-    @staticmethod
-    def auto_delete_watched(autodelete_days):
-        """delete watched videos after x days"""
-        now = int(datetime.now().strftime("%s"))
-        now_lte = now - autodelete_days * 24 * 60 * 60
-        data = {
-            "query": {"range": {"player.watched_date": {"lte": now_lte}}},
-            "sort": [{"player.watched_date": {"order": "asc"}}],
-        }
-        all_to_delete = IndexPaginate("ta_video", data).get_results()
-        all_youtube_ids = [i["youtube_id"] for i in all_to_delete]
-        if not all_youtube_ids:
-            return
-
-        for youtube_id in all_youtube_ids:
-            print(f"autodelete {youtube_id}")
-            YoutubeVideo(youtube_id).delete_media_file()
-
-        print("add deleted to ignore list")
-        pending_handler = PendingList()
-        pending_handler.add_to_pending(all_youtube_ids, ignore=True)

tubearchivist/home/src/es/connect.py

@@ -93,11 +93,12 @@ class IndexPaginate:
 
     DEFAULT_SIZE = 500
 
-    def __init__(self, index_name, data, size=False):
+    def __init__(self, index_name, data, size=False, keep_source=False):
         self.index_name = index_name
         self.data = data
         self.pit_id = False
         self.size = size
+        self.keep_source = keep_source
 
     def get_results(self):
         """get all results"""
@@ -132,7 +133,10 @@ class IndexPaginate:
             all_hits = response["hits"]["hits"]
             if all_hits:
                 for hit in all_hits:
-                    source = hit["_source"]
+                    if self.keep_source:
+                        source = hit
+                    else:
+                        source = hit["_source"]
                     search_after = hit["sort"]
                     all_results.append(source)
                 # update search_after with last hit data

tubearchivist/home/src/es/index_mapping.json

@@ -39,6 +39,19 @@
                 "channel_last_refresh": {
                     "type": "date",
                     "format": "epoch_second"
+                },
+                "channel_overwrites": {
+                    "properties": {
+                        "download_format": {
+                            "type": "text"
+                        },
+                        "autodelete_days": {
+                            "type": "long"
+                        },
+                        "index_playlists": {
+                            "type": "boolean"
+                        }
+                    }
                 }
             },
             "expected_set": {
@@ -102,6 +115,19 @@
                         "channel_last_refresh": {
                             "type": "date",
                             "format": "epoch_second"
+                        },
+                        "channel_overwrites": {
+                            "properties": {
+                                "download_format": {
+                                    "type": "text"
+                                },
+                                "autodelete_days": {
+                                    "type": "long"
+                                },
+                                "index_playlists": {
+                                    "type": "boolean"
+                                }
+                            }
                         }
                     }
                 },

tubearchivist/home/src/es/index_setup.py

@@ -10,7 +10,7 @@ import os
 import zipfile
 from datetime import datetime
 
-import requests
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import ignore_filelist
 
@@ -20,11 +20,6 @@ class ElasticIndex:
     handle mapping and settings on elastic search for a given index
     """
 
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
-    HEADERS = {"Content-type": "application/json"}
-
     def __init__(self, index_name, expected_map, expected_set):
         self.index_name = index_name
         self.expected_map = expected_map
@@ -33,15 +28,9 @@ class ElasticIndex:
 
     def index_exists(self):
         """check if index already exists and return mapping if it does"""
-        index_name = self.index_name
+        response, status_code = ElasticWrap(f"ta_{self.index_name}").get()
-        url = f"{self.ES_URL}/ta_{index_name}"
+        exists = status_code == 200
-        response = requests.get(url, auth=self.ES_AUTH)
+        details = response.get(f"ta_{self.index_name}", False)
-        exists = response.ok
-
-        if exists:
-            details = response.json()[f"ta_{index_name}"]
-        else:
-            details = False
 
         return exists, details
 
@@ -65,14 +54,15 @@ class ElasticIndex:
 
     def validate_mappings(self):
         """check if all mappings are as expected"""
-
-        expected_map = self.expected_map
         now_map = self.details["mappings"]["properties"]
 
-        for key, value in expected_map.items():
+        for key, value in self.expected_map.items():
             # nested
             if list(value.keys()) == ["properties"]:
                 for key_n, value_n in value["properties"].items():
+                    if key not in now_map:
+                        print(key_n, value_n)
+                        return True
                     if key_n not in now_map[key]["properties"].keys():
                         print(key_n, value_n)
                         return True
@@ -110,63 +100,41 @@ class ElasticIndex:
 
     def rebuild_index(self):
         """rebuild with new mapping"""
-        # backup
         self.reindex("backup")
-        # delete original
         self.delete_index(backup=False)
-        # create new
         self.create_blank()
         self.reindex("restore")
-        # delete backup
         self.delete_index()
 
     def reindex(self, method):
         """create on elastic search"""
-        index_name = self.index_name
         if method == "backup":
-            source = f"ta_{index_name}"
+            source = f"ta_{self.index_name}"
-            destination = f"ta_{index_name}_backup"
+            destination = f"ta_{self.index_name}_backup"
         elif method == "restore":
-            source = f"ta_{index_name}_backup"
+            source = f"ta_{self.index_name}_backup"
-            destination = f"ta_{index_name}"
+            destination = f"ta_{self.index_name}"
 
-        query = {"source": {"index": source}, "dest": {"index": destination}}
+        data = {"source": {"index": source}, "dest": {"index": destination}}
-        data = json.dumps(query)
+        _, _ = ElasticWrap("_reindex?refresh=true").post(data=data)
-        url = self.ES_URL + "/_reindex?refresh=true"
-        response = requests.post(
-            url=url, data=data, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not response.ok:
-            print(response.text)
 
     def delete_index(self, backup=True):
         """delete index passed as argument"""
+        path = f"ta_{self.index_name}"
         if backup:
-            url = f"{self.ES_URL}/ta_{self.index_name}_backup"
+            path = path + "_backup"
-        else:
+
-            url = f"{self.ES_URL}/ta_{self.index_name}"
+        _, _ = ElasticWrap(path).delete()
-        response = requests.delete(url, auth=self.ES_AUTH)
-        if not response.ok:
-            print(response.text)
 
     def create_blank(self):
         """apply new mapping and settings for blank new index"""
-        expected_map = self.expected_map
+        data = {}
-        expected_set = self.expected_set
+        if self.expected_set:
-        # stich payload
+            data.update({"settings": self.expected_set})
-        payload = {}
+        if self.expected_map:
-        if expected_set:
+            data.update({"mappings": {"properties": self.expected_map}})
-            payload.update({"settings": expected_set})
+
-        if expected_map:
+        _, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
-            payload.update({"mappings": {"properties": expected_map}})
-        # create
-        url = f"{self.ES_URL}/ta_{self.index_name}"
-        data = json.dumps(payload)
-        response = requests.put(
-            url=url, data=data, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not response.ok:
-            print(response.text)
 
 
 class ElasticBackup:
@@ -174,52 +142,21 @@ class ElasticBackup:
 
     def __init__(self, index_config, reason):
         self.config = AppConfig().config
+        self.cache_dir = self.config["application"]["cache_dir"]
         self.index_config = index_config
         self.reason = reason
         self.timestamp = datetime.now().strftime("%Y%m%d")
         self.backup_files = []
 
-    def get_all_documents(self, index_name):
+    @staticmethod
+    def get_all_documents(index_name):
         """export all documents of a single index"""
-        headers = {"Content-type": "application/json"}
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        # get PIT ID
-        url = f"{es_url}/ta_{index_name}/_pit?keep_alive=1m"
-        response = requests.post(url, auth=es_auth)
-        json_data = json.loads(response.text)
-        pit_id = json_data["id"]
-        # build query
         data = {
             "query": {"match_all": {}},
-            "size": 100,
+            "sort": [{"_doc": {"order": "desc"}}],
-            "pit": {"id": pit_id, "keep_alive": "1m"},
-            "sort": [{"_id": {"order": "asc"}}],
         }
-        query_str = json.dumps(data)
+        paginate = IndexPaginate(f"ta_{index_name}", data, keep_source=True)
-        url = es_url + "/_search"
+        all_results = paginate.get_results()
-        # loop until nothing left
-        all_results = []
-        while True:
-            response = requests.get(
-                url, data=query_str, headers=headers, auth=es_auth
-            )
-            json_data = json.loads(response.text)
-            all_hits = json_data["hits"]["hits"]
-            if all_hits:
-                for hit in all_hits:
-                    search_after = hit["sort"]
-                    all_results.append(hit)
-                # update search_after with last hit data
-                data["search_after"] = search_after
-                query_str = json.dumps(data)
-            else:
-                break
-        # clean up PIT
-        query_str = json.dumps({"id": pit_id})
-        requests.delete(
-            es_url + "/_pit", data=query_str, headers=headers, auth=es_auth
-        )
 
         return all_results
 
@@ -244,9 +181,8 @@ class ElasticBackup:
 
     def write_es_json(self, file_content, index_name):
         """write nd-json file for es _bulk API to disk"""
-        cache_dir = self.config["application"]["cache_dir"]
         file_name = f"es_{index_name}-{self.timestamp}.json"
-        file_path = os.path.join(cache_dir, "backup", file_name)
+        file_path = os.path.join(self.cache_dir, "backup", file_name)
         with open(file_path, "w", encoding="utf-8") as f:
             f.write(file_content)
 
@@ -254,9 +190,8 @@ class ElasticBackup:
 
     def write_ta_json(self, all_results, index_name):
         """write generic json file to disk"""
-        cache_dir = self.config["application"]["cache_dir"]
         file_name = f"ta_{index_name}-{self.timestamp}.json"
-        file_path = os.path.join(cache_dir, "backup", file_name)
+        file_path = os.path.join(self.cache_dir, "backup", file_name)
         to_write = [i["_source"] for i in all_results]
         file_content = json.dumps(to_write)
         with open(file_path, "w", encoding="utf-8") as f:
@@ -266,9 +201,8 @@ class ElasticBackup:
 
     def zip_it(self):
         """pack it up into single zip file"""
-        cache_dir = self.config["application"]["cache_dir"]
         file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
-        backup_folder = os.path.join(cache_dir, "backup")
+        backup_folder = os.path.join(self.cache_dir, "backup")
         backup_file = os.path.join(backup_folder, file_name)
 
         with zipfile.ZipFile(
@@ -283,29 +217,18 @@ class ElasticBackup:
 
     def post_bulk_restore(self, file_name):
         """send bulk to es"""
-        cache_dir = self.config["application"]["cache_dir"]
+        file_path = os.path.join(self.cache_dir, file_name)
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        headers = {"Content-type": "application/x-ndjson"}
-        file_path = os.path.join(cache_dir, file_name)
-
         with open(file_path, "r", encoding="utf-8") as f:
-            query_str = f.read()
+            data = f.read()
 
-        if not query_str.strip():
+        if not data.strip():
             return
 
-        url = es_url + "/_bulk"
+        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=es_auth
-        )
-        if not request.ok:
-            print(request.text)
 
     def get_all_backup_files(self):
         """build all available backup files for view"""
-        cache_dir = self.config["application"]["cache_dir"]
+        backup_dir = os.path.join(self.cache_dir, "backup")
-        backup_dir = os.path.join(cache_dir, "backup")
         backup_files = os.listdir(backup_dir)
         all_backup_files = ignore_filelist(backup_files)
         all_available_backups = [
@@ -336,8 +259,7 @@ class ElasticBackup:
 
     def unpack_zip_backup(self, filename):
         """extract backup zip and return filelist"""
-        cache_dir = self.config["application"]["cache_dir"]
+        backup_dir = os.path.join(self.cache_dir, "backup")
-        backup_dir = os.path.join(cache_dir, "backup")
         file_path = os.path.join(backup_dir, filename)
 
         with zipfile.ZipFile(file_path, "r") as z:
@@ -348,9 +270,7 @@ class ElasticBackup:
 
     def restore_json_files(self, zip_content):
         """go through the unpacked files and restore"""
-
+        backup_dir = os.path.join(self.cache_dir, "backup")
-        cache_dir = self.config["application"]["cache_dir"]
-        backup_dir = os.path.join(cache_dir, "backup")
 
         for json_f in zip_content:
 
@@ -364,14 +284,13 @@ class ElasticBackup:
             self.post_bulk_restore(file_name)
             os.remove(file_name)
 
-    def index_exists(self, index_name):
+    @staticmethod
+    def index_exists(index_name):
         """check if index already exists to skip"""
-        es_url = self.config["application"]["es_url"]
+        _, status_code = ElasticWrap(f"ta_{index_name}").get()
-        es_auth = self.config["application"]["es_auth"]
+        exists = status_code == 200
-        url = f"{es_url}/ta_{index_name}"
-        response = requests.get(url, auth=es_auth)
 
-        return response.ok
+        return exists
 
     def rotate_backup(self):
         """delete old backups if needed"""
@@ -386,8 +305,7 @@ class ElasticBackup:
             print("no backup files to rotate")
             return
 
-        cache_dir = self.config["application"]["cache_dir"]
+        backup_dir = os.path.join(self.cache_dir, "backup")
-        backup_dir = os.path.join(cache_dir, "backup")
 
         all_to_delete = auto[rotate:]
         for to_delete in all_to_delete:

tubearchivist/home/src/frontend/api_calls.py

@@ -4,7 +4,7 @@ Functionality:
 - called via user input
 """
 
-from home.src.download.queue import PendingList
+from home.src.download.queue import PendingInteract
 from home.src.download.subscriptions import (
     ChannelSubscription,
     PlaylistSubscription,
@@ -110,12 +110,11 @@ class PostData:
 
     def _ignore(self):
         """ignore from download queue"""
-        id_to_ignore = self.exec_val
+        video_id = self.exec_val
-        print("ignore video " + id_to_ignore)
+        print(f"ignore video {video_id}")
-        handler = PendingList()
+        PendingInteract(video_id=video_id, status="ignore").update_status()
-        handler.ignore_from_pending([id_to_ignore])
         # also clear from redis queue
-        RedisQueue("dl_queue").clear_item(id_to_ignore)
+        RedisQueue().clear_item(video_id)
         return {"success": True}
 
     @staticmethod
@@ -133,7 +132,7 @@ class PostData:
         to_execute = self.exec_val
         if to_execute == "stop":
             print("stopping download queue")
-            RedisQueue("dl_queue").clear()
+            RedisQueue().clear()
         elif to_execute == "kill":
             task_id = RedisArchivist().get_message("dl_queue_id")
             if not isinstance(task_id, str):
@@ -222,28 +221,25 @@ class PostData:
 
     def _forget_ignore(self):
         """delete from ta_download index"""
-        youtube_id = self.exec_val
+        video_id = self.exec_val
-        print("forgetting from download index: " + youtube_id)
+        print(f"forgetting from download index: {video_id}")
-        PendingList().delete_from_pending(youtube_id)
+        PendingInteract(video_id=video_id).delete_item()
         return {"success": True}
 
     def _add_single(self):
         """add single youtube_id to download queue"""
-        youtube_id = self.exec_val
+        video_id = self.exec_val
-        print("add vid to dl queue: " + youtube_id)
+        print(f"add vid to dl queue: {video_id}")
-        PendingList().delete_from_pending(youtube_id)
+        PendingInteract(video_id=video_id).delete_item()
-        youtube_ids = UrlListParser(youtube_id).process_list()
+        video_ids = UrlListParser(video_id).process_list()
-        extrac_dl.delay(youtube_ids)
+        extrac_dl.delay(video_ids)
         return {"success": True}
 
     def _delete_queue(self):
         """delete download queue"""
         status = self.exec_val
         print("deleting from download queue: " + status)
-        if status == "pending":
+        PendingInteract(status=status).delete_by_status()
-            PendingList().delete_pending("pending")
-        elif status == "ignore":
-            PendingList().delete_pending("ignore")
         return {"success": True}
 
     @staticmethod

tubearchivist/home/src/frontend/forms.py

@@ -178,3 +178,19 @@ class SubscribeToPlaylistForm(forms.Form):
             }
         ),
     )
+
+
+class ChannelOverwriteForm(forms.Form):
+    """custom overwrites for channel settings"""
+
+    PLAYLIST_INDEX = [
+        ("", "-- change playlist index --"),
+        ("0", "Disable playlist index"),
+        ("1", "Enable playlist index"),
+    ]
+
+    download_format = forms.CharField(label=False, required=False)
+    autodelete_days = forms.IntegerField(label=False, required=False)
+    index_playlists = forms.ChoiceField(
+        widget=forms.Select, choices=PLAYLIST_INDEX, required=False
+    )

tubearchivist/home/src/frontend/watched.py

@@ -3,22 +3,15 @@ functionality:
 - handle watched state for videos, channels and playlists
 """
 
-import json
 from datetime import datetime
 
-import requests
+from home.src.es.connect import ElasticWrap
-from home.src.ta.config import AppConfig
 from home.src.ta.helper import UrlListParser
 
 
 class WatchState:
     """handle watched checkbox for videos and channels"""
 
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
-    HEADERS = {"Content-type": "application/json"}
-
     def __init__(self, youtube_id):
         self.youtube_id = youtube_id
         self.stamp = int(datetime.now().strftime("%s"))
@@ -33,7 +26,7 @@ class WatchState:
         elif url_type == "playlist":
             self.mark_playlist_watched()
 
-        print(f"marked {self.youtube_id} as watched")
+        print(f"{self.youtube_id}: marked as watched")
 
     def mark_as_unwatched(self):
         """revert watched state to false"""
@@ -41,7 +34,7 @@ class WatchState:
         if url_type == "video":
             self.mark_vid_watched(revert=True)
 
-        print(f"revert {self.youtube_id} as unwatched")
+        print(f"{self.youtube_id}: revert as unwatched")
 
     def dedect_type(self):
         """find youtube id type"""
@@ -52,77 +45,54 @@ class WatchState:
 
     def mark_vid_watched(self, revert=False):
         """change watched status of single video"""
-        url = self.ES_URL + "/ta_video/_update/" + self.youtube_id
+        path = f"ta_video/_update/{self.youtube_id}"
         data = {
             "doc": {"player": {"watched": True, "watched_date": self.stamp}}
         }
         if revert:
             data["doc"]["player"]["watched"] = False
 
-        payload = json.dumps(data)
+        response, status_code = ElasticWrap(path).post(data=data)
-        request = requests.post(
+        if status_code != 200:
-            url, data=payload, headers=self.HEADERS, auth=self.ES_AUTH
+            print(response)
-        )
-        if not request.ok:
-            print(request.text)
             raise ValueError("failed to mark video as watched")
 
     def mark_channel_watched(self):
         """change watched status of every video in channel"""
+        path = "ta_video/_update_by_query"
+        must_list = [
+            {"term": {"channel.channel_id": {"value": self.youtube_id}}},
+            {"term": {"player.watched": {"value": False}}},
+        ]
         data = {
-            "query": {
+            "query": {"bool": {"must": must_list}},
-                "bool": {
-                    "must": [
-                        {
-                            "term": {
-                                "channel.channel_id": {
-                                    "value": self.youtube_id
-                                }
-                            }
-                        },
-                        {"term": {"player.watched": {"value": False}}},
-                    ]
-                }
-            },
             "script": {
                 "source": "ctx._source.player['watched'] = true",
                 "lang": "painless",
             },
         }
-        payload = json.dumps(data)
+
-        url = f"{self.ES_URL}/ta_video/_update_by_query"
+        response, status_code = ElasticWrap(path).post(data=data)
-        request = requests.post(
+        if status_code != 200:
-            url, data=payload, headers=self.HEADERS, auth=self.ES_AUTH
+            print(response)
-        )
-        if not request.ok:
-            print(request.text)
             raise ValueError("failed mark channel as watched")
 
     def mark_playlist_watched(self):
         """change watched state of all videos in playlist"""
+        path = "ta_video/_update_by_query"
+        must_list = [
+            {"term": {"playlist.keyword": {"value": self.youtube_id}}},
+            {"term": {"player.watched": {"value": False}}},
+        ]
         data = {
-            "query": {
+            "query": {"bool": {"must": must_list}},
-                "bool": {
-                    "must": [
-                        {
-                            "term": {
-                                "playlist.keyword": {"value": self.youtube_id}
-                            }
-                        },
-                        {"term": {"player.watched": {"value": False}}},
-                    ]
-                }
-            },
             "script": {
                 "source": "ctx._source.player['watched'] = true",
                 "lang": "painless",
             },
         }
-        payload = json.dumps(data)
+
-        url = f"{self.ES_URL}/ta_video/_update_by_query"
+        response, status_code = ElasticWrap(path).post(data=data)
-        request = requests.post(
+        if status_code != 200:
-            url, data=payload, headers=self.HEADERS, auth=self.ES_AUTH
+            print(response)
-        )
-        if not request.ok:
-            print(request.text)
             raise ValueError("failed mark playlist as watched")

tubearchivist/home/src/index/channel.py

@@ -12,11 +12,13 @@ from datetime import datetime
 import requests
 import yt_dlp
 from bs4 import BeautifulSoup
+from home.src.download import queue  # partial import
 from home.src.download.thumbnails import ThumbManager
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.generic import YouTubeItem
 from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.helper import clean_string
+from home.src.ta.helper import clean_string, requests_headers
+from home.src.ta.ta_redis import RedisArchivist
 
 
 class ChannelScraper:
@@ -44,7 +46,9 @@ class ChannelScraper:
         print(f"{self.channel_id}: scrape channel data from youtube")
         url = f"https://www.youtube.com/channel/{self.channel_id}/about?hl=en"
         cookies = {"CONSENT": "YES+xxxxxxxxxxxxxxxxxxxxxxxxxxx"}
-        response = requests.get(url, cookies=cookies)
+        response = requests.get(
+            url, cookies=cookies, headers=requests_headers()
+        )
         if response.ok:
             channel_page = response.text
         else:
@@ -153,6 +157,7 @@ class YoutubeChannel(YouTubeItem):
     def __init__(self, youtube_id):
         super().__init__(youtube_id)
         self.es_path = f"{self.index_name}/_doc/{youtube_id}"
+        self.all_playlists = False
 
     def build_json(self, upload=False):
         """get from es or from youtube"""
@@ -241,6 +246,68 @@ class YoutubeChannel(YouTubeItem):
         self.delete_es_videos()
         self.del_in_es()
 
+    def index_channel_playlists(self):
+        """add all playlists of channel to index"""
+        print(f"{self.youtube_id}: index all playlists")
+        self.get_from_es()
+        channel_name = self.json_data["channel_name"]
+        mess_dict = {
+            "status": "message:playlistscan",
+            "level": "info",
+            "title": "Looking for playlists",
+            "message": f"{channel_name}: Scanning channel in progress",
+        }
+        RedisArchivist().set_message("message:playlistscan", mess_dict)
+        self.get_all_playlists()
+        if not self.all_playlists:
+            print(f"{self.youtube_id}: no playlists found.")
+            return
+
+        all_youtube_ids = self.get_all_video_ids()
+        for idx, playlist in enumerate(self.all_playlists):
+            self._notify_single_playlist(idx, playlist)
+            self._index_single_playlist(playlist, all_youtube_ids)
+
+    def _notify_single_playlist(self, idx, playlist):
+        """send notification"""
+        channel_name = self.json_data["channel_name"]
+        mess_dict = {
+            "status": "message:playlistscan",
+            "level": "info",
+            "title": f"{channel_name}: Scanning channel for playlists",
+            "message": f"Progress: {idx + 1}/{len(self.all_playlists)}",
+        }
+        RedisArchivist().set_message("message:playlistscan", mess_dict)
+        print("add playlist: " + playlist[1])
+
+    @staticmethod
+    def _index_single_playlist(playlist, all_youtube_ids):
+        """add single playlist if needed"""
+        playlist = YoutubePlaylist(playlist[0])
+        playlist.all_youtube_ids = all_youtube_ids
+        playlist.build_json()
+        if not playlist.json_data:
+            return
+
+        entries = playlist.json_data["playlist_entries"]
+        downloaded = [i for i in entries if i["downloaded"]]
+        if not downloaded:
+            return
+
+        playlist.upload_to_es()
+        playlist.add_vids_to_playlist()
+        playlist.get_playlist_art()
+
+    @staticmethod
+    def get_all_video_ids():
+        """match all playlists with videos"""
+        handler = queue.PendingList()
+        handler.get_download()
+        handler.get_indexed()
+        all_youtube_ids = [i["youtube_id"] for i in handler.all_videos]
+
+        return all_youtube_ids
+
     def get_all_playlists(self):
         """get all playlists owned by this channel"""
         url = (
@@ -254,8 +321,7 @@ class YoutubeChannel(YouTubeItem):
         }
         playlists = yt_dlp.YoutubeDL(obs).extract_info(url)
         all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
-
+        self.all_playlists = all_entries
-        return all_entries
 
     def get_indexed_playlists(self):
         """get all indexed playlists from channel"""
@@ -267,3 +333,35 @@ class YoutubeChannel(YouTubeItem):
         }
         all_playlists = IndexPaginate("ta_playlist", data).get_results()
         return all_playlists
+
+    def get_overwrites(self):
+        """get all per channel overwrites"""
+        return self.json_data.get("channel_overwrites", False)
+
+    def set_overwrites(self, overwrites):
+        """set per channel overwrites"""
+        valid_keys = ["download_format", "autodelete_days", "index_playlists"]
+
+        to_write = self.json_data.get("channel_overwrites", {})
+        for key, value in overwrites.items():
+            if key not in valid_keys:
+                raise ValueError(f"invalid overwrite key: {key}")
+            if value in [0, "0"]:
+                del to_write[key]
+                continue
+            if value == "1":
+                to_write[key] = True
+                continue
+            if value:
+                to_write.update({key: value})
+
+        self.json_data["channel_overwrites"] = to_write
+
+
+def channel_overwrites(channel_id, overwrites):
+    """collection to overwrite settings per channel"""
+    channel = YoutubeChannel(channel_id)
+    channel.build_json()
+    channel.set_overwrites(overwrites)
+    channel.upload_to_es()
+    channel.sync_to_videos()

tubearchivist/home/src/index/filesystem.py

@@ -10,11 +10,10 @@ import os
 import re
 import shutil
 import subprocess
-from datetime import datetime
 
-import requests
 from home.src.download.queue import PendingList
 from home.src.download.yt_dlp_handler import VideoDownloader
+from home.src.es.connect import ElasticWrap
 from home.src.index.reindex import Reindex
 from home.src.index.video import index_new_video
 from home.src.ta.config import AppConfig
@@ -26,8 +25,6 @@ class FilesystemScanner:
     """handle scanning and fixing from filesystem"""
 
     CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
     VIDEOS = CONFIG["application"]["videos"]
 
     def __init__(self):
@@ -59,9 +56,11 @@ class FilesystemScanner:
     def get_all_indexed():
         """get a list of all indexed videos"""
         index_handler = PendingList()
-        all_indexed_raw = index_handler.get_all_indexed()
+        index_handler.get_download()
+        index_handler.get_indexed()
+
         all_indexed = []
-        for video in all_indexed_raw:
+        for video in index_handler.all_videos:
             youtube_id = video["youtube_id"]
             media_url = video["media_url"]
             published = video["published"]
@@ -146,25 +145,16 @@ class FilesystemScanner:
             bulk_list.append(json.dumps(source))
         # add last newline
         bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
+        data = "\n".join(bulk_list)
-        # make the call
+        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
-        headers = {"Content-type": "application/x-ndjson"}
-        url = self.ES_URL + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request.text)
 
     def delete_from_index(self):
         """find indexed but deleted mediafile"""
         for indexed in self.to_delete:
             youtube_id = indexed[0]
             print(f"deleting {youtube_id} from index")
-            url = self.ES_URL + "/ta_video/_doc/" + youtube_id
+            path = f"ta_video/_doc/{youtube_id}"
-            request = requests.delete(url, auth=self.ES_AUTH)
+            _, _ = ElasticWrap(path).delete()
-            if not request.ok:
-                print(request.text)
 
 
 class ManualImport:
@@ -317,10 +307,7 @@ def scan_filesystem():
 
 def reindex_old_documents():
     """daily refresh of old documents"""
-    # continue if needed
+    handler = Reindex()
-    reindex_handler = Reindex()
+    handler.check_outdated()
-    reindex_handler.check_outdated()
+    handler.reindex()
-    reindex_handler.reindex()
+    RedisArchivist().set_message("last_reindex", handler.now, expire=False)
-    # set timestamp
-    now = int(datetime.now().strftime("%s"))
-    RedisArchivist().set_message("last_reindex", now, expire=False)

tubearchivist/home/src/index/playlist.py

@@ -35,8 +35,11 @@ class YoutubePlaylist(YouTubeItem):
 
     def build_json(self, scrape=False):
         """collection to create json_data"""
-        if not scrape:
+        self.get_from_es()
-            self.get_from_es()
+        if self.json_data:
+            subscribed = self.json_data.get("playlist_subscribed")
+        else:
+            subscribed = False
 
         if scrape or not self.json_data:
             self.get_from_youtube()
@@ -44,13 +47,13 @@ class YoutubePlaylist(YouTubeItem):
             self.get_entries()
             self.json_data["playlist_entries"] = self.all_members
             self.get_playlist_art()
+            self.json_data["playlist_subscribed"] = subscribed
 
     def process_youtube_meta(self):
         """extract relevant fields from youtube"""
         self.json_data = {
             "playlist_id": self.youtube_id,
             "playlist_active": True,
-            "playlist_subscribed": False,
             "playlist_name": self.youtube_meta["title"],
             "playlist_channel": self.youtube_meta["channel"],
             "playlist_channel_id": self.youtube_meta["channel_id"],

tubearchivist/home/src/index/reindex.py

@@ -4,86 +4,60 @@ functionality:
 - index and update in es
 """
 
-import json
 from datetime import datetime
 from math import ceil
 from time import sleep
 
-import requests
 from home.src.download.queue import PendingList
-from home.src.download.subscriptions import ChannelSubscription
 from home.src.download.thumbnails import ThumbManager
+from home.src.es.connect import ElasticWrap
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_total_hits
 
 
 class Reindex:
     """check for outdated documents and refresh data from youtube"""
 
+    MATCH_FIELD = {
+        "ta_video": "active",
+        "ta_channel": "channel_active",
+        "ta_playlist": "playlist_active",
+    }
+    MULTIPLY = 1.2
+
     def __init__(self):
         # config
-        config = AppConfig().config
+        self.now = int(datetime.now().strftime("%s"))
-        self.sleep_interval = config["downloads"]["sleep_interval"]
+        self.config = AppConfig().config
-        self.es_url = config["application"]["es_url"]
+        self.interval = self.config["scheduler"]["check_reindex_days"]
-        self.es_auth = config["application"]["es_auth"]
-        self.refresh_interval = config["scheduler"]["check_reindex_days"]
-        self.integrate_ryd = config["downloads"]["integrate_ryd"]
         # scan
         self.all_youtube_ids = False
         self.all_channel_ids = False
         self.all_playlist_ids = False
 
-    def get_daily(self):
+    def _get_daily(self):
         """get daily refresh values"""
-        total_videos = get_total_hits(
+        total_videos = self._get_total_hits("ta_video")
-            "ta_video", self.es_url, self.es_auth, "active"
+        video_daily = ceil(total_videos / self.interval * self.MULTIPLY)
-        )
+        total_channels = self._get_total_hits("ta_channel")
-        video_daily = ceil(total_videos / self.refresh_interval * 1.2)
+        channel_daily = ceil(total_channels / self.interval * self.MULTIPLY)
-        total_channels = get_total_hits(
+        total_playlists = self._get_total_hits("ta_playlist")
-            "ta_channel", self.es_url, self.es_auth, "channel_active"
+        playlist_daily = ceil(total_playlists / self.interval * self.MULTIPLY)
-        )
-        channel_daily = ceil(total_channels / self.refresh_interval * 1.2)
-        total_playlists = get_total_hits(
-            "ta_playlist", self.es_url, self.es_auth, "playlist_active"
-        )
-        playlist_daily = ceil(total_playlists / self.refresh_interval * 1.2)
         return (video_daily, channel_daily, playlist_daily)
 
-    def get_outdated_vids(self, size):
+    def _get_total_hits(self, index):
-        """get daily videos to refresh"""
+        """get total hits from index"""
-        headers = {"Content-type": "application/json"}
+        match_field = self.MATCH_FIELD[index]
-        now = int(datetime.now().strftime("%s"))
+        path = f"{index}/_search?filter_path=hits.total"
-        now_lte = now - self.refresh_interval * 24 * 60 * 60
+        data = {"query": {"match": {match_field: True}}}
-        data = {
+        response, _ = ElasticWrap(path).post(data=data)
-            "size": size,
+        total_hits = response["hits"]["total"]["value"]
-            "query": {
+        return total_hits
-                "bool": {
-                    "must": [
-                        {"match": {"active": True}},
-                        {"range": {"vid_last_refresh": {"lte": now_lte}}},
-                    ]
-                }
-            },
-            "sort": [{"vid_last_refresh": {"order": "asc"}}],
-            "_source": False,
-        }
-        query_str = json.dumps(data)
-        url = self.es_url + "/ta_video/_search"
-        response = requests.get(
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        all_youtube_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
-        return all_youtube_ids
 
-    def get_unrated_vids(self):
+    def _get_unrated_vids(self):
-        """get all videos without rating if ryd integration is enabled"""
+        """get max 200 videos without rating if ryd integration is enabled"""
-        headers = {"Content-type": "application/json"}
         data = {
             "size": 200,
             "query": {
@@ -92,104 +66,78 @@ class Reindex:
                 }
             },
         }
-        query_str = json.dumps(data)
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
-        url = self.es_url + "/ta_video/_search"
+
-        response = requests.get(
+        missing_rating = [i["_id"] for i in response["hits"]["hits"]]
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        missing_rating = [i["_id"] for i in response_dict["hits"]["hits"]]
         self.all_youtube_ids = self.all_youtube_ids + missing_rating
 
-    def get_outdated_channels(self, size):
+    def _get_outdated_vids(self, size):
-        """get daily channels to refresh"""
+        """get daily videos to refresh"""
-        headers = {"Content-type": "application/json"}
+        now_lte = self.now - self.interval * 24 * 60 * 60
-        now = int(datetime.now().strftime("%s"))
+        must_list = [
-        now_lte = now - self.refresh_interval * 24 * 60 * 60
+            {"match": {"active": True}},
+            {"range": {"vid_last_refresh": {"lte": now_lte}}},
+        ]
         data = {
             "size": size,
-            "query": {
+            "query": {"bool": {"must": must_list}},
-                "bool": {
+            "sort": [{"vid_last_refresh": {"order": "asc"}}],
-                    "must": [
+            "_source": False,
-                        {"match": {"channel_active": True}},
+        }
-                        {"range": {"channel_last_refresh": {"lte": now_lte}}},
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
-                    ]
+
-                }
+        all_youtube_ids = [i["_id"] for i in response["hits"]["hits"]]
-            },
+        return all_youtube_ids
+
+    def _get_outdated_channels(self, size):
+        """get daily channels to refresh"""
+        now_lte = self.now - self.interval * 24 * 60 * 60
+        must_list = [
+            {"match": {"channel_active": True}},
+            {"range": {"channel_last_refresh": {"lte": now_lte}}},
+        ]
+        data = {
+            "size": size,
+            "query": {"bool": {"must": must_list}},
             "sort": [{"channel_last_refresh": {"order": "asc"}}],
             "_source": False,
         }
-        query_str = json.dumps(data)
+        response, _ = ElasticWrap("ta_channel/_search").get(data=data)
-        url = self.es_url + "/ta_channel/_search"
+
-        response = requests.get(
+        all_channel_ids = [i["_id"] for i in response["hits"]["hits"]]
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        all_channel_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
         return all_channel_ids
 
-    def get_outdated_playlists(self, size):
+    def _get_outdated_playlists(self, size):
         """get daily outdated playlists to refresh"""
-        headers = {"Content-type": "application/json"}
+        now_lte = self.now - self.interval * 24 * 60 * 60
-        now = int(datetime.now().strftime("%s"))
+        must_list = [
-        now_lte = now - self.refresh_interval * 24 * 60 * 60
+            {"match": {"playlist_active": True}},
+            {"range": {"playlist_last_refresh": {"lte": now_lte}}},
+        ]
         data = {
             "size": size,
-            "query": {
+            "query": {"bool": {"must": must_list}},
-                "bool": {
-                    "must": [
-                        {"match": {"playlist_active": True}},
-                        {"range": {"playlist_last_refresh": {"lte": now_lte}}},
-                    ]
-                }
-            },
             "sort": [{"playlist_last_refresh": {"order": "asc"}}],
             "_source": False,
         }
-        query_str = json.dumps(data)
+        response, _ = ElasticWrap("ta_playlist/_search").get(data=data)
-        url = self.es_url + "/ta_playlist/_search"
+
-        response = requests.get(
+        all_playlist_ids = [i["_id"] for i in response["hits"]["hits"]]
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        all_playlist_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
         return all_playlist_ids
 
     def check_outdated(self):
         """add missing vids and channels"""
-        video_daily, channel_daily, playlist_daily = self.get_daily()
+        video_daily, channel_daily, playlist_daily = self._get_daily()
-        self.all_youtube_ids = self.get_outdated_vids(video_daily)
+        self.all_youtube_ids = self._get_outdated_vids(video_daily)
-        self.all_channel_ids = self.get_outdated_channels(channel_daily)
+        self.all_channel_ids = self._get_outdated_channels(channel_daily)
-        self.all_playlist_ids = self.get_outdated_playlists(playlist_daily)
+        self.all_playlist_ids = self._get_outdated_playlists(playlist_daily)
-        if self.integrate_ryd:
-            self.get_unrated_vids()
 
-    def rescrape_all_channels(self):
+        integrate_ryd = self.config["downloads"]["integrate_ryd"]
-        """sync new data from channel to all matching videos"""
+        if integrate_ryd:
-        sleep_interval = self.sleep_interval
+            self._get_unrated_vids()
-        channel_sub_handler = ChannelSubscription()
-        all_channels = channel_sub_handler.get_channels(subscribed_only=False)
-        all_channel_ids = [i["channel_id"] for i in all_channels]
-
-        for channel_id in all_channel_ids:
-            channel = YoutubeChannel(channel_id)
-            subscribed = channel.json_data["channel_subscribed"]
-            channel.get_from_youtube()
-            channel.json_data["channel_subscribed"] = subscribed
-            channel.upload_to_es()
-            channel.sync_to_videos()
-
-            if sleep_interval:
-                sleep(sleep_interval)
 
     @staticmethod
-    def reindex_single_video(youtube_id):
+    def _reindex_single_video(youtube_id):
         """refresh data for single video"""
         video = YoutubeVideo(youtube_id)
 
@@ -207,6 +155,8 @@ class Reindex:
             return
 
         video.delete_subtitles()
+        video.check_subtitles()
+
         # add back
         video.json_data["player"] = player
         video.json_data["date_downloaded"] = date_downloaded
@@ -223,18 +173,21 @@ class Reindex:
         return
 
     @staticmethod
-    def reindex_single_channel(channel_id):
+    def _reindex_single_channel(channel_id):
         """refresh channel data and sync to videos"""
         channel = YoutubeChannel(channel_id)
         channel.get_from_es()
         subscribed = channel.json_data["channel_subscribed"]
+        overwrites = channel.json_data.get("channel_overwrites", False)
         channel.get_from_youtube()
         channel.json_data["channel_subscribed"] = subscribed
+        if overwrites:
+            channel.json_data["channel_overwrites"] = overwrites
         channel.upload_to_es()
         channel.sync_to_videos()
 
     @staticmethod
-    def reindex_single_playlist(playlist_id, all_indexed_ids):
+    def _reindex_single_playlist(playlist_id, all_indexed_ids):
         """refresh playlist data"""
         playlist = YoutubePlaylist(playlist_id)
         playlist.get_from_es()
@@ -251,24 +204,27 @@ class Reindex:
 
     def reindex(self):
         """reindex what's needed"""
+        sleep_interval = self.config["downloads"]["sleep_interval"]
         # videos
         print(f"reindexing {len(self.all_youtube_ids)} videos")
         for youtube_id in self.all_youtube_ids:
-            self.reindex_single_video(youtube_id)
+            self._reindex_single_video(youtube_id)
-            if self.sleep_interval:
+            if sleep_interval:
-                sleep(self.sleep_interval)
+                sleep(sleep_interval)
         # channels
         print(f"reindexing {len(self.all_channel_ids)} channels")
         for channel_id in self.all_channel_ids:
-            self.reindex_single_channel(channel_id)
+            self._reindex_single_channel(channel_id)
-            if self.sleep_interval:
+            if sleep_interval:
-                sleep(self.sleep_interval)
+                sleep(sleep_interval)
         # playlist
         print(f"reindexing {len(self.all_playlist_ids)} playlists")
         if self.all_playlist_ids:
-            all_indexed = PendingList().get_all_indexed()
+            handler = PendingList()
-            all_indexed_ids = [i["youtube_id"] for i in all_indexed]
+            handler.get_download()
+            handler.get_indexed()
+            all_indexed_ids = [i["youtube_id"] for i in handler.all_videos]
             for playlist_id in self.all_playlist_ids:
-                self.reindex_single_playlist(playlist_id, all_indexed_ids)
+                self._reindex_single_playlist(playlist_id, all_indexed_ids)
-                if self.sleep_interval:
+                if sleep_interval:
-                    sleep(self.sleep_interval)
+                    sleep(sleep_interval)

tubearchivist/home/src/index/video.py

@@ -6,14 +6,17 @@ functionality:
 
 import json
 import os
-import re
 from datetime import datetime
 
 import requests
 from home.src.es.connect import ElasticWrap
 from home.src.index import channel as ta_channel
 from home.src.index.generic import YouTubeItem
-from home.src.ta.helper import DurationConverter, clean_string
+from home.src.ta.helper import (
+    DurationConverter,
+    clean_string,
+    requests_headers,
+)
 from ryd_client import ryd_client
 
 
@@ -24,7 +27,7 @@ class YoutubeSubtitle:
         self.video = video
         self.languages = False
 
-    def sub_conf_parse(self):
+    def _sub_conf_parse(self):
         """add additional conf values to self"""
         languages_raw = self.video.config["downloads"]["subtitle"]
         if languages_raw:
@@ -32,26 +35,26 @@ class YoutubeSubtitle:
 
     def get_subtitles(self):
         """check what to do"""
-        self.sub_conf_parse()
+        self._sub_conf_parse()
         if not self.languages:
             # no subtitles
             return False
 
         relevant_subtitles = []
         for lang in self.languages:
-            user_sub = self.get_user_subtitles(lang)
+            user_sub = self._get_user_subtitles(lang)
             if user_sub:
                 relevant_subtitles.append(user_sub)
                 continue
 
             if self.video.config["downloads"]["subtitle_source"] == "auto":
-                auto_cap = self.get_auto_caption(lang)
+                auto_cap = self._get_auto_caption(lang)
                 if auto_cap:
                     relevant_subtitles.append(auto_cap)
 
         return relevant_subtitles
 
-    def get_auto_caption(self, lang):
+    def _get_auto_caption(self, lang):
         """get auto_caption subtitles"""
         print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
         all_subtitles = self.video.youtube_meta.get("automatic_captions")
@@ -65,7 +68,7 @@ class YoutubeSubtitle:
         if not all_formats:
             return False
 
-        subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
         subtitle.update(
             {"lang": lang, "source": "auto", "media_url": media_url}
         )
@@ -88,7 +91,7 @@ class YoutubeSubtitle:
 
         return all_subtitles
 
-    def get_user_subtitles(self, lang):
+    def _get_user_subtitles(self, lang):
         """get subtitles uploaded from channel owner"""
         print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
         all_subtitles = self._normalize_lang()
@@ -102,7 +105,7 @@ class YoutubeSubtitle:
             # no user subtitles found
             return False
 
-        subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
         subtitle.update(
             {"lang": lang, "source": "user", "media_url": media_url}
         )
@@ -115,12 +118,16 @@ class YoutubeSubtitle:
         for subtitle in relevant_subtitles:
             dest_path = os.path.join(videos_base, subtitle["media_url"])
             source = subtitle["source"]
-            response = requests.get(subtitle["url"])
+            lang = subtitle.get("lang")
+            response = requests.get(
+                subtitle["url"], headers=requests_headers()
+            )
             if not response.ok:
                 print(f"{self.video.youtube_id}: failed to download subtitle")
+                print(response.text)
                 continue
 
-            parser = SubtitleParser(response.text, subtitle.get("lang"))
+            parser = SubtitleParser(response.text, lang, source)
             parser.process()
             subtitle_str = parser.get_subtitle_str()
             self._write_subtitle_file(dest_path, subtitle_str)
@@ -145,109 +152,94 @@ class YoutubeSubtitle:
 class SubtitleParser:
     """parse subtitle str from youtube"""
 
-    time_reg = r"^([0-9]{2}:?){3}\.[0-9]{3} --> ([0-9]{2}:?){3}\.[0-9]{3}"
+    def __init__(self, subtitle_str, lang, source):
-    stamp_reg = r"<([0-9]{2}:?){3}\.[0-9]{3}>"
+        self.subtitle_raw = json.loads(subtitle_str)
-    tag_reg = r"</?c>"
-
-    def __init__(self, subtitle_str, lang):
-        self.subtitle_str = subtitle_str
         self.lang = lang
-        self.header = False
+        self.source = source
-        self.parsed_cue_list = False
+        self.all_cues = False
-        self.all_text_lines = False
-        self.matched = False
 
     def process(self):
-        """collection to process subtitle string"""
+        """extract relevant que data"""
-        self._parse_cues()
+        all_events = self.subtitle_raw.get("events")
-        self._match_text_lines()
+        if self.source == "auto":
-        self._add_id()
+            all_events = self._flat_auto_caption(all_events)
-        self._timestamp_check()
 
-    def _parse_cues(self):
+        self.all_cues = []
-        """split into cues"""
+        for idx, event in enumerate(all_events):
-        all_cues = self.subtitle_str.replace("\n \n", "\n").split("\n\n")
+            cue = {
-        self.header = all_cues[0]
+                "start": self._ms_conv(event["tStartMs"]),
-        self.all_text_lines = []
+                "end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
-        self.parsed_cue_list = [self._cue_cleaner(i) for i in all_cues[1:]]
+                "text": "".join([i.get("utf8") for i in event["segs"]]),
+                "idx": idx + 1,
+            }
+            self.all_cues.append(cue)
 
-    def _cue_cleaner(self, cue):
+    @staticmethod
-        """parse single cue"""
+    def _flat_auto_caption(all_events):
-        all_lines = cue.split("\n")
+        """flatten autocaption segments"""
-        cue_dict = {"lines": []}
+        flatten = []
-
+        for event in all_events:
-        for line in all_lines:
+            if "segs" not in event.keys():
-            if re.match(self.time_reg, line):
+                continue
-                clean = re.search(self.time_reg, line).group()
+            text = "".join([i.get("utf8") for i in event.get("segs")])
-                start, end = clean.split(" --> ")
+            if not text.strip():
-                cue_dict.update({"start": start, "end": end})
-            else:
-                clean = re.sub(self.stamp_reg, "", line)
-                clean = re.sub(self.tag_reg, "", clean)
-                cue_dict["lines"].append(clean)
-                if clean.strip() and clean not in self.all_text_lines[-4:]:
-                    # remove immediate duplicates
-                    self.all_text_lines.append(clean)
-
-        return cue_dict
-
-    def _match_text_lines(self):
-        """match unique text lines with timestamps"""
-
-        self.matched = []
-
-        while self.all_text_lines:
-            check = self.all_text_lines[0]
-            matches = [i for i in self.parsed_cue_list if check in i["lines"]]
-            new_cue = matches[-1]
-            new_cue["start"] = matches[0]["start"]
-
-            for line in new_cue["lines"]:
-                try:
-                    self.all_text_lines.remove(line)
-                except ValueError:
-                    continue
-
-            self.matched.append(new_cue)
-
-    def _timestamp_check(self):
-        """check if end timestamp is bigger than start timestamp"""
-        for idx, cue in enumerate(self.matched):
-            # this
-            end = int(re.sub("[^0-9]", "", cue.get("end")))
-            # next
-            try:
-                next_cue = self.matched[idx + 1]
-            except IndexError:
                 continue
 
-            start_next = int(re.sub("[^0-9]", "", next_cue.get("start")))
+            if flatten:
-            if end > start_next:
+                # fix overlapping retiming issue
-                self.matched[idx]["end"] = next_cue.get("start")
+                last_end = flatten[-1]["tStartMs"] + flatten[-1]["dDurationMs"]
+                if event["tStartMs"] < last_end:
+                    joined = flatten[-1]["segs"][0]["utf8"] + "\n" + text
+                    flatten[-1]["segs"][0]["utf8"] = joined
+                    continue
 
-    def _add_id(self):
+            event.update({"segs": [{"utf8": text}]})
-        """add id to matched cues"""
+            flatten.append(event)
-        for idx, _ in enumerate(self.matched):
+
-            self.matched[idx]["id"] = idx + 1
+        return flatten
+
+    @staticmethod
+    def _ms_conv(ms):
+        """convert ms to timestamp"""
+        hours = str((ms // (1000 * 60 * 60)) % 24).zfill(2)
+        minutes = str((ms // (1000 * 60)) % 60).zfill(2)
+        secs = str((ms // 1000) % 60).zfill(2)
+        millis = str(ms % 1000).zfill(3)
+
+        return f"{hours}:{minutes}:{secs}.{millis}"
 
     def get_subtitle_str(self):
-        """stitch cues and return processed new string"""
+        """create vtt text str from cues"""
-        new_subtitle_str = self.header + "\n\n"
+        subtitle_str = f"WEBVTT\nKind: captions\nLanguage: {self.lang}"
 
-        for cue in self.matched:
+        for cue in self.all_cues:
-            timestamp = f"{cue.get('start')} --> {cue.get('end')}"
+            stamp = f"{cue.get('start')} --> {cue.get('end')}"
-            lines = "\n".join(cue.get("lines"))
+            cue_text = f"\n\n{cue.get('idx')}\n{stamp}\n{cue.get('text')}"
-            cue_text = f"{cue.get('id')}\n{timestamp}\n{lines}\n\n"
+            subtitle_str = subtitle_str + cue_text
-            new_subtitle_str = new_subtitle_str + cue_text
 
-        return new_subtitle_str
+        return subtitle_str
 
     def create_bulk_import(self, video, source):
-        """process matched for es import"""
+        """subtitle lines for es import"""
+        documents = self._create_documents(video, source)
         bulk_list = []
-        channel = video.json_data.get("channel")
 
-        document = {
+        for document in documents:
+            document_id = document.get("subtitle_fragment_id")
+            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(document))
+
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+
+        return query_str
+
+    def _create_documents(self, video, source):
+        """process documents"""
+        documents = self._chunk_list(video.youtube_id)
+        channel = video.json_data.get("channel")
+        meta_dict = {
             "youtube_id": video.youtube_id,
             "title": video.json_data.get("title"),
             "subtitle_channel": channel.get("channel_name"),
@@ -257,26 +249,35 @@ class SubtitleParser:
             "subtitle_source": source,
         }
 
-        for match in self.matched:
+        _ = [i.update(meta_dict) for i in documents]
-            match_id = match.get("id")
+
-            document_id = f"{video.youtube_id}-{self.lang}-{match_id}"
+        return documents
-            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
+
-            document.update(
+    def _chunk_list(self, youtube_id):
-                {
+        """join cues for bulk import"""
-                    "subtitle_fragment_id": document_id,
+        chunk_list = []
-                    "subtitle_start": match.get("start"),
+
-                    "subtitle_end": match.get("end"),
+        chunk = {}
-                    "subtitle_index": match_id,
+        for cue in self.all_cues:
-                    "subtitle_line": " ".join(match.get("lines")),
+            if chunk:
+                text = f"{chunk.get('subtitle_line')} {cue.get('text')}\n"
+                chunk["subtitle_line"] = text
+            else:
+                idx = len(chunk_list) + 1
+                chunk = {
+                    "subtitle_index": idx,
+                    "subtitle_line": cue.get("text"),
+                    "subtitle_start": cue.get("start"),
                 }
-            )
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(document))
 
-        bulk_list.append("\n")
+            chunk["subtitle_fragment_id"] = f"{youtube_id}-{self.lang}-{idx}"
-        query_str = "\n".join(bulk_list)
 
-        return query_str
+            if cue["idx"] % 5 == 0:
+                chunk["subtitle_end"] = cue.get("end")
+                chunk_list.append(chunk)
+                chunk = {}
+
+        return chunk_list
 
 
 class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
@@ -302,7 +303,6 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         self._add_stats()
         self.add_file_path()
         self.add_player()
-        self._check_subtitles()
         if self.config["downloads"]["integrate_ryd"]:
             self._get_ryd_stats()
 
@@ -375,7 +375,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             channel_dir = os.path.join(self.app_conf["videos"], channel)
             all_files = os.listdir(channel_dir)
             for file in all_files:
-                if self.youtube_id in file:
+                if self.youtube_id in file and file.endswith(".mp4"):
                     vid_path = os.path.join(channel_dir, file)
                     break
             else:
@@ -447,7 +447,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
         return True
 
-    def _check_subtitles(self):
+    def check_subtitles(self):
         """optionally add subtitles"""
         handler = YoutubeSubtitle(self)
         subtitles = handler.get_subtitles()
@@ -457,8 +457,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
     def delete_subtitles(self):
         """delete indexed subtitles"""
+        path = "ta_subtitle/_delete_by_query?refresh=true"
         data = {"query": {"term": {"youtube_id": {"value": self.youtube_id}}}}
-        _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data=data)
+        _, _ = ElasticWrap(path).post(data=data)
 
 
 def index_new_video(youtube_id):
@@ -468,5 +469,6 @@ def index_new_video(youtube_id):
     if not video.json_data:
         raise ValueError("failed to get metadata for " + youtube_id)
 
+    video.check_subtitles()
     video.upload_to_es()
     return video.json_data

tubearchivist/home/src/ta/helper.py

@@ -3,31 +3,16 @@ Loose collection of helper functions
 - don't import AppConfig class here to avoid circular imports
 """
 
-import json
+import random
 import re
 import string
 import subprocess
 import unicodedata
 from urllib.parse import parse_qs, urlparse
 
-import requests
 import yt_dlp
 
 
-def get_total_hits(index, es_url, es_auth, match_field):
-    """get total hits from index"""
-    headers = {"Content-type": "application/json"}
-    data = {"query": {"match": {match_field: True}}}
-    payload = json.dumps(data)
-    url = f"{es_url}/{index}/_search?filter_path=hits.total"
-    request = requests.post(url, data=payload, headers=headers, auth=es_auth)
-    if not request.ok:
-        print(request.text)
-    total_json = json.loads(request.text)
-    total_hits = total_json["hits"]["total"]["value"]
-    return total_hits
-
-
 def clean_string(file_name):
     """clean string to only asci characters"""
     whitelist = "-_.() " + string.ascii_letters + string.digits
@@ -51,6 +36,58 @@ def ignore_filelist(filelist):
     return cleaned
 
 
+def requests_headers():
+    """build header with random user agent for requests outside of yt-dlp"""
+
+    chrome_versions = (
+        "90.0.4430.212",
+        "90.0.4430.24",
+        "90.0.4430.70",
+        "90.0.4430.72",
+        "90.0.4430.85",
+        "90.0.4430.93",
+        "91.0.4472.101",
+        "91.0.4472.106",
+        "91.0.4472.114",
+        "91.0.4472.124",
+        "91.0.4472.164",
+        "91.0.4472.19",
+        "91.0.4472.77",
+        "92.0.4515.107",
+        "92.0.4515.115",
+        "92.0.4515.131",
+        "92.0.4515.159",
+        "92.0.4515.43",
+        "93.0.4556.0",
+        "93.0.4577.15",
+        "93.0.4577.63",
+        "93.0.4577.82",
+        "94.0.4606.41",
+        "94.0.4606.54",
+        "94.0.4606.61",
+        "94.0.4606.71",
+        "94.0.4606.81",
+        "94.0.4606.85",
+        "95.0.4638.17",
+        "95.0.4638.50",
+        "95.0.4638.54",
+        "95.0.4638.69",
+        "95.0.4638.74",
+        "96.0.4664.18",
+        "96.0.4664.45",
+        "96.0.4664.55",
+        "96.0.4664.93",
+        "97.0.4692.20",
+    )
+    template = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        + "AppleWebKit/537.36 (KHTML, like Gecko) "
+        + f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
+    )
+
+    return {"User-Agent": template}
+
+
 class UrlListParser:
     """take a multi line string and detect valid youtube ids"""
 

tubearchivist/home/src/ta/ta_redis.py

@@ -11,12 +11,20 @@ import redis
 from home.src.ta.helper import ignore_filelist
 
 
-class RedisArchivist:
+class RedisBase:
-    """collection of methods to interact with redis"""
+    """connection base for redis"""
 
     REDIS_HOST = os.environ.get("REDIS_HOST")
     REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
     NAME_SPACE = "ta:"
+
+    def __init__(self):
+        self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
+
+
+class RedisArchivist(RedisBase):
+    """collection of methods to interact with redis"""
+
     CHANNELS = [
         "download",
         "add",
@@ -27,14 +35,9 @@ class RedisArchivist:
         "setting",
     ]
 
-    def __init__(self):
-        self.redis_connection = redis.Redis(
-            host=self.REDIS_HOST, port=self.REDIS_PORT
-        )
-
     def set_message(self, key, message, expire=True):
         """write new message to redis"""
-        self.redis_connection.execute_command(
+        self.conn.execute_command(
             "JSON.SET", self.NAME_SPACE + key, ".", json.dumps(message)
         )
 
@@ -43,15 +46,11 @@ class RedisArchivist:
                 secs = 20
             else:
                 secs = expire
-            self.redis_connection.execute_command(
+            self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
-                "EXPIRE", self.NAME_SPACE + key, secs
-            )
 
     def get_message(self, key):
         """get message dict from redis"""
-        reply = self.redis_connection.execute_command(
+        reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
-            "JSON.GET", self.NAME_SPACE + key
-        )
         if reply:
             json_str = json.loads(reply)
         else:
@@ -59,16 +58,27 @@ class RedisArchivist:
 
         return json_str
 
+    def list_items(self, query):
+        """list all matches"""
+        reply = self.conn.execute_command(
+            "KEYS", self.NAME_SPACE + query + "*"
+        )
+        all_matches = [i.decode().lstrip(self.NAME_SPACE) for i in reply]
+        all_results = []
+        for match in all_matches:
+            json_str = self.get_message(match)
+            all_results.append(json_str)
+
+        return all_results
+
     def del_message(self, key):
         """delete key from redis"""
-        response = self.redis_connection.execute_command(
+        response = self.conn.execute_command("DEL", self.NAME_SPACE + key)
-            "DEL", self.NAME_SPACE + key
-        )
         return response
 
     def get_lock(self, lock_key):
         """handle lock for task management"""
-        redis_lock = self.redis_connection.lock(self.NAME_SPACE + lock_key)
+        redis_lock = self.conn.lock(self.NAME_SPACE + lock_key)
         return redis_lock
 
     def get_progress(self):
@@ -76,7 +86,7 @@ class RedisArchivist:
         all_messages = []
         for channel in self.CHANNELS:
             key = "message:" + channel
-            reply = self.redis_connection.execute_command(
+            reply = self.conn.execute_command(
                 "JSON.GET", self.NAME_SPACE + key
             )
             if reply:
@@ -107,19 +117,12 @@ class RedisArchivist:
         return mess_dict
 
 
-class RedisQueue:
+class RedisQueue(RedisBase):
     """dynamically interact with the download queue in redis"""
 
-    REDIS_HOST = os.environ.get("REDIS_HOST")
+    def __init__(self):
-    REDIS_PORT = os.environ.get("REDIS_PORT")
+        super().__init__()
-    NAME_SPACE = "ta:"
+        self.key = self.NAME_SPACE + "dl_queue"
-
-    if not REDIS_PORT:
-        REDIS_PORT = 6379
-
-    def __init__(self, key):
-        self.key = self.NAME_SPACE + key
-        self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
 
     def get_all(self):
         """return all elements in list"""

tubearchivist/home/tasks.py

@@ -8,8 +8,8 @@ Functionality:
 
 import os
 
-import home.apps as startup_apps
 from celery import Celery, shared_task
+from home.apps import StartupCheck
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import (
     ChannelSubscription,
@@ -24,7 +24,6 @@ from home.src.index.filesystem import (
     reindex_old_documents,
     scan_filesystem,
 )
-from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.config import AppConfig, ScheduleBuilder
 from home.src.ta.helper import UrlListParser
 from home.src.ta.ta_redis import RedisArchivist, RedisQueue
@@ -63,9 +62,11 @@ def update_subscribed():
             missing_from_playlists = playlist_handler.find_missing()
             missing = missing_from_channels + missing_from_playlists
             if missing:
-                pending_handler = PendingList()
+                youtube_ids = [{"type": "video", "url": i} for i in missing]
-                all_videos_added = pending_handler.add_to_pending(missing)
+                pending_handler = PendingList(youtube_ids=youtube_ids)
-                ThumbManager().download_vid(all_videos_added)
+                pending_handler.parse_url_list()
+                pending_handler.add_to_pending()
+
         else:
             print("Did not acquire rescan lock.")
 
@@ -86,7 +87,6 @@ def download_pending():
             downloader = VideoDownloader()
             downloader.add_pending()
             downloader.run_queue()
-            downloader.validate_playlists()
         else:
             print("Did not acquire download lock.")
 
@@ -98,7 +98,7 @@ def download_pending():
 @shared_task
 def download_single(youtube_id):
     """start download single video now"""
-    queue = RedisQueue("dl_queue")
+    queue = RedisQueue()
     queue.add_priority(youtube_id)
     print("Added to queue with priority: " + youtube_id)
     # start queue if needed
@@ -128,19 +128,9 @@ def download_single(youtube_id):
 @shared_task
 def extrac_dl(youtube_ids):
     """parse list passed and add to pending"""
-    pending_handler = PendingList()
+    pending_handler = PendingList(youtube_ids=youtube_ids)
-    missing_videos = pending_handler.parse_url_list(youtube_ids)
+    pending_handler.parse_url_list()
-    all_videos_added = pending_handler.add_to_pending(missing_videos)
+    pending_handler.add_to_pending()
-    missing_playlists = pending_handler.missing_from_playlists
-
-    thumb_handler = ThumbManager()
-    if missing_playlists:
-        new_thumbs = PlaylistSubscription().process_url_str(
-            missing_playlists, subscribed=False
-        )
-        thumb_handler.download_playlist(new_thumbs)
-
-    thumb_handler.download_vid(all_videos_added)
 
 
 @shared_task(name="check_reindex")
@@ -191,7 +181,7 @@ def kill_dl(task_id):
         app.control.revoke(task_id, terminate=True)
 
     _ = RedisArchivist().del_message("dl_queue_id")
-    RedisQueue("dl_queue").clear()
+    RedisQueue().clear()
 
     # clear cache
     cache_dir = os.path.join(CONFIG["application"]["cache_dir"], "download")
@@ -277,55 +267,12 @@ def index_channel_playlists(channel_id):
         "message": f'Scanning channel "{channel.youtube_id}" in progress',
     }
     RedisArchivist().set_message("message:playlistscan", mess_dict)
-    all_playlists = channel.get_all_playlists()
+    channel.index_channel_playlists()
-
-    if not all_playlists:
-        print(f"no playlists found for channel {channel_id}")
-        return
-
-    all_indexed = PendingList().get_all_indexed()
-    all_youtube_ids = [i["youtube_id"] for i in all_indexed]
-
-    for idx, (playlist_id, playlist_title) in enumerate(all_playlists):
-        # notify
-        mess_dict = {
-            "status": "message:playlistscan",
-            "level": "info",
-            "title": "Scanning channel for playlists",
-            "message": f"Progress: {idx + 1}/{len(all_playlists)}",
-        }
-        RedisArchivist().set_message("message:playlistscan", mess_dict)
-        print("add playlist: " + playlist_title)
-
-        playlist = YoutubePlaylist(playlist_id)
-        playlist.all_youtube_ids = all_youtube_ids
-        playlist.build_json()
-
-        if not playlist.json_data:
-            # skip if not available
-            continue
-
-        # don't add if no videos downloaded
-        downloaded = [
-            i
-            for i in playlist.json_data["playlist_entries"]
-            if i["downloaded"]
-        ]
-        if not downloaded:
-            continue
-
-        playlist.upload_to_es()
-        playlist.add_vids_to_playlist()
-
-    if all_playlists:
-        playlist.get_playlist_art()
-
-    return
 
 
 try:
     app.conf.beat_schedule = ScheduleBuilder().build_schedule()
 except KeyError:
     # update path from v0.0.8 to v0.0.9 to load new defaults
-    startup_apps.sync_redis_state()
+    StartupCheck().sync_redis_state()
     app.conf.beat_schedule = ScheduleBuilder().build_schedule()

tubearchivist/home/templates/home/base.html

@@ -132,7 +132,7 @@
     </div>
     <div class="footer">
         <div class="boxed-content">
-            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist v0.1.1 </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
+            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist v0.1.3 </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
         </div>
     </div>
 </body>

tubearchivist/home/templates/home/channel.html

@@ -40,40 +40,40 @@
     <div class="channel-list {{ view_style }}">
         {% if results %}
             {% for channel in results %}
-            <div class="channel-item {{ view_style }}">
+                <div class="channel-item {{ view_style }}">
-                <div class="channel-banner {{ view_style }}">
+                    <div class="channel-banner {{ view_style }}">
-                    <a href="{% url 'channel_id' channel.source.channel_id %}">
+                        <a href="{% url 'channel_id' channel.source.channel_id %}">
-                        <img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
+                            <img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
-                    </a>
+                        </a>
-                </div>
-                <div class="info-box info-box-2 {{ view_style }}">
-                    <div class="info-box-item">
-                        <div class="round-img">
-                            <a href="{% url 'channel_id' channel.source.channel_id %}">
-                                <img src="/cache/channels/{{ channel.source.channel_id }}_thumb.jpg" alt="channel-thumb">
-                            </a>
-                        </div>
-                        <div>
-                            <h3><a href="{% url 'channel_id' channel.source.channel_id %}">{{ channel.source.channel_name }}</a></h3>
-                            {% if channel.source.channel_subs >= 1000000 %}
-                                <p>Subscribers: {{ channel.source.channel_subs|intword }}</p>
-                            {% else %}
-                                <p>Subscribers: {{ channel.source.channel_subs|intcomma }}</p>
-                            {% endif %}
-                        </div>
                     </div>
-                    <div class="info-box-item">
+                    <div class="info-box info-box-2 {{ view_style }}">
-                        <div>
+                        <div class="info-box-item">
-                            <p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
+                            <div class="round-img">
-                            {% if channel.source.channel_subscribed %}
+                                <a href="{% url 'channel_id' channel.source.channel_id %}">
-                                <button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
+                                    <img src="/cache/channels/{{ channel.source.channel_id }}_thumb.jpg" alt="channel-thumb">
-                            {% else %}
+                                </a>
-                                <button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
+                            </div>
-                            {% endif %}
+                            <div>
+                                <h3><a href="{% url 'channel_id' channel.source.channel_id %}">{{ channel.source.channel_name }}</a></h3>
+                                {% if channel.source.channel_subs >= 1000000 %}
+                                    <p>Subscribers: {{ channel.source.channel_subs|intword }}</p>
+                                {% else %}
+                                    <p>Subscribers: {{ channel.source.channel_subs|intcomma }}</p>
+                                {% endif %}
+                            </div>
+                        </div>
+                        <div class="info-box-item">
+                            <div>
+                                <p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
+                                {% if channel.source.channel_subscribed %}
+                                    <button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
+                                {% else %}
+                                    <button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
+                                {% endif %}
+                            </div>
                         </div>
                     </div>
                 </div>
-            </div>
             {% endfor %}
         {% else %}
             <h2>No channels found...</h2>

tubearchivist/home/templates/home/channel_id.html

@@ -6,6 +6,7 @@
     <div class="channel-banner">
         <a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
     </div>
+    <div id="notifications" data="channel_id"></div>
     <div class="info-box info-box-3">
         <div class="info-box-item">
             <div class="round-img">
@@ -52,11 +53,47 @@
                     <p>Total Videos archived: {{ max_hits }}</p>
                     <p>Watched: <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
                 {% endif %}
-                <button title="Search for playlists on YouTube for {{ channel_info.channel_name }}" type="button" id="find-playlists-button" data-id="{{ channel_info.channel_id }}" onclick="findPlaylists(this)">Find Playlists</button>
+                <button {% if channel_info.channel_overwrites %} class="danger-button"{% endif %} onclick="showOverwrite()" title="Overwrite settings for channel {{ channel_info.channel_name }}">Configure</button>
                 <a href="/playlist/?search={{ channel_info.channel_id }}" title="Show all playlists belonging to {{ channel_info.channel_name }}"><button>Show Playlists</button></a>
             </div>
         </div>
     </div>
+    <div id="overwrite-form" class="info-box{% if not channel_info.channel_overwrites %} hidden-overwrite{% endif %}">
+        <div class="info-box-item">
+            <form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/" method="POST">
+                {% csrf_token %}
+                <div class="overwrite-form-item">
+                    <p>Download format: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.download_format %}
+                            {{ channel_info.channel_overwrites.download_format }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.download_format }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Auto delete watched videos after x days: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.autodelete_days %}
+                            {{ channel_info.channel_overwrites.autodelete_days }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.autodelete_days }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Index playlists: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.index_playlists %}
+                            {{ channel_info.channel_overwrites.index_playlists }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.index_playlists }}<br>
+                </div>
+                <div class="overwrite-form-item"></div>
+                <button type="submit">Save Channel Overwrites</button>
+            </form>
+        </div>
+    </div>
     {% if channel_info.channel_description %}
         <div class="info-box-item description-box">
             <p>Description: <button onclick="textReveal()" id="text-reveal-button">Show</button></p>
@@ -65,7 +102,6 @@
             </div>
         </div>
     {% endif %}
-    <div id="notifications" data="channel_id"></div>
     <div class="view-controls">
         <div class="toggle">
             <span>Hide watched videos:</span>
@@ -110,6 +146,11 @@
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
                                 <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.source.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                {% else %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                {% endif %}
                             </div>
                             <div class="video-play">
                                 <img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
@@ -119,9 +160,9 @@
                     <div class="video-desc {{ view_style }}">
                         <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
                             {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
                             <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
                         </div>

tubearchivist/home/templates/home/home.html

@@ -2,6 +2,46 @@
 {% block content %}
 {% load static %}
 <div class="boxed-content">
+    {% if continue_vids %}
+        <div class="title-bar">
+            <h1>Continue Watching</h1>
+        </div>
+        <div class="video-list {{ view_style }}">
+            {% for video in continue_vids %}
+                <div class="video-item {{ view_style }}">
+                    <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
+                        <div class="video-thumb-wrap {{ view_style }}">
+                            <div class="video-thumb">
+                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.source.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                {% else %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                {% endif %}
+                            </div>
+                            <div class="video-play">
+                                <img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
+                            </div>
+                        </div>
+                    </a>
+                    <div class="video-desc {{ view_style }}">
+                        <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
+                            {% if video.source.player.watched %}
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
+                            {% else %}
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
+                            {% endif %}
+                            <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
+                        </div>
+                        <div>
+                            <a href="{% url 'channel_id' video.source.channel.channel_id %}"><h3>{{ video.source.channel.channel_name }}</h3></a>
+                            <a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
+                        </div>
+                    </div>
+                </div>
+            {% endfor %}
+        </div>
+    {% endif %}
     <div class="title-bar">
         <h1>Recent Videos</h1>
     </div>
@@ -49,6 +89,11 @@
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
                                 <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.source.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                {% else %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                {% endif %}
                             </div>
                             <div class="video-play">
                                 <img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
@@ -58,9 +103,9 @@
                     <div class="video-desc {{ view_style }}">
                         <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
                             {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
                             <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
                         </div>

tubearchivist/home/templates/home/playlist_id.html

@@ -49,8 +49,8 @@
         <div class="info-box-item">
             <div>
                 {% if max_hits %}
-                <p>Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}</p>
+                    <p>Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}</p>
-                <p>Watched: <button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
+                    <p>Watched: <button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
                 {% endif %}
             </div>
         </div>
@@ -91,6 +91,11 @@
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
                                 <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.source.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                {% else %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                {% endif %}
                             </div>
                             <div class="video-play">
                                 <img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
@@ -100,9 +105,9 @@
                     <div class="video-desc {{ view_style }}">
                         <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
                             {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
                             <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
                         </div>

tubearchivist/home/templates/home/video.html

@@ -2,17 +2,7 @@
 {% block content %}
 {% load static %}
 {% load humanize %}
-<div class="video-main">
+<div class="video-main"></div>
-    <video poster="/cache/{{ video.vid_thumb_url }}" controls preload="false" width="100%" playsinline 
-    ontimeupdate="onVideoProgress('{{ video.youtube_id }}')" onloadedmetadata="setVideoProgress(0)" id="video-item">
-        <source src="/media/{{ video.media_url }}" type="video/mp4" id="video-source">
-        {% if video.subtitles %}
-            {% for subtitle in video.subtitles %}
-                <track label="{{subtitle.name}}" kind="subtitles" srclang="{{subtitle.lang}}" src="/media/{{subtitle.media_url}}">
-            {% endfor %}
-        {% endif %}
-    </video>
-</div>
 <div class="boxed-content">
     <div class="title-bar">
         {% if cast %}
@@ -42,9 +32,9 @@
                 <p>Last refreshed: {{ video.vid_last_refresh }}</p>
                 <p class="video-info-watched">Watched:
                     {% if video.player.watched %}
-                        <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                        <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                     {% else %}
-                        <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                        <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                     {% endif %}
                 </p>
                 {% if video.active %}
@@ -122,4 +112,9 @@
         {% endfor %}
     {% endif %}
 </div>
+<script>
+    var videoData = getVideoData('{{ video.youtube_id }}');
+    var videoProgress = getVideoProgress('{{ video.youtube_id }}').position;
+    window.onload = insertVideoTag(videoData, videoProgress);
+</script>
 {% endblock content %}

tubearchivist/home/views.py

@@ -19,6 +19,7 @@ from home.src.frontend.api_calls import PostData
 from home.src.frontend.forms import (
     AddToQueueForm,
     ApplicationSettingsForm,
+    ChannelOverwriteForm,
     CustomAuthForm,
     MultiSearchForm,
     SchedulerSettingsForm,
@@ -27,12 +28,13 @@ from home.src.frontend.forms import (
     UserSettingsForm,
 )
 from home.src.frontend.searching import SearchHandler
+from home.src.index.channel import channel_overwrites
 from home.src.index.generic import Pagination
 from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.config import AppConfig, ScheduleBuilder
 from home.src.ta.helper import UrlListParser
 from home.src.ta.ta_redis import RedisArchivist
-from home.tasks import extrac_dl, subscribe_to
+from home.tasks import extrac_dl, index_channel_playlists, subscribe_to
 from rest_framework.authtoken.models import Token
 
 
@@ -169,6 +171,50 @@ class ArchivistResultsView(ArchivistViewConfig):
         }
         self.data = data
 
+    def match_progress(self):
+        """add video progress to result context"""
+        results = RedisArchivist().list_items(f"{self.user_id}:progress:")
+        if not results or not self.context["results"]:
+            return
+
+        self.context["continue_vids"] = self.get_in_progress(results)
+
+        in_progress = {i["youtube_id"]: i["position"] for i in results}
+        for hit in self.context["results"]:
+            video = hit["source"]
+            if video["youtube_id"] in in_progress:
+                played_sec = in_progress.get(video["youtube_id"])
+                total = video["player"]["duration"]
+                if not total:
+                    total = played_sec * 2
+                video["player"]["progress"] = 100 * (played_sec / total)
+
+    def get_in_progress(self, results):
+        """get all videos in progress"""
+        ids = [{"match": {"youtube_id": i.get("youtube_id")}} for i in results]
+        data = {
+            "size": self.default_conf["archive"]["page_size"],
+            "query": {"bool": {"should": ids}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        search = SearchHandler(
+            "ta_video/_search", self.default_conf, data=data
+        )
+        videos = search.get_data()
+        if not videos:
+            return False
+
+        for video in videos:
+            youtube_id = video["source"]["youtube_id"]
+            matched = [i for i in results if i["youtube_id"] == youtube_id]
+            played_sec = matched[0]["position"]
+            total = video["source"]["player"]["duration"]
+            if not total:
+                total = matched[0].get("position") * 2
+            video["source"]["player"]["progress"] = 100 * (played_sec / total)
+
+        return videos
+
     def single_lookup(self, es_path):
         """retrieve a single item from url"""
         search = SearchHandler(es_path, config=self.default_conf)
@@ -212,11 +258,19 @@ class HomeView(ArchivistResultsView):
         self.initiate_vars(request)
         self._update_view_data()
         self.find_results()
+        self.match_progress()
 
         return render(request, "home/home.html", self.context)
 
     def _update_view_data(self):
         """update view specific data dict"""
+        self.data["sort"].extend(
+            [
+                {"channel.channel_name.keyword": {"order": "asc"}},
+                {"title.keyword": {"order": "asc"}},
+            ]
+        )
+
         if self.context["hide_watched"]:
             self.data["query"] = {"term": {"player.watched": {"value": False}}}
         if self.search_get:
@@ -355,6 +409,7 @@ class ChannelIdView(ArchivistResultsView):
         self.initiate_vars(request)
         self._update_view_data(channel_id)
         self.find_results()
+        self.match_progress()
 
         if self.context["results"]:
             channel_info = self.context["results"][0]["source"]["channel"]
@@ -369,6 +424,7 @@ class ChannelIdView(ArchivistResultsView):
             {
                 "title": "Channel: " + channel_name,
                 "channel_info": channel_info,
+                "channel_overwrite_form": ChannelOverwriteForm,
             }
         )
 
@@ -376,19 +432,34 @@ class ChannelIdView(ArchivistResultsView):
 
     def _update_view_data(self, channel_id):
         """update view specific data dict"""
-        query = {
+        self.data["query"] = {
             "bool": {
                 "must": [
                     {"term": {"channel.channel_id": {"value": channel_id}}}
                 ]
             }
         }
-        self.data["query"] = query
+        self.data["sort"].append({"title.keyword": {"order": "asc"}})
 
         if self.context["hide_watched"]:
             to_append = {"term": {"player.watched": {"value": False}}}
             self.data["query"]["bool"]["must"].append(to_append)
 
+    @staticmethod
+    def post(request, channel_id):
+        """handle post request"""
+        print(f"handle post from {channel_id}")
+        channel_overwrite_form = ChannelOverwriteForm(request.POST)
+        if channel_overwrite_form.is_valid():
+            overwrites = channel_overwrite_form.cleaned_data
+            print(f"{channel_id}: set overwrites {overwrites}")
+            channel_overwrites(channel_id, overwrites=overwrites)
+            if overwrites.get("index_playlists") == "1":
+                index_channel_playlists.delay(channel_id)
+
+        sleep(1)
+        return redirect("channel_id", channel_id, permanent=True)
+
 
 class ChannelView(ArchivistResultsView):
     """resolves to /channel/
@@ -456,6 +527,7 @@ class PlaylistIdView(ArchivistResultsView):
         playlist_name = playlist_info["playlist_name"]
         self._update_view_data(playlist_id, playlist_info)
         self.find_results()
+        self.match_progress()
         self.context.update(
             {
                 "title": "Playlist: " + playlist_name,

tubearchivist/requirements.txt

@@ -1,12 +1,12 @@
 beautifulsoup4==4.10.0
 celery==5.2.3
-Django==4.0.2
+Django==4.0.3
 django-cors-headers==3.11.0
 djangorestframework==3.13.1
 Pillow==9.0.1
-redis==4.1.3
+redis==4.2.0
 requests==2.27.1
 ryd-client==0.0.3
 uWSGI==2.0.20
 whitenoise==6.0.0
-yt_dlp==2022.2.4
+yt_dlp==2022.3.8.2

tubearchivist/static/cast-videos.js

@@ -13,6 +13,16 @@ function initializeCastApi() {
             castConnectionChange(player) 
         }
     );
+    playerController.addEventListener(
+        cast.framework.RemotePlayerEventType.CURRENT_TIME_CHANGED, function() { 
+            castVideoProgress(player) 
+        }
+    );
+    playerController.addEventListener(
+        cast.framework.RemotePlayerEventType.IS_PAUSED_CHANGED, function() { 
+            castVideoPaused(player) 
+        }
+    );
 }
 
 
@@ -26,32 +36,65 @@ function castConnectionChange(player) {
     }
 }
 
+function castVideoProgress(player) {
+    var videoId = getVideoPlayerVideoId();
+    if (player.mediaInfo.contentId.includes(videoId)) {
+        var currentTime = player.currentTime;
+        var duration = player.duration;
+        if ((currentTime % 10) <= 1.0 && currentTime != 0 && duration != 0) { // Check progress every 10 seconds or else progress is checked a few times a second
+            postVideoProgress(videoId, currentTime);
+            setProgressBar(videoId, currentTime, duration);
+            if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
+                if (watchedThreshold(currentTime, duration)) {
+                    isWatched(videoId);
+                }
+            }
+        }
+    }
+}
+
+function castVideoPaused(player) {
+    var videoId = getVideoPlayerVideoId();
+    var currentTime = player.currentTime;
+    var duration = player.duration;
+    if (player.mediaInfo != null) {
+        if (player.mediaInfo.contentId.includes(videoId)) {
+            if (currentTime != 0 && duration != 0) {
+                postVideoProgress(videoId, currentTime);
+            }
+        }
+    }
+}
+
 function castStart() {
     var castSession = cast.framework.CastContext.getInstance().getCurrentSession();
-    
     // Check if there is already media playing on the cast target to prevent recasting on page reload or switching to another video page
     if (!castSession.getMediaSession()) { 
-        contentId = document.getElementById("video-source").src; // Get video URL
+        var videoId = getVideoPlayerVideoId();
-        contentTitle = document.getElementById('video-title').innerHTML; // Get video title
+        var videoData = getVideoData(videoId);
-        contentImage = document.getElementById("video-item").poster; // Get video thumbnail URL
+        var contentId = getURL() + videoData.data.media_url;
+        var contentTitle = videoData.data.title;
+        var contentImage = getURL() + videoData.data.vid_thumb_url;
+
         contentType = 'video/mp4'; // Set content type, only videos right now so it is hard coded
-        contentCurrentTime = document.getElementById("video-item").currentTime; // Get video's current position
+        contentCurrentTime = getVideoPlayerCurrentTime(); // Get video's current position
         contentActiveSubtitle = [];
         // Check if a subtitle is turned on.
-        for (var i = 0; i < document.getElementById("video-item").textTracks.length; i++) {
+        for (var i = 0; i < getVideoPlayer().textTracks.length; i++) {
-            if (document.getElementById("video-item").textTracks[i].mode == "showing") {
+            if (getVideoPlayer().textTracks[i].mode == "showing") {
                 contentActiveSubtitle =[i + 1];
             }
         }
         contentSubtitles = [];
-        for (var i = 0; i < document.getElementById("video-item").children.length; i++) {
+        var videoSubtitles = videoData.data.subtitles; // Array of subtitles
-            if (document.getElementById("video-item").children[i].tagName == "TRACK") {
+        if (typeof(videoSubtitles) != 'undefined' && videoData.config.downloads.subtitle) {
+            for (var i = 0; i < videoSubtitles.length; i++) {
                 subtitle = new chrome.cast.media.Track(i, chrome.cast.media.TrackType.TEXT);
-                subtitle.trackContentId = document.getElementById("video-item").children[i].src;
+                subtitle.trackContentId = videoSubtitles[i].media_url;
                 subtitle.trackContentType = 'text/vtt';
                 subtitle.subtype = chrome.cast.media.TextTrackType.SUBTITLES;
-                subtitle.name = document.getElementById("video-item").children[i].label;
+                subtitle.name = videoSubtitles[i].name;
-                subtitle.language = document.getElementById("video-item").children[i].srclang;
+                subtitle.language = videoSubtitles[i].lang;
                 subtitle.customData = null;
                 contentSubtitles.push(subtitle);
             }
@@ -91,7 +134,7 @@ function shiftCurrentTime(contentCurrentTime) { // Shift media back 3 seconds to
 
 function castSuccessful() {
     // console.log('Cast Successful.');
-    document.getElementById("video-item").pause(); // Pause browser video on successful cast
+    getVideoPlayer().pause(); // Pause browser video on successful cast
 }
 
 function castFailed(errorCode) {

tubearchivist/static/css/style.css

@@ -370,16 +370,18 @@ button:hover {
 }
 
 /* video list */
-.video-list.grid {
+.video-list {
     display: grid;
-    grid-template-columns: 1fr 1fr 1fr;
     grid-gap: 1rem;
+    margin-top: 1rem;
+}
+
+.video-list.grid {
+    grid-template-columns: 1fr 1fr 1fr;
 }
 
 .video-list.list {
-    display: grid;
     grid-template-columns: unset;
-    grid-gap: 1rem;
 }
 
 .video-item {
@@ -391,8 +393,17 @@ button:hover {
     grid-template-columns: 25% auto;
 }
 
+.video-progress-bar {
+    position: absolute;
+    background-color: var(--accent-font-dark);
+    height: 7px;
+    left: 0;
+    bottom: 3px;
+}
+
 .video-thumb img {
     width: 100%;
+    position: relative;
 }
 
 .video-play img {
@@ -473,8 +484,7 @@ button:hover {
     align-items: center;
 }
 
-.unseen-icon,
+.watch-button,
-.seen-icon,
 .close-button {
     cursor: pointer;
     filter: var(--img-filter);
@@ -556,6 +566,28 @@ button:hover {
     margin-bottom: 10px;
 }
 
+.overwrite-form {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    width: 100%;
+}
+
+.overwrite-form button {
+    width: 200px;
+}
+
+.overwrite-form-item {
+    margin-bottom: 1rem;
+}
+
+.overwrite-form-item input {
+    width: 90%;
+}
+
+.hidden-overwrite {
+    display: none;
+}
+
 /* login */
 .login-page {
     display: flex;
@@ -1052,7 +1084,10 @@ button:hover {
     .channel-list.grid,
     .video-item.list,
     .playlist-list.list,
-    .playlist-list.grid {
+    .playlist-list.grid,
+    .info-box-2,
+    .info-box-3,
+    .overwrite-form {
         grid-template-columns: 1fr;
     }
     .playlist-item.list {
@@ -1091,10 +1126,6 @@ button:hover {
     .sort select {
         margin: unset;
     }
-    .info-box-2,
-    .info-box-3 {
-        grid-template-columns: 1fr;
-    }
     .description-box {
         display: block;
     }

tubearchivist/static/progress.js

@@ -7,7 +7,7 @@ checkMessages()
 
 // page map to notification status
 const messageTypes = {
-    "download": ["message:download", "message:add", "message:rescan"],
+    "download": ["message:download", "message:add", "message:rescan", "message:playlistscan"],
     "channel": ["message:subchannel"],
     "channel_id": ["message:playlistscan"],
     "playlist": ["message:subplaylist"],

tubearchivist/static/script.js

@@ -8,18 +8,65 @@ function sortChange(sortValue) {
     }, 500);
 }
 
-function isWatched(youtube_id) {
+// Updates video watch status when passed a video id and it's current state (ex if the video was unwatched but you want to mark it as watched you will pass "unwatched")
-    // sendVideoProgress(youtube_id, 0); // Reset video progress on watched;
+function updateVideoWatchStatus(input1, videoCurrentWatchStatus) {
-    var payload = JSON.stringify({'watched': youtube_id});
+    if (videoCurrentWatchStatus) {
-    sendPost(payload);
+        videoId = input1;
-    var seenIcon = document.createElement('img');
+    } else if (input1.getAttribute("data-id")) {
-    seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
+        videoId = input1.getAttribute("data-id");
-    seenIcon.setAttribute('alt', 'seen-icon');
+        videoCurrentWatchStatus = input1.getAttribute("data-status");
-    seenIcon.setAttribute('id', youtube_id);
+    }
-    seenIcon.setAttribute('title', "Mark as unwatched");
+
-    seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
+    postVideoProgress(videoId, 0); // Reset video progress on watched/unwatched;
-    seenIcon.classList = 'seen-icon';
+    removeProgressBar(videoId);
-    document.getElementById(youtube_id).replaceWith(seenIcon);
+
+    if (videoCurrentWatchStatus == "watched") {
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "unwatched");
+        var payload = JSON.stringify({'un_watched': videoId});
+        sendPost(payload);
+    } else if (videoCurrentWatchStatus == "unwatched") {
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "watched");
+        var payload = JSON.stringify({'watched': videoId});
+        sendPost(payload);
+    }
+
+    var watchButtons = document.getElementsByClassName("watch-button");
+    for (let i = 0; i < watchButtons.length; i++) {
+        if (watchButtons[i].getAttribute("data-id") == videoId) {
+            watchButtons[i].outerHTML = watchStatusIndicator;
+        }
+    }
+}
+
+// Creates a watch status indicator when passed a video id and the videos watch status
+function createWatchStatusIndicator(videoId, videoWatchStatus) {
+    if (videoWatchStatus == "watched") {
+        var seen = "seen";
+        var title = "Mark as unwatched";
+    } else if (videoWatchStatus == "unwatched") {
+        var seen = "unseen";
+        var title = "Mark as watched";
+    }
+    var watchStatusIndicator = `<img src="/static/img/icon-${seen}.svg" alt="${seen}-icon" data-id="${videoId}" data-status="${videoWatchStatus}" onclick="updateVideoWatchStatus(this)" class="watch-button" title="${title}">`;
+    return watchStatusIndicator;
+}
+
+// function isWatched(youtube_id) {
+//     var payload = JSON.stringify({'watched': youtube_id});
+//     sendPost(payload);
+//     var seenIcon = document.createElement('img');
+//     seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
+//     seenIcon.setAttribute('alt', 'seen-icon');
+//     seenIcon.setAttribute('id', youtube_id);
+//     seenIcon.setAttribute('title', "Mark as unwatched");
+//     seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
+//     seenIcon.classList = 'seen-icon';
+//     document.getElementById(youtube_id).replaceWith(seenIcon);
+// }
+
+// Removes the progress bar when passed a video id
+function removeProgressBar(videoId) {
+    setProgressBar(videoId, 0, 1);
 }
 
 function isWatchedButton(button) {
@@ -33,19 +80,19 @@ function isWatchedButton(button) {
     }, 1000);
 }
 
-function isUnwatched(youtube_id) {
+// function isUnwatched(youtube_id) {
-    // sendVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
+//     postVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
-    var payload = JSON.stringify({'un_watched': youtube_id});
+//     var payload = JSON.stringify({'un_watched': youtube_id});
-    sendPost(payload);
+//     sendPost(payload);
-    var unseenIcon = document.createElement('img');
+//     var unseenIcon = document.createElement('img');
-    unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
+//     unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
-    unseenIcon.setAttribute('alt', 'unseen-icon');
+//     unseenIcon.setAttribute('alt', 'unseen-icon');
-    unseenIcon.setAttribute('id', youtube_id);
+//     unseenIcon.setAttribute('id', youtube_id);
-    unseenIcon.setAttribute('title', "Mark as watched");
+//     unseenIcon.setAttribute('title', "Mark as watched");
-    unseenIcon.setAttribute('onclick', "isWatched(this.id)");
+//     unseenIcon.setAttribute('onclick', "isWatched(this.id)");
-    unseenIcon.classList = 'unseen-icon';
+//     unseenIcon.classList = 'unseen-icon';
-    document.getElementById(youtube_id).replaceWith(unseenIcon);
+//     document.getElementById(youtube_id).replaceWith(unseenIcon);
-}
+// }
 
 function unsubscribe(id_unsub) {
     var payload = JSON.stringify({'unsubscribe': id_unsub});
@@ -145,7 +192,6 @@ function addSingle(button) {
 function deleteQueue(button) {
     var to_delete = button.getAttribute('data-id');
     var payload = JSON.stringify({'deleteQueue': to_delete});
-    console.log(payload);
     sendPost(payload);
     setTimeout(function(){
         location.reload();
@@ -222,19 +268,6 @@ function fsRescan() {
     toReplace.appendChild(message);
 }
 
-function findPlaylists(button) {
-    var channel_id = button.getAttribute("data-id");
-    var payload = JSON.stringify({'find-playlists': channel_id});
-    sendPost(payload);
-    // clear button
-    var message = document.createElement('p');
-    message.innerText = 'Scraping for playlists in progress';
-    document.getElementById("find-playlists-button").replaceWith(message);
-    setTimeout(function(){
-        checkMessages();
-    }, 500);
-}
-
 function resetToken() {
     var payload = JSON.stringify({'reset-token': true});
     sendPost(payload);
@@ -281,7 +314,6 @@ function deletePlaylist(button) {
             "playlist-action": playlist_action
         }
     });
-    console.log(payload);
     sendPost(payload);
     setTimeout(function(){
         window.location.replace("/playlist/");
@@ -298,20 +330,13 @@ function cancelDelete() {
 function createPlayer(button) {
     var videoId = button.getAttribute('data-id');
     var videoData = getVideoData(videoId);
-    var videoUrl = videoData.media_url;
+    var videoProgress = getVideoProgress(videoId).position;
-    var videoThumbUrl = videoData.vid_thumb_url;
+    var videoName = videoData.data.title;
-    var videoName = videoData.title;
 
-    var subtitles = '';
+    var videoTag = createVideoTag(videoData, videoProgress);
-    var videoSubtitles = videoData.subtitles; // Array of subtitles
-    if (typeof(videoSubtitles) != 'undefined') {
-        for (var i = 0; i < videoSubtitles.length; i++) {
-            subtitles += `<track label="${videoSubtitles[i].name}" kind="subtitles" srclang="${videoSubtitles[i].lang}" src="${videoSubtitles[i].media_url}">`;
-        }
-    }
 
     var playlist = '';
-    var videoPlaylists = videoData.playlist; // Array of playlists the video is in
+    var videoPlaylists = videoData.data.playlist; // Array of playlists the video is in
     if (typeof(videoPlaylists) != 'undefined') {
         var subbedPlaylists = getSubbedPlaylists(videoPlaylists); // Array of playlist the video is in that are subscribed
         if (subbedPlaylists.length != 0) {
@@ -322,51 +347,45 @@ function createPlayer(button) {
         }
     }
 
-    var videoProgress = videoData.player.progress; // Groundwork for saving video position, change once progress variable is added to API
+    var videoViews = formatNumbers(videoData.data.stats.view_count);
-    var videoViews = formatNumbers(videoData.stats.view_count);
 
-    var channelId = videoData.channel.channel_id;
+    var channelId = videoData.data.channel.channel_id;
-    var channelName = videoData.channel.channel_name;
+    var channelName = videoData.data.channel.channel_name;
 
     removePlayer();
-    document.getElementById(videoId).outerHTML = ''; // Remove watch indicator from video info
+    // document.getElementById(videoId).outerHTML = ''; // Remove watch indicator from video info
 
     // If cast integration is enabled create cast button
-    var castButton = ``;
+    var castButton = '';
-    var castScript = document.getElementById('cast-script');
+    if (videoData.config.application.enable_cast) {
-    if (typeof(castScript) != 'undefined' && castScript != null) {
         var castButton = `<google-cast-launcher id="castbutton"></google-cast-launcher>`;
     }
 
     // Watched indicator
-    if (videoData.player.watched) {
+    if (videoData.data.player.watched) {
-        var playerState = "seen";
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "watched");
-        var watchedFunction = "Unwatched";
     } else {
-        var playerState = "unseen";
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "unwatched");
-        var watchedFunction = "Watched";  
     }
+    
 
     var playerStats = `<div class="thumb-icon player-stats"><img src="/static/img/icon-eye.svg" alt="views icon"><span>${videoViews}</span>`;
-    if (videoData.stats.like_count) {
+    if (videoData.data.stats.like_count) {
-        var likes = formatNumbers(videoData.stats.like_count);
+        var likes = formatNumbers(videoData.data.stats.like_count);
         playerStats += `<span>|</span><img src="/static/img/icon-thumb.svg" alt="thumbs-up"><span>${likes}</span>`;
     }
-    if (videoData.stats.dislike_count) {
+    if (videoData.data.stats.dislike_count && videoData.config.downloads.integrate_ryd) {
-        var dislikes = formatNumbers(videoData.stats.dislike_count);
+        var dislikes = formatNumbers(videoData.data.stats.dislike_count);
         playerStats += `<span>|</span><img class="dislike" src="/static/img/icon-thumb.svg" alt="thumbs-down"><span>${dislikes}</span>`;
     }
     playerStats += "</div>";
 
     const markup = `
     <div class="video-player" data-id="${videoId}">
-        <video poster="${videoThumbUrl}" ontimeupdate="onVideoProgress('${videoId}')" controls autoplay width="100%" playsinline id="video-item">
+        ${videoTag}
-            <source src="${videoUrl}#t=${videoProgress}" type="video/mp4" id="video-source">
-            ${subtitles}
-        </video>
         <div class="player-title boxed-content">
             <img class="close-button" src="/static/img/icon-close.svg" alt="close-icon" data="${videoId}" onclick="removePlayer()" title="Close player">
-            <img src="/static/img/icon-${playerState}.svg" alt="${playerState}-icon" id="${videoId}" onclick="is${watchedFunction}(this.id)" class="${playerState}-icon" title="Mark as ${watchedFunction}">
+            ${watchStatusIndicator}
             ${castButton}
             ${playerStats}
             <div class="player-channel-playlist">
@@ -377,47 +396,135 @@ function createPlayer(button) {
         </div>
     </div>
     `;
-    const divPlayer =  document.getElementById("player");
+    const divPlayer = document.getElementById("player");
     divPlayer.innerHTML = markup;
 }
 
-// Set video progress in seconds
+// Add video tag to video page when passed a video id, function loaded on page load `video.html (115-117)`
-function setVideoProgress(videoProgress) {
+function insertVideoTag(videoData, videoProgress) {
-    if (isNaN(videoProgress)) {
+    var videoTag = createVideoTag(videoData, videoProgress);
-        videoProgress = 0;
+    var videoMain = document.getElementsByClassName("video-main");
-    }
+    videoMain[0].innerHTML = videoTag;
-    var videoElement = document.getElementById("video-item");
-    videoElement.currentTime = videoProgress;
 }
 
-// Runs on video playback, marks video as watched if video gets to 90% or higher, WIP sends position to api
+// Generates a video tag with subtitles when passed videoData and videoProgress.
-function onVideoProgress(videoId) {
+function createVideoTag(videoData, videoProgress) {
+    var videoId = videoData.data.youtube_id;
+    var videoUrl = videoData.data.media_url;
+    var videoThumbUrl = videoData.data.vid_thumb_url;
+    var subtitles = '';
+    var videoSubtitles = videoData.data.subtitles; // Array of subtitles
+    if (typeof(videoSubtitles) != 'undefined' && videoData.config.downloads.subtitle) {
+        for (var i = 0; i < videoSubtitles.length; i++) {
+            let label = videoSubtitles[i].name;
+            if (videoSubtitles[i].source == "auto") {
+                label += " - auto";
+            }
+            subtitles += `<track label="${label}" kind="subtitles" srclang="${videoSubtitles[i].lang}" src="${videoSubtitles[i].media_url}">`;
+        }
+    }
+
+    var videoTag = `
+    <video poster="${videoThumbUrl}" ontimeupdate="onVideoProgress()" onpause="onVideoPause()" onended="onVideoEnded()" controls autoplay width="100%" playsinline id="video-item">
+        <source src="${videoUrl}#t=${videoProgress}" type="video/mp4" id="video-source" videoid="${videoId}">
+        ${subtitles}
+    </video>
+    `;
+    return videoTag;
+}
+
+// Gets video tag
+function getVideoPlayer() {
     var videoElement = document.getElementById("video-item");
+    return videoElement;
+}
+
+// Gets the video source tag
+function getVideoPlayerVideoSource() {
+    var videoPlayerVideoSource = document.getElementById("video-source");
+    return videoPlayerVideoSource;
+}
+
+// Gets the current progress of the video currently in the player
+function getVideoPlayerCurrentTime() {
+    var videoElement = getVideoPlayer();
     if (videoElement != null) {
-        if ((videoElement.currentTime % 10).toFixed(1) <= 0.2) { // Check progress every 10 seconds or else progress is checked a few times a second
+        return videoElement.currentTime;
-            // sendVideoProgress(videoId, videoElement.currentTime); // Groundwork for saving video position
+    }
-            if (((videoElement.currentTime / videoElement.duration) >= 0.90) && document.getElementById(videoId).className == "unseen-icon") {
+}
-                isWatched(videoId);
+
+// Gets the video id of the video currently in the player
+function getVideoPlayerVideoId() {
+    var videoPlayerVideoSource = getVideoPlayerVideoSource();
+    if (videoPlayerVideoSource != null) {
+        return videoPlayerVideoSource.getAttribute("videoid");
+    }
+}
+
+// Gets the duration of the video currently in the player
+function getVideoPlayerDuration() {
+    var videoElement = getVideoPlayer();
+    if (videoElement != null) {
+        return videoElement.duration;
+    }
+}
+
+// Gets current watch status of video based on watch button
+function getVideoPlayerWatchStatus() {
+    var videoId = getVideoPlayerVideoId();
+    var watched = false;
+
+    var watchButtons = document.getElementsByClassName("watch-button");
+    for (let i = 0; i < watchButtons.length; i++) {
+        if (watchButtons[i].getAttribute("data-id") == videoId && watchButtons[i].getAttribute("data-status") == "watched") {
+            watched = true;
+        }
+    }
+    return watched;
+}
+
+// Runs on video playback, marks video as watched if video gets to 90% or higher, sends position to api
+function onVideoProgress() {
+    var videoId = getVideoPlayerVideoId();
+    var currentTime = getVideoPlayerCurrentTime();
+    var duration = getVideoPlayerDuration();
+    if ((currentTime % 10).toFixed(1) <= 0.2) { // Check progress every 10 seconds or else progress is checked a few times a second
+        postVideoProgress(videoId, currentTime);
+        if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
+            if (watchedThreshold(currentTime, duration)) {
+                updateVideoWatchStatus(videoId, "unwatched");
             }
         }
     }
 }
 
-// Groundwork for saving video position
+// Runs on video end, marks video as watched
-function sendVideoProgress(videoId, videoProgress) {
+function onVideoEnded() {
-    var apiEndpoint = "/api/video/";
+    var videoId = getVideoPlayerVideoId();
-    if (isNaN(videoProgress)) {
+    if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
-        videoProgress = 0;
+        updateVideoWatchStatus(videoId, "unwatched");
     }
-    var data = {
+}
-        "data": [{
+
-            "youtube_id": videoId,
+function watchedThreshold(currentTime, duration) {
-            "player": {
+    var watched = false;
-                "progress": videoProgress 
+    if (duration <= 1800){ // If video is less than 30 min
-            }
+        if ((currentTime / duration) >= 0.90) { // Mark as watched at 90%
-        }]
+            var watched = true;
-    };
+        }
-    videoData = apiRequest(apiEndpoint, "POST", data);
+    } else { // If video is more than 30 min
+        if (currentTime >= (duration - 120)) { // Mark as watched if there is two minutes left
+            var watched = true;
+        }
+    }
+    return watched;
+}
+
+// Runs on video pause. Sends current position.
+function onVideoPause() {
+    var videoId = getVideoPlayerVideoId();
+    var currentTime = getVideoPlayerCurrentTime();
+    postVideoProgress(videoId, currentTime);
 }
 
 // Format numbers for frontend
@@ -435,27 +542,34 @@ function formatNumbers(number) {
     return numberFormatted;
 }
 
-// Gets video data in JSON format when passed video ID
+// Gets video data when passed video ID
 function getVideoData(videoId) {
     var apiEndpoint = "/api/video/" + videoId + "/";
-    videoData = apiRequest(apiEndpoint, "GET");
+    var videoData = apiRequest(apiEndpoint, "GET");
-    return videoData.data;
+    return videoData;
 }
 
-// Gets channel data in JSON format when passed channel ID
+// Gets channel data when passed channel ID
 function getChannelData(channelId) {
     var apiEndpoint = "/api/channel/" + channelId + "/";
-    channelData = apiRequest(apiEndpoint, "GET");
+    var channelData = apiRequest(apiEndpoint, "GET");
     return channelData.data;
 }
 
-// Gets playlist data in JSON format when passed playlist ID
+// Gets playlist data when passed playlist ID
 function getPlaylistData(playlistId) {
     var apiEndpoint = "/api/playlist/" + playlistId + "/";
-    playlistData = apiRequest(apiEndpoint, "GET");
+    var playlistData = apiRequest(apiEndpoint, "GET");
     return playlistData.data;
 }
 
+// Get video progress data when passed video ID
+function getVideoProgress(videoId) {
+    var apiEndpoint = "/api/video/" + videoId + "/progress/";
+    var videoProgress = apiRequest(apiEndpoint, "GET");
+    return videoProgress;
+}
+
 // Given an array of playlist ids it returns an array of subbed playlist ids from that list
 function getSubbedPlaylists(videoPlaylists) {
     var subbedPlaylists = [];
@@ -467,18 +581,47 @@ function getSubbedPlaylists(videoPlaylists) {
     return subbedPlaylists;
 }
 
-// Makes api requests when passed an endpoint and method ("GET" or "POST")
+// Send video position when given video id and progress in seconds
+function postVideoProgress(videoId, videoProgress) {
+    var apiEndpoint = "/api/video/" + videoId + "/progress/";
+    var duartion = getVideoPlayerDuration();
+    if (!isNaN(videoProgress) && duartion != 'undefined') {
+        var data = {
+            "position": videoProgress
+        };
+        if (videoProgress == 0) {
+            apiRequest(apiEndpoint, "DELETE");
+            // console.log("Deleting Video Progress for Video ID: " + videoId + ", Progress: " + videoProgress);
+        } else if (!getVideoPlayerWatchStatus()) {
+            apiRequest(apiEndpoint, "POST", data);
+            // console.log("Saving Video Progress for Video ID: " + videoId + ", Progress: " + videoProgress);
+        }
+    }
+}
+
+// Makes api requests when passed an endpoint and method ("GET", "POST", "DELETE")
 function apiRequest(apiEndpoint, method, data) {
     const xhttp = new XMLHttpRequest();
     var sessionToken = getCookie("sessionid");
     xhttp.open(method, apiEndpoint, false);
+    xhttp.setRequestHeader("X-CSRFToken", getCookie("csrftoken")); // Used for video progress POST requests
     xhttp.setRequestHeader("Authorization", "Token " + sessionToken);
     xhttp.setRequestHeader("Content-Type", "application/json");
     xhttp.send(JSON.stringify(data));
     return JSON.parse(xhttp.responseText);
 }
 
+// Gets origin URL
+function getURL() {
+    return window.location.origin;
+}
+
 function removePlayer() {
+    var currentTime = getVideoPlayerCurrentTime();
+    var duration = getVideoPlayerDuration();
+    var videoId = getVideoPlayerVideoId();
+    postVideoProgress(videoId, currentTime);
+    setProgressBar(videoId, currentTime, duration);
     var playerElement = document.getElementById('player');
     if (playerElement.hasChildNodes()) {
         var youtubeId = playerElement.childNodes[1].getAttribute("data-id");
@@ -494,6 +637,24 @@ function removePlayer() {
     }
 }
 
+// Sets the progress bar when passed a video id, video progress and video duration
+function setProgressBar(videoId, currentTime, duration) {
+    var progressBarWidth = (currentTime / duration) * 100 + "%";
+    var progressBars = document.getElementsByClassName("video-progress-bar");
+    for (let i = 0; i < progressBars.length; i++) {
+        if (progressBars[i].id == "progress-" + videoId) {
+            if (!getVideoPlayerWatchStatus()) {
+                progressBars[i].style.width = progressBarWidth;
+            } else {
+                progressBars[i].style.width = "0%";
+            }
+        }
+    }
+
+    // progressBar = document.getElementById("progress-" + videoId);
+    
+    
+}
 
 // multi search form
 function searchMulti(query) {
@@ -561,9 +722,9 @@ function createVideo(video, viewStyle) {
     const videoPublished = video.published;
     const videoDuration = video.player.duration_str;
     if (video.player.watched) {
-        var playerState = "seen";
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "watched");
     } else {
-        var playerState = "unseen";
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "unwatched");
     };
     const channelId = video.channel.channel_id;
     const channelName = video.channel.channel_name;
@@ -581,7 +742,7 @@ function createVideo(video, viewStyle) {
     </a>
     <div class="video-desc ${viewStyle}">
         <div class="video-desc-player" id="video-info-${videoId}">
-                <img src="/static/img/icon-${playerState}.svg" alt="${playerState}-icon" id="${videoId}" onclick="isWatched(this.id)" class="${playerState}-icon">
+                ${watchStatusIndicator}
             <span>${videoPublished} | ${videoDuration}</span>
         </div>
         <div>
@@ -722,6 +883,15 @@ function showForm() {
     animate('animate-icon', 'pulse-img');
 }
 
+function showOverwrite() {
+    var overwriteDiv = document.getElementById("overwrite-form");
+    if (overwriteDiv.classList.contains("hidden-overwrite")) {
+        overwriteDiv.classList.remove("hidden-overwrite");
+    } else {
+        overwriteDiv.classList.add("hidden-overwrite")
+    }
+}
+
 function animate(elementId, animationClass) {
     var toAnimate = document.getElementById(elementId);
     if (toAnimate.className !== animationClass) {

tubearchivist/testing.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-# install debug and testing tools into slim container
-
-apt update && apt install -y vim htop bmon net-tools iputils-ping procps
-
-pip install ipython
-
-##
-exit 0