increase version to v0.1.3

Changed:
- layout of the channel configure form
- hide if no overwrites set
- validate min es version 7.17.x
- improved build to incl debug tools

.github/workflows/lint_python.yml

@@ -11,7 +11,7 @@ jobs:
                          flake8-comprehensions isort
       - run: black --check --diff --line-length 79 .
       - run: codespell
-      - run: flake8 . --count --max-complexity=12 --max-line-length=79
+      - run: flake8 . --count --max-complexity=10 --max-line-length=79
                       --show-source --statistics
       - run: isort --check-only --line-length 79 --profile black .
       # - run: pip install -r tubearchivist/requirements.txt

CONTRIBUTING.md

@@ -25,7 +25,7 @@ This is my setup I have landed on, YMMV:
 ```bash
 ./deploy.sh test
 ```
-- The command above will also copy the file `tubarchivist/testing.sh` into the working folder of the container. Running this script will install additional debugging tools I regularly use in testing. 
+- The command above will call the docker build command with `--build-arg INSTALL_DEBUG=1` to install additional useful debug tools.
 - The `test` argument takes another optional argument to build for a specific architecture valid options are: `amd64`, `arm64` and `multi`, default is `amd64`.
 - This `deploy.sh` file is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking. 
 

Dockerfile

@@ -1,7 +1,8 @@
 # build the tube archivist image from default python slim image
 
-FROM python:3.10.2-slim-bullseye
+FROM python:3.10.3-slim-bullseye
 ARG TARGETPLATFORM
+ARG INSTALL_DEBUG
 
 ENV PYTHONUNBUFFERED 1
 
@@ -26,6 +27,13 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
         apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
     ; fi
 
+# install debug tools for testing environment
+RUN if [ "$INSTALL_DEBUG" ] ; then \
+        apt-get -y update && apt-get -y install --no-install-recommends \
+        vim htop bmon net-tools iputils-ping procps \
+        && pip install --no-cache-dir ipython --src /usr/local/src \
+    ; fi
+
 # make folders
 RUN mkdir /cache
 RUN mkdir /youtube

deploy.sh

@@ -43,7 +43,10 @@ function sync_test {
     # pass argument to build for specific platform
 
     host="tubearchivist.local"
+    # make base folder
+    ssh "$host" "mkdir -p docker"
 
+    # copy project files to build image
     rsync -a --progress --delete-after \
         --exclude ".git" \
         --exclude ".gitignore" \
@@ -52,8 +55,8 @@ function sync_test {
         --exclude "db.sqlite3" \
         . -e ssh "$host":tubearchivist
 
-    # uncomment or copy your own docker-compose file
-    # rsync -r --progress --delete docker-compose.yml -e ssh "$host":docker
+    # copy default docker-compose file if not exist
+    rsync --progress --ignore-existing docker-compose.yml -e ssh "$host":docker
 
     if [[ $1 = "amd64" ]]; then
         platform="linux/amd64"
@@ -65,12 +68,9 @@ function sync_test {
         platform="linux/amd64"
     fi
 
-    ssh "$host" "docker buildx build --platform $platform -t bbilly1/tubearchivist:latest tubearchivist --load"
+    ssh "$host" "docker buildx build --build-arg INSTALL_DEBUG=1 --platform $platform -t bbilly1/tubearchivist:latest tubearchivist --load"
     ssh "$host" 'docker-compose -f docker/docker-compose.yml up -d'
 
-    ssh "$host" 'docker cp tubearchivist/tubearchivist/testing.sh tubearchivist:/app/testing.sh'
-    ssh "$host" 'docker exec tubearchivist chmod +x /app/testing.sh'
-
 }
 
 
@@ -91,7 +91,7 @@ function validate {
     echo "running codespell"
     codespell --skip="./.git" "$check_path"
     echo "running flake8"
-    flake8 "$check_path" --count --max-complexity=12 --max-line-length=79 \
+    flake8 "$check_path" --count --max-complexity=10 --max-line-length=79 \
         --show-source --statistics
     echo "running isort"
     isort --check-only --diff --profile black -l 79 "$check_path"

docker-compose.yml

@@ -33,7 +33,7 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
     container_name: archivist-es
     restart: always
     environment:

docs/Channels.md

@@ -22,5 +22,10 @@ Each channel will get a dedicated channel detail page accessible at `/channel/<c
 - You can *Show* the channel description, that matches with the *About* tab on YouTube.
 - The **Mark as Watched** button will mark all videos of this channel as watched.
 - The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata additionally this will also delete playlists metadata belonging to that channel.
-- The button **Find Playlists** will go out to youtube and look for playlists belonging to this channel and index all with at least one video downloaded. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.
 - The button **Show Playlists** will go to the [playlists](Playlists) page and filter the list to only show playlists from this channel.
+
+### Channel Customize
+Clicking on the *Configure* button will open a form with options to configure settings on a per channel basis. Any configurations here will overwrite your settings from the [settings](Settings) page.
+- **Download Format**: Overwrite the download qualities for videos from this channel.
+- **Auto Delete**: Automatically delete watched videos from this channel after selected days.
+- **Index Playlists**: Automatically add all Playlists with at least a video downloaded to your index. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.

tubearchivist/api/views.py

@@ -1,7 +1,7 @@
 """all API views"""
 
-import requests
 from home.src.download.thumbnails import ThumbManager
+from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import UrlListParser
 from home.src.ta.ta_redis import RedisArchivist
@@ -24,31 +24,21 @@ class ApiBaseView(APIView):
 
     def __init__(self):
         super().__init__()
-        self.response = {"data": False}
+        self.response = {"data": False, "config": AppConfig().config}
         self.status_code = False
         self.context = False
-        self.default_conf = AppConfig().config
-
-    def config_builder(self):
-        """build confic context"""
-        self.context = {
-            "es_url": self.default_conf["application"]["es_url"],
-            "es_auth": self.default_conf["application"]["es_auth"],
-        }
-        self.response["config"] = self.default_conf
 
     def get_document(self, document_id):
         """get single document from es"""
-        es_url = self.context["es_url"]
-        url = f"{es_url}{self.search_base}{document_id}"
-        print(url)
-        response = requests.get(url, auth=self.context["es_auth"])
+        path = f"{self.search_base}{document_id}"
+        print(path)
+        response, status_code = ElasticWrap(path).get()
         try:
-            self.response["data"] = response.json()["_source"]
+            self.response["data"] = response["_source"]
         except KeyError:
             print(f"item not found: {document_id}")
             self.response["data"] = False
-        self.status_code = response.status_code
+        self.status_code = status_code
 
     def process_keys(self):
         """process keys for frontend"""
@@ -59,7 +49,7 @@ class ApiBaseView(APIView):
         if "vid_thumb_url" in all_keys:
             youtube_id = self.response["data"]["youtube_id"]
             vid_thumb_url = ThumbManager().vid_thumb_path(youtube_id)
-            cache_dir = self.default_conf["application"]["cache_dir"]
+            cache_dir = self.response["config"]["application"]["cache_dir"]
             new_thumb = f"{cache_dir}/{vid_thumb_url}"
             self.response["data"]["vid_thumb_url"] = new_thumb
         if "subtitles" in all_keys:
@@ -75,13 +65,11 @@ class ApiBaseView(APIView):
 
     def get_document_list(self, data):
         """get a list of results"""
-        es_url = self.context["es_url"]
-        url = f"{es_url}{self.search_base}"
-        print(url)
-        response = requests.get(url, json=data, auth=self.context["es_auth"])
-        all_hits = response.json()["hits"]["hits"]
+        print(self.search_base)
+        response, status_code = ElasticWrap(self.search_base).get(data=data)
+        all_hits = response["hits"]["hits"]
         self.response["data"] = [i["_source"] for i in all_hits]
-        self.status_code = response.status_code
+        self.status_code = status_code
 
 
 class VideoApiView(ApiBaseView):
@@ -89,12 +77,11 @@ class VideoApiView(ApiBaseView):
     GET: returns metadata dict of video
     """
 
-    search_base = "/ta_video/_doc/"
+    search_base = "ta_video/_doc/"
 
     def get(self, request, video_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(video_id)
         self.process_keys()
         return Response(self.response, status=self.status_code)
@@ -143,12 +130,11 @@ class ChannelApiView(ApiBaseView):
     GET: returns metadata dict of channel
     """
 
-    search_base = "/ta_channel/_doc/"
+    search_base = "ta_channel/_doc/"
 
     def get(self, request, channel_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(channel_id)
         return Response(self.response, status=self.status_code)
 
@@ -159,13 +145,12 @@ class ChannelApiListView(ApiBaseView):
     POST: edit a list of channels
     """
 
-    search_base = "/ta_channel/_search/"
+    search_base = "ta_channel/_search/"
 
     def get(self, request):
         # pylint: disable=unused-argument
         """get request"""
         data = {"query": {"match_all": {}}}
-        self.config_builder()
         self.get_document_list(data)
         self.get_paginate()
 
@@ -194,12 +179,11 @@ class PlaylistApiView(ApiBaseView):
     GET: returns metadata dict of playlist
     """
 
-    search_base = "/ta_playlist/_doc/"
+    search_base = "ta_playlist/_doc/"
 
     def get(self, request, playlist_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(playlist_id)
         return Response(self.response, status=self.status_code)
 
@@ -209,12 +193,11 @@ class DownloadApiView(ApiBaseView):
     GET: returns metadata dict of an item in the download queue
     """
 
-    search_base = "/ta_download/_doc/"
+    search_base = "ta_download/_doc/"
 
     def get(self, request, video_id):
         # pylint: disable=unused-argument
         """get request"""
-        self.config_builder()
         self.get_document(video_id)
         return Response(self.response, status=self.status_code)
 
@@ -225,13 +208,12 @@ class DownloadApiListView(ApiBaseView):
     POST: add a list of videos to download queue
     """
 
-    search_base = "/ta_download/_search/"
+    search_base = "ta_download/_search/"
 
     def get(self, request):
         # pylint: disable=unused-argument
         """get request"""
         data = {"query": {"match_all": {}}}
-        self.config_builder()
         self.get_document_list(data)
         self.get_paginate()
         return Response(self.response)

tubearchivist/home/apps.py

@@ -1,53 +1,109 @@
 """handle custom startup functions"""
 
 import os
+import sys
 
 from django.apps import AppConfig
+from home.src.es.connect import ElasticWrap
 from home.src.es.index_setup import index_check
 from home.src.ta.config import AppConfig as ArchivistConfig
 from home.src.ta.ta_redis import RedisArchivist
 
 
-def sync_redis_state():
-    """make sure redis gets new config.json values"""
-    print("sync redis")
-    config_handler = ArchivistConfig()
-    config_handler.load_new_defaults()
+class StartupCheck:
+    """checks to run at application startup"""
 
+    MIN_MAJOR, MAX_MAJOR = 7, 7
+    MIN_MINOR = 17
 
-def make_folders():
-    """make needed cache folders here so docker doesn't mess it up"""
-    folders = [
-        "download",
-        "channels",
-        "videos",
-        "playlists",
-        "import",
-        "backup",
-    ]
-    config = ArchivistConfig().config
-    cache_dir = config["application"]["cache_dir"]
-    for folder in folders:
-        folder_path = os.path.join(cache_dir, folder)
-        try:
-            os.makedirs(folder_path)
-        except FileExistsError:
-            continue
+    def __init__(self):
+        self.config_handler = ArchivistConfig()
+        self.redis_con = RedisArchivist()
+        self.has_run = self.get_has_run()
 
+    def run(self):
+        """run all startup checks"""
+        print("run startup checks")
+        self.es_version_check()
+        self.release_lock()
+        index_check()
+        self.sync_redis_state()
+        self.make_folders()
+        self.set_has_run()
 
-def release_lock():
-    """make sure there are no leftover locks set in redis on container start"""
-    all_locks = [
-        "manual_import",
-        "downloading",
-        "dl_queue",
-        "dl_queue_id",
-        "rescan",
-    ]
-    for lock in all_locks:
-        response = RedisArchivist().del_message(lock)
-        if response:
-            print("deleted leftover key from redis: " + lock)
+    def get_has_run(self):
+        """validate if check has already executed"""
+        return self.redis_con.get_message("startup_check")
+
+    def set_has_run(self):
+        """startup checks run"""
+        message = {"status": True}
+        self.redis_con.set_message("startup_check", message, expire=120)
+
+    def sync_redis_state(self):
+        """make sure redis gets new config.json values"""
+        print("sync redis")
+        self.config_handler.load_new_defaults()
+
+    def make_folders(self):
+        """make needed cache folders here so docker doesn't mess it up"""
+        folders = [
+            "download",
+            "channels",
+            "videos",
+            "playlists",
+            "import",
+            "backup",
+        ]
+        cache_dir = self.config_handler.config["application"]["cache_dir"]
+        for folder in folders:
+            folder_path = os.path.join(cache_dir, folder)
+            try:
+                os.makedirs(folder_path)
+            except FileExistsError:
+                continue
+
+    def release_lock(self):
+        """make sure there are no leftover locks set in redis"""
+        all_locks = [
+            "startup_check",
+            "manual_import",
+            "downloading",
+            "dl_queue",
+            "dl_queue_id",
+            "rescan",
+        ]
+        for lock in all_locks:
+            response = self.redis_con.del_message(lock)
+            if response:
+                print("deleted leftover key from redis: " + lock)
+
+    def is_invalid(self, version):
+        """return true if es version is invalid, false if ok"""
+        major, minor = [int(i) for i in version.split(".")[:2]]
+        if not self.MIN_MAJOR <= major <= self.MAX_MAJOR:
+            return True
+
+        if minor >= self.MIN_MINOR:
+            return False
+
+        return True
+
+    def es_version_check(self):
+        """check for minimal elasticsearch version"""
+        response, _ = ElasticWrap("/").get()
+        version = response["version"]["number"]
+        invalid = self.is_invalid(version)
+
+        if invalid:
+            print(
+                "minial required elasticsearch version: "
+                + f"{self.MIN_MAJOR}.{self.MIN_MINOR}, "
+                + "please update to recommended version."
+            )
+            sys.exit(1)
+
+        print("elasticsearch version check passed")
 
 
 class HomeConfig(AppConfig):
@@ -57,7 +113,9 @@ class HomeConfig(AppConfig):
     name = "home"
 
     def ready(self):
-        release_lock()
-        index_check()
-        sync_redis_state()
-        make_folders()
+        startup = StartupCheck()
+        if startup.has_run["status"]:
+            print("startup checks run in other thread")
+            return
+
+        startup.run()

tubearchivist/home/src/download/queue.py

@@ -5,36 +5,132 @@ Functionality:
 """
 
 import json
-import os
 from datetime import datetime
 
-import requests
 import yt_dlp
-from home.src.download.subscriptions import ChannelSubscription
-from home.src.es.connect import IndexPaginate
+from home.src.download.subscriptions import (
+    ChannelSubscription,
+    PlaylistSubscription,
+)
+from home.src.download.thumbnails import ThumbManager
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import DurationConverter, ignore_filelist
+from home.src.ta.helper import DurationConverter
 from home.src.ta.ta_redis import RedisArchivist
 
 
-class PendingList:
-    """manage the pending videos list"""
-
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
-    VIDEOS = CONFIG["application"]["videos"]
+class PendingIndex:
+    """base class holding all export methods"""
 
     def __init__(self):
-        self.all_channel_ids = False
-        self.all_downloaded = False
-        self.missing_from_playlists = []
+        self.all_pending = False
+        self.all_ignored = False
+        self.all_videos = False
+        self.all_channels = False
+        self.channel_overwrites = False
+        self.video_overwrites = False
+        self.to_skip = False
 
-    def parse_url_list(self, youtube_ids):
+    def get_download(self):
+        """get a list of all pending videos in ta_download"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"timestamp": {"order": "asc"}}],
+        }
+        all_results = IndexPaginate("ta_download", data).get_results()
+
+        self.all_pending = []
+        self.all_ignored = []
+        self.to_skip = []
+
+        for result in all_results:
+            self.to_skip.append(result["youtube_id"])
+            if result["status"] == "pending":
+                self.all_pending.append(result)
+            elif result["status"] == "ignore":
+                self.all_ignored.append(result)
+
+    def get_indexed(self):
+        """get a list of all videos indexed"""
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        self.all_videos = IndexPaginate("ta_video", data).get_results()
+        for video in self.all_videos:
+            self.to_skip.append(video["youtube_id"])
+
+    def get_channels(self):
+        """get a list of all channels indexed"""
+        self.all_channels = []
+        self.channel_overwrites = {}
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"channel_id": {"order": "asc"}}],
+        }
+        channels = IndexPaginate("ta_channel", data).get_results()
+
+        for channel in channels:
+            channel_id = channel["channel_id"]
+            self.all_channels.append(channel_id)
+            if channel.get("channel_overwrites"):
+                self.channel_overwrites.update(
+                    {channel_id: channel.get("channel_overwrites")}
+                )
+
+        self._map_overwrites()
+
+    def _map_overwrites(self):
+        """map video ids to channel ids overwrites"""
+        self.video_overwrites = {}
+        for video in self.all_pending:
+            video_id = video["youtube_id"]
+            channel_id = video["channel_id"]
+            overwrites = self.channel_overwrites.get(channel_id, False)
+            if overwrites:
+                self.video_overwrites.update({video_id: overwrites})
+
+
+class PendingInteract:
+    """interact with items in download queue"""
+
+    def __init__(self, video_id=False, status=False):
+        self.video_id = video_id
+        self.status = status
+
+    def delete_item(self):
+        """delete single item from pending"""
+        path = f"ta_download/_doc/{self.video_id}"
+        _, _ = ElasticWrap(path).delete()
+
+    def delete_by_status(self):
+        """delete all matching item by status"""
+        data = {"query": {"term": {"status": {"value": self.status}}}}
+        path = "ta_download/_delete_by_query"
+        _, _ = ElasticWrap(path).post(data=data)
+
+    def update_status(self):
+        """update status field of pending item"""
+        data = {"doc": {"status": self.status}}
+        path = f"ta_download/_update/{self.video_id}"
+        _, _ = ElasticWrap(path).post(data=data)
+
+
+class PendingList(PendingIndex):
+    """manage the pending videos list"""
+
+    def __init__(self, youtube_ids=False):
+        super().__init__()
+        self.youtube_ids = youtube_ids
+        self.to_skip = False
+        self.missing_videos = False
+
+    def parse_url_list(self):
         """extract youtube ids from list"""
-        missing_videos = []
-        for entry in youtube_ids:
+        self.missing_videos = []
+        self.get_download()
+        self.get_indexed()
+        for entry in self.youtube_ids:
             # notify
             mess_dict = {
                 "status": "message:add",
@@ -43,97 +139,89 @@ class PendingList:
                 "message": "Extracting lists",
             }
             RedisArchivist().set_message("message:add", mess_dict)
-            # extract
-            url = entry["url"]
-            url_type = entry["type"]
-            if url_type == "video":
-                missing_videos.append(url)
-            elif url_type == "channel":
-                video_results = ChannelSubscription().get_last_youtube_videos(
-                    url, limit=False
-                )
-                youtube_ids = [i[0] for i in video_results]
-                missing_videos = missing_videos + youtube_ids
-            elif url_type == "playlist":
-                self.missing_from_playlists.append(entry)
-                playlist = YoutubePlaylist(url)
-                playlist.build_json()
-                video_results = playlist.json_data.get("playlist_entries")
-                youtube_ids = [i["youtube_id"] for i in video_results]
-                missing_videos = missing_videos + youtube_ids
+            self._process_entry(entry)
 
-        return missing_videos
+    def _process_entry(self, entry):
+        """process single entry from url list"""
+        if entry["type"] == "video":
+            self._add_video(entry["url"])
+        elif entry["type"] == "channel":
+            self._parse_channel(entry["url"])
+        elif entry["type"] == "playlist":
+            self._parse_playlist(entry["url"])
+            new_thumbs = PlaylistSubscription().process_url_str(
+                [entry], subscribed=False
+            )
+            ThumbManager().download_playlist(new_thumbs)
+        else:
+            raise ValueError(f"invalid url_type: {entry}")
 
-    def add_to_pending(self, missing_videos, ignore=False):
-        """build the bulk json data from pending"""
-        # check if channel is indexed
-        channel_handler = ChannelSubscription()
-        all_indexed = channel_handler.get_channels(subscribed_only=False)
-        self.all_channel_ids = [i["channel_id"] for i in all_indexed]
-        # check if already there
-        self.all_downloaded = self.get_all_downloaded()
+    def _add_video(self, url):
+        """add video to list"""
+        if url not in self.missing_videos and url not in self.to_skip:
+            self.missing_videos.append(url)
+
+    def _parse_channel(self, url):
+        """add all videos of channel to list"""
+        video_results = ChannelSubscription().get_last_youtube_videos(
+            url, limit=False
+        )
+        youtube_ids = [i[0] for i in video_results]
+        for video_id in youtube_ids:
+            self._add_video(video_id)
+
+    def _parse_playlist(self, url):
+        """add all videos of playlist to list"""
+        playlist = YoutubePlaylist(url)
+        playlist.build_json()
+        video_results = playlist.json_data.get("playlist_entries")
+        youtube_ids = [i["youtube_id"] for i in video_results]
+        for video_id in youtube_ids:
+            self._add_video(video_id)
+
+    def add_to_pending(self, status="pending"):
+        """add missing videos to pending list"""
+        self.get_channels()
+        bulk_list = []
+
+        thumb_handler = ThumbManager()
+        for idx, youtube_id in enumerate(self.missing_videos):
+            video_details = self.get_youtube_details(youtube_id)
+            if not video_details:
+                continue
+
+            video_details["status"] = status
+            action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(video_details))
+
+            thumb_needed = [(youtube_id, video_details["vid_thumb_url"])]
+            thumb_handler.download_vid(thumb_needed)
+            self._notify_add(idx)
 
-        bulk_list, all_videos_added = self.build_bulk(missing_videos, ignore)
         # add last newline
         bulk_list.append("\n")
         query_str = "\n".join(bulk_list)
-        headers = {"Content-type": "application/x-ndjson"}
-        url = self.ES_URL + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request)
-            raise ValueError("failed to add video to download queue")
+        _, _ = ElasticWrap("_bulk").post(query_str, ndjson=True)
 
-        return all_videos_added
+    def _notify_add(self, idx):
+        """send notification for adding videos to download queue"""
+        progress = f"{idx + 1}/{len(self.missing_videos)}"
+        mess_dict = {
+            "status": "message:add",
+            "level": "info",
+            "title": "Adding new videos to download queue.",
+            "message": "Progress: " + progress,
+        }
+        if idx + 1 == len(self.missing_videos):
+            RedisArchivist().set_message("message:add", mess_dict, expire=4)
+        else:
+            RedisArchivist().set_message("message:add", mess_dict)
 
-    def build_bulk(self, missing_videos, ignore=False):
-        """build the bulk lists"""
-        bulk_list = []
-        all_videos_added = []
+        if idx + 1 % 25 == 0:
+            print("adding to queue progress: " + progress)
 
-        for idx, youtube_id in enumerate(missing_videos):
-            # check if already downloaded
-            if youtube_id in self.all_downloaded:
-                continue
-
-            video = self.get_youtube_details(youtube_id)
-            # skip on download error
-            if not video:
-                continue
-
-            channel_indexed = video["channel_id"] in self.all_channel_ids
-            video["channel_indexed"] = channel_indexed
-            if ignore:
-                video["status"] = "ignore"
-            else:
-                video["status"] = "pending"
-            action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(video))
-            all_videos_added.append((youtube_id, video["vid_thumb_url"]))
-            # notify
-            progress = f"{idx + 1}/{len(missing_videos)}"
-            mess_dict = {
-                "status": "message:add",
-                "level": "info",
-                "title": "Adding new videos to download queue.",
-                "message": "Progress: " + progress,
-            }
-            if idx + 1 == len(missing_videos):
-                RedisArchivist().set_message(
-                    "message:add", mess_dict, expire=4
-                )
-            else:
-                RedisArchivist().set_message("message:add", mess_dict)
-            if idx + 1 % 25 == 0:
-                print("adding to queue progress: " + progress)
-
-        return bulk_list, all_videos_added
-
-    @staticmethod
-    def get_youtube_details(youtube_id):
+    def get_youtube_details(self, youtube_id):
         """get details from youtubedl for single pending video"""
         obs = {
             "default_search": "ytsearch",
@@ -151,113 +239,29 @@ class PendingList:
         # stop if video is streaming live now
         if vid["is_live"]:
             return False
-        # parse response
-        seconds = vid["duration"]
-        duration_str = DurationConverter.get_str(seconds)
+
+        return self._parse_youtube_details(vid)
+
+    def _parse_youtube_details(self, vid):
+        """parse response"""
+        vid_id = vid.get("id")
+        duration_str = DurationConverter.get_str(vid["duration"])
         if duration_str == "NA":
-            print(f"skip extracting duration for: {youtube_id}")
-        upload_date = vid["upload_date"]
-        upload_dt = datetime.strptime(upload_date, "%Y%m%d")
-        published = upload_dt.strftime("%Y-%m-%d")
+            print(f"skip extracting duration for: {vid_id}")
+        published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
+            "%Y-%m-%d"
+        )
+
         # build dict
         youtube_details = {
-            "youtube_id": youtube_id,
+            "youtube_id": vid_id,
             "channel_name": vid["channel"],
             "vid_thumb_url": vid["thumbnail"],
             "title": vid["title"],
             "channel_id": vid["channel_id"],
+            "channel_indexed": vid["channel_id"] in self.all_channels,
             "duration": duration_str,
             "published": published,
             "timestamp": int(datetime.now().strftime("%s")),
         }
         return youtube_details
-
-    @staticmethod
-    def get_all_pending():
-        """get a list of all pending videos in ta_download"""
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"timestamp": {"order": "asc"}}],
-        }
-        all_results = IndexPaginate("ta_download", data).get_results()
-
-        all_pending = []
-        all_ignore = []
-
-        for result in all_results:
-            if result["status"] == "pending":
-                all_pending.append(result)
-            elif result["status"] == "ignore":
-                all_ignore.append(result)
-
-        return all_pending, all_ignore
-
-    @staticmethod
-    def get_all_indexed():
-        """get a list of all videos indexed"""
-
-        data = {
-            "query": {"match_all": {}},
-            "sort": [{"published": {"order": "desc"}}],
-        }
-        all_indexed = IndexPaginate("ta_video", data).get_results()
-
-        return all_indexed
-
-    def get_all_downloaded(self):
-        """get a list of all videos in archive"""
-        channel_folders = os.listdir(self.VIDEOS)
-        all_channel_folders = ignore_filelist(channel_folders)
-        all_downloaded = []
-        for channel_folder in all_channel_folders:
-            channel_path = os.path.join(self.VIDEOS, channel_folder)
-            videos = os.listdir(channel_path)
-            all_videos = ignore_filelist(videos)
-            youtube_vids = [i[9:20] for i in all_videos]
-            for youtube_id in youtube_vids:
-                all_downloaded.append(youtube_id)
-        return all_downloaded
-
-    def delete_from_pending(self, youtube_id):
-        """delete the youtube_id from ta_download"""
-        url = f"{self.ES_URL}/ta_download/_doc/{youtube_id}"
-        response = requests.delete(url, auth=self.ES_AUTH)
-        if not response.ok:
-            print(response.text)
-
-    def delete_pending(self, status):
-        """delete download queue based on status value"""
-        data = {"query": {"term": {"status": {"value": status}}}}
-        payload = json.dumps(data)
-        url = self.ES_URL + "/ta_download/_delete_by_query"
-        headers = {"Content-type": "application/json"}
-        response = requests.post(
-            url, data=payload, headers=headers, auth=self.ES_AUTH
-        )
-        if not response.ok:
-            print(response.text)
-
-    def ignore_from_pending(self, ignore_list):
-        """build the bulk query string"""
-
-        stamp = int(datetime.now().strftime("%s"))
-        bulk_list = []
-
-        for youtube_id in ignore_list:
-            action = {"update": {"_id": youtube_id, "_index": "ta_download"}}
-            source = {"doc": {"status": "ignore", "timestamp": stamp}}
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(source))
-
-        # add last newline
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-
-        headers = {"Content-type": "application/x-ndjson"}
-        url = self.ES_URL + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request)
-            raise ValueError("failed to set video to ignore")

tubearchivist/home/src/download/subscriptions.py

@@ -61,11 +61,9 @@ class ChannelSubscription:
     def find_missing(self):
         """add missing videos from subscribed channels to pending"""
         all_channels = self.get_channels()
-        pending_handler = queue.PendingList()
-        all_pending, all_ignore = pending_handler.get_all_pending()
-        all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
-        all_downloaded = pending_handler.get_all_downloaded()
-        to_ignore = all_ids + all_downloaded
+        pending = queue.PendingList()
+        pending.get_download()
+        pending.get_indexed()
 
         missing_videos = []
 
@@ -75,7 +73,7 @@ class ChannelSubscription:
 
             if last_videos:
                 for video in last_videos:
-                    if video[0] not in to_ignore:
+                    if video[0] not in pending.to_skip:
                         missing_videos.append(video[0])
             # notify
             message = {
@@ -129,7 +127,11 @@ class PlaylistSubscription:
 
     def process_url_str(self, new_playlists, subscribed=True):
         """process playlist subscribe form url_str"""
-        all_indexed = queue.PendingList().get_all_indexed()
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        all_indexed = IndexPaginate("ta_video", data).get_results()
         all_youtube_ids = [i["youtube_id"] for i in all_indexed]
 
         new_thumbs = []
@@ -179,12 +181,11 @@ class PlaylistSubscription:
     @staticmethod
     def get_to_ignore():
         """get all youtube_ids already downloaded or ignored"""
-        pending_handler = queue.PendingList()
-        all_pending, all_ignore = pending_handler.get_all_pending()
-        all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
-        all_downloaded = pending_handler.get_all_downloaded()
-        to_ignore = all_ids + all_downloaded
-        return to_ignore
+        pending = queue.PendingList()
+        pending.get_download()
+        pending.get_indexed()
+
+        return pending.to_skip
 
     def find_missing(self):
         """find videos in subscribed playlists not downloaded yet"""

tubearchivist/home/src/download/thumbnails.py

@@ -58,11 +58,13 @@ class ThumbManager:
     def get_needed_thumbs(self, missing_only=False):
         """get a list of all missing thumbnails"""
         all_thumbs = self.get_all_thumbs()
-        all_indexed = queue.PendingList().get_all_indexed()
-        all_in_queue, all_ignored = queue.PendingList().get_all_pending()
+
+        pending = queue.PendingList()
+        pending.get_download()
+        pending.get_indexed()
 
         needed_thumbs = []
-        for video in all_indexed:
+        for video in pending.all_videos:
             youtube_id = video["youtube_id"]
             thumb_url = video["vid_thumb_url"]
             if missing_only:
@@ -71,7 +73,7 @@ class ThumbManager:
             else:
                 needed_thumbs.append((youtube_id, thumb_url))
 
-        for video in all_in_queue + all_ignored:
+        for video in pending.all_pending + pending.all_ignored:
             youtube_id = video["youtube_id"]
             thumb_url = video["vid_thumb_url"]
             if missing_only:
@@ -276,9 +278,11 @@ class ThumbManager:
 
     def get_thumb_list(self):
         """get list of mediafiles and matching thumbnails"""
-        all_indexed = queue.PendingList().get_all_indexed()
+        pending = queue.PendingList()
+        pending.get_indexed()
+
         video_list = []
-        for video in all_indexed:
+        for video in pending.all_videos:
             youtube_id = video["youtube_id"]
             media_url = os.path.join(self.MEDIA_DIR, video["media_url"])
             thumb_path = os.path.join(

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -11,11 +11,10 @@ import shutil
 from datetime import datetime
 from time import sleep
 
-import requests
 import yt_dlp
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import PlaylistSubscription
-from home.src.es.connect import IndexPaginate
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo, index_new_video
@@ -24,6 +23,123 @@ from home.src.ta.helper import clean_string, ignore_filelist
 from home.src.ta.ta_redis import RedisArchivist, RedisQueue
 
 
+class DownloadPostProcess:
+    """handle task to run after download queue finishes"""
+
+    def __init__(self, download):
+        self.download = download
+        self.now = int(datetime.now().strftime("%s"))
+        self.pending = False
+
+    def run(self):
+        """run all functions"""
+        self.pending = PendingList()
+        self.pending.get_download()
+        self.pending.get_channels()
+        self.pending.get_indexed()
+        self.auto_delete_all()
+        self.auto_delete_overwrites()
+        self.validate_playlists()
+
+    def auto_delete_all(self):
+        """handle auto delete"""
+        autodelete_days = self.download.config["downloads"]["autodelete_days"]
+        if not autodelete_days:
+            return
+
+        print(f"auto delete older than {autodelete_days} days")
+        now_lte = self.now - autodelete_days * 24 * 60 * 60
+        data = {
+            "query": {"range": {"player.watched_date": {"lte": now_lte}}},
+            "sort": [{"player.watched_date": {"order": "asc"}}],
+        }
+        self._auto_delete_watched(data)
+
+    def auto_delete_overwrites(self):
+        """handle per channel auto delete from overwrites"""
+        for channel_id, value in self.pending.channel_overwrites.items():
+            if "autodelete_days" in value:
+                autodelete_days = value.get("autodelete_days")
+                print(f"{channel_id}: delete older than {autodelete_days}d")
+                now_lte = self.now - autodelete_days * 24 * 60 * 60
+                must_list = [
+                    {"range": {"player.watched_date": {"lte": now_lte}}},
+                    {"term": {"channel.channel_id": {"value": channel_id}}},
+                ]
+                data = {
+                    "query": {"bool": {"must": must_list}},
+                    "sort": [{"player.watched_date": {"order": "desc"}}],
+                }
+                self._auto_delete_watched(data)
+
+    @staticmethod
+    def _auto_delete_watched(data):
+        """delete watched videos after x days"""
+        to_delete = IndexPaginate("ta_video", data).get_results()
+        if not to_delete:
+            return
+
+        for video in to_delete:
+            youtube_id = video["youtube_id"]
+            print(f"{youtube_id}: auto delete video")
+            YoutubeVideo(youtube_id).delete_media_file()
+
+        print("add deleted to ignore list")
+        vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
+        pending = PendingList(youtube_ids=vids)
+        pending.parse_url_list()
+        pending.add_to_pending(status="ignore")
+
+    def validate_playlists(self):
+        """look for playlist needing to update"""
+        for id_c, channel_id in enumerate(self.download.channels):
+            channel = YoutubeChannel(channel_id)
+            overwrites = self.pending.channel_overwrites.get(channel_id, False)
+            if overwrites and overwrites.get("index_playlists"):
+                # validate from remote
+                channel.index_channel_playlists()
+                continue
+
+            # validate from local
+            playlists = channel.get_indexed_playlists()
+            all_channel_playlist = [i["playlist_id"] for i in playlists]
+            self._validate_channel_playlist(all_channel_playlist, id_c)
+
+    def _validate_channel_playlist(self, all_channel_playlist, id_c):
+        """scan channel for playlist needing update"""
+        all_youtube_ids = [i["youtube_id"] for i in self.pending.all_videos]
+        for id_p, playlist_id in enumerate(all_channel_playlist):
+            playlist = YoutubePlaylist(playlist_id)
+            playlist.all_youtube_ids = all_youtube_ids
+            playlist.build_json(scrape=True)
+            if not playlist.json_data:
+                playlist.deactivate()
+
+            playlist.add_vids_to_playlist()
+            playlist.upload_to_es()
+            self._notify_playlist_progress(all_channel_playlist, id_c, id_p)
+
+    def _notify_playlist_progress(self, all_channel_playlist, id_c, id_p):
+        """notify to UI"""
+        title = (
+            "Processing playlists for channels: "
+            + f"{id_c + 1}/{len(self.download.channels)}"
+        )
+        message = f"Progress: {id_p + 1}/{len(all_channel_playlist)}"
+        mess_dict = {
+            "status": "message:download",
+            "level": "info",
+            "title": title,
+            "message": message,
+        }
+        if id_p + 1 == len(all_channel_playlist):
+            RedisArchivist().set_message(
+                "message:download", mess_dict, expire=4
+            )
+        else:
+            RedisArchivist().set_message("message:download", mess_dict)
+
+
 class VideoDownloader:
     """
     handle the video download functionality
@@ -32,6 +148,7 @@ class VideoDownloader:
 
     def __init__(self, youtube_id_list=False):
         self.obs = False
+        self.video_overwrites = False
         self.youtube_id_list = youtube_id_list
         self.config = AppConfig().config
         self._build_obs()
@@ -39,7 +156,12 @@ class VideoDownloader:
 
     def run_queue(self):
         """setup download queue in redis loop until no more items"""
-        queue = RedisQueue("dl_queue")
+        pending = PendingList()
+        pending.get_download()
+        pending.get_channels()
+        self.video_overwrites = pending.video_overwrites
+
+        queue = RedisQueue()
 
         limit_queue = self.config["downloads"]["limit_count"]
         if limit_queue:
@@ -60,10 +182,9 @@ class VideoDownloader:
             self.move_to_archive(vid_dict)
             self._delete_from_pending(youtube_id)
 
-        autodelete_days = self.config["downloads"]["autodelete_days"]
-        if autodelete_days:
-            print(f"auto delete older than {autodelete_days} days")
-            self.auto_delete_watched(autodelete_days)
+        # post processing
+        self._add_subscribed_channels()
+        DownloadPostProcess(self).run()
 
     @staticmethod
     def add_pending():
@@ -75,8 +196,9 @@ class VideoDownloader:
             "message": "Scanning your download queue.",
         }
         RedisArchivist().set_message("message:download", mess_dict)
-        all_pending, _ = PendingList().get_all_pending()
-        to_add = [i["youtube_id"] for i in all_pending]
+        pending = PendingList()
+        pending.get_download()
+        to_add = [i["youtube_id"] for i in pending.all_pending]
         if not to_add:
             # there is nothing pending
             print("download queue is empty")
@@ -89,8 +211,7 @@ class VideoDownloader:
             RedisArchivist().set_message("message:download", mess_dict)
             return
 
-        queue = RedisQueue("dl_queue")
-        queue.add_list(to_add)
+        RedisQueue().add_list(to_add)
 
     @staticmethod
     def _progress_hook(response):
@@ -181,16 +302,30 @@ class VideoDownloader:
 
         self.obs["postprocessors"] = postprocessors
 
+    def get_format_overwrites(self, youtube_id):
+        """get overwrites from single video"""
+        overwrites = self.video_overwrites.get(youtube_id, False)
+        if overwrites:
+            return overwrites.get("download_format", False)
+
+        return False
+
     def _dl_single_vid(self, youtube_id):
         """download single video"""
+        obs = self.obs.copy()
+        format_overwrite = self.get_format_overwrites(youtube_id)
+        if format_overwrite:
+            obs["format"] = format_overwrite
+
         dl_cache = self.config["application"]["cache_dir"] + "/download/"
 
         # check if already in cache to continue from there
         all_cached = ignore_filelist(os.listdir(dl_cache))
         for file_name in all_cached:
             if youtube_id in file_name:
-                self.obs["outtmpl"] = os.path.join(dl_cache, file_name)
-        with yt_dlp.YoutubeDL(self.obs) as ydl:
+                obs["outtmpl"] = os.path.join(dl_cache, file_name)
+
+        with yt_dlp.YoutubeDL(obs) as ydl:
             try:
                 ydl.download([youtube_id])
             except yt_dlp.utils.DownloadError:
@@ -234,14 +369,11 @@ class VideoDownloader:
         if host_uid and host_gid:
             os.chown(new_file_path, host_uid, host_gid)
 
-    def _delete_from_pending(self, youtube_id):
+    @staticmethod
+    def _delete_from_pending(youtube_id):
         """delete downloaded video from pending index if its there"""
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        url = f"{es_url}/ta_download/_doc/{youtube_id}"
-        response = requests.delete(url, auth=es_auth)
-        if not response.ok and not response.status_code == 404:
-            print(response.text)
+        path = f"ta_download/_doc/{youtube_id}"
+        _, _ = ElasticWrap(path).delete()
 
     def _add_subscribed_channels(self):
         """add all channels subscribed to refresh"""
@@ -254,63 +386,3 @@ class VideoDownloader:
             self.channels.add(channel_id)
 
         return
-
-    def validate_playlists(self):
-        """look for playlist needing to update"""
-        print("sync playlists")
-        self._add_subscribed_channels()
-        all_indexed = PendingList().get_all_indexed()
-        all_youtube_ids = [i["youtube_id"] for i in all_indexed]
-        for id_c, channel_id in enumerate(self.channels):
-            playlists = YoutubeChannel(channel_id).get_indexed_playlists()
-            all_playlist_ids = [i["playlist_id"] for i in playlists]
-            for id_p, playlist_id in enumerate(all_playlist_ids):
-                playlist = YoutubePlaylist(playlist_id)
-                playlist.all_youtube_ids = all_youtube_ids
-                playlist.build_json(scrape=True)
-                if not playlist.json_data:
-                    playlist.deactivate()
-
-                playlist.add_vids_to_playlist()
-                playlist.upload_to_es()
-
-                # notify
-                title = (
-                    "Processing playlists for channels: "
-                    + f"{id_c + 1}/{len(self.channels)}"
-                )
-                message = f"Progress: {id_p + 1}/{len(all_playlist_ids)}"
-                mess_dict = {
-                    "status": "message:download",
-                    "level": "info",
-                    "title": title,
-                    "message": message,
-                }
-                if id_p + 1 == len(all_playlist_ids):
-                    RedisArchivist().set_message(
-                        "message:download", mess_dict, expire=4
-                    )
-                else:
-                    RedisArchivist().set_message("message:download", mess_dict)
-
-    @staticmethod
-    def auto_delete_watched(autodelete_days):
-        """delete watched videos after x days"""
-        now = int(datetime.now().strftime("%s"))
-        now_lte = now - autodelete_days * 24 * 60 * 60
-        data = {
-            "query": {"range": {"player.watched_date": {"lte": now_lte}}},
-            "sort": [{"player.watched_date": {"order": "asc"}}],
-        }
-        all_to_delete = IndexPaginate("ta_video", data).get_results()
-        all_youtube_ids = [i["youtube_id"] for i in all_to_delete]
-        if not all_youtube_ids:
-            return
-
-        for youtube_id in all_youtube_ids:
-            print(f"autodelete {youtube_id}")
-            YoutubeVideo(youtube_id).delete_media_file()
-
-        print("add deleted to ignore list")
-        pending_handler = PendingList()
-        pending_handler.add_to_pending(all_youtube_ids, ignore=True)

tubearchivist/home/src/es/connect.py

@@ -93,11 +93,12 @@ class IndexPaginate:
 
     DEFAULT_SIZE = 500
 
-    def __init__(self, index_name, data, size=False):
+    def __init__(self, index_name, data, size=False, keep_source=False):
         self.index_name = index_name
         self.data = data
         self.pit_id = False
         self.size = size
+        self.keep_source = keep_source
 
     def get_results(self):
         """get all results"""
@@ -132,7 +133,10 @@ class IndexPaginate:
             all_hits = response["hits"]["hits"]
             if all_hits:
                 for hit in all_hits:
-                    source = hit["_source"]
+                    if self.keep_source:
+                        source = hit
+                    else:
+                        source = hit["_source"]
                     search_after = hit["sort"]
                     all_results.append(source)
                 # update search_after with last hit data

tubearchivist/home/src/es/index_mapping.json

@@ -39,6 +39,19 @@
                 "channel_last_refresh": {
                     "type": "date",
                     "format": "epoch_second"
+                },
+                "channel_overwrites": {
+                    "properties": {
+                        "download_format": {
+                            "type": "text"
+                        },
+                        "autodelete_days": {
+                            "type": "long"
+                        },
+                        "index_playlists": {
+                            "type": "boolean"
+                        }
+                    }
                 }
             },
             "expected_set": {
@@ -102,6 +115,19 @@
                         "channel_last_refresh": {
                             "type": "date",
                             "format": "epoch_second"
+                        },
+                        "channel_overwrites": {
+                            "properties": {
+                                "download_format": {
+                                    "type": "text"
+                                },
+                                "autodelete_days": {
+                                    "type": "long"
+                                },
+                                "index_playlists": {
+                                    "type": "boolean"
+                                }
+                            }
                         }
                     }
                 },

tubearchivist/home/src/es/index_setup.py

@@ -10,7 +10,7 @@ import os
 import zipfile
 from datetime import datetime
 
-import requests
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import ignore_filelist
 
@@ -20,11 +20,6 @@ class ElasticIndex:
     handle mapping and settings on elastic search for a given index
     """
 
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
-    HEADERS = {"Content-type": "application/json"}
-
     def __init__(self, index_name, expected_map, expected_set):
         self.index_name = index_name
         self.expected_map = expected_map
@@ -33,15 +28,9 @@ class ElasticIndex:
 
     def index_exists(self):
         """check if index already exists and return mapping if it does"""
-        index_name = self.index_name
-        url = f"{self.ES_URL}/ta_{index_name}"
-        response = requests.get(url, auth=self.ES_AUTH)
-        exists = response.ok
-
-        if exists:
-            details = response.json()[f"ta_{index_name}"]
-        else:
-            details = False
+        response, status_code = ElasticWrap(f"ta_{self.index_name}").get()
+        exists = status_code == 200
+        details = response.get(f"ta_{self.index_name}", False)
 
         return exists, details
 
@@ -65,14 +54,15 @@ class ElasticIndex:
 
     def validate_mappings(self):
         """check if all mappings are as expected"""
-
-        expected_map = self.expected_map
         now_map = self.details["mappings"]["properties"]
 
-        for key, value in expected_map.items():
+        for key, value in self.expected_map.items():
             # nested
             if list(value.keys()) == ["properties"]:
                 for key_n, value_n in value["properties"].items():
+                    if key not in now_map:
+                        print(key_n, value_n)
+                        return True
                     if key_n not in now_map[key]["properties"].keys():
                         print(key_n, value_n)
                         return True
@@ -110,63 +100,41 @@ class ElasticIndex:
 
     def rebuild_index(self):
         """rebuild with new mapping"""
-        # backup
         self.reindex("backup")
-        # delete original
         self.delete_index(backup=False)
-        # create new
         self.create_blank()
         self.reindex("restore")
-        # delete backup
         self.delete_index()
 
     def reindex(self, method):
         """create on elastic search"""
-        index_name = self.index_name
         if method == "backup":
-            source = f"ta_{index_name}"
-            destination = f"ta_{index_name}_backup"
+            source = f"ta_{self.index_name}"
+            destination = f"ta_{self.index_name}_backup"
         elif method == "restore":
-            source = f"ta_{index_name}_backup"
-            destination = f"ta_{index_name}"
+            source = f"ta_{self.index_name}_backup"
+            destination = f"ta_{self.index_name}"
 
-        query = {"source": {"index": source}, "dest": {"index": destination}}
-        data = json.dumps(query)
-        url = self.ES_URL + "/_reindex?refresh=true"
-        response = requests.post(
-            url=url, data=data, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not response.ok:
-            print(response.text)
+        data = {"source": {"index": source}, "dest": {"index": destination}}
+        _, _ = ElasticWrap("_reindex?refresh=true").post(data=data)
 
     def delete_index(self, backup=True):
         """delete index passed as argument"""
+        path = f"ta_{self.index_name}"
         if backup:
-            url = f"{self.ES_URL}/ta_{self.index_name}_backup"
-        else:
-            url = f"{self.ES_URL}/ta_{self.index_name}"
-        response = requests.delete(url, auth=self.ES_AUTH)
-        if not response.ok:
-            print(response.text)
+            path = path + "_backup"
+
+        _, _ = ElasticWrap(path).delete()
 
     def create_blank(self):
         """apply new mapping and settings for blank new index"""
-        expected_map = self.expected_map
-        expected_set = self.expected_set
-        # stich payload
-        payload = {}
-        if expected_set:
-            payload.update({"settings": expected_set})
-        if expected_map:
-            payload.update({"mappings": {"properties": expected_map}})
-        # create
-        url = f"{self.ES_URL}/ta_{self.index_name}"
-        data = json.dumps(payload)
-        response = requests.put(
-            url=url, data=data, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not response.ok:
-            print(response.text)
+        data = {}
+        if self.expected_set:
+            data.update({"settings": self.expected_set})
+        if self.expected_map:
+            data.update({"mappings": {"properties": self.expected_map}})
+
+        _, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
 
 
 class ElasticBackup:
@@ -174,52 +142,21 @@ class ElasticBackup:
 
     def __init__(self, index_config, reason):
         self.config = AppConfig().config
+        self.cache_dir = self.config["application"]["cache_dir"]
         self.index_config = index_config
         self.reason = reason
         self.timestamp = datetime.now().strftime("%Y%m%d")
         self.backup_files = []
 
-    def get_all_documents(self, index_name):
+    @staticmethod
+    def get_all_documents(index_name):
         """export all documents of a single index"""
-        headers = {"Content-type": "application/json"}
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        # get PIT ID
-        url = f"{es_url}/ta_{index_name}/_pit?keep_alive=1m"
-        response = requests.post(url, auth=es_auth)
-        json_data = json.loads(response.text)
-        pit_id = json_data["id"]
-        # build query
         data = {
             "query": {"match_all": {}},
-            "size": 100,
-            "pit": {"id": pit_id, "keep_alive": "1m"},
-            "sort": [{"_id": {"order": "asc"}}],
+            "sort": [{"_doc": {"order": "desc"}}],
         }
-        query_str = json.dumps(data)
-        url = es_url + "/_search"
-        # loop until nothing left
-        all_results = []
-        while True:
-            response = requests.get(
-                url, data=query_str, headers=headers, auth=es_auth
-            )
-            json_data = json.loads(response.text)
-            all_hits = json_data["hits"]["hits"]
-            if all_hits:
-                for hit in all_hits:
-                    search_after = hit["sort"]
-                    all_results.append(hit)
-                # update search_after with last hit data
-                data["search_after"] = search_after
-                query_str = json.dumps(data)
-            else:
-                break
-        # clean up PIT
-        query_str = json.dumps({"id": pit_id})
-        requests.delete(
-            es_url + "/_pit", data=query_str, headers=headers, auth=es_auth
-        )
+        paginate = IndexPaginate(f"ta_{index_name}", data, keep_source=True)
+        all_results = paginate.get_results()
 
         return all_results
 
@@ -244,9 +181,8 @@ class ElasticBackup:
 
     def write_es_json(self, file_content, index_name):
         """write nd-json file for es _bulk API to disk"""
-        cache_dir = self.config["application"]["cache_dir"]
         file_name = f"es_{index_name}-{self.timestamp}.json"
-        file_path = os.path.join(cache_dir, "backup", file_name)
+        file_path = os.path.join(self.cache_dir, "backup", file_name)
         with open(file_path, "w", encoding="utf-8") as f:
             f.write(file_content)
 
@@ -254,9 +190,8 @@ class ElasticBackup:
 
     def write_ta_json(self, all_results, index_name):
         """write generic json file to disk"""
-        cache_dir = self.config["application"]["cache_dir"]
         file_name = f"ta_{index_name}-{self.timestamp}.json"
-        file_path = os.path.join(cache_dir, "backup", file_name)
+        file_path = os.path.join(self.cache_dir, "backup", file_name)
         to_write = [i["_source"] for i in all_results]
         file_content = json.dumps(to_write)
         with open(file_path, "w", encoding="utf-8") as f:
@@ -266,9 +201,8 @@ class ElasticBackup:
 
     def zip_it(self):
         """pack it up into single zip file"""
-        cache_dir = self.config["application"]["cache_dir"]
         file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
-        backup_folder = os.path.join(cache_dir, "backup")
+        backup_folder = os.path.join(self.cache_dir, "backup")
         backup_file = os.path.join(backup_folder, file_name)
 
         with zipfile.ZipFile(
@@ -283,29 +217,18 @@ class ElasticBackup:
 
     def post_bulk_restore(self, file_name):
         """send bulk to es"""
-        cache_dir = self.config["application"]["cache_dir"]
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        headers = {"Content-type": "application/x-ndjson"}
-        file_path = os.path.join(cache_dir, file_name)
-
+        file_path = os.path.join(self.cache_dir, file_name)
         with open(file_path, "r", encoding="utf-8") as f:
-            query_str = f.read()
+            data = f.read()
 
-        if not query_str.strip():
+        if not data.strip():
             return
 
-        url = es_url + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=es_auth
-        )
-        if not request.ok:
-            print(request.text)
+        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
 
     def get_all_backup_files(self):
         """build all available backup files for view"""
-        cache_dir = self.config["application"]["cache_dir"]
-        backup_dir = os.path.join(cache_dir, "backup")
+        backup_dir = os.path.join(self.cache_dir, "backup")
         backup_files = os.listdir(backup_dir)
         all_backup_files = ignore_filelist(backup_files)
         all_available_backups = [
@@ -336,8 +259,7 @@ class ElasticBackup:
 
     def unpack_zip_backup(self, filename):
         """extract backup zip and return filelist"""
-        cache_dir = self.config["application"]["cache_dir"]
-        backup_dir = os.path.join(cache_dir, "backup")
+        backup_dir = os.path.join(self.cache_dir, "backup")
         file_path = os.path.join(backup_dir, filename)
 
         with zipfile.ZipFile(file_path, "r") as z:
@@ -348,9 +270,7 @@ class ElasticBackup:
 
     def restore_json_files(self, zip_content):
         """go through the unpacked files and restore"""
-
-        cache_dir = self.config["application"]["cache_dir"]
-        backup_dir = os.path.join(cache_dir, "backup")
+        backup_dir = os.path.join(self.cache_dir, "backup")
 
         for json_f in zip_content:
 
@@ -364,14 +284,13 @@ class ElasticBackup:
             self.post_bulk_restore(file_name)
             os.remove(file_name)
 
-    def index_exists(self, index_name):
+    @staticmethod
+    def index_exists(index_name):
         """check if index already exists to skip"""
-        es_url = self.config["application"]["es_url"]
-        es_auth = self.config["application"]["es_auth"]
-        url = f"{es_url}/ta_{index_name}"
-        response = requests.get(url, auth=es_auth)
+        _, status_code = ElasticWrap(f"ta_{index_name}").get()
+        exists = status_code == 200
 
-        return response.ok
+        return exists
 
     def rotate_backup(self):
         """delete old backups if needed"""
@@ -386,8 +305,7 @@ class ElasticBackup:
             print("no backup files to rotate")
             return
 
-        cache_dir = self.config["application"]["cache_dir"]
-        backup_dir = os.path.join(cache_dir, "backup")
+        backup_dir = os.path.join(self.cache_dir, "backup")
 
         all_to_delete = auto[rotate:]
         for to_delete in all_to_delete:

tubearchivist/home/src/frontend/api_calls.py

@@ -4,7 +4,7 @@ Functionality:
 - called via user input
 """
 
-from home.src.download.queue import PendingList
+from home.src.download.queue import PendingInteract
 from home.src.download.subscriptions import (
     ChannelSubscription,
     PlaylistSubscription,
@@ -110,12 +110,11 @@ class PostData:
 
     def _ignore(self):
         """ignore from download queue"""
-        id_to_ignore = self.exec_val
-        print("ignore video " + id_to_ignore)
-        handler = PendingList()
-        handler.ignore_from_pending([id_to_ignore])
+        video_id = self.exec_val
+        print(f"ignore video {video_id}")
+        PendingInteract(video_id=video_id, status="ignore").update_status()
         # also clear from redis queue
-        RedisQueue("dl_queue").clear_item(id_to_ignore)
+        RedisQueue().clear_item(video_id)
         return {"success": True}
 
     @staticmethod
@@ -133,7 +132,7 @@ class PostData:
         to_execute = self.exec_val
         if to_execute == "stop":
             print("stopping download queue")
-            RedisQueue("dl_queue").clear()
+            RedisQueue().clear()
         elif to_execute == "kill":
             task_id = RedisArchivist().get_message("dl_queue_id")
             if not isinstance(task_id, str):
@@ -222,28 +221,25 @@ class PostData:
 
     def _forget_ignore(self):
         """delete from ta_download index"""
-        youtube_id = self.exec_val
-        print("forgetting from download index: " + youtube_id)
-        PendingList().delete_from_pending(youtube_id)
+        video_id = self.exec_val
+        print(f"forgetting from download index: {video_id}")
+        PendingInteract(video_id=video_id).delete_item()
         return {"success": True}
 
     def _add_single(self):
         """add single youtube_id to download queue"""
-        youtube_id = self.exec_val
-        print("add vid to dl queue: " + youtube_id)
-        PendingList().delete_from_pending(youtube_id)
-        youtube_ids = UrlListParser(youtube_id).process_list()
-        extrac_dl.delay(youtube_ids)
+        video_id = self.exec_val
+        print(f"add vid to dl queue: {video_id}")
+        PendingInteract(video_id=video_id).delete_item()
+        video_ids = UrlListParser(video_id).process_list()
+        extrac_dl.delay(video_ids)
         return {"success": True}
 
     def _delete_queue(self):
         """delete download queue"""
         status = self.exec_val
         print("deleting from download queue: " + status)
-        if status == "pending":
-            PendingList().delete_pending("pending")
-        elif status == "ignore":
-            PendingList().delete_pending("ignore")
+        PendingInteract(status=status).delete_by_status()
         return {"success": True}
 
     @staticmethod

tubearchivist/home/src/frontend/forms.py

@@ -178,3 +178,19 @@ class SubscribeToPlaylistForm(forms.Form):
             }
         ),
     )
+
+
+class ChannelOverwriteForm(forms.Form):
+    """custom overwrites for channel settings"""
+
+    PLAYLIST_INDEX = [
+        ("", "-- change playlist index --"),
+        ("0", "Disable playlist index"),
+        ("1", "Enable playlist index"),
+    ]
+
+    download_format = forms.CharField(label=False, required=False)
+    autodelete_days = forms.IntegerField(label=False, required=False)
+    index_playlists = forms.ChoiceField(
+        widget=forms.Select, choices=PLAYLIST_INDEX, required=False
+    )

tubearchivist/home/src/frontend/watched.py

@@ -3,22 +3,15 @@ functionality:
 - handle watched state for videos, channels and playlists
 """
 
-import json
 from datetime import datetime
 
-import requests
-from home.src.ta.config import AppConfig
+from home.src.es.connect import ElasticWrap
 from home.src.ta.helper import UrlListParser
 
 
 class WatchState:
     """handle watched checkbox for videos and channels"""
 
-    CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
-    HEADERS = {"Content-type": "application/json"}
-
     def __init__(self, youtube_id):
         self.youtube_id = youtube_id
         self.stamp = int(datetime.now().strftime("%s"))
@@ -33,7 +26,7 @@ class WatchState:
         elif url_type == "playlist":
             self.mark_playlist_watched()
 
-        print(f"marked {self.youtube_id} as watched")
+        print(f"{self.youtube_id}: marked as watched")
 
     def mark_as_unwatched(self):
         """revert watched state to false"""
@@ -41,7 +34,7 @@ class WatchState:
         if url_type == "video":
             self.mark_vid_watched(revert=True)
 
-        print(f"revert {self.youtube_id} as unwatched")
+        print(f"{self.youtube_id}: revert as unwatched")
 
     def dedect_type(self):
         """find youtube id type"""
@@ -52,77 +45,54 @@ class WatchState:
 
     def mark_vid_watched(self, revert=False):
         """change watched status of single video"""
-        url = self.ES_URL + "/ta_video/_update/" + self.youtube_id
+        path = f"ta_video/_update/{self.youtube_id}"
         data = {
             "doc": {"player": {"watched": True, "watched_date": self.stamp}}
         }
         if revert:
             data["doc"]["player"]["watched"] = False
 
-        payload = json.dumps(data)
-        request = requests.post(
-            url, data=payload, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request.text)
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code != 200:
+            print(response)
             raise ValueError("failed to mark video as watched")
 
     def mark_channel_watched(self):
         """change watched status of every video in channel"""
+        path = "ta_video/_update_by_query"
+        must_list = [
+            {"term": {"channel.channel_id": {"value": self.youtube_id}}},
+            {"term": {"player.watched": {"value": False}}},
+        ]
         data = {
-            "query": {
-                "bool": {
-                    "must": [
-                        {
-                            "term": {
-                                "channel.channel_id": {
-                                    "value": self.youtube_id
-                                }
-                            }
-                        },
-                        {"term": {"player.watched": {"value": False}}},
-                    ]
-                }
-            },
+            "query": {"bool": {"must": must_list}},
             "script": {
                 "source": "ctx._source.player['watched'] = true",
                 "lang": "painless",
             },
         }
-        payload = json.dumps(data)
-        url = f"{self.ES_URL}/ta_video/_update_by_query"
-        request = requests.post(
-            url, data=payload, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request.text)
+
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code != 200:
+            print(response)
             raise ValueError("failed mark channel as watched")
 
     def mark_playlist_watched(self):
         """change watched state of all videos in playlist"""
+        path = "ta_video/_update_by_query"
+        must_list = [
+            {"term": {"playlist.keyword": {"value": self.youtube_id}}},
+            {"term": {"player.watched": {"value": False}}},
+        ]
         data = {
-            "query": {
-                "bool": {
-                    "must": [
-                        {
-                            "term": {
-                                "playlist.keyword": {"value": self.youtube_id}
-                            }
-                        },
-                        {"term": {"player.watched": {"value": False}}},
-                    ]
-                }
-            },
+            "query": {"bool": {"must": must_list}},
             "script": {
                 "source": "ctx._source.player['watched'] = true",
                 "lang": "painless",
             },
         }
-        payload = json.dumps(data)
-        url = f"{self.ES_URL}/ta_video/_update_by_query"
-        request = requests.post(
-            url, data=payload, headers=self.HEADERS, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request.text)
+
+        response, status_code = ElasticWrap(path).post(data=data)
+        if status_code != 200:
+            print(response)
             raise ValueError("failed mark playlist as watched")

tubearchivist/home/src/index/channel.py

@@ -12,11 +12,13 @@ from datetime import datetime
 import requests
 import yt_dlp
 from bs4 import BeautifulSoup
+from home.src.download import queue  # partial import
 from home.src.download.thumbnails import ThumbManager
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.generic import YouTubeItem
 from home.src.index.playlist import YoutubePlaylist
-from home.src.ta.helper import clean_string
+from home.src.ta.helper import clean_string, requests_headers
+from home.src.ta.ta_redis import RedisArchivist
 
 
 class ChannelScraper:
@@ -44,7 +46,9 @@ class ChannelScraper:
         print(f"{self.channel_id}: scrape channel data from youtube")
         url = f"https://www.youtube.com/channel/{self.channel_id}/about?hl=en"
         cookies = {"CONSENT": "YES+xxxxxxxxxxxxxxxxxxxxxxxxxxx"}
-        response = requests.get(url, cookies=cookies)
+        response = requests.get(
+            url, cookies=cookies, headers=requests_headers()
+        )
         if response.ok:
             channel_page = response.text
         else:
@@ -153,6 +157,7 @@ class YoutubeChannel(YouTubeItem):
     def __init__(self, youtube_id):
         super().__init__(youtube_id)
         self.es_path = f"{self.index_name}/_doc/{youtube_id}"
+        self.all_playlists = False
 
     def build_json(self, upload=False):
         """get from es or from youtube"""
@@ -241,6 +246,68 @@ class YoutubeChannel(YouTubeItem):
         self.delete_es_videos()
         self.del_in_es()
 
+    def index_channel_playlists(self):
+        """add all playlists of channel to index"""
+        print(f"{self.youtube_id}: index all playlists")
+        self.get_from_es()
+        channel_name = self.json_data["channel_name"]
+        mess_dict = {
+            "status": "message:playlistscan",
+            "level": "info",
+            "title": "Looking for playlists",
+            "message": f"{channel_name}: Scanning channel in progress",
+        }
+        RedisArchivist().set_message("message:playlistscan", mess_dict)
+        self.get_all_playlists()
+        if not self.all_playlists:
+            print(f"{self.youtube_id}: no playlists found.")
+            return
+
+        all_youtube_ids = self.get_all_video_ids()
+        for idx, playlist in enumerate(self.all_playlists):
+            self._notify_single_playlist(idx, playlist)
+            self._index_single_playlist(playlist, all_youtube_ids)
+
+    def _notify_single_playlist(self, idx, playlist):
+        """send notification"""
+        channel_name = self.json_data["channel_name"]
+        mess_dict = {
+            "status": "message:playlistscan",
+            "level": "info",
+            "title": f"{channel_name}: Scanning channel for playlists",
+            "message": f"Progress: {idx + 1}/{len(self.all_playlists)}",
+        }
+        RedisArchivist().set_message("message:playlistscan", mess_dict)
+        print("add playlist: " + playlist[1])
+
+    @staticmethod
+    def _index_single_playlist(playlist, all_youtube_ids):
+        """add single playlist if needed"""
+        playlist = YoutubePlaylist(playlist[0])
+        playlist.all_youtube_ids = all_youtube_ids
+        playlist.build_json()
+        if not playlist.json_data:
+            return
+
+        entries = playlist.json_data["playlist_entries"]
+        downloaded = [i for i in entries if i["downloaded"]]
+        if not downloaded:
+            return
+
+        playlist.upload_to_es()
+        playlist.add_vids_to_playlist()
+        playlist.get_playlist_art()
+
+    @staticmethod
+    def get_all_video_ids():
+        """match all playlists with videos"""
+        handler = queue.PendingList()
+        handler.get_download()
+        handler.get_indexed()
+        all_youtube_ids = [i["youtube_id"] for i in handler.all_videos]
+
+        return all_youtube_ids
+
     def get_all_playlists(self):
         """get all playlists owned by this channel"""
         url = (
@@ -254,8 +321,7 @@ class YoutubeChannel(YouTubeItem):
         }
         playlists = yt_dlp.YoutubeDL(obs).extract_info(url)
         all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
-
-        return all_entries
+        self.all_playlists = all_entries
 
     def get_indexed_playlists(self):
         """get all indexed playlists from channel"""
@@ -267,3 +333,35 @@ class YoutubeChannel(YouTubeItem):
         }
         all_playlists = IndexPaginate("ta_playlist", data).get_results()
         return all_playlists
+
+    def get_overwrites(self):
+        """get all per channel overwrites"""
+        return self.json_data.get("channel_overwrites", False)
+
+    def set_overwrites(self, overwrites):
+        """set per channel overwrites"""
+        valid_keys = ["download_format", "autodelete_days", "index_playlists"]
+
+        to_write = self.json_data.get("channel_overwrites", {})
+        for key, value in overwrites.items():
+            if key not in valid_keys:
+                raise ValueError(f"invalid overwrite key: {key}")
+            if value in [0, "0"]:
+                del to_write[key]
+                continue
+            if value == "1":
+                to_write[key] = True
+                continue
+            if value:
+                to_write.update({key: value})
+
+        self.json_data["channel_overwrites"] = to_write
+
+
+def channel_overwrites(channel_id, overwrites):
+    """collection to overwrite settings per channel"""
+    channel = YoutubeChannel(channel_id)
+    channel.build_json()
+    channel.set_overwrites(overwrites)
+    channel.upload_to_es()
+    channel.sync_to_videos()

tubearchivist/home/src/index/filesystem.py

@@ -10,11 +10,10 @@ import os
 import re
 import shutil
 import subprocess
-from datetime import datetime
 
-import requests
 from home.src.download.queue import PendingList
 from home.src.download.yt_dlp_handler import VideoDownloader
+from home.src.es.connect import ElasticWrap
 from home.src.index.reindex import Reindex
 from home.src.index.video import index_new_video
 from home.src.ta.config import AppConfig
@@ -26,8 +25,6 @@ class FilesystemScanner:
     """handle scanning and fixing from filesystem"""
 
     CONFIG = AppConfig().config
-    ES_URL = CONFIG["application"]["es_url"]
-    ES_AUTH = CONFIG["application"]["es_auth"]
     VIDEOS = CONFIG["application"]["videos"]
 
     def __init__(self):
@@ -59,9 +56,11 @@ class FilesystemScanner:
     def get_all_indexed():
         """get a list of all indexed videos"""
         index_handler = PendingList()
-        all_indexed_raw = index_handler.get_all_indexed()
+        index_handler.get_download()
+        index_handler.get_indexed()
+
         all_indexed = []
-        for video in all_indexed_raw:
+        for video in index_handler.all_videos:
             youtube_id = video["youtube_id"]
             media_url = video["media_url"]
             published = video["published"]
@@ -146,25 +145,16 @@ class FilesystemScanner:
             bulk_list.append(json.dumps(source))
         # add last newline
         bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
-        # make the call
-        headers = {"Content-type": "application/x-ndjson"}
-        url = self.ES_URL + "/_bulk"
-        request = requests.post(
-            url, data=query_str, headers=headers, auth=self.ES_AUTH
-        )
-        if not request.ok:
-            print(request.text)
+        data = "\n".join(bulk_list)
+        _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
 
     def delete_from_index(self):
         """find indexed but deleted mediafile"""
         for indexed in self.to_delete:
             youtube_id = indexed[0]
             print(f"deleting {youtube_id} from index")
-            url = self.ES_URL + "/ta_video/_doc/" + youtube_id
-            request = requests.delete(url, auth=self.ES_AUTH)
-            if not request.ok:
-                print(request.text)
+            path = f"ta_video/_doc/{youtube_id}"
+            _, _ = ElasticWrap(path).delete()
 
 
 class ManualImport:
@@ -317,10 +307,7 @@ def scan_filesystem():
 
 def reindex_old_documents():
     """daily refresh of old documents"""
-    # continue if needed
-    reindex_handler = Reindex()
-    reindex_handler.check_outdated()
-    reindex_handler.reindex()
-    # set timestamp
-    now = int(datetime.now().strftime("%s"))
-    RedisArchivist().set_message("last_reindex", now, expire=False)
+    handler = Reindex()
+    handler.check_outdated()
+    handler.reindex()
+    RedisArchivist().set_message("last_reindex", handler.now, expire=False)

tubearchivist/home/src/index/reindex.py

@@ -4,86 +4,60 @@ functionality:
 - index and update in es
 """
 
-import json
 from datetime import datetime
 from math import ceil
 from time import sleep
 
-import requests
 from home.src.download.queue import PendingList
-from home.src.download.subscriptions import ChannelSubscription
 from home.src.download.thumbnails import ThumbManager
+from home.src.es.connect import ElasticWrap
 from home.src.index.channel import YoutubeChannel
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo
 from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_total_hits
 
 
 class Reindex:
     """check for outdated documents and refresh data from youtube"""
 
+    MATCH_FIELD = {
+        "ta_video": "active",
+        "ta_channel": "channel_active",
+        "ta_playlist": "playlist_active",
+    }
+    MULTIPLY = 1.2
+
     def __init__(self):
         # config
-        config = AppConfig().config
-        self.sleep_interval = config["downloads"]["sleep_interval"]
-        self.es_url = config["application"]["es_url"]
-        self.es_auth = config["application"]["es_auth"]
-        self.refresh_interval = config["scheduler"]["check_reindex_days"]
-        self.integrate_ryd = config["downloads"]["integrate_ryd"]
+        self.now = int(datetime.now().strftime("%s"))
+        self.config = AppConfig().config
+        self.interval = self.config["scheduler"]["check_reindex_days"]
         # scan
         self.all_youtube_ids = False
         self.all_channel_ids = False
         self.all_playlist_ids = False
 
-    def get_daily(self):
+    def _get_daily(self):
         """get daily refresh values"""
-        total_videos = get_total_hits(
-            "ta_video", self.es_url, self.es_auth, "active"
-        )
-        video_daily = ceil(total_videos / self.refresh_interval * 1.2)
-        total_channels = get_total_hits(
-            "ta_channel", self.es_url, self.es_auth, "channel_active"
-        )
-        channel_daily = ceil(total_channels / self.refresh_interval * 1.2)
-        total_playlists = get_total_hits(
-            "ta_playlist", self.es_url, self.es_auth, "playlist_active"
-        )
-        playlist_daily = ceil(total_playlists / self.refresh_interval * 1.2)
+        total_videos = self._get_total_hits("ta_video")
+        video_daily = ceil(total_videos / self.interval * self.MULTIPLY)
+        total_channels = self._get_total_hits("ta_channel")
+        channel_daily = ceil(total_channels / self.interval * self.MULTIPLY)
+        total_playlists = self._get_total_hits("ta_playlist")
+        playlist_daily = ceil(total_playlists / self.interval * self.MULTIPLY)
         return (video_daily, channel_daily, playlist_daily)
 
-    def get_outdated_vids(self, size):
-        """get daily videos to refresh"""
-        headers = {"Content-type": "application/json"}
-        now = int(datetime.now().strftime("%s"))
-        now_lte = now - self.refresh_interval * 24 * 60 * 60
-        data = {
-            "size": size,
-            "query": {
-                "bool": {
-                    "must": [
-                        {"match": {"active": True}},
-                        {"range": {"vid_last_refresh": {"lte": now_lte}}},
-                    ]
-                }
-            },
-            "sort": [{"vid_last_refresh": {"order": "asc"}}],
-            "_source": False,
-        }
-        query_str = json.dumps(data)
-        url = self.es_url + "/ta_video/_search"
-        response = requests.get(
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        all_youtube_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
-        return all_youtube_ids
+    def _get_total_hits(self, index):
+        """get total hits from index"""
+        match_field = self.MATCH_FIELD[index]
+        path = f"{index}/_search?filter_path=hits.total"
+        data = {"query": {"match": {match_field: True}}}
+        response, _ = ElasticWrap(path).post(data=data)
+        total_hits = response["hits"]["total"]["value"]
+        return total_hits
 
-    def get_unrated_vids(self):
-        """get all videos without rating if ryd integration is enabled"""
-        headers = {"Content-type": "application/json"}
+    def _get_unrated_vids(self):
+        """get max 200 videos without rating if ryd integration is enabled"""
         data = {
             "size": 200,
             "query": {
@@ -92,104 +66,78 @@ class Reindex:
                 }
             },
         }
-        query_str = json.dumps(data)
-        url = self.es_url + "/ta_video/_search"
-        response = requests.get(
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        missing_rating = [i["_id"] for i in response_dict["hits"]["hits"]]
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+
+        missing_rating = [i["_id"] for i in response["hits"]["hits"]]
         self.all_youtube_ids = self.all_youtube_ids + missing_rating
 
-    def get_outdated_channels(self, size):
-        """get daily channels to refresh"""
-        headers = {"Content-type": "application/json"}
-        now = int(datetime.now().strftime("%s"))
-        now_lte = now - self.refresh_interval * 24 * 60 * 60
+    def _get_outdated_vids(self, size):
+        """get daily videos to refresh"""
+        now_lte = self.now - self.interval * 24 * 60 * 60
+        must_list = [
+            {"match": {"active": True}},
+            {"range": {"vid_last_refresh": {"lte": now_lte}}},
+        ]
         data = {
             "size": size,
-            "query": {
-                "bool": {
-                    "must": [
-                        {"match": {"channel_active": True}},
-                        {"range": {"channel_last_refresh": {"lte": now_lte}}},
-                    ]
-                }
-            },
+            "query": {"bool": {"must": must_list}},
+            "sort": [{"vid_last_refresh": {"order": "asc"}}],
+            "_source": False,
+        }
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+
+        all_youtube_ids = [i["_id"] for i in response["hits"]["hits"]]
+        return all_youtube_ids
+
+    def _get_outdated_channels(self, size):
+        """get daily channels to refresh"""
+        now_lte = self.now - self.interval * 24 * 60 * 60
+        must_list = [
+            {"match": {"channel_active": True}},
+            {"range": {"channel_last_refresh": {"lte": now_lte}}},
+        ]
+        data = {
+            "size": size,
+            "query": {"bool": {"must": must_list}},
             "sort": [{"channel_last_refresh": {"order": "asc"}}],
             "_source": False,
         }
-        query_str = json.dumps(data)
-        url = self.es_url + "/ta_channel/_search"
-        response = requests.get(
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        all_channel_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
+        response, _ = ElasticWrap("ta_channel/_search").get(data=data)
+
+        all_channel_ids = [i["_id"] for i in response["hits"]["hits"]]
         return all_channel_ids
 
-    def get_outdated_playlists(self, size):
+    def _get_outdated_playlists(self, size):
         """get daily outdated playlists to refresh"""
-        headers = {"Content-type": "application/json"}
-        now = int(datetime.now().strftime("%s"))
-        now_lte = now - self.refresh_interval * 24 * 60 * 60
+        now_lte = self.now - self.interval * 24 * 60 * 60
+        must_list = [
+            {"match": {"playlist_active": True}},
+            {"range": {"playlist_last_refresh": {"lte": now_lte}}},
+        ]
         data = {
             "size": size,
-            "query": {
-                "bool": {
-                    "must": [
-                        {"match": {"playlist_active": True}},
-                        {"range": {"playlist_last_refresh": {"lte": now_lte}}},
-                    ]
-                }
-            },
+            "query": {"bool": {"must": must_list}},
             "sort": [{"playlist_last_refresh": {"order": "asc"}}],
             "_source": False,
         }
-        query_str = json.dumps(data)
-        url = self.es_url + "/ta_playlist/_search"
-        response = requests.get(
-            url, data=query_str, headers=headers, auth=self.es_auth
-        )
-        if not response.ok:
-            print(response.text)
-        response_dict = json.loads(response.text)
-        all_playlist_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
+        response, _ = ElasticWrap("ta_playlist/_search").get(data=data)
+
+        all_playlist_ids = [i["_id"] for i in response["hits"]["hits"]]
         return all_playlist_ids
 
     def check_outdated(self):
         """add missing vids and channels"""
-        video_daily, channel_daily, playlist_daily = self.get_daily()
-        self.all_youtube_ids = self.get_outdated_vids(video_daily)
-        self.all_channel_ids = self.get_outdated_channels(channel_daily)
-        self.all_playlist_ids = self.get_outdated_playlists(playlist_daily)
-        if self.integrate_ryd:
-            self.get_unrated_vids()
+        video_daily, channel_daily, playlist_daily = self._get_daily()
+        self.all_youtube_ids = self._get_outdated_vids(video_daily)
+        self.all_channel_ids = self._get_outdated_channels(channel_daily)
+        self.all_playlist_ids = self._get_outdated_playlists(playlist_daily)
 
-    def rescrape_all_channels(self):
-        """sync new data from channel to all matching videos"""
-        sleep_interval = self.sleep_interval
-        channel_sub_handler = ChannelSubscription()
-        all_channels = channel_sub_handler.get_channels(subscribed_only=False)
-        all_channel_ids = [i["channel_id"] for i in all_channels]
-
-        for channel_id in all_channel_ids:
-            channel = YoutubeChannel(channel_id)
-            subscribed = channel.json_data["channel_subscribed"]
-            channel.get_from_youtube()
-            channel.json_data["channel_subscribed"] = subscribed
-            channel.upload_to_es()
-            channel.sync_to_videos()
-
-            if sleep_interval:
-                sleep(sleep_interval)
+        integrate_ryd = self.config["downloads"]["integrate_ryd"]
+        if integrate_ryd:
+            self._get_unrated_vids()
 
     @staticmethod
-    def reindex_single_video(youtube_id):
+    def _reindex_single_video(youtube_id):
         """refresh data for single video"""
         video = YoutubeVideo(youtube_id)
 
@@ -207,6 +155,8 @@ class Reindex:
             return
 
         video.delete_subtitles()
+        video.check_subtitles()
+
         # add back
         video.json_data["player"] = player
         video.json_data["date_downloaded"] = date_downloaded
@@ -223,18 +173,21 @@ class Reindex:
         return
 
     @staticmethod
-    def reindex_single_channel(channel_id):
+    def _reindex_single_channel(channel_id):
         """refresh channel data and sync to videos"""
         channel = YoutubeChannel(channel_id)
         channel.get_from_es()
         subscribed = channel.json_data["channel_subscribed"]
+        overwrites = channel.json_data.get("channel_overwrites", False)
         channel.get_from_youtube()
         channel.json_data["channel_subscribed"] = subscribed
+        if overwrites:
+            channel.json_data["channel_overwrites"] = overwrites
         channel.upload_to_es()
         channel.sync_to_videos()
 
     @staticmethod
-    def reindex_single_playlist(playlist_id, all_indexed_ids):
+    def _reindex_single_playlist(playlist_id, all_indexed_ids):
         """refresh playlist data"""
         playlist = YoutubePlaylist(playlist_id)
         playlist.get_from_es()
@@ -251,24 +204,27 @@ class Reindex:
 
     def reindex(self):
         """reindex what's needed"""
+        sleep_interval = self.config["downloads"]["sleep_interval"]
         # videos
         print(f"reindexing {len(self.all_youtube_ids)} videos")
         for youtube_id in self.all_youtube_ids:
-            self.reindex_single_video(youtube_id)
-            if self.sleep_interval:
-                sleep(self.sleep_interval)
+            self._reindex_single_video(youtube_id)
+            if sleep_interval:
+                sleep(sleep_interval)
         # channels
         print(f"reindexing {len(self.all_channel_ids)} channels")
         for channel_id in self.all_channel_ids:
-            self.reindex_single_channel(channel_id)
-            if self.sleep_interval:
-                sleep(self.sleep_interval)
+            self._reindex_single_channel(channel_id)
+            if sleep_interval:
+                sleep(sleep_interval)
         # playlist
         print(f"reindexing {len(self.all_playlist_ids)} playlists")
         if self.all_playlist_ids:
-            all_indexed = PendingList().get_all_indexed()
-            all_indexed_ids = [i["youtube_id"] for i in all_indexed]
+            handler = PendingList()
+            handler.get_download()
+            handler.get_indexed()
+            all_indexed_ids = [i["youtube_id"] for i in handler.all_videos]
             for playlist_id in self.all_playlist_ids:
-                self.reindex_single_playlist(playlist_id, all_indexed_ids)
-                if self.sleep_interval:
-                    sleep(self.sleep_interval)
+                self._reindex_single_playlist(playlist_id, all_indexed_ids)
+                if sleep_interval:
+                    sleep(sleep_interval)

tubearchivist/home/src/index/video.py

@@ -6,14 +6,17 @@ functionality:
 
 import json
 import os
-import re
 from datetime import datetime
 
 import requests
 from home.src.es.connect import ElasticWrap
 from home.src.index import channel as ta_channel
 from home.src.index.generic import YouTubeItem
-from home.src.ta.helper import DurationConverter, clean_string
+from home.src.ta.helper import (
+    DurationConverter,
+    clean_string,
+    requests_headers,
+)
 from ryd_client import ryd_client
 
 
@@ -24,7 +27,7 @@ class YoutubeSubtitle:
         self.video = video
         self.languages = False
 
-    def sub_conf_parse(self):
+    def _sub_conf_parse(self):
         """add additional conf values to self"""
         languages_raw = self.video.config["downloads"]["subtitle"]
         if languages_raw:
@@ -32,26 +35,26 @@ class YoutubeSubtitle:
 
     def get_subtitles(self):
         """check what to do"""
-        self.sub_conf_parse()
+        self._sub_conf_parse()
         if not self.languages:
             # no subtitles
             return False
 
         relevant_subtitles = []
         for lang in self.languages:
-            user_sub = self.get_user_subtitles(lang)
+            user_sub = self._get_user_subtitles(lang)
             if user_sub:
                 relevant_subtitles.append(user_sub)
                 continue
 
             if self.video.config["downloads"]["subtitle_source"] == "auto":
-                auto_cap = self.get_auto_caption(lang)
+                auto_cap = self._get_auto_caption(lang)
                 if auto_cap:
                     relevant_subtitles.append(auto_cap)
 
         return relevant_subtitles
 
-    def get_auto_caption(self, lang):
+    def _get_auto_caption(self, lang):
         """get auto_caption subtitles"""
         print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
         all_subtitles = self.video.youtube_meta.get("automatic_captions")
@@ -65,7 +68,7 @@ class YoutubeSubtitle:
         if not all_formats:
             return False
 
-        subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
         subtitle.update(
             {"lang": lang, "source": "auto", "media_url": media_url}
         )
@@ -88,7 +91,7 @@ class YoutubeSubtitle:
 
         return all_subtitles
 
-    def get_user_subtitles(self, lang):
+    def _get_user_subtitles(self, lang):
         """get subtitles uploaded from channel owner"""
         print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
         all_subtitles = self._normalize_lang()
@@ -102,7 +105,7 @@ class YoutubeSubtitle:
             # no user subtitles found
             return False
 
-        subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
+        subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
         subtitle.update(
             {"lang": lang, "source": "user", "media_url": media_url}
         )
@@ -115,12 +118,16 @@ class YoutubeSubtitle:
         for subtitle in relevant_subtitles:
             dest_path = os.path.join(videos_base, subtitle["media_url"])
             source = subtitle["source"]
-            response = requests.get(subtitle["url"])
+            lang = subtitle.get("lang")
+            response = requests.get(
+                subtitle["url"], headers=requests_headers()
+            )
             if not response.ok:
                 print(f"{self.video.youtube_id}: failed to download subtitle")
+                print(response.text)
                 continue
 
-            parser = SubtitleParser(response.text, subtitle.get("lang"))
+            parser = SubtitleParser(response.text, lang, source)
             parser.process()
             subtitle_str = parser.get_subtitle_str()
             self._write_subtitle_file(dest_path, subtitle_str)
@@ -145,109 +152,94 @@ class YoutubeSubtitle:
 class SubtitleParser:
     """parse subtitle str from youtube"""
 
-    time_reg = r"^([0-9]{2}:?){3}\.[0-9]{3} --> ([0-9]{2}:?){3}\.[0-9]{3}"
-    stamp_reg = r"<([0-9]{2}:?){3}\.[0-9]{3}>"
-    tag_reg = r"</?c>"
-
-    def __init__(self, subtitle_str, lang):
-        self.subtitle_str = subtitle_str
+    def __init__(self, subtitle_str, lang, source):
+        self.subtitle_raw = json.loads(subtitle_str)
         self.lang = lang
-        self.header = False
-        self.parsed_cue_list = False
-        self.all_text_lines = False
-        self.matched = False
+        self.source = source
+        self.all_cues = False
 
     def process(self):
-        """collection to process subtitle string"""
-        self._parse_cues()
-        self._match_text_lines()
-        self._add_id()
-        self._timestamp_check()
+        """extract relevant que data"""
+        all_events = self.subtitle_raw.get("events")
+        if self.source == "auto":
+            all_events = self._flat_auto_caption(all_events)
 
-    def _parse_cues(self):
-        """split into cues"""
-        all_cues = self.subtitle_str.replace("\n \n", "\n").split("\n\n")
-        self.header = all_cues[0]
-        self.all_text_lines = []
-        self.parsed_cue_list = [self._cue_cleaner(i) for i in all_cues[1:]]
+        self.all_cues = []
+        for idx, event in enumerate(all_events):
+            cue = {
+                "start": self._ms_conv(event["tStartMs"]),
+                "end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
+                "text": "".join([i.get("utf8") for i in event["segs"]]),
+                "idx": idx + 1,
+            }
+            self.all_cues.append(cue)
 
-    def _cue_cleaner(self, cue):
-        """parse single cue"""
-        all_lines = cue.split("\n")
-        cue_dict = {"lines": []}
-
-        for line in all_lines:
-            if re.match(self.time_reg, line):
-                clean = re.search(self.time_reg, line).group()
-                start, end = clean.split(" --> ")
-                cue_dict.update({"start": start, "end": end})
-            else:
-                clean = re.sub(self.stamp_reg, "", line)
-                clean = re.sub(self.tag_reg, "", clean)
-                cue_dict["lines"].append(clean)
-                if clean.strip() and clean not in self.all_text_lines[-4:]:
-                    # remove immediate duplicates
-                    self.all_text_lines.append(clean)
-
-        return cue_dict
-
-    def _match_text_lines(self):
-        """match unique text lines with timestamps"""
-
-        self.matched = []
-
-        while self.all_text_lines:
-            check = self.all_text_lines[0]
-            matches = [i for i in self.parsed_cue_list if check in i["lines"]]
-            new_cue = matches[-1]
-            new_cue["start"] = matches[0]["start"]
-
-            for line in new_cue["lines"]:
-                try:
-                    self.all_text_lines.remove(line)
-                except ValueError:
-                    continue
-
-            self.matched.append(new_cue)
-
-    def _timestamp_check(self):
-        """check if end timestamp is bigger than start timestamp"""
-        for idx, cue in enumerate(self.matched):
-            # this
-            end = int(re.sub("[^0-9]", "", cue.get("end")))
-            # next
-            try:
-                next_cue = self.matched[idx + 1]
-            except IndexError:
+    @staticmethod
+    def _flat_auto_caption(all_events):
+        """flatten autocaption segments"""
+        flatten = []
+        for event in all_events:
+            if "segs" not in event.keys():
+                continue
+            text = "".join([i.get("utf8") for i in event.get("segs")])
+            if not text.strip():
                 continue
 
-            start_next = int(re.sub("[^0-9]", "", next_cue.get("start")))
-            if end > start_next:
-                self.matched[idx]["end"] = next_cue.get("start")
+            if flatten:
+                # fix overlapping retiming issue
+                last_end = flatten[-1]["tStartMs"] + flatten[-1]["dDurationMs"]
+                if event["tStartMs"] < last_end:
+                    joined = flatten[-1]["segs"][0]["utf8"] + "\n" + text
+                    flatten[-1]["segs"][0]["utf8"] = joined
+                    continue
 
-    def _add_id(self):
-        """add id to matched cues"""
-        for idx, _ in enumerate(self.matched):
-            self.matched[idx]["id"] = idx + 1
+            event.update({"segs": [{"utf8": text}]})
+            flatten.append(event)
+
+        return flatten
+
+    @staticmethod
+    def _ms_conv(ms):
+        """convert ms to timestamp"""
+        hours = str((ms // (1000 * 60 * 60)) % 24).zfill(2)
+        minutes = str((ms // (1000 * 60)) % 60).zfill(2)
+        secs = str((ms // 1000) % 60).zfill(2)
+        millis = str(ms % 1000).zfill(3)
+
+        return f"{hours}:{minutes}:{secs}.{millis}"
 
     def get_subtitle_str(self):
-        """stitch cues and return processed new string"""
-        new_subtitle_str = self.header + "\n\n"
+        """create vtt text str from cues"""
+        subtitle_str = f"WEBVTT\nKind: captions\nLanguage: {self.lang}"
 
-        for cue in self.matched:
-            timestamp = f"{cue.get('start')} --> {cue.get('end')}"
-            lines = "\n".join(cue.get("lines"))
-            cue_text = f"{cue.get('id')}\n{timestamp}\n{lines}\n\n"
-            new_subtitle_str = new_subtitle_str + cue_text
+        for cue in self.all_cues:
+            stamp = f"{cue.get('start')} --> {cue.get('end')}"
+            cue_text = f"\n\n{cue.get('idx')}\n{stamp}\n{cue.get('text')}"
+            subtitle_str = subtitle_str + cue_text
 
-        return new_subtitle_str
+        return subtitle_str
 
     def create_bulk_import(self, video, source):
-        """process matched for es import"""
+        """subtitle lines for es import"""
+        documents = self._create_documents(video, source)
         bulk_list = []
-        channel = video.json_data.get("channel")
 
-        document = {
+        for document in documents:
+            document_id = document.get("subtitle_fragment_id")
+            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
+            bulk_list.append(json.dumps(action))
+            bulk_list.append(json.dumps(document))
+
+        bulk_list.append("\n")
+        query_str = "\n".join(bulk_list)
+
+        return query_str
+
+    def _create_documents(self, video, source):
+        """process documents"""
+        documents = self._chunk_list(video.youtube_id)
+        channel = video.json_data.get("channel")
+        meta_dict = {
             "youtube_id": video.youtube_id,
             "title": video.json_data.get("title"),
             "subtitle_channel": channel.get("channel_name"),
@@ -257,26 +249,35 @@ class SubtitleParser:
             "subtitle_source": source,
         }
 
-        for match in self.matched:
-            match_id = match.get("id")
-            document_id = f"{video.youtube_id}-{self.lang}-{match_id}"
-            action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
-            document.update(
-                {
-                    "subtitle_fragment_id": document_id,
-                    "subtitle_start": match.get("start"),
-                    "subtitle_end": match.get("end"),
-                    "subtitle_index": match_id,
-                    "subtitle_line": " ".join(match.get("lines")),
+        _ = [i.update(meta_dict) for i in documents]
+
+        return documents
+
+    def _chunk_list(self, youtube_id):
+        """join cues for bulk import"""
+        chunk_list = []
+
+        chunk = {}
+        for cue in self.all_cues:
+            if chunk:
+                text = f"{chunk.get('subtitle_line')} {cue.get('text')}\n"
+                chunk["subtitle_line"] = text
+            else:
+                idx = len(chunk_list) + 1
+                chunk = {
+                    "subtitle_index": idx,
+                    "subtitle_line": cue.get("text"),
+                    "subtitle_start": cue.get("start"),
                 }
-            )
-            bulk_list.append(json.dumps(action))
-            bulk_list.append(json.dumps(document))
 
-        bulk_list.append("\n")
-        query_str = "\n".join(bulk_list)
+            chunk["subtitle_fragment_id"] = f"{youtube_id}-{self.lang}-{idx}"
 
-        return query_str
+            if cue["idx"] % 5 == 0:
+                chunk["subtitle_end"] = cue.get("end")
+                chunk_list.append(chunk)
+                chunk = {}
+
+        return chunk_list
 
 
 class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
@@ -302,7 +303,6 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         self._add_stats()
         self.add_file_path()
         self.add_player()
-        self._check_subtitles()
         if self.config["downloads"]["integrate_ryd"]:
             self._get_ryd_stats()
 
@@ -375,7 +375,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             channel_dir = os.path.join(self.app_conf["videos"], channel)
             all_files = os.listdir(channel_dir)
             for file in all_files:
-                if self.youtube_id in file:
+                if self.youtube_id in file and file.endswith(".mp4"):
                     vid_path = os.path.join(channel_dir, file)
                     break
             else:
@@ -447,7 +447,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
         return True
 
-    def _check_subtitles(self):
+    def check_subtitles(self):
         """optionally add subtitles"""
         handler = YoutubeSubtitle(self)
         subtitles = handler.get_subtitles()
@@ -457,8 +457,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
     def delete_subtitles(self):
         """delete indexed subtitles"""
+        path = "ta_subtitle/_delete_by_query?refresh=true"
         data = {"query": {"term": {"youtube_id": {"value": self.youtube_id}}}}
-        _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data=data)
+        _, _ = ElasticWrap(path).post(data=data)
 
 
 def index_new_video(youtube_id):
@@ -468,5 +469,6 @@ def index_new_video(youtube_id):
     if not video.json_data:
         raise ValueError("failed to get metadata for " + youtube_id)
 
+    video.check_subtitles()
     video.upload_to_es()
     return video.json_data

tubearchivist/home/src/ta/helper.py

@@ -3,31 +3,16 @@ Loose collection of helper functions
 - don't import AppConfig class here to avoid circular imports
 """
 
-import json
+import random
 import re
 import string
 import subprocess
 import unicodedata
 from urllib.parse import parse_qs, urlparse
 
-import requests
 import yt_dlp
 
 
-def get_total_hits(index, es_url, es_auth, match_field):
-    """get total hits from index"""
-    headers = {"Content-type": "application/json"}
-    data = {"query": {"match": {match_field: True}}}
-    payload = json.dumps(data)
-    url = f"{es_url}/{index}/_search?filter_path=hits.total"
-    request = requests.post(url, data=payload, headers=headers, auth=es_auth)
-    if not request.ok:
-        print(request.text)
-    total_json = json.loads(request.text)
-    total_hits = total_json["hits"]["total"]["value"]
-    return total_hits
-
-
 def clean_string(file_name):
     """clean string to only asci characters"""
     whitelist = "-_.() " + string.ascii_letters + string.digits
@@ -51,6 +36,58 @@ def ignore_filelist(filelist):
     return cleaned
 
 
+def requests_headers():
+    """build header with random user agent for requests outside of yt-dlp"""
+
+    chrome_versions = (
+        "90.0.4430.212",
+        "90.0.4430.24",
+        "90.0.4430.70",
+        "90.0.4430.72",
+        "90.0.4430.85",
+        "90.0.4430.93",
+        "91.0.4472.101",
+        "91.0.4472.106",
+        "91.0.4472.114",
+        "91.0.4472.124",
+        "91.0.4472.164",
+        "91.0.4472.19",
+        "91.0.4472.77",
+        "92.0.4515.107",
+        "92.0.4515.115",
+        "92.0.4515.131",
+        "92.0.4515.159",
+        "92.0.4515.43",
+        "93.0.4556.0",
+        "93.0.4577.15",
+        "93.0.4577.63",
+        "93.0.4577.82",
+        "94.0.4606.41",
+        "94.0.4606.54",
+        "94.0.4606.61",
+        "94.0.4606.71",
+        "94.0.4606.81",
+        "94.0.4606.85",
+        "95.0.4638.17",
+        "95.0.4638.50",
+        "95.0.4638.54",
+        "95.0.4638.69",
+        "95.0.4638.74",
+        "96.0.4664.18",
+        "96.0.4664.45",
+        "96.0.4664.55",
+        "96.0.4664.93",
+        "97.0.4692.20",
+    )
+    template = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        + "AppleWebKit/537.36 (KHTML, like Gecko) "
+        + f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
+    )
+
+    return {"User-Agent": template}
+
+
 class UrlListParser:
     """take a multi line string and detect valid youtube ids"""
 

tubearchivist/home/src/ta/ta_redis.py

@@ -11,12 +11,20 @@ import redis
 from home.src.ta.helper import ignore_filelist
 
 
-class RedisArchivist:
-    """collection of methods to interact with redis"""
+class RedisBase:
+    """connection base for redis"""
 
     REDIS_HOST = os.environ.get("REDIS_HOST")
     REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
     NAME_SPACE = "ta:"
+
+    def __init__(self):
+        self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
+
+
+class RedisArchivist(RedisBase):
+    """collection of methods to interact with redis"""
+
     CHANNELS = [
         "download",
         "add",
@@ -27,14 +35,9 @@ class RedisArchivist:
         "setting",
     ]
 
-    def __init__(self):
-        self.redis_connection = redis.Redis(
-            host=self.REDIS_HOST, port=self.REDIS_PORT
-        )
-
     def set_message(self, key, message, expire=True):
         """write new message to redis"""
-        self.redis_connection.execute_command(
+        self.conn.execute_command(
             "JSON.SET", self.NAME_SPACE + key, ".", json.dumps(message)
         )
 
@@ -43,15 +46,11 @@ class RedisArchivist:
                 secs = 20
             else:
                 secs = expire
-            self.redis_connection.execute_command(
-                "EXPIRE", self.NAME_SPACE + key, secs
-            )
+            self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
 
     def get_message(self, key):
         """get message dict from redis"""
-        reply = self.redis_connection.execute_command(
-            "JSON.GET", self.NAME_SPACE + key
-        )
+        reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
         if reply:
             json_str = json.loads(reply)
         else:
@@ -61,7 +60,7 @@ class RedisArchivist:
 
     def list_items(self, query):
         """list all matches"""
-        reply = self.redis_connection.execute_command(
+        reply = self.conn.execute_command(
             "KEYS", self.NAME_SPACE + query + "*"
         )
         all_matches = [i.decode().lstrip(self.NAME_SPACE) for i in reply]
@@ -74,14 +73,12 @@ class RedisArchivist:
 
     def del_message(self, key):
         """delete key from redis"""
-        response = self.redis_connection.execute_command(
-            "DEL", self.NAME_SPACE + key
-        )
+        response = self.conn.execute_command("DEL", self.NAME_SPACE + key)
         return response
 
     def get_lock(self, lock_key):
         """handle lock for task management"""
-        redis_lock = self.redis_connection.lock(self.NAME_SPACE + lock_key)
+        redis_lock = self.conn.lock(self.NAME_SPACE + lock_key)
         return redis_lock
 
     def get_progress(self):
@@ -89,7 +86,7 @@ class RedisArchivist:
         all_messages = []
         for channel in self.CHANNELS:
             key = "message:" + channel
-            reply = self.redis_connection.execute_command(
+            reply = self.conn.execute_command(
                 "JSON.GET", self.NAME_SPACE + key
             )
             if reply:
@@ -120,19 +117,12 @@ class RedisArchivist:
         return mess_dict
 
 
-class RedisQueue:
+class RedisQueue(RedisBase):
     """dynamically interact with the download queue in redis"""
 
-    REDIS_HOST = os.environ.get("REDIS_HOST")
-    REDIS_PORT = os.environ.get("REDIS_PORT")
-    NAME_SPACE = "ta:"
-
-    if not REDIS_PORT:
-        REDIS_PORT = 6379
-
-    def __init__(self, key):
-        self.key = self.NAME_SPACE + key
-        self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
+    def __init__(self):
+        super().__init__()
+        self.key = self.NAME_SPACE + "dl_queue"
 
     def get_all(self):
         """return all elements in list"""

tubearchivist/home/tasks.py

@@ -8,8 +8,8 @@ Functionality:
 
 import os
 
-import home.apps as startup_apps
 from celery import Celery, shared_task
+from home.apps import StartupCheck
 from home.src.download.queue import PendingList
 from home.src.download.subscriptions import (
     ChannelSubscription,
@@ -24,7 +24,6 @@ from home.src.index.filesystem import (
     reindex_old_documents,
     scan_filesystem,
 )
-from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.config import AppConfig, ScheduleBuilder
 from home.src.ta.helper import UrlListParser
 from home.src.ta.ta_redis import RedisArchivist, RedisQueue
@@ -63,9 +62,11 @@ def update_subscribed():
             missing_from_playlists = playlist_handler.find_missing()
             missing = missing_from_channels + missing_from_playlists
             if missing:
-                pending_handler = PendingList()
-                all_videos_added = pending_handler.add_to_pending(missing)
-                ThumbManager().download_vid(all_videos_added)
+                youtube_ids = [{"type": "video", "url": i} for i in missing]
+                pending_handler = PendingList(youtube_ids=youtube_ids)
+                pending_handler.parse_url_list()
+                pending_handler.add_to_pending()
+
         else:
             print("Did not acquire rescan lock.")
 
@@ -86,7 +87,6 @@ def download_pending():
             downloader = VideoDownloader()
             downloader.add_pending()
             downloader.run_queue()
-            downloader.validate_playlists()
         else:
             print("Did not acquire download lock.")
 
@@ -98,7 +98,7 @@ def download_pending():
 @shared_task
 def download_single(youtube_id):
     """start download single video now"""
-    queue = RedisQueue("dl_queue")
+    queue = RedisQueue()
     queue.add_priority(youtube_id)
     print("Added to queue with priority: " + youtube_id)
     # start queue if needed
@@ -128,19 +128,9 @@ def download_single(youtube_id):
 @shared_task
 def extrac_dl(youtube_ids):
     """parse list passed and add to pending"""
-    pending_handler = PendingList()
-    missing_videos = pending_handler.parse_url_list(youtube_ids)
-    all_videos_added = pending_handler.add_to_pending(missing_videos)
-    missing_playlists = pending_handler.missing_from_playlists
-
-    thumb_handler = ThumbManager()
-    if missing_playlists:
-        new_thumbs = PlaylistSubscription().process_url_str(
-            missing_playlists, subscribed=False
-        )
-        thumb_handler.download_playlist(new_thumbs)
-
-    thumb_handler.download_vid(all_videos_added)
+    pending_handler = PendingList(youtube_ids=youtube_ids)
+    pending_handler.parse_url_list()
+    pending_handler.add_to_pending()
 
 
 @shared_task(name="check_reindex")
@@ -191,7 +181,7 @@ def kill_dl(task_id):
         app.control.revoke(task_id, terminate=True)
 
     _ = RedisArchivist().del_message("dl_queue_id")
-    RedisQueue("dl_queue").clear()
+    RedisQueue().clear()
 
     # clear cache
     cache_dir = os.path.join(CONFIG["application"]["cache_dir"], "download")
@@ -277,55 +267,12 @@ def index_channel_playlists(channel_id):
         "message": f'Scanning channel "{channel.youtube_id}" in progress',
     }
     RedisArchivist().set_message("message:playlistscan", mess_dict)
-    all_playlists = channel.get_all_playlists()
-
-    if not all_playlists:
-        print(f"no playlists found for channel {channel_id}")
-        return
-
-    all_indexed = PendingList().get_all_indexed()
-    all_youtube_ids = [i["youtube_id"] for i in all_indexed]
-
-    for idx, (playlist_id, playlist_title) in enumerate(all_playlists):
-        # notify
-        mess_dict = {
-            "status": "message:playlistscan",
-            "level": "info",
-            "title": "Scanning channel for playlists",
-            "message": f"Progress: {idx + 1}/{len(all_playlists)}",
-        }
-        RedisArchivist().set_message("message:playlistscan", mess_dict)
-        print("add playlist: " + playlist_title)
-
-        playlist = YoutubePlaylist(playlist_id)
-        playlist.all_youtube_ids = all_youtube_ids
-        playlist.build_json()
-
-        if not playlist.json_data:
-            # skip if not available
-            continue
-
-        # don't add if no videos downloaded
-        downloaded = [
-            i
-            for i in playlist.json_data["playlist_entries"]
-            if i["downloaded"]
-        ]
-        if not downloaded:
-            continue
-
-        playlist.upload_to_es()
-        playlist.add_vids_to_playlist()
-
-    if all_playlists:
-        playlist.get_playlist_art()
-
-    return
+    channel.index_channel_playlists()
 
 
 try:
     app.conf.beat_schedule = ScheduleBuilder().build_schedule()
 except KeyError:
     # update path from v0.0.8 to v0.0.9 to load new defaults
-    startup_apps.sync_redis_state()
+    StartupCheck().sync_redis_state()
     app.conf.beat_schedule = ScheduleBuilder().build_schedule()

tubearchivist/home/templates/home/base.html

@@ -132,7 +132,7 @@
     </div>
     <div class="footer">
         <div class="boxed-content">
-            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist v0.1.2 </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
+            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist v0.1.3 </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
         </div>
     </div>
 </body>

tubearchivist/home/templates/home/channel.html

@@ -40,40 +40,40 @@
     <div class="channel-list {{ view_style }}">
         {% if results %}
             {% for channel in results %}
-            <div class="channel-item {{ view_style }}">
-                <div class="channel-banner {{ view_style }}">
-                    <a href="{% url 'channel_id' channel.source.channel_id %}">
-                        <img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
-                    </a>
-                </div>
-                <div class="info-box info-box-2 {{ view_style }}">
-                    <div class="info-box-item">
-                        <div class="round-img">
-                            <a href="{% url 'channel_id' channel.source.channel_id %}">
-                                <img src="/cache/channels/{{ channel.source.channel_id }}_thumb.jpg" alt="channel-thumb">
-                            </a>
-                        </div>
-                        <div>
-                            <h3><a href="{% url 'channel_id' channel.source.channel_id %}">{{ channel.source.channel_name }}</a></h3>
-                            {% if channel.source.channel_subs >= 1000000 %}
-                                <p>Subscribers: {{ channel.source.channel_subs|intword }}</p>
-                            {% else %}
-                                <p>Subscribers: {{ channel.source.channel_subs|intcomma }}</p>
-                            {% endif %}
-                        </div>
+                <div class="channel-item {{ view_style }}">
+                    <div class="channel-banner {{ view_style }}">
+                        <a href="{% url 'channel_id' channel.source.channel_id %}">
+                            <img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
+                        </a>
                     </div>
-                    <div class="info-box-item">
-                        <div>
-                            <p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
-                            {% if channel.source.channel_subscribed %}
-                                <button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
-                            {% else %}
-                                <button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
-                            {% endif %}
+                    <div class="info-box info-box-2 {{ view_style }}">
+                        <div class="info-box-item">
+                            <div class="round-img">
+                                <a href="{% url 'channel_id' channel.source.channel_id %}">
+                                    <img src="/cache/channels/{{ channel.source.channel_id }}_thumb.jpg" alt="channel-thumb">
+                                </a>
+                            </div>
+                            <div>
+                                <h3><a href="{% url 'channel_id' channel.source.channel_id %}">{{ channel.source.channel_name }}</a></h3>
+                                {% if channel.source.channel_subs >= 1000000 %}
+                                    <p>Subscribers: {{ channel.source.channel_subs|intword }}</p>
+                                {% else %}
+                                    <p>Subscribers: {{ channel.source.channel_subs|intcomma }}</p>
+                                {% endif %}
+                            </div>
+                        </div>
+                        <div class="info-box-item">
+                            <div>
+                                <p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
+                                {% if channel.source.channel_subscribed %}
+                                    <button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
+                                {% else %}
+                                    <button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
+                                {% endif %}
+                            </div>
                         </div>
                     </div>
                 </div>
-            </div>
             {% endfor %}
         {% else %}
             <h2>No channels found...</h2>

tubearchivist/home/templates/home/channel_id.html

@@ -6,6 +6,7 @@
     <div class="channel-banner">
         <a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
     </div>
+    <div id="notifications" data="channel_id"></div>
     <div class="info-box info-box-3">
         <div class="info-box-item">
             <div class="round-img">
@@ -52,11 +53,47 @@
                     <p>Total Videos archived: {{ max_hits }}</p>
                     <p>Watched: <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
                 {% endif %}
-                <button title="Search for playlists on YouTube for {{ channel_info.channel_name }}" type="button" id="find-playlists-button" data-id="{{ channel_info.channel_id }}" onclick="findPlaylists(this)">Find Playlists</button>
+                <button {% if channel_info.channel_overwrites %} class="danger-button"{% endif %} onclick="showOverwrite()" title="Overwrite settings for channel {{ channel_info.channel_name }}">Configure</button>
                 <a href="/playlist/?search={{ channel_info.channel_id }}" title="Show all playlists belonging to {{ channel_info.channel_name }}"><button>Show Playlists</button></a>
             </div>
         </div>
     </div>
+    <div id="overwrite-form" class="info-box{% if not channel_info.channel_overwrites %} hidden-overwrite{% endif %}">
+        <div class="info-box-item">
+            <form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/" method="POST">
+                {% csrf_token %}
+                <div class="overwrite-form-item">
+                    <p>Download format: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.download_format %}
+                            {{ channel_info.channel_overwrites.download_format }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.download_format }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Auto delete watched videos after x days: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.autodelete_days %}
+                            {{ channel_info.channel_overwrites.autodelete_days }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.autodelete_days }}<br>
+                </div>
+                <div class="overwrite-form-item">
+                    <p>Index playlists: <span class="settings-current">
+                        {% if channel_info.channel_overwrites.index_playlists %}
+                            {{ channel_info.channel_overwrites.index_playlists }}
+                        {% else %}
+                            False
+                        {% endif %}</span></p>
+                    {{ channel_overwrite_form.index_playlists }}<br>
+                </div>
+                <div class="overwrite-form-item"></div>
+                <button type="submit">Save Channel Overwrites</button>
+            </form>
+        </div>
+    </div>
     {% if channel_info.channel_description %}
         <div class="info-box-item description-box">
             <p>Description: <button onclick="textReveal()" id="text-reveal-button">Show</button></p>
@@ -65,7 +102,6 @@
             </div>
         </div>
     {% endif %}
-    <div id="notifications" data="channel_id"></div>
     <div class="view-controls">
         <div class="toggle">
             <span>Hide watched videos:</span>
@@ -111,9 +147,9 @@
                             <div class="video-thumb">
                                 <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -124,9 +160,9 @@
                     <div class="video-desc {{ view_style }}">
                         <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
                             {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
                             <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
                         </div>

tubearchivist/home/templates/home/home.html

@@ -2,6 +2,46 @@
 {% block content %}
 {% load static %}
 <div class="boxed-content">
+    {% if continue_vids %}
+        <div class="title-bar">
+            <h1>Continue Watching</h1>
+        </div>
+        <div class="video-list {{ view_style }}">
+            {% for video in continue_vids %}
+                <div class="video-item {{ view_style }}">
+                    <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
+                        <div class="video-thumb-wrap {{ view_style }}">
+                            <div class="video-thumb">
+                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                {% if video.source.player.progress %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                {% else %}
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                {% endif %}
+                            </div>
+                            <div class="video-play">
+                                <img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
+                            </div>
+                        </div>
+                    </a>
+                    <div class="video-desc {{ view_style }}">
+                        <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
+                            {% if video.source.player.watched %}
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
+                            {% else %}
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
+                            {% endif %}
+                            <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
+                        </div>
+                        <div>
+                            <a href="{% url 'channel_id' video.source.channel.channel_id %}"><h3>{{ video.source.channel.channel_name }}</h3></a>
+                            <a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
+                        </div>
+                    </div>
+                </div>
+            {% endfor %}
+        </div>
+    {% endif %}
     <div class="title-bar">
         <h1>Recent Videos</h1>
     </div>
@@ -50,9 +90,9 @@
                             <div class="video-thumb">
                                 <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -63,9 +103,9 @@
                     <div class="video-desc {{ view_style }}">
                         <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
                             {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
                             <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
                         </div>

tubearchivist/home/templates/home/playlist_id.html

@@ -49,8 +49,8 @@
         <div class="info-box-item">
             <div>
                 {% if max_hits %}
-                <p>Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}</p>
-                <p>Watched: <button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
+                    <p>Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}</p>
+                    <p>Watched: <button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
                 {% endif %}
             </div>
         </div>
@@ -92,9 +92,9 @@
                             <div class="video-thumb">
                                 <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}
-                                <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
+                                    <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
                                 {% endif %}
                             </div>
                             <div class="video-play">
@@ -105,9 +105,9 @@
                     <div class="video-desc {{ view_style }}">
                         <div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
                             {% if video.source.player.watched %}
-                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                                <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                             {% else %}
-                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                                <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                             {% endif %}
                             <span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
                         </div>

tubearchivist/home/templates/home/video.html

@@ -32,9 +32,9 @@
                 <p>Last refreshed: {{ video.vid_last_refresh }}</p>
                 <p class="video-info-watched">Watched:
                     {% if video.player.watched %}
-                        <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
+                        <img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
                     {% else %}
-                        <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
+                        <img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
                     {% endif %}
                 </p>
                 {% if video.active %}

tubearchivist/home/views.py

@@ -19,6 +19,7 @@ from home.src.frontend.api_calls import PostData
 from home.src.frontend.forms import (
     AddToQueueForm,
     ApplicationSettingsForm,
+    ChannelOverwriteForm,
     CustomAuthForm,
     MultiSearchForm,
     SchedulerSettingsForm,
@@ -27,12 +28,13 @@ from home.src.frontend.forms import (
     UserSettingsForm,
 )
 from home.src.frontend.searching import SearchHandler
+from home.src.index.channel import channel_overwrites
 from home.src.index.generic import Pagination
 from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.config import AppConfig, ScheduleBuilder
 from home.src.ta.helper import UrlListParser
 from home.src.ta.ta_redis import RedisArchivist
-from home.tasks import extrac_dl, subscribe_to
+from home.tasks import extrac_dl, index_channel_playlists, subscribe_to
 from rest_framework.authtoken.models import Token
 
 
@@ -175,14 +177,44 @@ class ArchivistResultsView(ArchivistViewConfig):
         if not results or not self.context["results"]:
             return
 
-        progress = {i["youtube_id"]: i["position"] for i in results}
+        self.context["continue_vids"] = self.get_in_progress(results)
+
+        in_progress = {i["youtube_id"]: i["position"] for i in results}
         for hit in self.context["results"]:
             video = hit["source"]
-            if video["youtube_id"] in progress:
-                played_sec = progress.get(video["youtube_id"])
+            if video["youtube_id"] in in_progress:
+                played_sec = in_progress.get(video["youtube_id"])
                 total = video["player"]["duration"]
+                if not total:
+                    total = played_sec * 2
                 video["player"]["progress"] = 100 * (played_sec / total)
 
+    def get_in_progress(self, results):
+        """get all videos in progress"""
+        ids = [{"match": {"youtube_id": i.get("youtube_id")}} for i in results]
+        data = {
+            "size": self.default_conf["archive"]["page_size"],
+            "query": {"bool": {"should": ids}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        search = SearchHandler(
+            "ta_video/_search", self.default_conf, data=data
+        )
+        videos = search.get_data()
+        if not videos:
+            return False
+
+        for video in videos:
+            youtube_id = video["source"]["youtube_id"]
+            matched = [i for i in results if i["youtube_id"] == youtube_id]
+            played_sec = matched[0]["position"]
+            total = video["source"]["player"]["duration"]
+            if not total:
+                total = matched[0].get("position") * 2
+            video["source"]["player"]["progress"] = 100 * (played_sec / total)
+
+        return videos
+
     def single_lookup(self, es_path):
         """retrieve a single item from url"""
         search = SearchHandler(es_path, config=self.default_conf)
@@ -232,6 +264,13 @@ class HomeView(ArchivistResultsView):
 
     def _update_view_data(self):
         """update view specific data dict"""
+        self.data["sort"].extend(
+            [
+                {"channel.channel_name.keyword": {"order": "asc"}},
+                {"title.keyword": {"order": "asc"}},
+            ]
+        )
+
         if self.context["hide_watched"]:
             self.data["query"] = {"term": {"player.watched": {"value": False}}}
         if self.search_get:
@@ -385,6 +424,7 @@ class ChannelIdView(ArchivistResultsView):
             {
                 "title": "Channel: " + channel_name,
                 "channel_info": channel_info,
+                "channel_overwrite_form": ChannelOverwriteForm,
             }
         )
 
@@ -392,19 +432,34 @@ class ChannelIdView(ArchivistResultsView):
 
     def _update_view_data(self, channel_id):
         """update view specific data dict"""
-        query = {
+        self.data["query"] = {
             "bool": {
                 "must": [
                     {"term": {"channel.channel_id": {"value": channel_id}}}
                 ]
             }
         }
-        self.data["query"] = query
+        self.data["sort"].append({"title.keyword": {"order": "asc"}})
 
         if self.context["hide_watched"]:
             to_append = {"term": {"player.watched": {"value": False}}}
             self.data["query"]["bool"]["must"].append(to_append)
 
+    @staticmethod
+    def post(request, channel_id):
+        """handle post request"""
+        print(f"handle post from {channel_id}")
+        channel_overwrite_form = ChannelOverwriteForm(request.POST)
+        if channel_overwrite_form.is_valid():
+            overwrites = channel_overwrite_form.cleaned_data
+            print(f"{channel_id}: set overwrites {overwrites}")
+            channel_overwrites(channel_id, overwrites=overwrites)
+            if overwrites.get("index_playlists") == "1":
+                index_channel_playlists.delay(channel_id)
+
+        sleep(1)
+        return redirect("channel_id", channel_id, permanent=True)
+
 
 class ChannelView(ArchivistResultsView):
     """resolves to /channel/

tubearchivist/requirements.txt

@@ -1,12 +1,12 @@
 beautifulsoup4==4.10.0
 celery==5.2.3
-Django==4.0.2
+Django==4.0.3
 django-cors-headers==3.11.0
 djangorestframework==3.13.1
 Pillow==9.0.1
-redis==4.1.4
+redis==4.2.0
 requests==2.27.1
 ryd-client==0.0.3
 uWSGI==2.0.20
 whitenoise==6.0.0
-yt_dlp==2022.2.4
+yt_dlp==2022.3.8.2

tubearchivist/static/css/style.css

@@ -370,16 +370,18 @@ button:hover {
 }
 
 /* video list */
-.video-list.grid {
+.video-list {
     display: grid;
-    grid-template-columns: 1fr 1fr 1fr;
     grid-gap: 1rem;
+    margin-top: 1rem;
+}
+
+.video-list.grid {
+    grid-template-columns: 1fr 1fr 1fr;
 }
 
 .video-list.list {
-    display: grid;
     grid-template-columns: unset;
-    grid-gap: 1rem;
 }
 
 .video-item {
@@ -482,8 +484,7 @@ button:hover {
     align-items: center;
 }
 
-.unseen-icon,
-.seen-icon,
+.watch-button,
 .close-button {
     cursor: pointer;
     filter: var(--img-filter);
@@ -565,6 +566,28 @@ button:hover {
     margin-bottom: 10px;
 }
 
+.overwrite-form {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    width: 100%;
+}
+
+.overwrite-form button {
+    width: 200px;
+}
+
+.overwrite-form-item {
+    margin-bottom: 1rem;
+}
+
+.overwrite-form-item input {
+    width: 90%;
+}
+
+.hidden-overwrite {
+    display: none;
+}
+
 /* login */
 .login-page {
     display: flex;
@@ -1061,7 +1084,10 @@ button:hover {
     .channel-list.grid,
     .video-item.list,
     .playlist-list.list,
-    .playlist-list.grid {
+    .playlist-list.grid,
+    .info-box-2,
+    .info-box-3,
+    .overwrite-form {
         grid-template-columns: 1fr;
     }
     .playlist-item.list {
@@ -1100,10 +1126,6 @@ button:hover {
     .sort select {
         margin: unset;
     }
-    .info-box-2,
-    .info-box-3 {
-        grid-template-columns: 1fr;
-    }
     .description-box {
         display: block;
     }

tubearchivist/static/progress.js

@@ -7,7 +7,7 @@ checkMessages()
 
 // page map to notification status
 const messageTypes = {
-    "download": ["message:download", "message:add", "message:rescan"],
+    "download": ["message:download", "message:add", "message:rescan", "message:playlistscan"],
     "channel": ["message:subchannel"],
     "channel_id": ["message:playlistscan"],
     "playlist": ["message:subplaylist"],

tubearchivist/static/script.js

@@ -8,21 +8,62 @@ function sortChange(sortValue) {
     }, 500);
 }
 
-function isWatched(youtube_id) {
-    postVideoProgress(youtube_id, 0); // Reset video progress on watched;
-    removeProgressBar(youtube_id);
-    var payload = JSON.stringify({'watched': youtube_id});
-    sendPost(payload);
-    var seenIcon = document.createElement('img');
-    seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
-    seenIcon.setAttribute('alt', 'seen-icon');
-    seenIcon.setAttribute('id', youtube_id);
-    seenIcon.setAttribute('title', "Mark as unwatched");
-    seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
-    seenIcon.classList = 'seen-icon';
-    document.getElementById(youtube_id).replaceWith(seenIcon);
+// Updates video watch status when passed a video id and it's current state (ex if the video was unwatched but you want to mark it as watched you will pass "unwatched")
+function updateVideoWatchStatus(input1, videoCurrentWatchStatus) {
+    if (videoCurrentWatchStatus) {
+        videoId = input1;
+    } else if (input1.getAttribute("data-id")) {
+        videoId = input1.getAttribute("data-id");
+        videoCurrentWatchStatus = input1.getAttribute("data-status");
+    }
+
+    postVideoProgress(videoId, 0); // Reset video progress on watched/unwatched;
+    removeProgressBar(videoId);
+
+    if (videoCurrentWatchStatus == "watched") {
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "unwatched");
+        var payload = JSON.stringify({'un_watched': videoId});
+        sendPost(payload);
+    } else if (videoCurrentWatchStatus == "unwatched") {
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "watched");
+        var payload = JSON.stringify({'watched': videoId});
+        sendPost(payload);
+    }
+
+    var watchButtons = document.getElementsByClassName("watch-button");
+    for (let i = 0; i < watchButtons.length; i++) {
+        if (watchButtons[i].getAttribute("data-id") == videoId) {
+            watchButtons[i].outerHTML = watchStatusIndicator;
+        }
+    }
 }
 
+// Creates a watch status indicator when passed a video id and the videos watch status
+function createWatchStatusIndicator(videoId, videoWatchStatus) {
+    if (videoWatchStatus == "watched") {
+        var seen = "seen";
+        var title = "Mark as unwatched";
+    } else if (videoWatchStatus == "unwatched") {
+        var seen = "unseen";
+        var title = "Mark as watched";
+    }
+    var watchStatusIndicator = `<img src="/static/img/icon-${seen}.svg" alt="${seen}-icon" data-id="${videoId}" data-status="${videoWatchStatus}" onclick="updateVideoWatchStatus(this)" class="watch-button" title="${title}">`;
+    return watchStatusIndicator;
+}
+
+// function isWatched(youtube_id) {
+//     var payload = JSON.stringify({'watched': youtube_id});
+//     sendPost(payload);
+//     var seenIcon = document.createElement('img');
+//     seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
+//     seenIcon.setAttribute('alt', 'seen-icon');
+//     seenIcon.setAttribute('id', youtube_id);
+//     seenIcon.setAttribute('title', "Mark as unwatched");
+//     seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
+//     seenIcon.classList = 'seen-icon';
+//     document.getElementById(youtube_id).replaceWith(seenIcon);
+// }
+
 // Removes the progress bar when passed a video id
 function removeProgressBar(videoId) {
     setProgressBar(videoId, 0, 1);
@@ -39,19 +80,19 @@ function isWatchedButton(button) {
     }, 1000);
 }
 
-function isUnwatched(youtube_id) {
-    postVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
-    var payload = JSON.stringify({'un_watched': youtube_id});
-    sendPost(payload);
-    var unseenIcon = document.createElement('img');
-    unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
-    unseenIcon.setAttribute('alt', 'unseen-icon');
-    unseenIcon.setAttribute('id', youtube_id);
-    unseenIcon.setAttribute('title', "Mark as watched");
-    unseenIcon.setAttribute('onclick', "isWatched(this.id)");
-    unseenIcon.classList = 'unseen-icon';
-    document.getElementById(youtube_id).replaceWith(unseenIcon);
-}
+// function isUnwatched(youtube_id) {
+//     postVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
+//     var payload = JSON.stringify({'un_watched': youtube_id});
+//     sendPost(payload);
+//     var unseenIcon = document.createElement('img');
+//     unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
+//     unseenIcon.setAttribute('alt', 'unseen-icon');
+//     unseenIcon.setAttribute('id', youtube_id);
+//     unseenIcon.setAttribute('title', "Mark as watched");
+//     unseenIcon.setAttribute('onclick', "isWatched(this.id)");
+//     unseenIcon.classList = 'unseen-icon';
+//     document.getElementById(youtube_id).replaceWith(unseenIcon);
+// }
 
 function unsubscribe(id_unsub) {
     var payload = JSON.stringify({'unsubscribe': id_unsub});
@@ -151,7 +192,6 @@ function addSingle(button) {
 function deleteQueue(button) {
     var to_delete = button.getAttribute('data-id');
     var payload = JSON.stringify({'deleteQueue': to_delete});
-    console.log(payload);
     sendPost(payload);
     setTimeout(function(){
         location.reload();
@@ -228,19 +268,6 @@ function fsRescan() {
     toReplace.appendChild(message);
 }
 
-function findPlaylists(button) {
-    var channel_id = button.getAttribute("data-id");
-    var payload = JSON.stringify({'find-playlists': channel_id});
-    sendPost(payload);
-    // clear button
-    var message = document.createElement('p');
-    message.innerText = 'Scraping for playlists in progress';
-    document.getElementById("find-playlists-button").replaceWith(message);
-    setTimeout(function(){
-        checkMessages();
-    }, 500);
-}
-
 function resetToken() {
     var payload = JSON.stringify({'reset-token': true});
     sendPost(payload);
@@ -287,7 +314,6 @@ function deletePlaylist(button) {
             "playlist-action": playlist_action
         }
     });
-    console.log(payload);
     sendPost(payload);
     setTimeout(function(){
         window.location.replace("/playlist/");
@@ -327,7 +353,7 @@ function createPlayer(button) {
     var channelName = videoData.data.channel.channel_name;
 
     removePlayer();
-    document.getElementById(videoId).outerHTML = ''; // Remove watch indicator from video info
+    // document.getElementById(videoId).outerHTML = ''; // Remove watch indicator from video info
 
     // If cast integration is enabled create cast button
     var castButton = '';
@@ -337,12 +363,11 @@ function createPlayer(button) {
 
     // Watched indicator
     if (videoData.data.player.watched) {
-        var playerState = "seen";
-        var watchedFunction = "Unwatched";
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "watched");
     } else {
-        var playerState = "unseen";
-        var watchedFunction = "Watched";  
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "unwatched");
     }
+    
 
     var playerStats = `<div class="thumb-icon player-stats"><img src="/static/img/icon-eye.svg" alt="views icon"><span>${videoViews}</span>`;
     if (videoData.data.stats.like_count) {
@@ -360,7 +385,7 @@ function createPlayer(button) {
         ${videoTag}
         <div class="player-title boxed-content">
             <img class="close-button" src="/static/img/icon-close.svg" alt="close-icon" data="${videoId}" onclick="removePlayer()" title="Close player">
-            <img src="/static/img/icon-${playerState}.svg" alt="${playerState}-icon" id="${videoId}" onclick="is${watchedFunction}(this.id)" class="${playerState}-icon" title="Mark as ${watchedFunction}">
+            ${watchStatusIndicator}
             ${castButton}
             ${playerStats}
             <div class="player-channel-playlist">
@@ -391,7 +416,11 @@ function createVideoTag(videoData, videoProgress) {
     var videoSubtitles = videoData.data.subtitles; // Array of subtitles
     if (typeof(videoSubtitles) != 'undefined' && videoData.config.downloads.subtitle) {
         for (var i = 0; i < videoSubtitles.length; i++) {
-            subtitles += `<track label="${videoSubtitles[i].name}" kind="subtitles" srclang="${videoSubtitles[i].lang}" src="${videoSubtitles[i].media_url}">`;
+            let label = videoSubtitles[i].name;
+            if (videoSubtitles[i].source == "auto") {
+                label += " - auto";
+            }
+            subtitles += `<track label="${label}" kind="subtitles" srclang="${videoSubtitles[i].lang}" src="${videoSubtitles[i].media_url}">`;
         }
     }
 
@@ -444,8 +473,12 @@ function getVideoPlayerDuration() {
 function getVideoPlayerWatchStatus() {
     var videoId = getVideoPlayerVideoId();
     var watched = false;
-    if(document.getElementById(videoId).className != "unseen-icon") {
-        watched = true;
+
+    var watchButtons = document.getElementsByClassName("watch-button");
+    for (let i = 0; i < watchButtons.length; i++) {
+        if (watchButtons[i].getAttribute("data-id") == videoId && watchButtons[i].getAttribute("data-status") == "watched") {
+            watched = true;
+        }
     }
     return watched;
 }
@@ -459,7 +492,7 @@ function onVideoProgress() {
         postVideoProgress(videoId, currentTime);
         if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
             if (watchedThreshold(currentTime, duration)) {
-                isWatched(videoId);
+                updateVideoWatchStatus(videoId, "unwatched");
             }
         }
     }
@@ -469,7 +502,7 @@ function onVideoProgress() {
 function onVideoEnded() {
     var videoId = getVideoPlayerVideoId();
     if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
-        isWatched(videoId);
+        updateVideoWatchStatus(videoId, "unwatched");
     }
 }
 
@@ -606,11 +639,21 @@ function removePlayer() {
 
 // Sets the progress bar when passed a video id, video progress and video duration
 function setProgressBar(videoId, currentTime, duration) {
-    progressBar = document.getElementById("progress-" + videoId);
-    progressBarWidth = (currentTime / duration) * 100 + "%";
-    if (progressBar) {
-        progressBar.style.width = progressBarWidth;
+    var progressBarWidth = (currentTime / duration) * 100 + "%";
+    var progressBars = document.getElementsByClassName("video-progress-bar");
+    for (let i = 0; i < progressBars.length; i++) {
+        if (progressBars[i].id == "progress-" + videoId) {
+            if (!getVideoPlayerWatchStatus()) {
+                progressBars[i].style.width = progressBarWidth;
+            } else {
+                progressBars[i].style.width = "0%";
+            }
+        }
     }
+
+    // progressBar = document.getElementById("progress-" + videoId);
+    
+    
 }
 
 // multi search form
@@ -679,9 +722,9 @@ function createVideo(video, viewStyle) {
     const videoPublished = video.published;
     const videoDuration = video.player.duration_str;
     if (video.player.watched) {
-        var playerState = "seen";
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "watched");
     } else {
-        var playerState = "unseen";
+        var watchStatusIndicator = createWatchStatusIndicator(videoId, "unwatched");
     };
     const channelId = video.channel.channel_id;
     const channelName = video.channel.channel_name;
@@ -699,7 +742,7 @@ function createVideo(video, viewStyle) {
     </a>
     <div class="video-desc ${viewStyle}">
         <div class="video-desc-player" id="video-info-${videoId}">
-                <img src="/static/img/icon-${playerState}.svg" alt="${playerState}-icon" id="${videoId}" onclick="isWatched(this.id)" class="${playerState}-icon">
+                ${watchStatusIndicator}
             <span>${videoPublished} | ${videoDuration}</span>
         </div>
         <div>
@@ -840,6 +883,15 @@ function showForm() {
     animate('animate-icon', 'pulse-img');
 }
 
+function showOverwrite() {
+    var overwriteDiv = document.getElementById("overwrite-form");
+    if (overwriteDiv.classList.contains("hidden-overwrite")) {
+        overwriteDiv.classList.remove("hidden-overwrite");
+    } else {
+        overwriteDiv.classList.add("hidden-overwrite")
+    }
+}
+
 function animate(elementId, animationClass) {
     var toAnimate = document.getElementById(elementId);
     if (toAnimate.className !== animationClass) {

tubearchivist/testing.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-# install debug and testing tools into slim container
-
-apt update && apt install -y vim htop bmon net-tools iputils-ping procps
-
-pip install ipython
-
-##
-exit 0