add timeout for sponsorblock api requests, handle 503

Changed:
- added download filter dropdown
- fix for UnidentifiedImageError thumbnail error
- fix for _update_by_query channel missing watched_date
- fix for chrome compatibility text reveal

Dockerfile

@@ -3,7 +3,7 @@
 
 
 # First stage to build python wheel
-FROM python:3.10.7-slim-bullseye AS builder
+FROM python:3.10.8-slim-bullseye AS builder
 ARG TARGETPLATFORM
 
 RUN apt-get update
@@ -14,7 +14,7 @@ COPY ./tubearchivist/requirements.txt /requirements.txt
 RUN pip install --user -r requirements.txt
 
 # build final image
-FROM python:3.10.7-slim-bullseye as tubearchivist
+FROM python:3.10.8-slim-bullseye as tubearchivist
 
 ARG TARGETPLATFORM
 ARG INSTALL_DEBUG

README.md

@@ -63,6 +63,8 @@ Once your YouTube video collection grows, it becomes hard to search and find a s
 ## Installing and updating
 Take a look at the example `docker-compose.yml` file provided. Use the *latest* or the named semantic version tag. The *unstable* tag is for intermediate testing and as the name implies, is **unstable** and not be used on your main installation but in a [testing environment](CONTRIBUTING.md).  
 
+For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation.  
+
 Tube Archivist depends on three main components split up into separate docker containers:  
 
 ### Tube Archivist

deploy.sh

@@ -18,11 +18,7 @@ set -e
 
 function sync_blackhole {
 
-    # docker commands need sudo, only build amd64
     host="blackhole.local"
-
-    read -sp 'Password: ' remote_pw
-    export PASS=$remote_pw
     
     rsync -a --progress --delete-after \
         --exclude ".git" \
@@ -32,8 +28,8 @@ function sync_blackhole {
         --exclude "db.sqlite3" \
         . -e ssh "$host":tubearchivist
 
-    echo "$PASS" | ssh "$host" 'sudo -S docker buildx build --platform linux/amd64 -t bbilly1/tubearchivist:latest tubearchivist --load 2>/dev/null'
+    ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist'
-    echo "$PASS" | ssh "$host" 'sudo -S docker compose up -d 2>/dev/null'
+    ssh "$host" 'docker compose up -d'
 
 }
 

docker-compose.yml

@@ -34,7 +34,7 @@ services:
     depends_on:
       - archivist-es
   archivist-es:
-    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.3.3
+    image: bbilly1/tubearchivist-es         # only for amd64, or use official es 8.4.3
     container_name: archivist-es
     restart: unless-stopped
     environment:

docs/Channels.md

@@ -24,7 +24,7 @@ Each channel will get a dedicated channel detail page accessible at `/channel/<c
 
 Additionally there is a *Channel Playlist* page, accessible at `/channel/<channel-id>/playlist/` to show all indexed playlists from this channel. 
 
-On the *Channel About* page you can see additional metadata.
+On the *Channel About* page, accessible at `/channel/<channel-id>/about/`, you can see additional metadata.
 - The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata additionally this will also delete playlists metadata belonging to that channel.
 
 The channel customize form gives options to change settings on a per channel basis. Any configurations here will overwrite your configurations from the [settings](Settings) page.
@@ -32,3 +32,5 @@ The channel customize form gives options to change settings on a per channel bas
 - **Auto Delete**: Automatically delete watched videos from this channel after selected days.
 - **Index Playlists**: Automatically add all Playlists with at least a video downloaded to your index. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to.
 - **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. Customize per channel: You can *disable* or *enable* SponsorBlock for certain channels only to overwrite the behavior set on the [Settings](settings) page. Selecting *unset* will remove the overwrite and your setting will fall back to the default on the settings page.
+
+If you have any videos pending in the download queue, a *Downloads* link will show, bringing you directly to the [downloads](Downloads) page, filtering the list by the selected channel.

docs/Downloads.md

@@ -29,6 +29,8 @@ The **Add to Download Queue** icon <img src="assets/icon-add.png?raw=true" alt="
 ## The Download Queue
 Below the three buttons you find the download queue. New items will get added at the bottom of the queue, the next video to download once you click on **Start Download** will be the first in the list.
 
+You can filter the download queue with the **filter** dropdown box, the filter will show once you have more than one channel in the download queue. Select the channel to filter by name, the number in parentheses indicates how many videos you have pending from this channel. Reset the filter by selecting *all* from the dropdown. This will generate links for the top 30 channels with pending videos.
+
 Every video in the download queue has two buttons:
 - **Ignore**: This will remove that video from the download queue and this video will not get added again, even when you **Rescan Subscriptions**.
 - **Download now**: This will give priority to this video. If the download process is already running, the prioritized video will get downloaded as soon as the current video is finished. If there is no download process running, this will start downloading this single video and stop after that.  

tubearchivist/api/README.md

@@ -61,6 +61,7 @@ The list views return a paginate object with the following keys:
 - prev_pages: *array of ints* of previous pages, if available
 - current_page: *int* current page from query
 - max_hits: *bool* if max of 10k results is reached
+- params: *str* additional url encoded query parameters
 - last_page: *int* of last page link
 - next_pages: *array of ints* of next pages
 - total_hits: *int* total results
@@ -169,6 +170,7 @@ GET /api/download/
 
 Parameter:
 - filter: pending, ignore
+- channel: channel-id
 
 ### Add list of videos to download queue
 POST /api/download/

tubearchivist/api/views.py

@@ -53,9 +53,7 @@ class ApiBaseView(APIView):
 
     def initiate_pagination(self, request):
         """set initial pagination values"""
-        user_id = request.user.id
+        self.pagination_handler = Pagination(request)
-        page_get = int(request.GET.get("page", 0))
-        self.pagination_handler = Pagination(page_get, user_id)
         self.data.update(
             {
                 "size": self.pagination_handler.pagination["page_size"],
@@ -368,13 +366,23 @@ class DownloadApiListView(ApiBaseView):
         """get request"""
         query_filter = request.GET.get("filter", False)
         self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
+
+        must_list = []
         if query_filter:
             if query_filter not in self.valid_filter:
                 message = f"invalid url query filder: {query_filter}"
                 print(message)
                 return Response({"message": message}, status=400)
 
-            self.data["query"] = {"term": {"status": {"value": query_filter}}}
+            must_list.append({"term": {"status": {"value": query_filter}}})
+
+        filter_channel = request.GET.get("channel", False)
+        if filter_channel:
+            must_list.append(
+                {"term": {"channel_id": {"value": filter_channel}}}
+            )
+
+        self.data["query"] = {"bool": {"must": must_list}}
 
         self.get_document_list(request)
         return Response(self.response)

tubearchivist/config/settings.py

@@ -207,4 +207,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.2.2"
+TA_VERSION = "v0.2.3"

tubearchivist/home/src/download/queue.py

@@ -246,7 +246,7 @@ class PendingList(PendingIndex):
             print(f"{youtube_id}: skipping premium video, id not matching")
             return False
         # stop if video is streaming live now
-        if vid["is_live"]:
+        if vid["live_status"] in ["is_upcoming", "is_live"]:
             return False
 
         return self._parse_youtube_details(vid)

tubearchivist/home/src/download/subscriptions.py

@@ -63,6 +63,7 @@ class ChannelSubscription:
 
         for idx, channel in enumerate(all_channels):
             channel_id = channel["channel_id"]
+            print(f"{channel_id}: find missing videos.")
             last_videos = self.get_last_youtube_videos(channel_id)
 
             if last_videos:

tubearchivist/home/src/download/thumbnails.py

@@ -14,7 +14,7 @@ from home.src.download import queue  # partial import
 from home.src.es.connect import IndexPaginate
 from home.src.ta.config import AppConfig
 from mutagen.mp4 import MP4, MP4Cover
-from PIL import Image, ImageFile, ImageFilter
+from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
 
 ImageFile.LOAD_TRUNCATED_IMAGES = True
 
@@ -42,7 +42,12 @@ class ThumbManagerBase:
             try:
                 response = requests.get(url, stream=True, timeout=5)
                 if response.ok:
-                    return Image.open(response.raw)
+                    try:
+                        return Image.open(response.raw)
+                    except UnidentifiedImageError:
+                        print(f"failed to open thumbnail: {url}")
+                        return self.get_fallback()
+
                 if response.status_code == 404:
                     return self.get_fallback()
 

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -175,6 +175,14 @@ class VideoDownloader:
             if not success:
                 continue
 
+            mess_dict = {
+                "status": self.MSG,
+                "level": "info",
+                "title": "Indexing....",
+                "message": "Add video metadata to index.",
+            }
+            RedisArchivist().set_message(self.MSG, mess_dict, expire=60)
+
             vid_dict = index_new_video(
                 youtube_id, video_overwrites=self.video_overwrites
             )
@@ -187,12 +195,17 @@ class VideoDownloader:
             }
             RedisArchivist().set_message(self.MSG, mess_dict)
 
+            if queue.has_item():
+                message = "Continue with next video."
+            else:
+                message = "Download queue is finished."
+
             self.move_to_archive(vid_dict)
             mess_dict = {
                 "status": self.MSG,
                 "level": "info",
                 "title": "Completed",
-                "message": "",
+                "message": message,
             }
             RedisArchivist().set_message(self.MSG, mess_dict, expire=10)
             self._delete_from_pending(youtube_id)

tubearchivist/home/src/frontend/watched.py

@@ -57,6 +57,14 @@ class WatchState:
             print(response)
             raise ValueError("failed to mark video as watched")
 
+    def _get_source(self):
+        """build source line for update_by_query script"""
+        source = [
+            "ctx._source.player['watched'] = true",
+            f"ctx._source.player['watched_date'] = {self.stamp}",
+        ]
+        return "; ".join(source)
+
     def mark_channel_watched(self):
         """change watched status of every video in channel"""
         path = "ta_video/_update_by_query"
@@ -67,7 +75,7 @@ class WatchState:
         data = {
             "query": {"bool": {"must": must_list}},
             "script": {
-                "source": "ctx._source.player['watched'] = true",
+                "source": self._get_source(),
                 "lang": "painless",
             },
         }
@@ -87,7 +95,7 @@ class WatchState:
         data = {
             "query": {"bool": {"must": must_list}},
             "script": {
-                "source": "ctx._source.player['watched'] = true",
+                "source": self._get_source(),
                 "lang": "painless",
             },
         }

tubearchivist/home/src/index/generic.py

@@ -73,16 +73,25 @@ class Pagination:
     figure out the pagination based on page size and total_hits
     """
 
-    def __init__(self, page_get, user_id, search_get=False):
+    def __init__(self, request):
-        self.user_id = user_id
+        self.request = request
+        self.page_get = False
+        self.params = False
+        self.get_params()
         self.page_size = self.get_page_size()
-        self.page_get = page_get
-        self.search_get = search_get
         self.pagination = self.first_guess()
 
+    def get_params(self):
+        """process url query parameters"""
+        query_dict = self.request.GET.copy()
+        self.page_get = int(query_dict.get("page", 0))
+
+        _ = query_dict.pop("page", False)
+        self.params = query_dict.urlencode()
+
     def get_page_size(self):
         """get default or user modified page_size"""
-        key = f"{self.user_id}:page_size"
+        key = f"{self.request.user.id}:page_size"
         page_size = RedisArchivist().get_message(key)["status"]
         if not page_size:
             config = AppConfig().config
@@ -108,9 +117,9 @@ class Pagination:
             "prev_pages": prev_pages,
             "current_page": page_get,
             "max_hits": False,
+            "params": self.params,
         }
-        if self.search_get:
+
-            pagination.update({"search_get": self.search_get})
         return pagination
 
     def validate(self, total_hits):

tubearchivist/home/src/index/playlist.py

@@ -45,13 +45,19 @@ class YoutubePlaylist(YouTubeItem):
 
     def process_youtube_meta(self):
         """extract relevant fields from youtube"""
+        try:
+            playlist_thumbnail = self.youtube_meta["thumbnails"][-1]["url"]
+        except IndexError:
+            print(f"{self.youtube_id}: thumbnail extraction failed")
+            playlist_thumbnail = False
+
         self.json_data = {
             "playlist_id": self.youtube_id,
             "playlist_active": True,
             "playlist_name": self.youtube_meta["title"],
             "playlist_channel": self.youtube_meta["channel"],
             "playlist_channel_id": self.youtube_meta["channel_id"],
-            "playlist_thumbnail": self.youtube_meta["thumbnails"][-1]["url"],
+            "playlist_thumbnail": playlist_thumbnail,
             "playlist_description": self.youtube_meta["description"] or False,
             "playlist_last_refresh": int(datetime.now().strftime("%s")),
         }

tubearchivist/home/src/index/video.py

@@ -48,9 +48,17 @@ class SponsorBlock:
         url = f"{self.API}/skipSegments?videoID={youtube_id}"
         headers = {"User-Agent": self.user_agent}
         print(f"{youtube_id}: get sponsorblock timestamps")
-        response = requests.get(url, headers=headers)
+        try:
+            response = requests.get(url, headers=headers, timeout=10)
+        except requests.ReadTimeout:
+            print(f"{youtube_id}: sponsorblock API timeout")
+            return False
+
         if not response.ok:
-            print(f"{youtube_id}: sponsorblock failed: {response.text}")
+            print(f"{youtube_id}: sponsorblock failed: {response.status_code}")
+            if response.status_code == 503:
+                return False
+
             sponsor_dict = {
                 "last_refresh": self.last_refresh,
                 "is_enabled": True,

tubearchivist/home/src/ta/ta_redis.py

@@ -143,3 +143,8 @@ class RedisQueue(RedisBase):
     def trim(self, size):
         """trim the queue based on settings amount"""
         self.conn.execute_command("LTRIM", self.key, 0, size)
+
+    def has_item(self):
+        """check if queue as at least one pending item"""
+        result = self.conn.execute_command("LRANGE", self.key, 0, 0)
+        return bool(result)

tubearchivist/home/templates/home/base.html

@@ -79,16 +79,16 @@
             <div class="pagination">
                 {% if pagination %}
                     {% if pagination.current_page > 1 %}
-                        {% if pagination.search_get %}
+                        {% if pagination.params %}
-                        <a class="pagination-item" href="{{ request.path }}?search={{ pagination.search_get }}">First</a>
+                        <a class="pagination-item" href="{{ request.path }}?{{ pagination.params }}">First</a>
                         {% else %}
                         <a class="pagination-item" href="{{ request.path }}">First</a>
                         {% endif %}
                     {% endif %}
                     {% if pagination.prev_pages %}
                         {% for page in pagination.prev_pages %}
-                            {% if pagination.search_get %}
+                            {% if pagination.params %}
-                            <a class="pagination-item" href="?page={{ page }}&search={{ pagination.search_get }}">{{ page }}</a>
+                            <a class="pagination-item" href="?page={{ page }}&{{ pagination.params }}">{{ page }}</a>
                             {% else %}
                             <a class="pagination-item" href="?page={{ page }}">{{ page }}</a>
                             {% endif %}
@@ -100,16 +100,16 @@
                     {% if pagination.next_pages %}
                         <span> ></span>
                         {% for page in pagination.next_pages %}
-                            {% if pagination.search_get %}
+                            {% if pagination.params %}
-                            <a class="pagination-item" href="?page={{ page }}&search={{ pagination.search_get }}">{{ page }}</a>
+                            <a class="pagination-item" href="?page={{ page }}&{{ pagination.params }}">{{ page }}</a>
                             {% else %}
                             <a class="pagination-item" href="?page={{ page }}">{{ page }}</a>
                             {% endif %}
                         {% endfor %}
                     {% endif %}
                     {% if pagination.last_page > 0 %}
-                        {% if pagination.search_get %}
+                        {% if pagination.params %}
-                        <a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">
+                        <a class="pagination-item" href="?page={{ pagination.last_page }}&{{ pagination.params }}">
                             {% if pagination.max_hits %}
                                 Max ({{ pagination.last_page }})
                             {% else %}

tubearchivist/home/templates/home/channel_id.html

@@ -10,6 +10,9 @@
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
         <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
         <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+        {% if has_pending %}
+            <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
+        {% endif %}
     </div>
     <div id="notifications" data="channel_id"></div>
     <div class="info-box info-box-2">

tubearchivist/home/templates/home/channel_id_about.html

@@ -10,6 +10,9 @@
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
         <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
         <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+        {% if has_pending %}
+            <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
+        {% endif %}
     </div>
     <div class="info-box info-box-3">
         <div class="info-box-item">

tubearchivist/home/templates/home/channel_id_playlist.html

@@ -10,6 +10,9 @@
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
         <a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
         <a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
+        {% if has_pending %}
+            <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
+        {% endif %}
     </div>
     <div class="view-controls">
         <div class="toggle">

tubearchivist/home/templates/home/downloads.html

@@ -3,7 +3,7 @@
 {% block content %}
 <div class="boxed-content">
     <div class="title-bar">
-        <h1>Downloads</h1>
+        <h1>Downloads {% if channel_filter_id %} for {{ channel_filter_name }}{% endif %}</h1>
     </div>
     <div id="notifications" data="download"></div>
     <div id="downloadControl"></div>
@@ -41,6 +41,15 @@
             </div>
         </div>
         <div class="view-icons">
+            {% if channel_agg_list|length > 1 %}
+                <span>Filter:</span>
+                <select name="channel_filter" id="channel_filter" onchange="channelFilterDownload(this.value)">
+                    <option value="all" {% if not channel_filter_id %}selected{% endif %}>all</option>
+                    {% for channel in channel_agg_list %}
+                        <option {% if channel_filter_id == channel.id %}selected{% endif %} value="{{ channel.id }}">{{ channel.name }} ({{channel.count}})</option>
+                    {% endfor %}
+                </select>
+            {% endif %}
             {% if view_style == "grid" %}
                 <div class="grid-count">
                     {% if grid_items < 7 %}
@@ -55,7 +64,7 @@
             <img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="downloads" data-value="list" alt="list view">
         </div>
     </div>
-    <h3>Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %}</h3>
+    <h3>Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %} {% if channel_filter_id %} - from channel <i>{{ channel_filter_name }}</i>{% endif %}</h3>
 </div>
 <div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
     <div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">

tubearchivist/home/views.py

@@ -31,7 +31,7 @@ from home.src.frontend.forms import (
     UserSettingsForm,
 )
 from home.src.frontend.searching import SearchHandler
-from home.src.index.channel import channel_overwrites
+from home.src.index.channel import YoutubeChannel, channel_overwrites
 from home.src.index.generic import Pagination
 from home.src.index.playlist import YoutubePlaylist
 from home.src.ta.config import AppConfig, ScheduleBuilder
@@ -237,14 +237,10 @@ class ArchivistResultsView(ArchivistViewConfig):
 
     def initiate_vars(self, request):
         """search in es for vidoe hits"""
-        page_get = int(request.GET.get("page", 0))
         self.user_id = request.user.id
         self.config_builder(self.user_id)
         self.search_get = request.GET.get("search", False)
-        search_encoded = self._url_encode(self.search_get)
+        self.pagination_handler = Pagination(request)
-        self.pagination_handler = Pagination(
-            page_get=page_get, user_id=self.user_id, search_get=search_encoded
-        )
         self.sort_by = self._sort_by_overwrite()
         self._initial_data()
 
@@ -362,29 +358,80 @@ class DownloadView(ArchivistResultsView):
     def get(self, request):
         """handle get request"""
         self.initiate_vars(request)
-        self._update_view_data()
+        self._update_view_data(request)
         self.find_results()
         self.context.update(
             {
                 "title": "Downloads",
                 "add_form": AddToQueueForm(),
+                "channel_agg_list": self._get_channel_agg(),
             }
         )
         return render(request, "home/downloads.html", self.context)
 
-    def _update_view_data(self):
+    def _update_view_data(self, request):
         """update downloads view specific data dict"""
         if self.context["show_ignored_only"]:
             filter_view = "ignore"
         else:
             filter_view = "pending"
+
+        must_list = [{"term": {"status": {"value": filter_view}}}]
+
+        channel_filter = request.GET.get("channel", False)
+        if channel_filter:
+            must_list.append(
+                {"term": {"channel_id": {"value": channel_filter}}}
+            )
+
+            channel = YoutubeChannel(channel_filter)
+            channel.get_from_es()
+            self.context.update(
+                {
+                    "channel_filter_id": channel_filter,
+                    "channel_filter_name": channel.json_data["channel_name"],
+                }
+            )
+
         self.data.update(
             {
-                "query": {"term": {"status": {"value": filter_view}}},
+                "query": {"bool": {"must": must_list}},
                 "sort": [{"timestamp": {"order": "asc"}}],
             }
         )
 
+    def _get_channel_agg(self):
+        """get pending channel with count"""
+        data = {
+            "size": 0,
+            "query": {"term": {"status": {"value": "pending"}}},
+            "aggs": {
+                "channel_downloads": {
+                    "multi_terms": {
+                        "size": 30,
+                        "terms": [
+                            {"field": "channel_name.keyword"},
+                            {"field": "channel_id"},
+                        ],
+                        "order": {"_count": "desc"},
+                    }
+                }
+            },
+        }
+        response, _ = ElasticWrap(self.es_search).get(data=data)
+        buckets = response["aggregations"]["channel_downloads"]["buckets"]
+
+        buckets_sorted = []
+        for i in buckets:
+            bucket = {
+                "name": i["key"][0],
+                "id": i["key"][1],
+                "count": i["doc_count"],
+            }
+            buckets_sorted.append(bucket)
+
+        return buckets_sorted
+
     @staticmethod
     def post(request):
         """handle post requests"""
@@ -414,7 +461,37 @@ class DownloadView(ArchivistResultsView):
         return redirect("downloads", permanent=True)
 
 
-class ChannelIdView(ArchivistResultsView):
+class ChannelIdBaseView(ArchivistResultsView):
+    """base class for all channel-id views"""
+
+    def get_channel_meta(self, channel_id):
+        """get metadata for channel"""
+        path = f"ta_channel/_doc/{channel_id}"
+        response, _ = ElasticWrap(path).get()
+        channel_info = SearchProcess(response).process()
+
+        return channel_info
+
+    def channel_has_pending(self, channel_id):
+        """check if channel has pending videos in queue"""
+        path = "ta_download/_search"
+        data = {
+            "size": 1,
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {"status": {"value": "pending"}}},
+                        {"term": {"channel_id": {"value": channel_id}}},
+                    ]
+                }
+            },
+        }
+        response, _ = ElasticWrap(path).get(data=data)
+
+        self.context.update({"has_pending": bool(response["hits"]["hits"])})
+
+
+class ChannelIdView(ChannelIdBaseView):
     """resolves to /channel/<channel-id>/
     display single channel page from channel_id
     """
@@ -428,6 +505,7 @@ class ChannelIdView(ArchivistResultsView):
         self._update_view_data(channel_id)
         self.find_results()
         self.match_progress()
+        self.channel_has_pending(channel_id)
 
         if self.context["results"]:
             channel_info = self.context["results"][0]["source"]["channel"]
@@ -478,7 +556,7 @@ class ChannelIdView(ArchivistResultsView):
         return redirect("channel_id", channel_id, permanent=True)
 
 
-class ChannelIdAboutView(ArchivistResultsView):
+class ChannelIdAboutView(ChannelIdBaseView):
     """resolves to /channel/<channel-id>/about/
     show metadata, handle per channel conf
     """
@@ -488,6 +566,7 @@ class ChannelIdAboutView(ArchivistResultsView):
     def get(self, request, channel_id):
         """handle get request"""
         self.initiate_vars(request)
+        self.channel_has_pending(channel_id)
 
         path = f"ta_channel/_doc/{channel_id}"
         response, _ = ElasticWrap(path).get()
@@ -521,7 +600,7 @@ class ChannelIdAboutView(ArchivistResultsView):
         return redirect("channel_id_about", channel_id, permanent=True)
 
 
-class ChannelIdPlaylistView(ArchivistResultsView):
+class ChannelIdPlaylistView(ChannelIdBaseView):
     """resolves to /channel/<channel-id>/playlist/
     show all playlists of channel
     """
@@ -534,8 +613,9 @@ class ChannelIdPlaylistView(ArchivistResultsView):
         self.initiate_vars(request)
         self._update_view_data(channel_id)
         self.find_results()
+        self.channel_has_pending(channel_id)
 
-        channel_info = self._get_channel_meta(channel_id)
+        channel_info = self.get_channel_meta(channel_id)
         channel_name = channel_info["channel_name"]
         self.context.update(
             {
@@ -556,14 +636,6 @@ class ChannelIdPlaylistView(ArchivistResultsView):
 
         self.data["query"] = {"bool": {"must": must_list}}
 
-    def _get_channel_meta(self, channel_id):
-        """get metadata for channel"""
-        path = f"ta_channel/_doc/{channel_id}"
-        response, _ = ElasticWrap(path).get()
-        channel_info = SearchProcess(response).process()
-
-        return channel_info
-
 
 class ChannelView(ArchivistResultsView):
     """resolves to /channel/

tubearchivist/requirements.txt

@@ -1,13 +1,13 @@
 beautifulsoup4==4.11.1
 celery==5.2.7
-Django==4.0.6
+Django==4.1.2
 django-auth-ldap==4.1.0
 django-cors-headers==3.13.0
-djangorestframework==3.13.1
+djangorestframework==3.14.0
 Pillow==9.2.0
 redis==4.3.4
 requests==2.28.1
 ryd-client==0.0.6
 uWSGI==2.0.20
 whitenoise==6.2.0
-yt_dlp==2022.9.1
+yt_dlp==2022.10.4

tubearchivist/static/css/style.css

@@ -344,6 +344,7 @@ button:hover {
 .grid-count {
     display: flex;
     justify-content: end;
+    align-items: center;
 }
 
 .view-icons img {

tubearchivist/static/script.js

@@ -140,7 +140,7 @@ function toggleCheckbox(checkbox) {
     var payload = JSON.stringify(payloadDict);
     sendPost(payload);
     setTimeout(function(){
-        var currPage = window.location.pathname;
+        var currPage = window.location.pathname + window.location.search;
         window.location.replace(currPage);
         return false;
     }, 500);
@@ -1102,12 +1102,12 @@ function textReveal() {
 function textExpand() {
     var textBox = document.getElementById("text-expand");
     var button = document.getElementById("text-expand-button");
-    var textBoxLineClamp = textBox.style["-webkit-line-clamp"];
+    var style = window.getComputedStyle(textBox)
-    if (textBoxLineClamp === "none") {
+    if (style.webkitLineClamp === "none") {
         textBox.style["-webkit-line-clamp"] = "4";
         button.innerText = "Show more";
     } else {
-        textBox.style["-webkit-line-clamp"] = "none";
+        textBox.style["-webkit-line-clamp"] = "unset";
         button.innerText = "Show less";
     }
 }
@@ -1119,8 +1119,9 @@ function textExpandButtonVisibilityUpdate() {
     if (!textBox || !button)
         return;
 
-    var textBoxLineClamp = textBox.style["-webkit-line-clamp"];
+    var styles = window.getComputedStyle(textBox);
-    if (textBoxLineClamp === "none")
+    var textBoxLineClamp = styles.webkitLineClamp;
+    if (textBoxLineClamp === "unset")
         return; // text box is in revealed state
 
     if (textBox.offsetHeight < textBox.scrollHeight
@@ -1147,6 +1148,14 @@ function showForm() {
     animate('animate-icon', 'pulse-img');
 }
 
+function channelFilterDownload(value) {
+    if (value === "all") {
+        window.location = "/downloads/";
+    } else {
+        window.location.search = "?channel=" + value;
+    }
+}
+
 function showOverwrite() {
     var overwriteDiv = document.getElementById("overwrite-form");
     if (overwriteDiv.classList.contains("hidden-overwrite")) {