various improvements, #build

Changed: - Fixed partial refresh on UI toggle - Fixed channel extraction - Fixed showIgnored filter on downloads agg - Better debug messages in dev mode - Hide 0 comment linke count - better cookie validation, use caching - cache channel search result - mobile fixes - sleep on download queue
2026-08-04 21:29:36 +00:00 · 2025-01-28 18:10:48 +07:00
parent 05ff9df049 4d23b7dce4
commit baf0c568a8
17 changed files with 160 additions and 108 deletions
--- a/backend/appsettings/views.py
+++ b/backend/appsettings/views.py
@@ -5,6 +5,7 @@ from appsettings.src.config import AppConfig
 from appsettings.src.snapshot import ElasticSnapshot
 from common.src.ta_redis import RedisArchivist
 from common.views_base import AdminOnly, ApiBaseView
+from django.conf import settings
 from download.src.yt_dlp_base import CookieHandler, POTokenHandler
 from rest_framework.authtoken.models import Token
 from rest_framework.response import Response
@@ -220,13 +221,15 @@ class CookieView(ApiBaseView):
            print(message)
            return Response({"message": message}, status=400)

-        print(f"cookie preview:\n\n{cookie[:300]}")
+        if settings.DEBUG:
+            print(f"[cookie] preview:\n\n{cookie[:300]}")
+
        handler = CookieHandler(config)
        handler.set_cookie(cookie)
        validated = handler.validate()
        if not validated:
            handler.revoke()
-            print("cookie import failed, not valid")
+            print("[cookie]: import failed, not valid")
            status = 400
        else:
            status = 200
--- a/backend/channel/src/index.py
+++ b/backend/channel/src/index.py
@@ -42,7 +42,7 @@ class YoutubeChannel(YouTubeItem):
        if not self.youtube_meta and fallback:
            self._video_fallback(fallback)
        else:
-            if not self.json_data:
+            if not self.youtube_meta:
                message = f"{self.youtube_id}: Failed to get metadata"
                raise ValueError(message)

--- a/backend/common/src/urlparser.py
+++ b/backend/common/src/urlparser.py
@@ -6,15 +6,20 @@ Functionality:

 from urllib.parse import parse_qs, urlparse

+from common.src.ta_redis import RedisArchivist
 from download.src.yt_dlp_base import YtWrap
 from video.src.constants import VideoTypeEnum


 class Parser:
-    """take a multi line string and detect valid youtube ids"""
+    """
+    take a multi line string and detect valid youtube ids
+    channel handle lookup is cached, can be disabled for unittests
+    """

-    def __init__(self, url_str):
+    def __init__(self, url_str, use_cache=True):
        self.url_list = [i.strip() for i in url_str.split()]
+        self.use_cache = use_cache

    def parse(self):
        """parse the list"""
@@ -106,9 +111,13 @@ class Parser:

        return {"type": item_type, "url": id_str}

-    @staticmethod
-    def _extract_channel_name(url):
-        """find channel id from channel name with yt-dlp help"""
+    def _extract_channel_name(self, url):
+        """find channel id from channel name with yt-dlp help, cache result"""
+        if self.use_cache:
+            cached = self._get_cached(url)
+            if cached:
+                return cached
+
        obs_request = {
            "check_formats": None,
            "skip_download": True,
@@ -121,6 +130,9 @@ class Parser:

        channel_id = url_info.get("channel_id", False)
        if channel_id:
+            if self.use_cache:
+                self._set_cache(url, channel_id)
+
            return channel_id

        url = url_info.get("url", False)
@@ -133,6 +145,42 @@ class Parser:
        print(f"failed to extract channel id from {url}")
        raise ValueError

+    @staticmethod
+    def _get_cached(url) -> str | None:
+        """get cached channel ID, if available"""
+        path = urlparse(url).path.lstrip("/")
+        if not path.startswith("@"):
+            return None
+
+        handle = path.split("/")[0]
+        if not handle:
+            return None
+
+        cache_key = f"channel:handlesearch:{handle.lower()}"
+        cached = RedisArchivist().get_message_dict(cache_key)
+        if cached:
+            return cached["channel_id"]
+
+        return None
+
+    @staticmethod
+    def _set_cache(url, channel_id) -> None:
+        """set cache"""
+        path = urlparse(url).path.lstrip("/")
+        if not path.startswith("@"):
+            return
+
+        handle = path.split("/")[0]
+        if not handle:
+            return
+
+        cache_key = f"channel:handlesearch:{handle.lower()}"
+        message = {
+            "channel_id": channel_id,
+            "handle": handle,
+        }
+        RedisArchivist().set_message(cache_key, message, expire=3600 * 24 * 7)
+
    def _detect_vid_type(self, path):
        """try to match enum from path, needs to be serializable"""
        last = path.strip("/").split("/")[-1]
--- a/backend/common/tests/test_src/test_urlparser.py
+++ b/backend/common/tests/test_src/test_urlparser.py
@@ -110,7 +110,7 @@ PASSTING_TESTS.extend(PERSONAL_PLAYLISTS_TEST_CASES)
@pytest.mark.parametrize("url_str, expected_result", PASSTING_TESTS)
 def test_passing_parse(url_str, expected_result):
    """test parser"""
-    parser = Parser(url_str)
+    parser = Parser(url_str, use_cache=False)
    parsed = parser.parse()
    assert parsed == expected_result

@@ -127,7 +127,7 @@ INVALID_IDS_ERRORS = [
 def test_invalid_ids(invalid_value):
    """test for invalid IDs"""
    with pytest.raises(ValueError, match="not a valid id_str"):
-        parser = Parser(invalid_value)
+        parser = Parser(invalid_value, use_cache=False)
        parser.parse()


@@ -140,6 +140,6 @@ INVALID_DOMAINS = [
@pytest.mark.parametrize("invalid_value", INVALID_DOMAINS)
 def test_invalid_domains(invalid_value):
    """raise error on none YT domains"""
-    parser = Parser(invalid_value)
+    parser = Parser(invalid_value, use_cache=False)
    with pytest.raises(ValueError, match="invalid domain"):
        parser.parse()
--- a/backend/download/src/yt_dlp_base.py
+++ b/backend/download/src/yt_dlp_base.py
@@ -40,6 +40,7 @@ class YtWrap:
            self._add_potoken()

        if getattr(settings, "DEBUG", False):
+            del self.obs["quiet"]
            print(self.obs)

    def _add_cookie(self):
@@ -133,7 +134,7 @@ class CookieHandler:
        RedisArchivist().set_message("cookie", cookie, save=True)
        AppConfig().update_config({"downloads.cookie_import": True})
        self.config["downloads"]["cookie_import"] = True
-        print("cookie: activated and stored in Redis")
+        print("[cookie]: activated and stored in Redis")

    @staticmethod
    def revoke():
@@ -141,11 +142,16 @@ class CookieHandler:
        RedisArchivist().del_message("cookie")
        RedisArchivist().del_message("cookie:valid")
        AppConfig().update_config({"downloads.cookie_import": False})
-        print("cookie: revoked")
+        print("[cookie]: revoked")

    def validate(self):
        """validate cookie using the liked videos playlist"""
-        print("validating cookie")
+        validation = RedisArchivist().get_message_dict("cookie:valid")
+        if validation:
+            print("[cookie]: used cached cookie validation")
+            return True
+
+        print("[cookie] validating cookie")
        obs_request = {
            "skip_download": True,
            "extract_flat": True,
@@ -169,8 +175,9 @@ class CookieHandler:
            RedisArchivist().set_message(
                "message:download", mess_dict, expire=4
            )
-            print("cookie validation failed, exiting...")
+            print("[cookie]: validation failed, exiting...")

+        print(f"[cookie]: validation success: {response}")
        return response

    @staticmethod
@@ -182,7 +189,7 @@ class CookieHandler:
            "validated": int(now.timestamp()),
            "validated_str": now.strftime("%Y-%m-%d %H:%M"),
        }
-        RedisArchivist().set_message("cookie:valid", message)
+        RedisArchivist().set_message("cookie:valid", message, expire=3600)


 class POTokenHandler:
--- a/backend/download/src/yt_dlp_handler.py
+++ b/backend/download/src/yt_dlp_handler.py
@@ -64,6 +64,9 @@ class VideoDownloader(DownloaderBase):
                self._reset_auto()
                break

+            if downloaded > 0:
+                rand_sleep(self.config)
+
            youtube_id = video_data["youtube_id"]
            channel_id = video_data["channel_id"]
            print(f"{youtube_id}: Downloading video")
--- a/deploy.sh
+++ b/deploy.sh
@@ -18,20 +18,24 @@ set -e

 function sync_blackhole {

-    host="blackhole.local"
+    host="blackhole.lan"
    
    rsync -a --progress --delete-after \
        --exclude ".git" \
        --exclude ".gitignore" \
        --exclude "**/cache" \
        --exclude "**/__pycache__/" \
+        --exclude "**/.pytest_cache/" \
+        --exclude "**/static/" \
+        --exclude "**/node_modules/" \
+        --exclude "**/.env" \
        --exclude ".venv" \
        --exclude "db.sqlite3" \
        --exclude ".mypy_cache" \
        . -e ssh "$host":tubearchivist

-    ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist'
-    ssh "$host" 'docker compose up -d'
+    ssh "$host" 'docker build -t bbilly1/tubearchivist:unstable tubearchivist'
+    ssh "$host" 'docker compose -f docker/docker-compose.yml up -d'

 }

--- a/frontend/src/api/loader/loadDownloadAggs.ts
+++ b/frontend/src/api/loader/loadDownloadAggs.ts
@@ -14,8 +14,12 @@ export type DownloadAggsType = {
  };
 };

-const loadDownloadAggs = async (): Promise<DownloadAggsType> => {
-  return APIClient('/api/download/aggs/');
+const loadDownloadAggs = async (showIgnored: boolean): Promise<DownloadAggsType> => {
+  const searchParams = new URLSearchParams();
+  searchParams.append('filter', showIgnored ? 'ignore' : 'pending');
+  return APIClient(
+    `/api/download/aggs/${searchParams.toString() ? `?${searchParams.toString()}` : ''}`,
+  );
 };

 export default loadDownloadAggs;
--- a/frontend/src/components/CommentBox.tsx
+++ b/frontend/src/components/CommentBox.tsx
@@ -57,12 +57,15 @@ const CommentBox = ({ comment }: CommentBoxProps) => {
      <div className="comment-meta">
        <span>{formatDate(comment.comment_timestamp * 1000)}</span>

-        <span className="space-carrot">|</span>
-
-        <span className="thumb-icon">
-          <img src={iconThumb} />{' '}
-          {formatNumbers(comment.comment_likecount, { notation: 'compact' })}
-        </span>
+        {comment.comment_likecount > 0 && (
+          <>
+            <span className="space-carrot">|</span>
+            <span className="thumb-icon">
+              <img src={iconThumb} />{' '}
+              {formatNumbers(comment.comment_likecount, { notation: 'compact' })}
+            </span>
+          </>
+        )}

        {comment.comment_is_favorited && (
          <>
--- a/frontend/src/components/EmbeddableVideoPlayer.tsx
+++ b/frontend/src/components/EmbeddableVideoPlayer.tsx
@@ -1,4 +1,4 @@
-import { useEffect, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import { VideoResponseType } from '../pages/Video';
 import VideoPlayer from './VideoPlayer';
 import loadVideoById from '../api/loader/loadVideoById';
@@ -24,6 +24,8 @@ type EmbeddableVideoPlayerProps = {
 };

 const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {
+  const inlinePlayerRef = useRef<HTMLDivElement>(null);
+
  const [, setSearchParams] = useSearchParams();

  const [refresh, setRefresh] = useState(false);
@@ -63,8 +65,7 @@ const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {

      setVideoResponse(videoResponse);

-      const inlinePlayer = document.getElementById('inline-player');
-      inlinePlayer?.scrollIntoView();
+      inlinePlayerRef.current?.scrollIntoView();

      setRefresh(false);
      setLoading(false);
@@ -91,7 +92,7 @@ const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {

  return (
    <>
-      <div id="inline-player" className="player-wrapper">
+      <div ref={inlinePlayerRef} className="player-wrapper">
        <div className="video-player">
          {!loading && (
            <VideoPlayer
--- a/frontend/src/components/Filterbar.tsx
+++ b/frontend/src/components/Filterbar.tsx
@@ -11,10 +11,10 @@ import { ViewStyles } from '../configuration/constants/ViewStyle';
 type FilterbarProps = {
  hideToggleText: string;
  viewStyleName: string;
-  setRefresh?: (status: boolean) => void;
+  showSort?: boolean;
 };

-const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps) => {
+const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: FilterbarProps) => {
  const { userConfig, setPartialConfig } = useUserConfigStore();
  const [showHidden, setShowHidden] = useState(false);
  const isGridView = userConfig.config.view_style_home === ViewStyles.grid;
@@ -29,7 +29,6 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
            type="checkbox"
            checked={userConfig.config.hide_watched}
            onChange={() => {
-              setRefresh?.(true);
              setPartialConfig({ hide_watched: !userConfig.config.hide_watched });
            }}
          />
@@ -46,7 +45,7 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
        </div>
      </div>

-      {showHidden && (
+      {showHidden && showSort && (
        <div className="sort">
          <div id="form">
            <span>Sort by:</span>
@@ -55,7 +54,6 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
              id="sort"
              value={userConfig.config.sort_by}
              onChange={event => {
-                setRefresh?.(true);
                setPartialConfig({ sort_by: event.target.value as SortByType });
              }}
            >
@@ -71,7 +69,6 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
              id="sort-order"
              value={userConfig.config.sort_order}
              onChange={event => {
-                setRefresh?.(true);
                setPartialConfig({ sort_order: event.target.value as SortOrderType });
              }}
            >
@@ -81,9 +78,8 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
          </div>
        </div>
      )}
-
      <div className="view-icons">
-        {setShowHidden && (
+        {setShowHidden && showSort && (
          <img
            src={iconSort}
            alt="sort-icon"
--- a/frontend/src/pages/ChannelVideo.tsx
+++ b/frontend/src/pages/ChannelVideo.tsx
@@ -152,11 +152,7 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => {
        </div>
      </div>
      <div className={`boxed-content ${gridView}`}>
-        <Filterbar
-          hideToggleText={'Hide watched videos:'}
-          viewStyleName={ViewStyleNames.home}
-          setRefresh={setRefresh}
-        />
+        <Filterbar hideToggleText={'Hide watched videos:'} viewStyleName={ViewStyleNames.home} />
      </div>
      {showEmbeddedVideo && <EmbeddableVideoPlayer videoId={videoId} />}
      <div className={`boxed-content ${gridView}`}>
--- a/frontend/src/pages/Channels.tsx
+++ b/frontend/src/pages/Channels.tsx
@@ -65,16 +65,14 @@ const Channels = () => {

  useEffect(() => {
    (async () => {
-      if (refresh) {
-        const channelListResponse = await loadChannelList(
-          currentPage,
-          userConfig.config.show_subed_only,
-        );
+      const channelListResponse = await loadChannelList(
+        currentPage,
+        userConfig.config.show_subed_only,
+      );

-        setChannelListResponse(channelListResponse);
-        setShowNotification(false);
-        setRefresh(false);
-      }
+      setChannelListResponse(channelListResponse);
+      setShowNotification(false);
+      setRefresh(false);
    })();
  }, [refresh, userConfig.config.show_subed_only, currentPage, pagination?.current_page]);

--- a/frontend/src/pages/Download.tsx
+++ b/frontend/src/pages/Download.tsx
@@ -100,11 +100,11 @@ const Download = () => {

  useEffect(() => {
    (async () => {
-      const downloadAggs = await loadDownloadAggs();
+      const downloadAggs = await loadDownloadAggs(showIgnored);

      setDownloadAggsResponse(downloadAggs);
    })();
-  }, [lastVideoCount]);
+  }, [lastVideoCount, showIgnored]);

  useEffect(() => {
    setRefresh(true);
--- a/frontend/src/pages/Home.tsx
+++ b/frontend/src/pages/Home.tsx
@@ -129,30 +129,24 @@ const Home = () => {

  useEffect(() => {
    (async () => {
-      if (
-        refreshVideoList ||
-        pagination?.current_page === undefined ||
-        currentPage !== pagination?.current_page
-      ) {
-        const videos = await loadVideoListByFilter({
-          page: currentPage,
-          watch: userMeConfig.hide_watched ? 'unwatched' : undefined,
-          sort: userMeConfig.sort_by,
-          order: userMeConfig.sort_order,
-        });
+      const videos = await loadVideoListByFilter({
+        page: currentPage,
+        watch: userMeConfig.hide_watched ? 'unwatched' : undefined,
+        sort: userMeConfig.sort_by,
+        order: userMeConfig.sort_order,
+      });

-        try {
-          const continueVideoResponse = await loadVideoListByFilter({ watch: 'continue' });
-          setContinueVideoResponse(continueVideoResponse);
-        } catch (error) {
-          console.log('Server error on continue vids?');
-          console.error(error);
-        }
-
-        setVideoReponse(videos);
-
-        setRefreshVideoList(false);
+      try {
+        const continueVideoResponse = await loadVideoListByFilter({ watch: 'continue' });
+        setContinueVideoResponse(continueVideoResponse);
+      } catch (error) {
+        console.log('Server error on continue vids?');
+        console.error(error);
      }
+
+      setVideoReponse(videos);
+
+      setRefreshVideoList(false);
    })();
  }, [
    refreshVideoList,
@@ -190,11 +184,7 @@ const Home = () => {
          <h1>Recent Videos</h1>
        </div>

-        <Filterbar
-          hideToggleText="Hide watched:"
-          viewStyleName={ViewStyleNames.home}
-          setRefresh={setRefreshVideoList}
-        />
+        <Filterbar hideToggleText="Hide watched:" viewStyleName={ViewStyleNames.home} />
      </div>

      <div className={`boxed-content ${gridView}`}>
--- a/frontend/src/pages/Playlist.tsx
+++ b/frontend/src/pages/Playlist.tsx
@@ -90,34 +90,27 @@ const Playlist = () => {

  useEffect(() => {
    (async () => {
-      if (
-        refresh ||
-        pagination?.current_page === undefined ||
-        currentPage !== pagination?.current_page
-      ) {
-        const playlist = await loadPlaylistById(playlistId);
-        const video = await loadVideoListByFilter({
-          playlist: playlistId,
-          page: currentPage,
-          watch: hideWatched ? 'unwatched' : undefined,
-          sort: 'downloaded', // downloaded or published? or playlist sort order?
-        });
+      const playlist = await loadPlaylistById(playlistId);
+      const video = await loadVideoListByFilter({
+        playlist: playlistId,
+        page: currentPage,
+        watch: hideWatched ? 'unwatched' : undefined,
+        sort: 'downloaded', // downloaded or published? or playlist sort order?
+      });

-        const isCustomPlaylist = playlist?.data?.playlist_type === 'custom';
-        if (!isCustomPlaylist) {
-          const channel = await loadChannelById(playlist.data.playlist_channel_id);
+      const isCustomPlaylist = playlist?.data?.playlist_type === 'custom';
+      if (!isCustomPlaylist) {
+        const channel = await loadChannelById(playlist.data.playlist_channel_id);

-          setChannelResponse(channel);
-        }
-
-        setPlaylistResponse(playlist);
-        setVideoResponse(video);
-        setRefresh(false);
+        setChannelResponse(channel);
      }
+
+      setPlaylistResponse(playlist);
+      setVideoResponse(video);
+      setRefresh(false);
    })();
-    // Do not add hideWatched this will not work as expected!
    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [playlistId, refresh, currentPage, pagination?.current_page]);
+  }, [playlistId, userConfig.config.hide_watched, refresh, currentPage, pagination?.current_page]);

  if (!playlistId || !playlist) {
    return `Playlist ${playlistId} not found!`;
@@ -320,7 +313,7 @@ const Playlist = () => {
        <Filterbar
          hideToggleText="Hide watched videos:"
          viewStyleName={ViewStyleNames.playlist}
-          setRefresh={setRefresh}
+          showSort={false}
        />
      </div>

--- a/frontend/src/style.css
+++ b/frontend/src/style.css
@@ -1258,6 +1258,10 @@ video:-webkit-full-screen {
  .video-player video {
    width: 90%;
  }
+  .info-box-3,
+  .info-box-4 {
+    grid-template-columns: 1fr;
+  }
 }

 /* phone */
@@ -1275,11 +1279,13 @@ video:-webkit-full-screen {
  .playlist-list.list,
  .playlist-list.grid,
  .info-box-2,
-  .info-box-3,
-  .info-box-4,
  .overwrite-form {
    grid-template-columns: 1fr;
  }
+  .settings-box-wrapper {
+    grid-template-columns: 1fr;
+    padding: 1.2rem 0;
+  }
  .playlist-item.list {
    display: block;
  }