various improvements, #build

Changed:
- Fixed partial refresh on UI toggle
- Fixed channel extraction
- Fixed showIgnored filter on downloads agg
- Better debug messages in dev mode
- Hide 0 comment linke count
- better cookie validation, use caching
- cache channel search result
- mobile fixes
- sleep on download queue

backend/appsettings/views.py

@@ -5,6 +5,7 @@ from appsettings.src.config import AppConfig
 from appsettings.src.snapshot import ElasticSnapshot
 from common.src.ta_redis import RedisArchivist
 from common.views_base import AdminOnly, ApiBaseView
+from django.conf import settings
 from download.src.yt_dlp_base import CookieHandler, POTokenHandler
 from rest_framework.authtoken.models import Token
 from rest_framework.response import Response
@@ -220,13 +221,15 @@ class CookieView(ApiBaseView):
             print(message)
             return Response({"message": message}, status=400)
 
-        print(f"cookie preview:\n\n{cookie[:300]}")
+        if settings.DEBUG:
+            print(f"[cookie] preview:\n\n{cookie[:300]}")
+
         handler = CookieHandler(config)
         handler.set_cookie(cookie)
         validated = handler.validate()
         if not validated:
             handler.revoke()
-            print("cookie import failed, not valid")
+            print("[cookie]: import failed, not valid")
             status = 400
         else:
             status = 200

backend/channel/src/index.py

@@ -42,7 +42,7 @@ class YoutubeChannel(YouTubeItem):
         if not self.youtube_meta and fallback:
             self._video_fallback(fallback)
         else:
-            if not self.json_data:
+            if not self.youtube_meta:
                 message = f"{self.youtube_id}: Failed to get metadata"
                 raise ValueError(message)
 

backend/common/src/urlparser.py

@@ -6,15 +6,20 @@ Functionality:
 
 from urllib.parse import parse_qs, urlparse
 
+from common.src.ta_redis import RedisArchivist
 from download.src.yt_dlp_base import YtWrap
 from video.src.constants import VideoTypeEnum
 
 
 class Parser:
-    """take a multi line string and detect valid youtube ids"""
+    """
+    take a multi line string and detect valid youtube ids
+    channel handle lookup is cached, can be disabled for unittests
+    """
 
-    def __init__(self, url_str):
+    def __init__(self, url_str, use_cache=True):
         self.url_list = [i.strip() for i in url_str.split()]
+        self.use_cache = use_cache
 
     def parse(self):
         """parse the list"""
@@ -106,9 +111,13 @@ class Parser:
 
         return {"type": item_type, "url": id_str}
 
-    @staticmethod
+    def _extract_channel_name(self, url):
-    def _extract_channel_name(url):
+        """find channel id from channel name with yt-dlp help, cache result"""
-        """find channel id from channel name with yt-dlp help"""
+        if self.use_cache:
+            cached = self._get_cached(url)
+            if cached:
+                return cached
+
         obs_request = {
             "check_formats": None,
             "skip_download": True,
@@ -121,6 +130,9 @@ class Parser:
 
         channel_id = url_info.get("channel_id", False)
         if channel_id:
+            if self.use_cache:
+                self._set_cache(url, channel_id)
+
             return channel_id
 
         url = url_info.get("url", False)
@@ -133,6 +145,42 @@ class Parser:
         print(f"failed to extract channel id from {url}")
         raise ValueError
 
+    @staticmethod
+    def _get_cached(url) -> str | None:
+        """get cached channel ID, if available"""
+        path = urlparse(url).path.lstrip("/")
+        if not path.startswith("@"):
+            return None
+
+        handle = path.split("/")[0]
+        if not handle:
+            return None
+
+        cache_key = f"channel:handlesearch:{handle.lower()}"
+        cached = RedisArchivist().get_message_dict(cache_key)
+        if cached:
+            return cached["channel_id"]
+
+        return None
+
+    @staticmethod
+    def _set_cache(url, channel_id) -> None:
+        """set cache"""
+        path = urlparse(url).path.lstrip("/")
+        if not path.startswith("@"):
+            return
+
+        handle = path.split("/")[0]
+        if not handle:
+            return
+
+        cache_key = f"channel:handlesearch:{handle.lower()}"
+        message = {
+            "channel_id": channel_id,
+            "handle": handle,
+        }
+        RedisArchivist().set_message(cache_key, message, expire=3600 * 24 * 7)
+
     def _detect_vid_type(self, path):
         """try to match enum from path, needs to be serializable"""
         last = path.strip("/").split("/")[-1]

backend/common/tests/test_src/test_urlparser.py

@@ -110,7 +110,7 @@ PASSTING_TESTS.extend(PERSONAL_PLAYLISTS_TEST_CASES)
 @pytest.mark.parametrize("url_str, expected_result", PASSTING_TESTS)
 def test_passing_parse(url_str, expected_result):
     """test parser"""
-    parser = Parser(url_str)
+    parser = Parser(url_str, use_cache=False)
     parsed = parser.parse()
     assert parsed == expected_result
 
@@ -127,7 +127,7 @@ INVALID_IDS_ERRORS = [
 def test_invalid_ids(invalid_value):
     """test for invalid IDs"""
     with pytest.raises(ValueError, match="not a valid id_str"):
-        parser = Parser(invalid_value)
+        parser = Parser(invalid_value, use_cache=False)
         parser.parse()
 
 
@@ -140,6 +140,6 @@ INVALID_DOMAINS = [
 @pytest.mark.parametrize("invalid_value", INVALID_DOMAINS)
 def test_invalid_domains(invalid_value):
     """raise error on none YT domains"""
-    parser = Parser(invalid_value)
+    parser = Parser(invalid_value, use_cache=False)
     with pytest.raises(ValueError, match="invalid domain"):
         parser.parse()

backend/download/src/yt_dlp_base.py

@@ -40,6 +40,7 @@ class YtWrap:
             self._add_potoken()
 
         if getattr(settings, "DEBUG", False):
+            del self.obs["quiet"]
             print(self.obs)
 
     def _add_cookie(self):
@@ -133,7 +134,7 @@ class CookieHandler:
         RedisArchivist().set_message("cookie", cookie, save=True)
         AppConfig().update_config({"downloads.cookie_import": True})
         self.config["downloads"]["cookie_import"] = True
-        print("cookie: activated and stored in Redis")
+        print("[cookie]: activated and stored in Redis")
 
     @staticmethod
     def revoke():
@@ -141,11 +142,16 @@ class CookieHandler:
         RedisArchivist().del_message("cookie")
         RedisArchivist().del_message("cookie:valid")
         AppConfig().update_config({"downloads.cookie_import": False})
-        print("cookie: revoked")
+        print("[cookie]: revoked")
 
     def validate(self):
         """validate cookie using the liked videos playlist"""
-        print("validating cookie")
+        validation = RedisArchivist().get_message_dict("cookie:valid")
+        if validation:
+            print("[cookie]: used cached cookie validation")
+            return True
+
+        print("[cookie] validating cookie")
         obs_request = {
             "skip_download": True,
             "extract_flat": True,
@@ -169,8 +175,9 @@ class CookieHandler:
             RedisArchivist().set_message(
                 "message:download", mess_dict, expire=4
             )
-            print("cookie validation failed, exiting...")
+            print("[cookie]: validation failed, exiting...")
 
+        print(f"[cookie]: validation success: {response}")
         return response
 
     @staticmethod
@@ -182,7 +189,7 @@ class CookieHandler:
             "validated": int(now.timestamp()),
             "validated_str": now.strftime("%Y-%m-%d %H:%M"),
         }
-        RedisArchivist().set_message("cookie:valid", message)
+        RedisArchivist().set_message("cookie:valid", message, expire=3600)
 
 
 class POTokenHandler:

backend/download/src/yt_dlp_handler.py

@@ -64,6 +64,9 @@ class VideoDownloader(DownloaderBase):
                 self._reset_auto()
                 break
 
+            if downloaded > 0:
+                rand_sleep(self.config)
+
             youtube_id = video_data["youtube_id"]
             channel_id = video_data["channel_id"]
             print(f"{youtube_id}: Downloading video")

deploy.sh

@@ -18,20 +18,24 @@ set -e
 
 function sync_blackhole {
 
-    host="blackhole.local"
+    host="blackhole.lan"
     
     rsync -a --progress --delete-after \
         --exclude ".git" \
         --exclude ".gitignore" \
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
+        --exclude "**/.pytest_cache/" \
+        --exclude "**/static/" \
+        --exclude "**/node_modules/" \
+        --exclude "**/.env" \
         --exclude ".venv" \
         --exclude "db.sqlite3" \
         --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
 
-    ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist'
+    ssh "$host" 'docker build -t bbilly1/tubearchivist:unstable tubearchivist'
-    ssh "$host" 'docker compose up -d'
+    ssh "$host" 'docker compose -f docker/docker-compose.yml up -d'
 
 }
 

frontend/src/api/loader/loadDownloadAggs.ts

@@ -14,8 +14,12 @@ export type DownloadAggsType = {
   };
 };
 
-const loadDownloadAggs = async (): Promise<DownloadAggsType> => {
+const loadDownloadAggs = async (showIgnored: boolean): Promise<DownloadAggsType> => {
-  return APIClient('/api/download/aggs/');
+  const searchParams = new URLSearchParams();
+  searchParams.append('filter', showIgnored ? 'ignore' : 'pending');
+  return APIClient(
+    `/api/download/aggs/${searchParams.toString() ? `?${searchParams.toString()}` : ''}`,
+  );
 };
 
 export default loadDownloadAggs;

frontend/src/components/CommentBox.tsx

@@ -57,12 +57,15 @@ const CommentBox = ({ comment }: CommentBoxProps) => {
       <div className="comment-meta">
         <span>{formatDate(comment.comment_timestamp * 1000)}</span>
 
-        <span className="space-carrot">|</span>
+        {comment.comment_likecount > 0 && (
-
+          <>
-        <span className="thumb-icon">
+            <span className="space-carrot">|</span>
-          <img src={iconThumb} />{' '}
+            <span className="thumb-icon">
-          {formatNumbers(comment.comment_likecount, { notation: 'compact' })}
+              <img src={iconThumb} />{' '}
-        </span>
+              {formatNumbers(comment.comment_likecount, { notation: 'compact' })}
+            </span>
+          </>
+        )}
 
         {comment.comment_is_favorited && (
           <>

frontend/src/components/EmbeddableVideoPlayer.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import { VideoResponseType } from '../pages/Video';
 import VideoPlayer from './VideoPlayer';
 import loadVideoById from '../api/loader/loadVideoById';
@@ -24,6 +24,8 @@ type EmbeddableVideoPlayerProps = {
 };
 
 const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {
+  const inlinePlayerRef = useRef<HTMLDivElement>(null);
+
   const [, setSearchParams] = useSearchParams();
 
   const [refresh, setRefresh] = useState(false);
@@ -63,8 +65,7 @@ const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {
 
       setVideoResponse(videoResponse);
 
-      const inlinePlayer = document.getElementById('inline-player');
+      inlinePlayerRef.current?.scrollIntoView();
-      inlinePlayer?.scrollIntoView();
 
       setRefresh(false);
       setLoading(false);
@@ -91,7 +92,7 @@ const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {
 
   return (
     <>
-      <div id="inline-player" className="player-wrapper">
+      <div ref={inlinePlayerRef} className="player-wrapper">
         <div className="video-player">
           {!loading && (
             <VideoPlayer

frontend/src/components/Filterbar.tsx

@@ -11,10 +11,10 @@ import { ViewStyles } from '../configuration/constants/ViewStyle';
 type FilterbarProps = {
   hideToggleText: string;
   viewStyleName: string;
-  setRefresh?: (status: boolean) => void;
+  showSort?: boolean;
 };
 
-const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps) => {
+const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: FilterbarProps) => {
   const { userConfig, setPartialConfig } = useUserConfigStore();
   const [showHidden, setShowHidden] = useState(false);
   const isGridView = userConfig.config.view_style_home === ViewStyles.grid;
@@ -29,7 +29,6 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
             type="checkbox"
             checked={userConfig.config.hide_watched}
             onChange={() => {
-              setRefresh?.(true);
               setPartialConfig({ hide_watched: !userConfig.config.hide_watched });
             }}
           />
@@ -46,7 +45,7 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
         </div>
       </div>
 
-      {showHidden && (
+      {showHidden && showSort && (
         <div className="sort">
           <div id="form">
             <span>Sort by:</span>
@@ -55,7 +54,6 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
               id="sort"
               value={userConfig.config.sort_by}
               onChange={event => {
-                setRefresh?.(true);
                 setPartialConfig({ sort_by: event.target.value as SortByType });
               }}
             >
@@ -71,7 +69,6 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
               id="sort-order"
               value={userConfig.config.sort_order}
               onChange={event => {
-                setRefresh?.(true);
                 setPartialConfig({ sort_order: event.target.value as SortOrderType });
               }}
             >
@@ -81,9 +78,8 @@ const Filterbar = ({ hideToggleText, viewStyleName, setRefresh }: FilterbarProps
           </div>
         </div>
       )}
-
       <div className="view-icons">
-        {setShowHidden && (
+        {setShowHidden && showSort && (
           <img
             src={iconSort}
             alt="sort-icon"

frontend/src/pages/ChannelVideo.tsx

@@ -152,11 +152,7 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => {
         </div>
       </div>
       <div className={`boxed-content ${gridView}`}>
-        <Filterbar
+        <Filterbar hideToggleText={'Hide watched videos:'} viewStyleName={ViewStyleNames.home} />
-          hideToggleText={'Hide watched videos:'}
-          viewStyleName={ViewStyleNames.home}
-          setRefresh={setRefresh}
-        />
       </div>
       {showEmbeddedVideo && <EmbeddableVideoPlayer videoId={videoId} />}
       <div className={`boxed-content ${gridView}`}>

frontend/src/pages/Channels.tsx

@@ -65,16 +65,14 @@ const Channels = () => {
 
   useEffect(() => {
     (async () => {
-      if (refresh) {
+      const channelListResponse = await loadChannelList(
-        const channelListResponse = await loadChannelList(
+        currentPage,
-          currentPage,
+        userConfig.config.show_subed_only,
-          userConfig.config.show_subed_only,
+      );
-        );
 
-        setChannelListResponse(channelListResponse);
+      setChannelListResponse(channelListResponse);
-        setShowNotification(false);
+      setShowNotification(false);
-        setRefresh(false);
+      setRefresh(false);
-      }
     })();
   }, [refresh, userConfig.config.show_subed_only, currentPage, pagination?.current_page]);
 

frontend/src/pages/Download.tsx

@@ -100,11 +100,11 @@ const Download = () => {
 
   useEffect(() => {
     (async () => {
-      const downloadAggs = await loadDownloadAggs();
+      const downloadAggs = await loadDownloadAggs(showIgnored);
 
       setDownloadAggsResponse(downloadAggs);
     })();
-  }, [lastVideoCount]);
+  }, [lastVideoCount, showIgnored]);
 
   useEffect(() => {
     setRefresh(true);

frontend/src/pages/Home.tsx

@@ -129,30 +129,24 @@ const Home = () => {
 
   useEffect(() => {
     (async () => {
-      if (
+      const videos = await loadVideoListByFilter({
-        refreshVideoList ||
+        page: currentPage,
-        pagination?.current_page === undefined ||
+        watch: userMeConfig.hide_watched ? 'unwatched' : undefined,
-        currentPage !== pagination?.current_page
+        sort: userMeConfig.sort_by,
-      ) {
+        order: userMeConfig.sort_order,
-        const videos = await loadVideoListByFilter({
+      });
-          page: currentPage,
-          watch: userMeConfig.hide_watched ? 'unwatched' : undefined,
-          sort: userMeConfig.sort_by,
-          order: userMeConfig.sort_order,
-        });
 
-        try {
+      try {
-          const continueVideoResponse = await loadVideoListByFilter({ watch: 'continue' });
+        const continueVideoResponse = await loadVideoListByFilter({ watch: 'continue' });
-          setContinueVideoResponse(continueVideoResponse);
+        setContinueVideoResponse(continueVideoResponse);
-        } catch (error) {
+      } catch (error) {
-          console.log('Server error on continue vids?');
+        console.log('Server error on continue vids?');
-          console.error(error);
+        console.error(error);
-        }
-
-        setVideoReponse(videos);
-
-        setRefreshVideoList(false);
       }
+
+      setVideoReponse(videos);
+
+      setRefreshVideoList(false);
     })();
   }, [
     refreshVideoList,
@@ -190,11 +184,7 @@ const Home = () => {
           <h1>Recent Videos</h1>
         </div>
 
-        <Filterbar
+        <Filterbar hideToggleText="Hide watched:" viewStyleName={ViewStyleNames.home} />
-          hideToggleText="Hide watched:"
-          viewStyleName={ViewStyleNames.home}
-          setRefresh={setRefreshVideoList}
-        />
       </div>
 
       <div className={`boxed-content ${gridView}`}>

frontend/src/pages/Playlist.tsx

@@ -90,34 +90,27 @@ const Playlist = () => {
 
   useEffect(() => {
     (async () => {
-      if (
+      const playlist = await loadPlaylistById(playlistId);
-        refresh ||
+      const video = await loadVideoListByFilter({
-        pagination?.current_page === undefined ||
+        playlist: playlistId,
-        currentPage !== pagination?.current_page
+        page: currentPage,
-      ) {
+        watch: hideWatched ? 'unwatched' : undefined,
-        const playlist = await loadPlaylistById(playlistId);
+        sort: 'downloaded', // downloaded or published? or playlist sort order?
-        const video = await loadVideoListByFilter({
+      });
-          playlist: playlistId,
-          page: currentPage,
-          watch: hideWatched ? 'unwatched' : undefined,
-          sort: 'downloaded', // downloaded or published? or playlist sort order?
-        });
 
-        const isCustomPlaylist = playlist?.data?.playlist_type === 'custom';
+      const isCustomPlaylist = playlist?.data?.playlist_type === 'custom';
-        if (!isCustomPlaylist) {
+      if (!isCustomPlaylist) {
-          const channel = await loadChannelById(playlist.data.playlist_channel_id);
+        const channel = await loadChannelById(playlist.data.playlist_channel_id);
 
-          setChannelResponse(channel);
+        setChannelResponse(channel);
-        }
-
-        setPlaylistResponse(playlist);
-        setVideoResponse(video);
-        setRefresh(false);
       }
+
+      setPlaylistResponse(playlist);
+      setVideoResponse(video);
+      setRefresh(false);
     })();
-    // Do not add hideWatched this will not work as expected!
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [playlistId, refresh, currentPage, pagination?.current_page]);
+  }, [playlistId, userConfig.config.hide_watched, refresh, currentPage, pagination?.current_page]);
 
   if (!playlistId || !playlist) {
     return `Playlist ${playlistId} not found!`;
@@ -320,7 +313,7 @@ const Playlist = () => {
         <Filterbar
           hideToggleText="Hide watched videos:"
           viewStyleName={ViewStyleNames.playlist}
-          setRefresh={setRefresh}
+          showSort={false}
         />
       </div>
 

frontend/src/style.css

@@ -1258,6 +1258,10 @@ video:-webkit-full-screen {
   .video-player video {
     width: 90%;
   }
+  .info-box-3,
+  .info-box-4 {
+    grid-template-columns: 1fr;
+  }
 }
 
 /* phone */
@@ -1275,11 +1279,13 @@ video:-webkit-full-screen {
   .playlist-list.list,
   .playlist-list.grid,
   .info-box-2,
-  .info-box-3,
-  .info-box-4,
   .overwrite-form {
     grid-template-columns: 1fr;
   }
+  .settings-box-wrapper {
+    grid-template-columns: 1fr;
+    padding: 1.2rem 0;
+  }
   .playlist-item.list {
     display: block;
   }