diff --git a/backend/common/src/urlparser.py b/backend/common/src/urlparser.py index ae314791..3db07ad6 100644 --- a/backend/common/src/urlparser.py +++ b/backend/common/src/urlparser.py @@ -6,15 +6,20 @@ Functionality: from urllib.parse import parse_qs, urlparse +from common.src.ta_redis import RedisArchivist from download.src.yt_dlp_base import YtWrap from video.src.constants import VideoTypeEnum class Parser: - """take a multi line string and detect valid youtube ids""" + """ + take a multi line string and detect valid youtube ids + channel handle lookup is cached, can be disabled for unittests + """ - def __init__(self, url_str): + def __init__(self, url_str, use_cache=True): self.url_list = [i.strip() for i in url_str.split()] + self.use_cache = use_cache def parse(self): """parse the list""" @@ -106,9 +111,13 @@ class Parser: return {"type": item_type, "url": id_str} - @staticmethod - def _extract_channel_name(url): - """find channel id from channel name with yt-dlp help""" + def _extract_channel_name(self, url): + """find channel id from channel name with yt-dlp help, cache result""" + if self.use_cache: + cached = self._get_cached(url) + if cached: + return cached + obs_request = { "check_formats": None, "skip_download": True, @@ -121,6 +130,9 @@ class Parser: channel_id = url_info.get("channel_id", False) if channel_id: + if self.use_cache: + self._set_cache(url, channel_id) + return channel_id url = url_info.get("url", False) @@ -133,6 +145,42 @@ class Parser: print(f"failed to extract channel id from {url}") raise ValueError + @staticmethod + def _get_cached(url) -> str | None: + """get cached channel ID, if available""" + path = urlparse(url).path.lstrip("/") + if not path.startswith("@"): + return None + + handle = path.split("/")[0] + if not handle: + return None + + cache_key = f"channel:handlesearch:{handle.lower()}" + cached = RedisArchivist().get_message_dict(cache_key) + if cached: + return cached["channel_id"] + + return None + + @staticmethod + def _set_cache(url, channel_id) -> None: + """set cache""" + path = urlparse(url).path.lstrip("/") + if not path.startswith("@"): + return + + handle = path.split("/")[0] + if not handle: + return + + cache_key = f"channel:handlesearch:{handle.lower()}" + message = { + "channel_id": channel_id, + "handle": handle, + } + RedisArchivist().set_message(cache_key, message, expire=3600 * 24 * 7) + def _detect_vid_type(self, path): """try to match enum from path, needs to be serializable""" last = path.strip("/").split("/")[-1] diff --git a/backend/common/tests/test_src/test_urlparser.py b/backend/common/tests/test_src/test_urlparser.py index 2f54544e..866624ba 100644 --- a/backend/common/tests/test_src/test_urlparser.py +++ b/backend/common/tests/test_src/test_urlparser.py @@ -110,7 +110,7 @@ PASSTING_TESTS.extend(PERSONAL_PLAYLISTS_TEST_CASES) @pytest.mark.parametrize("url_str, expected_result", PASSTING_TESTS) def test_passing_parse(url_str, expected_result): """test parser""" - parser = Parser(url_str) + parser = Parser(url_str, use_cache=False) parsed = parser.parse() assert parsed == expected_result @@ -127,7 +127,7 @@ INVALID_IDS_ERRORS = [ def test_invalid_ids(invalid_value): """test for invalid IDs""" with pytest.raises(ValueError, match="not a valid id_str"): - parser = Parser(invalid_value) + parser = Parser(invalid_value, use_cache=False) parser.parse() @@ -140,6 +140,6 @@ INVALID_DOMAINS = [ @pytest.mark.parametrize("invalid_value", INVALID_DOMAINS) def test_invalid_domains(invalid_value): """raise error on none YT domains""" - parser = Parser(invalid_value) + parser = Parser(invalid_value, use_cache=False) with pytest.raises(ValueError, match="invalid domain"): parser.parse()