diff --git a/.github/workflows/lint_js.yml b/.github/workflows/lint_js.yml index 73038422..de2650b3 100644 --- a/.github/workflows/lint_js.yml +++ b/.github/workflows/lint_js.yml @@ -1,16 +1,22 @@ name: lint_js -on: [pull_request, push] +on: + push: + paths: + - '**/*.js' + pull_request: + paths: + - '**/*.js' jobs: check: name: lint_js runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: '16' + node-version: '22' - run: npm ci - run: npm run lint - run: npm run format -- --check diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 2b2c0930..0aae9597 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -1,14 +1,42 @@ name: lint_python -on: [pull_request, push] + +on: + push: + paths: + - '**/*.py' + pull_request: + paths: + - '**/*.py' + jobs: lint_python: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y gcc libldap2-dev libsasl2-dev libssl-dev + + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: '3.10' - - run: pip install --upgrade pip wheel - - run: pip install bandit black codespell flake8 flake8-bugbear - flake8-comprehensions isort requests - - run: ./deploy.sh validate + python-version: '3.11' + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + pip install -r tubearchivist/requirements-dev.txt + + - name: Run Linter + run: ./deploy.sh validate diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 00000000..33f3bd5f --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,43 @@ +name: python_unit_tests + +on: + push: + paths: + - '**/*.py' + pull_request: + paths: + - '**/*.py' + +jobs: + unit-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y gcc libldap2-dev libsasl2-dev libssl-dev + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r tubearchivist/requirements-dev.txt + + - name: Run unit tests + run: pytest tubearchivist diff --git a/Dockerfile b/Dockerfile index 7e8e0d14..33a6acff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,7 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm RUN if [ "$INSTALL_DEBUG" ] ; then \ apt-get -y update && apt-get -y install --no-install-recommends \ vim htop bmon net-tools iputils-ping procps \ - && pip install --user ipython \ + && pip install --user ipython pytest pytest-django \ ; fi # make folders diff --git a/deploy.sh b/deploy.sh index a3620bb0..6087b03f 100755 --- a/deploy.sh +++ b/deploy.sh @@ -50,6 +50,7 @@ function sync_test { --exclude ".gitignore" \ --exclude "**/cache" \ --exclude "**/__pycache__/" \ + --exclude "**/.pytest_cache/" \ --exclude ".venv" \ --exclude "db.sqlite3" \ --exclude ".mypy_cache" \ diff --git a/tubearchivist/api/src/search_processor.py b/tubearchivist/api/src/search_processor.py index 9f77b05e..a7891c96 100644 --- a/tubearchivist/api/src/search_processor.py +++ b/tubearchivist/api/src/search_processor.py @@ -7,7 +7,7 @@ Functionality: import urllib.parse from home.src.download.thumbnails import ThumbManager -from home.src.ta.helper import date_praser, get_duration_str +from home.src.ta.helper import date_parser, get_duration_str from home.src.ta.settings import EnvironmentSettings @@ -67,7 +67,7 @@ class SearchProcess: """run on single channel""" channel_id = channel_dict["channel_id"] art_base = f"/cache/channels/{channel_id}" - date_str = date_praser(channel_dict["channel_last_refresh"]) + date_str = date_parser(channel_dict["channel_last_refresh"]) channel_dict.update( { "channel_last_refresh": date_str, @@ -83,8 +83,8 @@ class SearchProcess: """run on single video dict""" video_id = video_dict["youtube_id"] media_url = urllib.parse.quote(video_dict["media_url"]) - vid_last_refresh = date_praser(video_dict["vid_last_refresh"]) - published = date_praser(video_dict["published"]) + vid_last_refresh = date_parser(video_dict["vid_last_refresh"]) + published = date_parser(video_dict["published"]) vid_thumb_url = ThumbManager(video_id).vid_thumb_path() channel = self._process_channel(video_dict["channel"]) @@ -109,7 +109,7 @@ class SearchProcess: def _process_playlist(playlist_dict): """run on single playlist dict""" playlist_id = playlist_dict["playlist_id"] - playlist_last_refresh = date_praser( + playlist_last_refresh = date_parser( playlist_dict["playlist_last_refresh"] ) playlist_dict.update( @@ -125,7 +125,7 @@ class SearchProcess: """run on single download item""" video_id = download_dict["youtube_id"] vid_thumb_url = ThumbManager(video_id).vid_thumb_path() - published = date_praser(download_dict["published"]) + published = date_parser(download_dict["published"]) download_dict.update( { diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 1daca9dc..898c721d 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -688,14 +688,7 @@ class DownloadApiListView(ApiBaseView): pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"] url_str = " ".join(pending) - try: - youtube_ids = Parser(url_str).parse() - except ValueError: - message = f"failed to parse: {url_str}" - print(message) - return Response({"message": message}, status=400) - - extrac_dl.delay(youtube_ids, auto_start=auto_start) + extrac_dl.delay(url_str, auto_start=auto_start) return Response(data) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index ac41eaa0..8ec78889 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -33,7 +33,9 @@ SECRET_KEY = PW_HASH.hexdigest() # SECURITY WARNING: don't run with debug turned on in production! DEBUG = bool(environ.get("DJANGO_DEBUG")) -ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(environ["TA_HOST"]) +ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser( + environ.get("TA_HOST", "localhost") +) # Application definition @@ -241,7 +243,11 @@ USE_TZ = True STATIC_URL = "/static/" STATICFILES_DIRS = (str(BASE_DIR.joinpath("static")),) STATIC_ROOT = str(BASE_DIR.joinpath("staticfiles")) -STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" +STORAGES = { + "staticfiles": { + "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage", + }, +} # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field diff --git a/tubearchivist/home/celery.py b/tubearchivist/home/celery.py index 9f65795b..1f3369f8 100644 --- a/tubearchivist/home/celery.py +++ b/tubearchivist/home/celery.py @@ -3,10 +3,8 @@ import os from celery import Celery -from home.src.ta.config import AppConfig from home.src.ta.settings import EnvironmentSettings -CONFIG = AppConfig().config REDIS_HOST = EnvironmentSettings.REDIS_HOST REDIS_PORT = EnvironmentSettings.REDIS_PORT diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 1db975e5..51563998 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -375,6 +375,7 @@ class Reindex(ReindexBase): channel.json_data["channel_overwrites"] = overwrites channel.upload_to_es() + channel.sync_to_videos() ChannelFullScan(channel_id).scan() self.processed["channels"] += 1 diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 05e1ef0e..767b4a00 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -93,12 +93,14 @@ def requests_headers() -> dict[str, str]: return {"User-Agent": template} -def date_praser(timestamp: int | str) -> str: +def date_parser(timestamp: int | str) -> str: """return formatted date string""" if isinstance(timestamp, int): date_obj = datetime.fromtimestamp(timestamp) elif isinstance(timestamp, str): date_obj = datetime.strptime(timestamp, "%Y-%m-%d") + else: + raise TypeError(f"invalid timestamp: {timestamp}") return date_obj.date().isoformat() @@ -138,8 +140,9 @@ def get_mapping() -> dict: def is_shorts(youtube_id: str) -> bool: """check if youtube_id is a shorts video, bot not it it's not a shorts""" shorts_url = f"https://www.youtube.com/shorts/{youtube_id}" + cookies = {"SOCS": "CAI"} response = requests.head( - shorts_url, headers=requests_headers(), timeout=10 + shorts_url, cookies=cookies, headers=requests_headers(), timeout=10 ) return response.status_code == 200 @@ -183,6 +186,8 @@ def get_duration_str(seconds: int) -> str: unit_count, seconds = divmod(seconds, unit_seconds) duration_parts.append(f"{unit_count:02}{unit_label}") + duration_parts[0] = duration_parts[0].lstrip("0") + return " ".join(duration_parts) diff --git a/tubearchivist/home/src/ta/urlparser.py b/tubearchivist/home/src/ta/urlparser.py index 743af988..924961ac 100644 --- a/tubearchivist/home/src/ta/urlparser.py +++ b/tubearchivist/home/src/ta/urlparser.py @@ -42,6 +42,10 @@ class Parser: youtube_id = parsed.path.strip("/") return self._validate_expected(youtube_id, "video") + if "youtube.com" not in parsed.netloc: + message = f"invalid domain: {parsed.netloc}" + raise ValueError(message) + query_parsed = parse_qs(parsed.query) if "v" in query_parsed: # video from v query str diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index 63e22865..2bfdef8f 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -60,7 +60,10 @@ gear-icon - + + exit-icon diff --git a/tubearchivist/home/tests/conftest.py b/tubearchivist/home/tests/conftest.py new file mode 100644 index 00000000..5e5cb7f4 --- /dev/null +++ b/tubearchivist/home/tests/conftest.py @@ -0,0 +1,11 @@ +"""test configs""" + +import os + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def change_test_dir(request): + """change directory to project folder""" + os.chdir(request.config.rootdir / "tubearchivist") diff --git a/tubearchivist/home/tests/test_ta/__init__.py b/tubearchivist/home/tests/test_ta/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/home/tests/test_ta/test_helper.py b/tubearchivist/home/tests/test_ta/test_helper.py new file mode 100644 index 00000000..7c1c4bbf --- /dev/null +++ b/tubearchivist/home/tests/test_ta/test_helper.py @@ -0,0 +1,113 @@ +"""tests for helper functions""" + +import pytest +from home.src.ta.helper import ( + date_parser, + get_duration_str, + get_mapping, + is_shorts, + randomizor, + time_parser, +) + + +def test_randomizor_with_positive_length(): + """test randomizer""" + length = 10 + result = randomizor(length) + assert len(result) == length + assert result.isalnum() + + +def test_date_parser_with_int(): + """unix timestamp""" + timestamp = 1621539600 + expected_date = "2021-05-20" + assert date_parser(timestamp) == expected_date + + +def test_date_parser_with_str(): + """iso timestamp""" + date_str = "2021-05-21" + expected_date = "2021-05-21" + assert date_parser(date_str) == expected_date + + +def test_date_parser_with_invalid_input(): + """invalid type""" + invalid_input = [1621539600] + with pytest.raises(TypeError): + date_parser(invalid_input) + + +def test_date_parser_with_invalid_string_format(): + """invalid date string""" + invalid_date_str = "21/05/2021" + with pytest.raises(ValueError): + date_parser(invalid_date_str) + + +def test_time_parser_with_numeric_string(): + """as number""" + timestamp = "100" + expected_seconds = 100 + assert time_parser(timestamp) == expected_seconds + + +def test_time_parser_with_hh_mm_ss_format(): + """to seconds""" + timestamp = "01:00:00" + expected_seconds = 3600.0 + assert time_parser(timestamp) == expected_seconds + + +def test_time_parser_with_empty_string(): + """handle empty""" + timestamp = "" + assert time_parser(timestamp) is False + + +def test_time_parser_with_invalid_format(): + """not enough to unpack""" + timestamp = "01:00" + with pytest.raises(ValueError): + time_parser(timestamp) + + +def test_time_parser_with_non_numeric_input(): + """non numeric""" + timestamp = "1a:00:00" + with pytest.raises(ValueError): + time_parser(timestamp) + + +def test_get_mapping(): + """test mappint""" + index_config = get_mapping() + assert isinstance(index_config, list) + assert all(isinstance(i, dict) for i in index_config) + + +def test_is_shorts(): + """is shorts id""" + youtube_id = "YG3-Pw3rixU" + assert is_shorts(youtube_id) + + +def test_is_not_shorts(): + """is not shorts id""" + youtube_id = "Ogr9kbypSNg" + assert is_shorts(youtube_id) is False + + +def test_get_duration_str(): + """only seconds""" + assert get_duration_str(None) == "NA" + assert get_duration_str(5) == "5s" + assert get_duration_str(10) == "10s" + assert get_duration_str(500) == "8m 20s" + assert get_duration_str(1000) == "16m 40s" + assert get_duration_str(5000) == "1h 23m 20s" + assert get_duration_str(500000) == "5d 18h 53m 20s" + assert get_duration_str(5000000) == "57d 20h 53m 20s" + assert get_duration_str(50000000) == "1y 213d 16h 53m 20s" diff --git a/tubearchivist/home/tests/test_ta/test_urlparser.py b/tubearchivist/home/tests/test_ta/test_urlparser.py new file mode 100644 index 00000000..6d4ab2f4 --- /dev/null +++ b/tubearchivist/home/tests/test_ta/test_urlparser.py @@ -0,0 +1,144 @@ +"""tests for url parser""" + +import pytest +from home.src.ta.urlparser import Parser + +# video id parsing +VIDEO_URL_IN = [ + "7DKv5H5Frt0", + "https://www.youtube.com/watch?v=7DKv5H5Frt0", + "https://www.youtube.com/watch?v=7DKv5H5Frt0&t=113&feature=shared", + "https://www.youtube.com/watch?v=7DKv5H5Frt0&list=PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5&index=1&pp=iAQB" # noqa: E501 + "https://youtu.be/7DKv5H5Frt0", +] +VIDEO_OUT = [{"type": "video", "url": "7DKv5H5Frt0", "vid_type": "unknown"}] +VIDEO_TEST_CASES = [(i, VIDEO_OUT) for i in VIDEO_URL_IN] + +# shorts id parsing +SHORTS_URL_IN = [ + "https://www.youtube.com/shorts/YG3-Pw3rixU", + "https://youtube.com/shorts/YG3-Pw3rixU?feature=shared", +] +SHORTS_OUT = [{"type": "video", "url": "YG3-Pw3rixU", "vid_type": "shorts"}] +SHORTS_TEST_CASES = [(i, SHORTS_OUT) for i in SHORTS_URL_IN] + +# channel id parsing +CHANNEL_URL_IN = [ + "UCBa659QWEk1AI4Tg--mrJ2A", + "@TomScottGo", + "https://www.youtube.com/channel/UCBa659QWEk1AI4Tg--mrJ2A", + "https://www.youtube.com/@TomScottGo", +] +CHANNEL_OUT = [ + { + "type": "channel", + "url": "UCBa659QWEk1AI4Tg--mrJ2A", + "vid_type": "unknown", + } +] +CHANNEL_TEST_CASES = [(i, CHANNEL_OUT) for i in CHANNEL_URL_IN] + +# channel vid type parsing +CHANNEL_VID_TYPES = [ + ( + "https://www.youtube.com/@IBRACORP/videos", + [ + { + "type": "channel", + "url": "UC7aW7chIafJG6ECYAd3N5uQ", + "vid_type": "videos", + } + ], + ), + ( + "https://www.youtube.com/@IBRACORP/shorts", + [ + { + "type": "channel", + "url": "UC7aW7chIafJG6ECYAd3N5uQ", + "vid_type": "shorts", + } + ], + ), + ( + "https://www.youtube.com/@IBRACORP/streams", + [ + { + "type": "channel", + "url": "UC7aW7chIafJG6ECYAd3N5uQ", + "vid_type": "streams", + } + ], + ), +] + +# playlist id parsing +PLAYLIST_URL_IN = [ + "PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5", + "https://www.youtube.com/playlist?list=PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5", +] +PLAYLIST_OUT = [ + { + "type": "playlist", + "url": "PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5", + "vid_type": "unknown", + } +] +PLAYLIST_TEST_CASES = [(i, PLAYLIST_OUT) for i in PLAYLIST_URL_IN] + +# personal playlists +EXPECTED_WL = [{"type": "playlist", "url": "WL", "vid_type": "unknown"}] +EXPECTED_LL = [{"type": "playlist", "url": "LL", "vid_type": "unknown"}] +PERSONAL_PLAYLISTS_TEST_CASES = [ + ("WL", EXPECTED_WL), + ("https://www.youtube.com/playlist?list=WL", EXPECTED_WL), + ("LL", EXPECTED_LL), + ("https://www.youtube.com/playlist?list=LL", EXPECTED_LL), +] + +# collect tests expected to pass +PASSTING_TESTS = [] +PASSTING_TESTS.extend(VIDEO_TEST_CASES) +PASSTING_TESTS.extend(SHORTS_TEST_CASES) +PASSTING_TESTS.extend(CHANNEL_TEST_CASES) +PASSTING_TESTS.extend(CHANNEL_VID_TYPES) +PASSTING_TESTS.extend(PLAYLIST_TEST_CASES) +PASSTING_TESTS.extend(PERSONAL_PLAYLISTS_TEST_CASES) + + +@pytest.mark.parametrize("url_str, expected_result", PASSTING_TESTS) +def test_passing_parse(url_str, expected_result): + """test parser""" + parser = Parser(url_str) + parsed = parser.parse() + assert parsed == expected_result + + +INVALID_IDS_ERRORS = [ + "aaaaa", + "https://www.youtube.com/playlist?list=AAAA", + "https://www.youtube.com/channel/UC9-y-6csu5WGm29I7Jiwpn", + "https://www.youtube.com/watch?v=CK3_zarXkw", +] + + +@pytest.mark.parametrize("invalid_value", INVALID_IDS_ERRORS) +def test_invalid_ids(invalid_value): + """test for invalid IDs""" + with pytest.raises(ValueError, match="not a valid id_str"): + parser = Parser(invalid_value) + parser.parse() + + +INVALID_DOMAINS = [ + "https://vimeo.com/32001208", + "https://peertube.tv/w/8RiJE2j2nw569FVgPNjDt7", +] + + +@pytest.mark.parametrize("invalid_value", INVALID_DOMAINS) +def test_invalid_domains(invalid_value): + """raise error on none YT domains""" + parser = Parser(invalid_value) + with pytest.raises(ValueError, match="invalid domain"): + parser.parse() diff --git a/tubearchivist/home/tests/test_views.py b/tubearchivist/home/tests/test_views.py deleted file mode 100644 index 0d3bd23f..00000000 --- a/tubearchivist/home/tests/test_views.py +++ /dev/null @@ -1,42 +0,0 @@ -"""All test classes""" - -from django.test import TestCase - - -class URLTests(TestCase): - """test if all expected URL are there""" - - def test_home_view(self): - """check homepage""" - response = self.client.get("/") - self.assertEqual(response.status_code, 200) - - def test_about_view(self): - """check about page""" - response = self.client.get("/about/") - self.assertEqual(response.status_code, 200) - - def test_downloads_view(self): - """check downloads page""" - response = self.client.get("/downloads/") - self.assertEqual(response.status_code, 200) - - def test_channel_view(self): - """check channel page""" - response = self.client.get("/channel/") - self.assertEqual(response.status_code, 200) - - def test_settings_view(self): - """check settings page""" - response = self.client.get("/settings/") - self.assertEqual(response.status_code, 200) - - def test_progress_view(self): - """check ajax progress endpoint""" - response = self.client.get("/downloads/progress/") - self.assertEqual(response.status_code, 200) - - def test_process_view(self): - """check process ajax endpoint""" - response = self.client.get("/process/") - self.assertEqual(response.status_code, 200) diff --git a/tubearchivist/requirements-dev.txt b/tubearchivist/requirements-dev.txt index 33c81ec8..02778fba 100644 --- a/tubearchivist/requirements-dev.txt +++ b/tubearchivist/requirements-dev.txt @@ -5,4 +5,6 @@ flake8 isort pylint pylint-django +pytest +pytest-django types-requests diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index be4b3e94..ac8782cb 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -7,8 +7,8 @@ django-cors-headers==4.3.1 djangorestframework==3.15.1 Pillow==10.3.0 redis==5.0.4 -requests==2.31.0 +requests==2.32.1 ryd-client==0.0.6 uWSGI==2.0.25.1 whitenoise==6.6.0 -yt-dlp @ git+https://github.com/bbilly1/yt-dlp@54b823be28f396608349cca69d52eb4c4b72b8b0 +yt-dlp @ git+https://github.com/yt-dlp/yt-dlp@8e15177b4113c355989881e4e030f695a9b59c3a