From e467beb1c70379050b9e37e17c5a837ad080897b Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 26 Aug 2022 17:07:55 +0700 Subject: [PATCH 01/11] fix manual import cleanup metadata, #331 --- tubearchivist/home/src/index/filesystem.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index e6042cca..0b9b9980 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -555,11 +555,13 @@ class ManualImport: def _cleanup(self, json_data): """cleanup leftover files""" - if os.path.exists(self.current_video["metadata"]): - os.remove(self.current_video["metadata"]) + meta_data = self.current_video["metadata"] + if meta_data and os.path.exists(meta_data): + os.remove(meta_data) - if os.path.exists(self.current_video["thumb"]): - os.remove(self.current_video["thumb"]) + thumb = self.current_video["thumb"] + if thumb and os.path.exists(thumb): + os.remove(thumb) for subtitle_file in self.current_video["subtitle"]: if os.path.exists(subtitle_file): From 06795a014cfdabd0c17000890226756ec4714c32 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 6 Sep 2022 21:11:18 +0700 Subject: [PATCH 02/11] explicitly set DJANGO_DEBUG env var on startup --- docker_assets/run.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker_assets/run.sh b/docker_assets/run.sh index b03c5203..e46b5168 100644 --- a/docker_assets/run.sh +++ b/docker_assets/run.sh @@ -5,6 +5,10 @@ if [[ -z "$ELASTIC_USER" ]]; then export ELASTIC_USER=elastic fi +if [[ -z "$DJANGO_DEBUG" ]]; then + export DJANGO_DEBUG=False +fi + cachedir=/cache [[ -d $cachedir ]] || cachedir=. lockfile=${cachedir}/initsu.lock From b2bf4ecc02e7f41c4f36a9193d5468fb080417cf Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 6 Sep 2022 21:20:29 +0700 Subject: [PATCH 03/11] bump libs --- tubearchivist/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 55856775..388338db 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4==4.11.1 celery==5.2.7 -Django==4.0.6 +Django==4.1.1 django-auth-ldap==4.1.0 django-cors-headers==3.13.0 djangorestframework==3.13.1 @@ -10,4 +10,4 @@ requests==2.28.1 ryd-client==0.0.6 uWSGI==2.0.20 whitenoise==6.2.0 -yt_dlp==2022.8.19 +yt_dlp==2022.9.1 From a7aead728cba97734de74aee5e785b7d0434275c Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 6 Sep 2022 21:22:30 +0700 Subject: [PATCH 04/11] add roadmap link to FR template --- .github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml index 592d5e3a..95290822 100644 --- a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml +++ b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml @@ -15,7 +15,7 @@ body: options: - label: I have read through the [wiki](https://github.com/tubearchivist/tubearchivist/wiki). required: true - - label: I understand the [scope](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) of this project and am aware of the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations). + - label: I understand the [scope](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) of this project and am aware of the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and my idea is not already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap). required: true - type: textarea From e51a662da9b0da2c36ed26d18bc4d44fb6680a0b Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 6 Sep 2022 22:27:18 +0700 Subject: [PATCH 05/11] fix manual import splitext matching invalid extensions, #311 --- tubearchivist/home/src/index/filesystem.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index 0b9b9980..8268806f 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -217,9 +217,7 @@ class ImportFolderScanner: last_base = False for file_path in all_files: - base_name_raw, ext = os.path.splitext(file_path) - base_name, _ = os.path.splitext(base_name_raw) - + base_name, ext = self._detect_base_name(file_path) key, file_path = self._detect_type(file_path, ext) if not key or not file_path: continue @@ -239,6 +237,18 @@ class ImportFolderScanner: if current_video.get("media"): self.to_import.append(current_video) + def _detect_base_name(self, file_path): + """extract base_name and ext for matching""" + base_name_raw, ext = os.path.splitext(file_path) + base_name, ext2 = os.path.splitext(base_name_raw) + + if ext2: + if ISO639Utils.short2long(ext2.strip(".")) or ext2 == ".info": + # valid secondary extension + return base_name, ext + + return base_name_raw, ext + def _detect_type(self, file_path, ext): """detect metadata type for file""" From 3c92de8e24452962302b2eae72fe2af73fbaec82 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 6 Sep 2022 22:42:55 +0700 Subject: [PATCH 06/11] improved logging for manual import --- tubearchivist/home/src/index/filesystem.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index 8268806f..768d7137 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -224,6 +224,7 @@ class ImportFolderScanner: if base_name != last_base: if last_base: + print(f"manual import: {current_video}") self.to_import.append(current_video) current_video = self._get_template() @@ -235,6 +236,7 @@ class ImportFolderScanner: current_video[key] = file_path if current_video.get("media"): + print(f"manual import: {current_video}") self.to_import.append(current_video) def _detect_base_name(self, file_path): @@ -270,12 +272,12 @@ class ImportFolderScanner: self._convert_thumb(current_video) self._get_subtitles(current_video) self._convert_video(current_video) + print(f"manual import: {current_video}") ManualImport(current_video, self.CONFIG).run() def _detect_youtube_id(self, current_video): """find video id from filename or json""" - print(current_video) youtube_id = self._extract_id_from_filename(current_video["media"]) if youtube_id: current_video["video_id"] = youtube_id @@ -286,7 +288,6 @@ class ImportFolderScanner: current_video["video_id"] = youtube_id return - print(current_video["media"]) raise ValueError("failed to find video id") @staticmethod From a6dd9278600aad57ec0b629c2d352b8fa09e87e2 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 6 Sep 2022 23:12:49 +0700 Subject: [PATCH 07/11] fix channel extraction to catch all alerts, #312 --- tubearchivist/home/src/index/channel.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 34e4adc7..520b1669 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -73,13 +73,14 @@ class ChannelScraper: def _is_deactivated(self): """check if channel is deactivated""" - alert_text = "This channel does not exist." alerts = self.yt_json.get("alerts") - if alerts and alert_text in str(alerts): - print(f"{self.channel_id}: {alert_text}") - return True + if not alerts: + return False - return False + for alert in alerts: + alert_text = alert["alertRenderer"]["text"]["simpleText"] + print(f"{self.channel_id}: failed to extract, {alert_text}") + return True def _parse_channel_main(self): """extract maintab values from scraped channel json data""" @@ -226,8 +227,8 @@ class YoutubeChannel(YouTubeItem): self.json_data.update( { - "channel_subs": content["channel_follower_count"], - "channel_description": content["description"], + "channel_subs": content.get("channel_follower_count", 0), + "channel_description": content.get("description", False), } ) From 1c3febc49dee7366eff6b67f37403c93f1ae9507 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 6 Sep 2022 23:13:29 +0700 Subject: [PATCH 08/11] force create thumb folder for manual import --- tubearchivist/home/src/download/thumbnails.py | 6 +++++- tubearchivist/home/src/index/filesystem.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 4e77453f..ff556c09 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -124,7 +124,7 @@ class ThumbManager(ThumbManagerBase): img_raw.convert("RGB").save(thumb_path) - def vid_thumb_path(self, absolute=False): + def vid_thumb_path(self, absolute=False, create_folder=False): """build expected path for video thumbnail from youtube_id""" folder_name = self.item_id[0].lower() folder_path = os.path.join("videos", folder_name) @@ -132,6 +132,10 @@ class ThumbManager(ThumbManagerBase): if absolute: thumb_path = os.path.join(self.CACHE_DIR, thumb_path) + if create_folder: + folder_path = os.path.join(self.CACHE_DIR, folder_path) + os.makedirs(folder_path, exist_ok=True) + return thumb_path def download_channel_art(self, urls, skip_existing=False): diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index 768d7137..bbe69fc4 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -527,7 +527,8 @@ class ManualImport: if video.offline_import and self.current_video["thumb"]: old_path = self.current_video["thumb"] - new_path = ThumbManager(video_id).vid_thumb_path(absolute=True) + thumbs = ThumbManager(video_id) + new_path = thumbs.vid_thumb_path(absolute=True, create_folder=True) shutil.move(old_path, new_path, copy_function=shutil.copyfile) else: url = video.json_data["vid_thumb_url"] From 301a7349055ddc98c158df69807e83e7d8bbf689 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 6 Sep 2022 23:26:17 +0700 Subject: [PATCH 09/11] fix DJANGO_DEBUG set to default empty string --- docker_assets/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_assets/run.sh b/docker_assets/run.sh index e46b5168..32d02635 100644 --- a/docker_assets/run.sh +++ b/docker_assets/run.sh @@ -6,7 +6,7 @@ if [[ -z "$ELASTIC_USER" ]]; then fi if [[ -z "$DJANGO_DEBUG" ]]; then - export DJANGO_DEBUG=False + export DJANGO_DEBUG="" fi cachedir=/cache From 17c178c8bf01316fc95bf25daa74aea40033d3a7 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 6 Sep 2022 23:27:05 +0700 Subject: [PATCH 10/11] add 400 response for wrong TA_HOST env var --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 53d852f9..55ef76b2 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ The main Python application that displays and serves your video collection, buil - And another volume to save application data at **/cache**. - The environment variables `ES_URL` and `REDIS_HOST` are needed to tell Tube Archivist where Elasticsearch and Redis respectively are located. - The environment variables `HOST_UID` and `HOST_GID` allows Tube Archivist to `chown` the video files to the main host system user instead of the container user. Those two variables are optional, not setting them will disable that functionality. That might be needed if the underlying filesystem doesn't support `chown` like *NFS*. - - Set the environment variable `TA_HOST` to match with the system running Tube Archivist. This can be a domain like *example.com*, a subdomain like *ta.example.com* or an IP address like *192.168.1.20*, add without the protocol and without the port. You can add multiple hostnames separated with a space. + - Set the environment variable `TA_HOST` to match with the system running Tube Archivist. This can be a domain like *example.com*, a subdomain like *ta.example.com* or an IP address like *192.168.1.20*, add without the protocol and without the port. You can add multiple hostnames separated with a space. Any wrong configurations here will result in a `Bad Request (400)` response. - Change the environment variables `TA_USERNAME` and `TA_PASSWORD` to create the initial credentials. - `ELASTIC_PASSWORD` is for the password for Elasticsearch. The environment variable `ELASTIC_USER` is optional, should you want to change the username from the default *elastic*. - For the scheduler to know what time it is, set your timezone with the `TZ` environment variable, defaults to *UTC*. From 6d22e62642d176dc3a278949e6df7e41e5a4db81 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 10 Sep 2022 11:22:49 +0700 Subject: [PATCH 11/11] downgrade django to fix django_debug env --- docker_assets/run.sh | 4 ---- tubearchivist/requirements.txt | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/docker_assets/run.sh b/docker_assets/run.sh index 32d02635..b03c5203 100644 --- a/docker_assets/run.sh +++ b/docker_assets/run.sh @@ -5,10 +5,6 @@ if [[ -z "$ELASTIC_USER" ]]; then export ELASTIC_USER=elastic fi -if [[ -z "$DJANGO_DEBUG" ]]; then - export DJANGO_DEBUG="" -fi - cachedir=/cache [[ -d $cachedir ]] || cachedir=. lockfile=${cachedir}/initsu.lock diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 388338db..e556893b 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4==4.11.1 celery==5.2.7 -Django==4.1.1 +Django==4.0.6 django-auth-ldap==4.1.0 django-cors-headers==3.13.0 djangorestframework==3.13.1