public/tubearchivist
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/tubearchivist.git synced 2026-08-04 20:29:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

context manager for yt-dlp extract, stricter worker recycle

Browse Source
This commit is contained in:
Simon
2025-02-02 15:45:40 +07:00
parent c7fc2666fa
commit 2ec81c7ac7
2 changed files with 20 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
backend/download/src/yt_dlp_base.py
View File

@@ -82,23 +82,24 @@ class YtWrap:
def extract(self, url): def extract(self, url):
"""make extract request""" """make extract request"""
try: with yt_dlp.YoutubeDL(self.obs) as ydl:
response = yt_dlp.YoutubeDL(self.obs).extract_info(url) try:
except cookiejar.LoadError as err: response = ydl.extract_info(url)
print(f"cookie file is invalid: {err}") except cookiejar.LoadError as err:
return False print(f"cookie file is invalid: {err}")
except yt_dlp.utils.ExtractorError as err:
print(f"{url}: failed to extract with message: {err}, continue...")
return False
except yt_dlp.utils.DownloadError as err:
if "This channel does not have a" in str(err):
return False return False
except yt_dlp.utils.ExtractorError as err:
print(f"{url}: failed to extract: {err}, continue...")
return False
except yt_dlp.utils.DownloadError as err:
if "This channel does not have a" in str(err):
return False
print(f"{url}: failed to get info from youtube with message {err}") print(f"{url}: failed to get info from youtube: {err}")
if "Temporary failure in name resolution" in str(err): if "Temporary failure in name resolution" in str(err):
raise ConnectionError("lost the internet, abort!") from err raise ConnectionError("lost the internet, abort!") from err
return False return False
self._validate_cookie() self._validate_cookie()

6
docker_assets/run.sh
View File

@@ -20,7 +20,11 @@ python manage.py ta_startup
# start all tasks # start all tasks
nginx & nginx &
celery -A task.celery worker --loglevel=INFO --max-tasks-per-child 10 & celery -A task.celery worker \
--loglevel=INFO \
--concurrency 4 \
--max-tasks-per-child 5 \
--max-memory-per-child 150000 &
celery -A task beat --loglevel=INFO \ celery -A task beat --loglevel=INFO \
--scheduler django_celery_beat.schedulers:DatabaseScheduler & --scheduler django_celery_beat.schedulers:DatabaseScheduler &
python backend_start.py python backend_start.py