mirror of
https://git.vectorsigma.ru/public/tubearchivist.git
synced 2026-08-04 21:39:49 +00:00
renamed django app folder to backend
This commit is contained in:
Simon
0
backend/download/__init__.py
Normal file
0
backend/download/__init__.py
Normal file
0
backend/download/migrations/__init__.py
Normal file
0
backend/download/migrations/__init__.py
Normal file
0
backend/download/src/__init__.py
Normal file
0
backend/download/src/__init__.py
Normal file
370
backend/download/src/queue.py
Normal file
370
backend/download/src/queue.py
Normal file
@@ -0,0 +1,370 @@
|
||||
"""
|
||||
Functionality:
|
||||
- handle download queue
|
||||
- linked with ta_dowload index
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import get_duration_str, is_shorts
|
||||
from download.src.subscriptions import ChannelSubscription
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
from video.src.constants import VideoTypeEnum
|
||||
|
||||
|
||||
class PendingIndex:
|
||||
"""base class holding all export methods"""
|
||||
|
||||
def __init__(self):
|
||||
self.all_pending = False
|
||||
self.all_ignored = False
|
||||
self.all_videos = False
|
||||
self.all_channels = False
|
||||
self.channel_overwrites = False
|
||||
self.video_overwrites = False
|
||||
self.to_skip = False
|
||||
|
||||
def get_download(self):
|
||||
"""get a list of all pending videos in ta_download"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"sort": [{"timestamp": {"order": "asc"}}],
|
||||
}
|
||||
all_results = IndexPaginate("ta_download", data).get_results()
|
||||
|
||||
self.all_pending = []
|
||||
self.all_ignored = []
|
||||
self.to_skip = []
|
||||
|
||||
for result in all_results:
|
||||
self.to_skip.append(result["youtube_id"])
|
||||
if result["status"] == "pending":
|
||||
self.all_pending.append(result)
|
||||
elif result["status"] == "ignore":
|
||||
self.all_ignored.append(result)
|
||||
|
||||
def get_indexed(self):
|
||||
"""get a list of all videos indexed"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"sort": [{"published": {"order": "desc"}}],
|
||||
}
|
||||
self.all_videos = IndexPaginate("ta_video", data).get_results()
|
||||
for video in self.all_videos:
|
||||
self.to_skip.append(video["youtube_id"])
|
||||
|
||||
def get_channels(self):
|
||||
"""get a list of all channels indexed"""
|
||||
self.all_channels = []
|
||||
self.channel_overwrites = {}
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"sort": [{"channel_id": {"order": "asc"}}],
|
||||
}
|
||||
channels = IndexPaginate("ta_channel", data).get_results()
|
||||
|
||||
for channel in channels:
|
||||
channel_id = channel["channel_id"]
|
||||
self.all_channels.append(channel_id)
|
||||
if channel.get("channel_overwrites"):
|
||||
self.channel_overwrites.update(
|
||||
{channel_id: channel.get("channel_overwrites")}
|
||||
)
|
||||
|
||||
self._map_overwrites()
|
||||
|
||||
def _map_overwrites(self):
|
||||
"""map video ids to channel ids overwrites"""
|
||||
self.video_overwrites = {}
|
||||
for video in self.all_pending:
|
||||
video_id = video["youtube_id"]
|
||||
channel_id = video["channel_id"]
|
||||
overwrites = self.channel_overwrites.get(channel_id, False)
|
||||
if overwrites:
|
||||
self.video_overwrites.update({video_id: overwrites})
|
||||
|
||||
|
||||
class PendingInteract:
|
||||
"""interact with items in download queue"""
|
||||
|
||||
def __init__(self, youtube_id=False, status=False):
|
||||
self.youtube_id = youtube_id
|
||||
self.status = status
|
||||
|
||||
def delete_item(self):
|
||||
"""delete single item from pending"""
|
||||
path = f"ta_download/_doc/{self.youtube_id}"
|
||||
_, _ = ElasticWrap(path).delete(refresh=True)
|
||||
|
||||
def delete_by_status(self):
|
||||
"""delete all matching item by status"""
|
||||
data = {"query": {"term": {"status": {"value": self.status}}}}
|
||||
path = "ta_download/_delete_by_query"
|
||||
_, _ = ElasticWrap(path).post(data=data)
|
||||
|
||||
def update_status(self):
|
||||
"""update status of pending item"""
|
||||
if self.status == "priority":
|
||||
data = {
|
||||
"doc": {
|
||||
"status": "pending",
|
||||
"auto_start": True,
|
||||
"message": None,
|
||||
}
|
||||
}
|
||||
else:
|
||||
data = {"doc": {"status": self.status}}
|
||||
|
||||
path = f"ta_download/_update/{self.youtube_id}/?refresh=true"
|
||||
_, _ = ElasticWrap(path).post(data=data)
|
||||
|
||||
def get_item(self):
|
||||
"""return pending item dict"""
|
||||
path = f"ta_download/_doc/{self.youtube_id}"
|
||||
response, status_code = ElasticWrap(path).get()
|
||||
return response["_source"], status_code
|
||||
|
||||
def get_channel(self):
|
||||
"""
|
||||
get channel metadata from queue to not depend on channel to be indexed
|
||||
"""
|
||||
data = {
|
||||
"size": 1,
|
||||
"query": {"term": {"channel_id": {"value": self.youtube_id}}},
|
||||
}
|
||||
response, _ = ElasticWrap("ta_download/_search").get(data=data)
|
||||
hits = response["hits"]["hits"]
|
||||
if not hits:
|
||||
channel_name = "NA"
|
||||
else:
|
||||
channel_name = hits[0]["_source"].get("channel_name", "NA")
|
||||
|
||||
return {
|
||||
"channel_id": self.youtube_id,
|
||||
"channel_name": channel_name,
|
||||
}
|
||||
|
||||
|
||||
class PendingList(PendingIndex):
|
||||
"""manage the pending videos list"""
|
||||
|
||||
yt_obs = {
|
||||
"noplaylist": True,
|
||||
"writethumbnail": True,
|
||||
"simulate": True,
|
||||
"check_formats": None,
|
||||
}
|
||||
|
||||
def __init__(self, youtube_ids=False, task=False):
|
||||
super().__init__()
|
||||
self.config = AppConfig().config
|
||||
self.youtube_ids = youtube_ids
|
||||
self.task = task
|
||||
self.to_skip = False
|
||||
self.missing_videos = False
|
||||
|
||||
def parse_url_list(self):
|
||||
"""extract youtube ids from list"""
|
||||
self.missing_videos = []
|
||||
self.get_download()
|
||||
self.get_indexed()
|
||||
total = len(self.youtube_ids)
|
||||
for idx, entry in enumerate(self.youtube_ids):
|
||||
self._process_entry(entry)
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
self.task.send_progress(
|
||||
message_lines=[f"Extracting items {idx + 1}/{total}"],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
|
||||
def _process_entry(self, entry):
|
||||
"""process single entry from url list"""
|
||||
vid_type = self._get_vid_type(entry)
|
||||
if entry["type"] == "video":
|
||||
self._add_video(entry["url"], vid_type)
|
||||
elif entry["type"] == "channel":
|
||||
self._parse_channel(entry["url"], vid_type)
|
||||
elif entry["type"] == "playlist":
|
||||
self._parse_playlist(entry["url"])
|
||||
else:
|
||||
raise ValueError(f"invalid url_type: {entry}")
|
||||
|
||||
@staticmethod
|
||||
def _get_vid_type(entry):
|
||||
"""add vid type enum if available"""
|
||||
vid_type_str = entry.get("vid_type")
|
||||
if not vid_type_str:
|
||||
return VideoTypeEnum.UNKNOWN
|
||||
|
||||
return VideoTypeEnum(vid_type_str)
|
||||
|
||||
def _add_video(self, url, vid_type):
|
||||
"""add video to list"""
|
||||
if url not in self.missing_videos and url not in self.to_skip:
|
||||
self.missing_videos.append((url, vid_type))
|
||||
else:
|
||||
print(f"{url}: skipped adding already indexed video to download.")
|
||||
|
||||
def _parse_channel(self, url, vid_type):
|
||||
"""add all videos of channel to list"""
|
||||
video_results = ChannelSubscription().get_last_youtube_videos(
|
||||
url, limit=False, query_filter=vid_type
|
||||
)
|
||||
for video_id, _, vid_type in video_results:
|
||||
self._add_video(video_id, vid_type)
|
||||
|
||||
def _parse_playlist(self, url):
|
||||
"""add all videos of playlist to list"""
|
||||
playlist = YoutubePlaylist(url)
|
||||
is_active = playlist.update_playlist()
|
||||
if not is_active:
|
||||
message = f"{playlist.youtube_id}: failed to extract metadata"
|
||||
print(message)
|
||||
raise ValueError(message)
|
||||
|
||||
entries = playlist.json_data["playlist_entries"]
|
||||
to_add = [i["youtube_id"] for i in entries if not i["downloaded"]]
|
||||
if not to_add:
|
||||
return
|
||||
|
||||
for video_id in to_add:
|
||||
# match vid_type later
|
||||
self._add_video(video_id, VideoTypeEnum.UNKNOWN)
|
||||
|
||||
def add_to_pending(self, status="pending", auto_start=False):
|
||||
"""add missing videos to pending list"""
|
||||
self.get_channels()
|
||||
bulk_list = []
|
||||
|
||||
total = len(self.missing_videos)
|
||||
videos_added = []
|
||||
for idx, (youtube_id, vid_type) in enumerate(self.missing_videos):
|
||||
if self.task and self.task.is_stopped():
|
||||
break
|
||||
|
||||
print(f"{youtube_id}: [{idx + 1}/{total}]: add to queue")
|
||||
self._notify_add(idx, total)
|
||||
video_details = self.get_youtube_details(youtube_id, vid_type)
|
||||
if not video_details:
|
||||
continue
|
||||
|
||||
video_details.update(
|
||||
{
|
||||
"status": status,
|
||||
"auto_start": auto_start,
|
||||
}
|
||||
)
|
||||
|
||||
action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
|
||||
bulk_list.append(json.dumps(action))
|
||||
bulk_list.append(json.dumps(video_details))
|
||||
|
||||
url = video_details["vid_thumb_url"]
|
||||
ThumbManager(youtube_id).download_video_thumb(url)
|
||||
videos_added.append(youtube_id)
|
||||
|
||||
if len(bulk_list) >= 20:
|
||||
self._ingest_bulk(bulk_list)
|
||||
bulk_list = []
|
||||
|
||||
self._ingest_bulk(bulk_list)
|
||||
|
||||
return videos_added
|
||||
|
||||
def _ingest_bulk(self, bulk_list):
|
||||
"""add items to queue in bulk"""
|
||||
if not bulk_list:
|
||||
return
|
||||
|
||||
# add last newline
|
||||
bulk_list.append("\n")
|
||||
query_str = "\n".join(bulk_list)
|
||||
_, _ = ElasticWrap("_bulk?refresh=true").post(query_str, ndjson=True)
|
||||
|
||||
def _notify_add(self, idx, total):
|
||||
"""send notification for adding videos to download queue"""
|
||||
if not self.task:
|
||||
return
|
||||
|
||||
self.task.send_progress(
|
||||
message_lines=[
|
||||
"Adding new videos to download queue.",
|
||||
f"Extracting items {idx + 1}/{total}",
|
||||
],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
|
||||
def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEOS):
|
||||
"""get details from youtubedl for single pending video"""
|
||||
vid = YtWrap(self.yt_obs, self.config).extract(youtube_id)
|
||||
if not vid:
|
||||
return False
|
||||
|
||||
if vid.get("id") != youtube_id:
|
||||
# skip premium videos with different id
|
||||
print(f"{youtube_id}: skipping premium video, id not matching")
|
||||
return False
|
||||
# stop if video is streaming live now
|
||||
if vid["live_status"] in ["is_upcoming", "is_live"]:
|
||||
print(f"{youtube_id}: skip is_upcoming or is_live")
|
||||
return False
|
||||
|
||||
if vid["live_status"] == "was_live":
|
||||
vid_type = VideoTypeEnum.STREAMS
|
||||
else:
|
||||
if self._check_shorts(vid):
|
||||
vid_type = VideoTypeEnum.SHORTS
|
||||
else:
|
||||
vid_type = VideoTypeEnum.VIDEOS
|
||||
|
||||
if not vid.get("channel"):
|
||||
print(f"{youtube_id}: skip video not part of channel")
|
||||
return False
|
||||
|
||||
return self._parse_youtube_details(vid, vid_type)
|
||||
|
||||
@staticmethod
|
||||
def _check_shorts(vid):
|
||||
"""check if vid is shorts video"""
|
||||
if vid["width"] > vid["height"]:
|
||||
return False
|
||||
|
||||
duration = vid.get("duration")
|
||||
if duration and isinstance(duration, int):
|
||||
if duration > 60:
|
||||
return False
|
||||
|
||||
return is_shorts(vid["id"])
|
||||
|
||||
def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS):
|
||||
"""parse response"""
|
||||
vid_id = vid.get("id")
|
||||
published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
|
||||
"%Y-%m-%d"
|
||||
)
|
||||
|
||||
# build dict
|
||||
youtube_details = {
|
||||
"youtube_id": vid_id,
|
||||
"channel_name": vid["channel"],
|
||||
"vid_thumb_url": vid["thumbnail"],
|
||||
"title": vid["title"],
|
||||
"channel_id": vid["channel_id"],
|
||||
"duration": get_duration_str(vid["duration"]),
|
||||
"published": published,
|
||||
"timestamp": int(datetime.now().timestamp()),
|
||||
# Pulling enum value out so it is serializable
|
||||
"vid_type": vid_type.value,
|
||||
}
|
||||
if self.all_channels:
|
||||
youtube_details.update(
|
||||
{"channel_indexed": vid["channel_id"] in self.all_channels}
|
||||
)
|
||||
return youtube_details
|
||||
436
backend/download/src/subscriptions.py
Normal file
436
backend/download/src/subscriptions.py
Normal file
@@ -0,0 +1,436 @@
|
||||
"""
|
||||
Functionality:
|
||||
- handle channel subscriptions
|
||||
- handle playlist subscriptions
|
||||
"""
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from channel.src.index import YoutubeChannel
|
||||
from common.src.es_connect import IndexPaginate
|
||||
from common.src.helper import is_missing
|
||||
from common.src.urlparser import Parser
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
from video.src.constants import VideoTypeEnum
|
||||
from video.src.index import YoutubeVideo
|
||||
|
||||
|
||||
class ChannelSubscription:
|
||||
"""manage the list of channels subscribed"""
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.config = AppConfig().config
|
||||
self.task = task
|
||||
|
||||
@staticmethod
|
||||
def get_channels(subscribed_only=True):
|
||||
"""get a list of all channels subscribed to"""
|
||||
data = {
|
||||
"sort": [{"channel_name.keyword": {"order": "asc"}}],
|
||||
}
|
||||
if subscribed_only:
|
||||
data["query"] = {"term": {"channel_subscribed": {"value": True}}}
|
||||
else:
|
||||
data["query"] = {"match_all": {}}
|
||||
|
||||
all_channels = IndexPaginate("ta_channel", data).get_results()
|
||||
|
||||
return all_channels
|
||||
|
||||
def get_last_youtube_videos(
|
||||
self,
|
||||
channel_id,
|
||||
limit=True,
|
||||
query_filter=None,
|
||||
channel_overwrites=None,
|
||||
):
|
||||
"""get a list of last videos from channel"""
|
||||
query_handler = VideoQueryBuilder(self.config, channel_overwrites)
|
||||
queries = query_handler.build_queries(query_filter)
|
||||
last_videos = []
|
||||
|
||||
for vid_type_enum, limit_amount in queries:
|
||||
obs = {
|
||||
"skip_download": True,
|
||||
"extract_flat": True,
|
||||
}
|
||||
vid_type = vid_type_enum.value
|
||||
|
||||
if limit:
|
||||
obs["playlistend"] = limit_amount
|
||||
|
||||
url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
|
||||
channel_query = YtWrap(obs, self.config).extract(url)
|
||||
if not channel_query:
|
||||
continue
|
||||
|
||||
last_videos.extend(
|
||||
[
|
||||
(i["id"], i["title"], vid_type)
|
||||
for i in channel_query["entries"]
|
||||
]
|
||||
)
|
||||
|
||||
return last_videos
|
||||
|
||||
def find_missing(self):
|
||||
"""add missing videos from subscribed channels to pending"""
|
||||
all_channels = self.get_channels()
|
||||
if not all_channels:
|
||||
return False
|
||||
|
||||
missing_videos = []
|
||||
|
||||
total = len(all_channels)
|
||||
for idx, channel in enumerate(all_channels):
|
||||
channel_id = channel["channel_id"]
|
||||
print(f"{channel_id}: find missing videos.")
|
||||
last_videos = self.get_last_youtube_videos(
|
||||
channel_id,
|
||||
channel_overwrites=channel.get("channel_overwrites"),
|
||||
)
|
||||
|
||||
if last_videos:
|
||||
ids_to_add = is_missing([i[0] for i in last_videos])
|
||||
for video_id, _, vid_type in last_videos:
|
||||
if video_id in ids_to_add:
|
||||
missing_videos.append((video_id, vid_type))
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
if self.task.is_stopped():
|
||||
self.task.send_progress(["Received Stop signal."])
|
||||
break
|
||||
|
||||
self.task.send_progress(
|
||||
message_lines=[f"Scanning Channel {idx + 1}/{total}"],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
|
||||
return missing_videos
|
||||
|
||||
@staticmethod
|
||||
def change_subscribe(channel_id, channel_subscribed):
|
||||
"""subscribe or unsubscribe from channel and update"""
|
||||
channel = YoutubeChannel(channel_id)
|
||||
channel.build_json()
|
||||
channel.json_data["channel_subscribed"] = channel_subscribed
|
||||
channel.upload_to_es()
|
||||
channel.sync_to_videos()
|
||||
|
||||
|
||||
class VideoQueryBuilder:
|
||||
"""Build queries for yt-dlp."""
|
||||
|
||||
def __init__(self, config: dict, channel_overwrites: dict | None = None):
|
||||
self.config = config
|
||||
self.channel_overwrites = channel_overwrites or {}
|
||||
|
||||
def build_queries(
|
||||
self, video_type: VideoTypeEnum | None, limit: bool = True
|
||||
) -> list[tuple[VideoTypeEnum, int | None]]:
|
||||
"""Build queries for all or specific video type."""
|
||||
query_methods = {
|
||||
VideoTypeEnum.VIDEOS: self.videos_query,
|
||||
VideoTypeEnum.STREAMS: self.streams_query,
|
||||
VideoTypeEnum.SHORTS: self.shorts_query,
|
||||
}
|
||||
|
||||
if video_type:
|
||||
# build query for specific type
|
||||
query_method = query_methods.get(video_type)
|
||||
if query_method:
|
||||
query = query_method(limit)
|
||||
if query[1] != 0:
|
||||
return [query]
|
||||
return []
|
||||
|
||||
# Build and return queries for all video types
|
||||
queries = []
|
||||
for build_query in query_methods.values():
|
||||
query = build_query(limit)
|
||||
if query[1] != 0:
|
||||
queries.append(query)
|
||||
|
||||
return queries
|
||||
|
||||
def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
|
||||
"""Build query for videos."""
|
||||
return self._build_generic_query(
|
||||
video_type=VideoTypeEnum.VIDEOS,
|
||||
overwrite_key="subscriptions_channel_size",
|
||||
config_key="channel_size",
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
|
||||
"""Build query for streams."""
|
||||
return self._build_generic_query(
|
||||
video_type=VideoTypeEnum.STREAMS,
|
||||
overwrite_key="subscriptions_live_channel_size",
|
||||
config_key="live_channel_size",
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
|
||||
"""Build query for shorts."""
|
||||
return self._build_generic_query(
|
||||
video_type=VideoTypeEnum.SHORTS,
|
||||
overwrite_key="subscriptions_shorts_channel_size",
|
||||
config_key="shorts_channel_size",
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def _build_generic_query(
|
||||
self,
|
||||
video_type: VideoTypeEnum,
|
||||
overwrite_key: str,
|
||||
config_key: str,
|
||||
limit: bool,
|
||||
) -> tuple[VideoTypeEnum, int | None]:
|
||||
"""Generic query for video page scraping."""
|
||||
if not limit:
|
||||
return (video_type, None)
|
||||
|
||||
if (
|
||||
overwrite_key in self.channel_overwrites
|
||||
and self.channel_overwrites[overwrite_key] is not None
|
||||
):
|
||||
overwrite = self.channel_overwrites[overwrite_key]
|
||||
return (video_type, overwrite)
|
||||
|
||||
if overwrite := self.config["subscriptions"].get(config_key):
|
||||
return (video_type, overwrite)
|
||||
|
||||
return (video_type, 0)
|
||||
|
||||
|
||||
class PlaylistSubscription:
|
||||
"""manage the playlist download functionality"""
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.config = AppConfig().config
|
||||
self.task = task
|
||||
|
||||
@staticmethod
|
||||
def get_playlists(subscribed_only=True):
|
||||
"""get a list of all active playlists"""
|
||||
data = {
|
||||
"sort": [{"playlist_channel.keyword": {"order": "desc"}}],
|
||||
}
|
||||
data["query"] = {
|
||||
"bool": {"must": [{"term": {"playlist_active": {"value": True}}}]}
|
||||
}
|
||||
if subscribed_only:
|
||||
data["query"]["bool"]["must"].append(
|
||||
{"term": {"playlist_subscribed": {"value": True}}}
|
||||
)
|
||||
|
||||
all_playlists = IndexPaginate("ta_playlist", data).get_results()
|
||||
|
||||
return all_playlists
|
||||
|
||||
def process_url_str(self, new_playlists, subscribed=True):
|
||||
"""process playlist subscribe form url_str"""
|
||||
for idx, playlist in enumerate(new_playlists):
|
||||
playlist_id = playlist["url"]
|
||||
if not playlist["type"] == "playlist":
|
||||
print(f"{playlist_id} not a playlist, skipping...")
|
||||
continue
|
||||
|
||||
playlist_h = YoutubePlaylist(playlist_id)
|
||||
playlist_h.build_json()
|
||||
if not playlist_h.json_data:
|
||||
message = f"{playlist_h.youtube_id}: failed to extract data"
|
||||
print(message)
|
||||
raise ValueError(message)
|
||||
|
||||
playlist_h.json_data["playlist_subscribed"] = subscribed
|
||||
playlist_h.upload_to_es()
|
||||
playlist_h.add_vids_to_playlist()
|
||||
self.channel_validate(playlist_h.json_data["playlist_channel_id"])
|
||||
|
||||
url = playlist_h.json_data["playlist_thumbnail"]
|
||||
thumb = ThumbManager(playlist_id, item_type="playlist")
|
||||
thumb.download_playlist_thumb(url)
|
||||
|
||||
if self.task:
|
||||
self.task.send_progress(
|
||||
message_lines=[
|
||||
f"Processing {idx + 1} of {len(new_playlists)}"
|
||||
],
|
||||
progress=(idx + 1) / len(new_playlists),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def channel_validate(channel_id):
|
||||
"""make sure channel of playlist is there"""
|
||||
channel = YoutubeChannel(channel_id)
|
||||
channel.build_json(upload=True)
|
||||
|
||||
@staticmethod
|
||||
def change_subscribe(playlist_id, subscribe_status):
|
||||
"""change the subscribe status of a playlist"""
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
playlist.build_json()
|
||||
playlist.json_data["playlist_subscribed"] = subscribe_status
|
||||
playlist.upload_to_es()
|
||||
|
||||
def find_missing(self):
|
||||
"""find videos in subscribed playlists not downloaded yet"""
|
||||
all_playlists = [i["playlist_id"] for i in self.get_playlists()]
|
||||
if not all_playlists:
|
||||
return False
|
||||
|
||||
missing_videos = []
|
||||
total = len(all_playlists)
|
||||
for idx, playlist_id in enumerate(all_playlists):
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
is_active = playlist.update_playlist()
|
||||
if not is_active:
|
||||
playlist.deactivate()
|
||||
continue
|
||||
|
||||
playlist_entries = playlist.json_data["playlist_entries"]
|
||||
size_limit = self.config["subscriptions"]["channel_size"]
|
||||
if size_limit:
|
||||
del playlist_entries[size_limit:]
|
||||
|
||||
to_check = [
|
||||
i["youtube_id"]
|
||||
for i in playlist_entries
|
||||
if i["downloaded"] is False
|
||||
]
|
||||
needs_downloading = is_missing(to_check)
|
||||
missing_videos.extend(needs_downloading)
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
if self.task.is_stopped():
|
||||
self.task.send_progress(["Received Stop signal."])
|
||||
break
|
||||
|
||||
self.task.send_progress(
|
||||
message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
|
||||
return missing_videos
|
||||
|
||||
|
||||
class SubscriptionScanner:
|
||||
"""add missing videos to queue"""
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.task = task
|
||||
self.missing_videos = False
|
||||
self.auto_start = AppConfig().config["subscriptions"].get("auto_start")
|
||||
|
||||
def scan(self):
|
||||
"""scan channels and playlists"""
|
||||
if self.task:
|
||||
self.task.send_progress(["Rescanning channels and playlists."])
|
||||
|
||||
self.missing_videos = []
|
||||
self.scan_channels()
|
||||
if self.task and not self.task.is_stopped():
|
||||
self.scan_playlists()
|
||||
|
||||
return self.missing_videos
|
||||
|
||||
def scan_channels(self):
|
||||
"""get missing from channels"""
|
||||
channel_handler = ChannelSubscription(task=self.task)
|
||||
missing = channel_handler.find_missing()
|
||||
if not missing:
|
||||
return
|
||||
|
||||
for vid_id, vid_type in missing:
|
||||
self.missing_videos.append(
|
||||
{"type": "video", "vid_type": vid_type, "url": vid_id}
|
||||
)
|
||||
|
||||
def scan_playlists(self):
|
||||
"""get missing from playlists"""
|
||||
playlist_handler = PlaylistSubscription(task=self.task)
|
||||
missing = playlist_handler.find_missing()
|
||||
if not missing:
|
||||
return
|
||||
|
||||
for i in missing:
|
||||
self.missing_videos.append(
|
||||
{
|
||||
"type": "video",
|
||||
"vid_type": VideoTypeEnum.VIDEOS.value,
|
||||
"url": i,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class SubscriptionHandler:
|
||||
"""subscribe to channels and playlists from url_str"""
|
||||
|
||||
def __init__(self, url_str, task=False):
|
||||
self.url_str = url_str
|
||||
self.task = task
|
||||
self.to_subscribe = False
|
||||
|
||||
def subscribe(self, expected_type=False):
|
||||
"""subscribe to url_str items"""
|
||||
if self.task:
|
||||
self.task.send_progress(["Processing form content."])
|
||||
self.to_subscribe = Parser(self.url_str).parse()
|
||||
|
||||
total = len(self.to_subscribe)
|
||||
for idx, item in enumerate(self.to_subscribe):
|
||||
if self.task:
|
||||
self._notify(idx, item, total)
|
||||
|
||||
self.subscribe_type(item, expected_type=expected_type)
|
||||
|
||||
def subscribe_type(self, item, expected_type):
|
||||
"""process single item"""
|
||||
if item["type"] == "playlist":
|
||||
if expected_type and expected_type != "playlist":
|
||||
raise TypeError(
|
||||
f"expected {expected_type} url but got {item.get('type')}"
|
||||
)
|
||||
|
||||
PlaylistSubscription().process_url_str([item])
|
||||
return
|
||||
|
||||
if item["type"] == "video":
|
||||
# extract channel id from video
|
||||
video = YoutubeVideo(item["url"])
|
||||
video.get_from_youtube()
|
||||
video.process_youtube_meta()
|
||||
channel_id = video.channel_id
|
||||
elif item["type"] == "channel":
|
||||
channel_id = item["url"]
|
||||
else:
|
||||
raise ValueError("failed to subscribe to: " + item["url"])
|
||||
|
||||
if expected_type and expected_type != "channel":
|
||||
raise TypeError(
|
||||
f"expected {expected_type} url but got {item.get('type')}"
|
||||
)
|
||||
|
||||
self._subscribe(channel_id)
|
||||
|
||||
def _subscribe(self, channel_id):
|
||||
"""subscribe to channel"""
|
||||
ChannelSubscription().change_subscribe(
|
||||
channel_id, channel_subscribed=True
|
||||
)
|
||||
|
||||
def _notify(self, idx, item, total):
|
||||
"""send notification message to redis"""
|
||||
subscribe_type = item["type"].title()
|
||||
message_lines = [
|
||||
f"Subscribe to {subscribe_type}",
|
||||
f"Progress: {idx + 1}/{total}",
|
||||
]
|
||||
self.task.send_progress(message_lines, progress=(idx + 1) / total)
|
||||
505
backend/download/src/thumbnails.py
Normal file
505
backend/download/src/thumbnails.py
Normal file
@@ -0,0 +1,505 @@
|
||||
"""
|
||||
functionality:
|
||||
- handle download and caching for thumbnails
|
||||
- check for missing thumbnails
|
||||
"""
|
||||
|
||||
import base64
|
||||
import os
|
||||
from io import BytesIO
|
||||
from time import sleep
|
||||
|
||||
import requests
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import is_missing
|
||||
from mutagen.mp4 import MP4, MP4Cover
|
||||
from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
|
||||
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = True
|
||||
|
||||
|
||||
class ThumbManagerBase:
|
||||
"""base class for thumbnail management"""
|
||||
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
|
||||
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
|
||||
PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
|
||||
|
||||
def __init__(self, item_id, item_type, fallback=False):
|
||||
self.item_id = item_id
|
||||
self.item_type = item_type
|
||||
self.fallback = fallback
|
||||
|
||||
def download_raw(self, url):
|
||||
"""download thumbnail for video"""
|
||||
if not url:
|
||||
return self.get_fallback()
|
||||
|
||||
for i in range(3):
|
||||
try:
|
||||
response = requests.get(url, stream=True, timeout=5)
|
||||
if response.ok:
|
||||
try:
|
||||
img = Image.open(response.raw)
|
||||
if isinstance(img, Image.Image):
|
||||
return img
|
||||
return self.get_fallback()
|
||||
|
||||
except (UnidentifiedImageError, OSError):
|
||||
print(f"failed to open thumbnail: {url}")
|
||||
return self.get_fallback()
|
||||
|
||||
if response.status_code == 404:
|
||||
return self.get_fallback()
|
||||
|
||||
except (
|
||||
requests.exceptions.RequestException,
|
||||
requests.exceptions.ReadTimeout,
|
||||
):
|
||||
print(f"{self.item_id}: retry thumbnail download {url}")
|
||||
sleep((i + 1) ** i)
|
||||
|
||||
return self.get_fallback()
|
||||
|
||||
def get_fallback(self):
|
||||
"""get fallback thumbnail if not available"""
|
||||
print(f"{self.item_id}: failed to extract thumbnail, use fallback")
|
||||
if self.fallback:
|
||||
img_raw = Image.open(self.fallback)
|
||||
return img_raw
|
||||
|
||||
app_root = EnvironmentSettings.APP_DIR
|
||||
default_map = {
|
||||
"video": os.path.join(
|
||||
app_root, "static/img/default-video-thumb.jpg"
|
||||
),
|
||||
"playlist": os.path.join(
|
||||
app_root, "static/img/default-playlist-thumb.jpg"
|
||||
),
|
||||
"icon": os.path.join(
|
||||
app_root, "static/img/default-channel-icon.jpg"
|
||||
),
|
||||
"banner": os.path.join(
|
||||
app_root, "static/img/default-channel-banner.jpg"
|
||||
),
|
||||
"tvart": os.path.join(
|
||||
app_root, "static/img/default-channel-art.jpg"
|
||||
),
|
||||
}
|
||||
|
||||
img_raw = Image.open(default_map[self.item_type])
|
||||
|
||||
return img_raw
|
||||
|
||||
|
||||
class ThumbManager(ThumbManagerBase):
|
||||
"""handle thumbnails related functions"""
|
||||
|
||||
def __init__(self, item_id, item_type="video", fallback=False):
|
||||
super().__init__(item_id, item_type, fallback=fallback)
|
||||
|
||||
def download(self, url):
|
||||
"""download thumbnail"""
|
||||
print(f"{self.item_id}: download {self.item_type} thumbnail")
|
||||
if self.item_type == "video":
|
||||
self.download_video_thumb(url)
|
||||
elif self.item_type == "channel":
|
||||
self.download_channel_art(url)
|
||||
elif self.item_type == "playlist":
|
||||
self.download_playlist_thumb(url)
|
||||
|
||||
def delete(self):
|
||||
"""delete thumbnail file"""
|
||||
print(f"{self.item_id}: delete {self.item_type} thumbnail")
|
||||
if self.item_type == "video":
|
||||
self.delete_video_thumb()
|
||||
elif self.item_type == "channel":
|
||||
self.delete_channel_thumb()
|
||||
elif self.item_type == "playlist":
|
||||
self.delete_playlist_thumb()
|
||||
|
||||
def download_video_thumb(self, url, skip_existing=False):
|
||||
"""pass url for video thumbnail"""
|
||||
folder_path = os.path.join(self.VIDEO_DIR, self.item_id[0].lower())
|
||||
thumb_path = self.vid_thumb_path(absolute=True)
|
||||
|
||||
if skip_existing and os.path.exists(thumb_path):
|
||||
return
|
||||
|
||||
os.makedirs(folder_path, exist_ok=True)
|
||||
img_raw = self.download_raw(url)
|
||||
width, height = img_raw.size
|
||||
|
||||
if not width / height == 16 / 9:
|
||||
new_height = width / 16 * 9
|
||||
offset = (height - new_height) / 2
|
||||
img_raw = img_raw.crop((0, offset, width, height - offset))
|
||||
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
def vid_thumb_path(self, absolute=False, create_folder=False):
|
||||
"""build expected path for video thumbnail from youtube_id"""
|
||||
folder_name = self.item_id[0].lower()
|
||||
folder_path = os.path.join("videos", folder_name)
|
||||
thumb_path = os.path.join(folder_path, f"{self.item_id}.jpg")
|
||||
if absolute:
|
||||
thumb_path = os.path.join(self.CACHE_DIR, thumb_path)
|
||||
|
||||
if create_folder:
|
||||
folder_path = os.path.join(self.CACHE_DIR, folder_path)
|
||||
os.makedirs(folder_path, exist_ok=True)
|
||||
|
||||
return thumb_path
|
||||
|
||||
def download_channel_art(self, urls, skip_existing=False):
|
||||
"""pass tuple of channel thumbnails"""
|
||||
channel_thumb, channel_banner, channel_tv = urls
|
||||
self._download_channel_thumb(channel_thumb, skip_existing)
|
||||
self._download_channel_banner(channel_banner, skip_existing)
|
||||
self._download_channel_tv(channel_tv, skip_existing)
|
||||
|
||||
def _download_channel_thumb(self, channel_thumb, skip_existing):
|
||||
"""download channel thumbnail"""
|
||||
|
||||
thumb_path = os.path.join(
|
||||
self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg"
|
||||
)
|
||||
self.item_type = "icon"
|
||||
|
||||
if skip_existing and os.path.exists(thumb_path):
|
||||
return
|
||||
|
||||
img_raw = self.download_raw(channel_thumb)
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
def _download_channel_banner(self, channel_banner, skip_existing):
|
||||
"""download channel banner"""
|
||||
|
||||
banner_path = os.path.join(
|
||||
self.CHANNEL_DIR, self.item_id + "_banner.jpg"
|
||||
)
|
||||
self.item_type = "banner"
|
||||
if skip_existing and os.path.exists(banner_path):
|
||||
return
|
||||
|
||||
img_raw = self.download_raw(channel_banner)
|
||||
img_raw.convert("RGB").save(banner_path)
|
||||
|
||||
def _download_channel_tv(self, channel_tv, skip_existing):
|
||||
"""download channel tv art"""
|
||||
art_path = os.path.join(self.CHANNEL_DIR, self.item_id + "_tvart.jpg")
|
||||
self.item_type = "tvart"
|
||||
if skip_existing and os.path.exists(art_path):
|
||||
return
|
||||
|
||||
img_raw = self.download_raw(channel_tv)
|
||||
img_raw.convert("RGB").save(art_path)
|
||||
|
||||
def download_playlist_thumb(self, url, skip_existing=False):
|
||||
"""pass thumbnail url"""
|
||||
thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
|
||||
if skip_existing and os.path.exists(thumb_path):
|
||||
return
|
||||
|
||||
img_raw = (
|
||||
self.download_raw(url)
|
||||
if not isinstance(url, str) or url.startswith("http")
|
||||
else Image.open(os.path.join(self.CACHE_DIR, url))
|
||||
)
|
||||
width, height = img_raw.size
|
||||
|
||||
if not width / height == 16 / 9:
|
||||
new_height = width / 16 * 9
|
||||
offset = (height - new_height) / 2
|
||||
img_raw = img_raw.crop((0, offset, width, height - offset))
|
||||
img_raw = img_raw.resize((336, 189))
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
def delete_video_thumb(self):
|
||||
"""delete video thumbnail if exists"""
|
||||
thumb_path = self.vid_thumb_path()
|
||||
to_delete = os.path.join(self.CACHE_DIR, thumb_path)
|
||||
if os.path.exists(to_delete):
|
||||
os.remove(to_delete)
|
||||
|
||||
def delete_channel_thumb(self):
|
||||
"""delete all artwork of channel"""
|
||||
thumb = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg")
|
||||
banner = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_banner.jpg")
|
||||
if os.path.exists(thumb):
|
||||
os.remove(thumb)
|
||||
if os.path.exists(banner):
|
||||
os.remove(banner)
|
||||
|
||||
def delete_playlist_thumb(self):
|
||||
"""delete playlist thumbnail"""
|
||||
thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
|
||||
if os.path.exists(thumb_path):
|
||||
os.remove(thumb_path)
|
||||
|
||||
def get_vid_base64_blur(self):
|
||||
"""return base64 encoded placeholder"""
|
||||
file_path = os.path.join(self.CACHE_DIR, self.vid_thumb_path())
|
||||
img_raw = Image.open(file_path)
|
||||
img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20))
|
||||
img_blur = img_raw.filter(ImageFilter.BLUR)
|
||||
buffer = BytesIO()
|
||||
img_blur.save(buffer, format="JPEG")
|
||||
img_data = buffer.getvalue()
|
||||
img_base64 = base64.b64encode(img_data).decode()
|
||||
data_url = f"data:image/jpg;base64,{img_base64}"
|
||||
|
||||
return data_url
|
||||
|
||||
|
||||
class ValidatorCallback:
|
||||
"""handle callback validate thumbnails page by page"""
|
||||
|
||||
def __init__(self, source, index_name, counter=0):
|
||||
self.source = source
|
||||
self.index_name = index_name
|
||||
self.counter = counter
|
||||
|
||||
def run(self):
|
||||
"""run the task for page"""
|
||||
print(f"{self.index_name}: validate artwork")
|
||||
if self.index_name == "ta_video":
|
||||
self._validate_videos()
|
||||
elif self.index_name == "ta_channel":
|
||||
self._validate_channels()
|
||||
elif self.index_name == "ta_playlist":
|
||||
self._validate_playlists()
|
||||
|
||||
def _validate_videos(self):
|
||||
"""check if video thumbnails are correct"""
|
||||
for video in self.source:
|
||||
url = video["_source"]["vid_thumb_url"]
|
||||
handler = ThumbManager(video["_source"]["youtube_id"])
|
||||
handler.download_video_thumb(url, skip_existing=True)
|
||||
|
||||
def _validate_channels(self):
|
||||
"""check if all channel artwork is there"""
|
||||
for channel in self.source:
|
||||
urls = (
|
||||
channel["_source"]["channel_thumb_url"],
|
||||
channel["_source"]["channel_banner_url"],
|
||||
channel["_source"].get("channel_tvart_url", False),
|
||||
)
|
||||
handler = ThumbManager(channel["_source"]["channel_id"])
|
||||
handler.download_channel_art(urls, skip_existing=True)
|
||||
|
||||
def _validate_playlists(self):
|
||||
"""check if all playlist artwork is there"""
|
||||
for playlist in self.source:
|
||||
url = playlist["_source"]["playlist_thumbnail"]
|
||||
handler = ThumbManager(playlist["_source"]["playlist_id"])
|
||||
handler.download_playlist_thumb(url, skip_existing=True)
|
||||
|
||||
|
||||
class ThumbValidator:
|
||||
"""validate thumbnails"""
|
||||
|
||||
INDEX = [
|
||||
{
|
||||
"data": {
|
||||
"query": {"term": {"active": {"value": True}}},
|
||||
"_source": ["vid_thumb_url", "youtube_id"],
|
||||
},
|
||||
"name": "ta_video",
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"query": {"term": {"channel_active": {"value": True}}},
|
||||
"_source": {
|
||||
"excludes": ["channel_description", "channel_overwrites"]
|
||||
},
|
||||
},
|
||||
"name": "ta_channel",
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"query": {"term": {"playlist_active": {"value": True}}},
|
||||
"_source": ["playlist_id", "playlist_thumbnail"],
|
||||
},
|
||||
"name": "ta_playlist",
|
||||
},
|
||||
]
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.task = task
|
||||
|
||||
def validate(self):
|
||||
"""validate all indexes"""
|
||||
for index in self.INDEX:
|
||||
total = self._get_total(index["name"])
|
||||
if not total:
|
||||
continue
|
||||
|
||||
paginate = IndexPaginate(
|
||||
index_name=index["name"],
|
||||
data=index["data"],
|
||||
size=1000,
|
||||
callback=ValidatorCallback,
|
||||
task=self.task,
|
||||
total=total,
|
||||
)
|
||||
_ = paginate.get_results()
|
||||
|
||||
def clean_up(self):
|
||||
"""clean up all thumbs"""
|
||||
self._clean_up_vids()
|
||||
self._clean_up_channels()
|
||||
self._clean_up_playlists()
|
||||
|
||||
def _clean_up_vids(self):
|
||||
"""clean unneeded vid thumbs"""
|
||||
video_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "videos")
|
||||
video_folders = os.listdir(video_dir)
|
||||
for video_folder in video_folders:
|
||||
folder_path = os.path.join(video_dir, video_folder)
|
||||
thumbs_is = {i.split(".")[0] for i in os.listdir(folder_path)}
|
||||
thumbs_should = self._get_vid_thumbs_should(video_folder)
|
||||
to_delete = thumbs_is - thumbs_should
|
||||
for thumb in to_delete:
|
||||
delete_path = os.path.join(folder_path, f"{thumb}.jpg")
|
||||
os.remove(delete_path)
|
||||
|
||||
if to_delete:
|
||||
message = (
|
||||
f"[thumbs][video][{video_folder}] "
|
||||
+ f"delete {len(to_delete)} unused thumbnails"
|
||||
)
|
||||
print(message)
|
||||
if self.task:
|
||||
self.task.send_progress([message])
|
||||
|
||||
@staticmethod
|
||||
def _get_vid_thumbs_should(video_folder: str) -> set[str]:
|
||||
"""get indexed"""
|
||||
should_list = [
|
||||
{"prefix": {"youtube_id": {"value": video_folder.lower()}}},
|
||||
{"prefix": {"youtube_id": {"value": video_folder.upper()}}},
|
||||
]
|
||||
data = {
|
||||
"query": {"bool": {"should": should_list}},
|
||||
"_source": ["youtube_id"],
|
||||
}
|
||||
result = IndexPaginate("ta_video,ta_download", data).get_results()
|
||||
thumbs_should = {i["youtube_id"] for i in result}
|
||||
|
||||
return thumbs_should
|
||||
|
||||
def _clean_up_channels(self):
|
||||
"""clean unneeded channel thumbs"""
|
||||
channel_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "channels")
|
||||
channel_art = os.listdir(channel_dir)
|
||||
thumbs_is = {"_".join(i.split("_")[:-1]) for i in channel_art}
|
||||
to_delete = is_missing(list(thumbs_is), "ta_channel", "channel_id")
|
||||
for channel_thumb in channel_art:
|
||||
if channel_thumb[:24] in to_delete:
|
||||
delete_path = os.path.join(channel_dir, channel_thumb)
|
||||
os.remove(delete_path)
|
||||
|
||||
if to_delete:
|
||||
message = (
|
||||
"[thumbs][channel] "
|
||||
+ f"delete {len(to_delete)} unused channel art"
|
||||
)
|
||||
print(message)
|
||||
if self.task:
|
||||
self.task.send_progress([message])
|
||||
|
||||
def _clean_up_playlists(self):
|
||||
"""clean up unneeded playlist thumbs"""
|
||||
playlist_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "playlists")
|
||||
playlist_art = os.listdir(playlist_dir)
|
||||
thumbs_is = {i.split(".")[0] for i in playlist_art}
|
||||
to_delete = is_missing(list(thumbs_is), "ta_playlist", "playlist_id")
|
||||
for playlist_id in to_delete:
|
||||
delete_path = os.path.join(playlist_dir, f"{playlist_id}.jpg")
|
||||
os.remove(delete_path)
|
||||
|
||||
if to_delete:
|
||||
message = (
|
||||
"[thumbs][playlist] "
|
||||
+ f"delete {len(to_delete)} unused playlist art"
|
||||
)
|
||||
print(message)
|
||||
if self.task:
|
||||
self.task.send_progress([message])
|
||||
|
||||
@staticmethod
|
||||
def _get_total(index_name):
|
||||
"""get total documents in index"""
|
||||
path = f"{index_name}/_count"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
|
||||
return response.get("count")
|
||||
|
||||
|
||||
class ThumbFilesystem:
|
||||
"""sync thumbnail files to media files"""
|
||||
|
||||
INDEX_NAME = "ta_video"
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.task = task
|
||||
|
||||
def embed(self):
|
||||
"""entry point"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"_source": ["media_url", "youtube_id"],
|
||||
}
|
||||
paginate = IndexPaginate(
|
||||
index_name=self.INDEX_NAME,
|
||||
data=data,
|
||||
size=200,
|
||||
callback=EmbedCallback,
|
||||
task=self.task,
|
||||
total=self._get_total(),
|
||||
)
|
||||
_ = paginate.get_results()
|
||||
|
||||
def _get_total(self):
|
||||
"""get total documents in index"""
|
||||
path = f"{self.INDEX_NAME}/_count"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
|
||||
return response.get("count")
|
||||
|
||||
|
||||
class EmbedCallback:
|
||||
"""callback class to embed thumbnails"""
|
||||
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
|
||||
FORMAT = MP4Cover.FORMAT_JPEG
|
||||
|
||||
def __init__(self, source, index_name, counter=0):
|
||||
self.source = source
|
||||
self.index_name = index_name
|
||||
self.counter = counter
|
||||
|
||||
def run(self):
|
||||
"""run embed"""
|
||||
for video in self.source:
|
||||
video_id = video["_source"]["youtube_id"]
|
||||
media_url = os.path.join(
|
||||
self.MEDIA_DIR, video["_source"]["media_url"]
|
||||
)
|
||||
thumb_path = os.path.join(
|
||||
self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path()
|
||||
)
|
||||
if os.path.exists(thumb_path):
|
||||
self.embed(media_url, thumb_path)
|
||||
|
||||
def embed(self, media_url, thumb_path):
|
||||
"""embed thumb in single media file"""
|
||||
video = MP4(media_url)
|
||||
with open(thumb_path, "rb") as f:
|
||||
video["covr"] = [MP4Cover(f.read(), imageformat=self.FORMAT)]
|
||||
|
||||
video.save()
|
||||
173
backend/download/src/yt_dlp_base.py
Normal file
173
backend/download/src/yt_dlp_base.py
Normal file
@@ -0,0 +1,173 @@
|
||||
"""
|
||||
functionality:
|
||||
- base class to make all calls to yt-dlp
|
||||
- handle yt-dlp errors
|
||||
"""
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
from http import cookiejar
|
||||
from io import StringIO
|
||||
|
||||
import yt_dlp
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
|
||||
|
||||
class YtWrap:
|
||||
"""wrap calls to yt"""
|
||||
|
||||
OBS_BASE = {
|
||||
"default_search": "ytsearch",
|
||||
"quiet": True,
|
||||
"check_formats": "selected",
|
||||
"socket_timeout": 10,
|
||||
"extractor_retries": 3,
|
||||
"retries": 10,
|
||||
}
|
||||
|
||||
def __init__(self, obs_request, config=False):
|
||||
self.obs_request = obs_request
|
||||
self.config = config
|
||||
self.build_obs()
|
||||
|
||||
def build_obs(self):
|
||||
"""build yt-dlp obs"""
|
||||
self.obs = self.OBS_BASE.copy()
|
||||
self.obs.update(self.obs_request)
|
||||
if self.config:
|
||||
self.add_cookie()
|
||||
|
||||
def add_cookie(self):
|
||||
"""add cookie if enabled"""
|
||||
if self.config["downloads"]["cookie_import"]:
|
||||
cookie_io = CookieHandler(self.config).get()
|
||||
self.obs["cookiefile"] = cookie_io
|
||||
|
||||
def download(self, url):
|
||||
"""make download request"""
|
||||
with yt_dlp.YoutubeDL(self.obs) as ydl:
|
||||
try:
|
||||
ydl.download([url])
|
||||
except yt_dlp.utils.DownloadError as err:
|
||||
print(f"{url}: failed to download with message {err}")
|
||||
if "Temporary failure in name resolution" in str(err):
|
||||
raise ConnectionError("lost the internet, abort!") from err
|
||||
|
||||
return False, str(err)
|
||||
|
||||
return True, True
|
||||
|
||||
def extract(self, url):
|
||||
"""make extract request"""
|
||||
try:
|
||||
response = yt_dlp.YoutubeDL(self.obs).extract_info(url)
|
||||
except cookiejar.LoadError as err:
|
||||
print(f"cookie file is invalid: {err}")
|
||||
return False
|
||||
except yt_dlp.utils.ExtractorError as err:
|
||||
print(f"{url}: failed to extract with message: {err}, continue...")
|
||||
return False
|
||||
except yt_dlp.utils.DownloadError as err:
|
||||
if "This channel does not have a" in str(err):
|
||||
return False
|
||||
|
||||
print(f"{url}: failed to get info from youtube with message {err}")
|
||||
if "Temporary failure in name resolution" in str(err):
|
||||
raise ConnectionError("lost the internet, abort!") from err
|
||||
|
||||
return False
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class CookieHandler:
|
||||
"""handle youtube cookie for yt-dlp"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.cookie_io = False
|
||||
self.config = config
|
||||
self.cache_dir = EnvironmentSettings.CACHE_DIR
|
||||
|
||||
def get(self):
|
||||
"""get cookie io stream"""
|
||||
cookie = RedisArchivist().get_message("cookie")
|
||||
self.cookie_io = StringIO(cookie)
|
||||
return self.cookie_io
|
||||
|
||||
def import_cookie(self):
|
||||
"""import cookie from file"""
|
||||
import_path = os.path.join(
|
||||
self.cache_dir, "import", "cookies.google.txt"
|
||||
)
|
||||
|
||||
try:
|
||||
with open(import_path, encoding="utf-8") as cookie_file:
|
||||
cookie = cookie_file.read()
|
||||
except FileNotFoundError as err:
|
||||
print(f"cookie: {import_path} file not found")
|
||||
raise err
|
||||
|
||||
self.set_cookie(cookie)
|
||||
|
||||
os.remove(import_path)
|
||||
print("cookie: import successful")
|
||||
|
||||
def set_cookie(self, cookie):
|
||||
"""set cookie str and activate in config"""
|
||||
RedisArchivist().set_message("cookie", cookie, save=True)
|
||||
path = ".downloads.cookie_import"
|
||||
RedisArchivist().set_message("config", True, path=path, save=True)
|
||||
self.config["downloads"]["cookie_import"] = True
|
||||
print("cookie: activated and stored in Redis")
|
||||
|
||||
@staticmethod
|
||||
def revoke():
|
||||
"""revoke cookie"""
|
||||
RedisArchivist().del_message("cookie")
|
||||
RedisArchivist().del_message("cookie:valid")
|
||||
RedisArchivist().set_message(
|
||||
"config", False, path=".downloads.cookie_import"
|
||||
)
|
||||
print("cookie: revoked")
|
||||
|
||||
def validate(self):
|
||||
"""validate cookie using the liked videos playlist"""
|
||||
print("validating cookie")
|
||||
obs_request = {
|
||||
"skip_download": True,
|
||||
"extract_flat": True,
|
||||
}
|
||||
validator = YtWrap(obs_request, self.config)
|
||||
response = bool(validator.extract("LL"))
|
||||
self.store_validation(response)
|
||||
|
||||
# update in redis to avoid expiring
|
||||
modified = validator.obs["cookiefile"].getvalue()
|
||||
if modified:
|
||||
RedisArchivist().set_message("cookie", modified)
|
||||
|
||||
if not response:
|
||||
mess_dict = {
|
||||
"status": "message:download",
|
||||
"level": "error",
|
||||
"title": "Cookie validation failed, exiting...",
|
||||
"message": "",
|
||||
}
|
||||
RedisArchivist().set_message(
|
||||
"message:download", mess_dict, expire=4
|
||||
)
|
||||
print("cookie validation failed, exiting...")
|
||||
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def store_validation(response):
|
||||
"""remember last validation"""
|
||||
now = datetime.now()
|
||||
message = {
|
||||
"status": response,
|
||||
"validated": int(now.timestamp()),
|
||||
"validated_str": now.strftime("%Y-%m-%d %H:%M"),
|
||||
}
|
||||
RedisArchivist().set_message("cookie:valid", message)
|
||||
451
backend/download/src/yt_dlp_handler.py
Normal file
451
backend/download/src/yt_dlp_handler.py
Normal file
@@ -0,0 +1,451 @@
|
||||
"""
|
||||
functionality:
|
||||
- handle yt_dlp
|
||||
- build options and post processor
|
||||
- download video files
|
||||
- move to archive
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from channel.src.index import YoutubeChannel
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import get_channel_overwrites, ignore_filelist
|
||||
from common.src.ta_redis import RedisQueue
|
||||
from download.src.queue import PendingList
|
||||
from download.src.subscriptions import PlaylistSubscription
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
from video.src.comments import CommentList
|
||||
from video.src.constants import VideoTypeEnum
|
||||
from video.src.index import YoutubeVideo, index_new_video
|
||||
|
||||
|
||||
class DownloaderBase:
|
||||
"""base class for shared config"""
|
||||
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
|
||||
CHANNEL_QUEUE = "download:channel"
|
||||
PLAYLIST_QUEUE = "download:playlist:full"
|
||||
PLAYLIST_QUICK = "download:playlist:quick"
|
||||
VIDEO_QUEUE = "download:video"
|
||||
|
||||
def __init__(self, task):
|
||||
self.task = task
|
||||
self.config = AppConfig().config
|
||||
self.channel_overwrites = get_channel_overwrites()
|
||||
self.now = int(datetime.now().timestamp())
|
||||
|
||||
|
||||
class VideoDownloader(DownloaderBase):
|
||||
"""handle the video download functionality"""
|
||||
|
||||
def __init__(self, task=False):
|
||||
super().__init__(task)
|
||||
self.obs = False
|
||||
self._build_obs()
|
||||
|
||||
def run_queue(self, auto_only=False) -> tuple[int, int]:
|
||||
"""setup download queue in redis loop until no more items"""
|
||||
downloaded = 0
|
||||
failed = 0
|
||||
while True:
|
||||
video_data = self._get_next(auto_only)
|
||||
if self.task.is_stopped() or not video_data:
|
||||
self._reset_auto()
|
||||
break
|
||||
|
||||
youtube_id = video_data["youtube_id"]
|
||||
channel_id = video_data["channel_id"]
|
||||
print(f"{youtube_id}: Downloading video")
|
||||
self._notify(video_data, "Validate download format")
|
||||
|
||||
success = self._dl_single_vid(youtube_id, channel_id)
|
||||
if not success:
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
self._notify(video_data, "Add video metadata to index", progress=1)
|
||||
video_type = VideoTypeEnum(video_data["vid_type"])
|
||||
vid_dict = index_new_video(youtube_id, video_type=video_type)
|
||||
RedisQueue(self.CHANNEL_QUEUE).add(channel_id)
|
||||
RedisQueue(self.VIDEO_QUEUE).add(youtube_id)
|
||||
|
||||
self._notify(video_data, "Move downloaded file to archive")
|
||||
self.move_to_archive(vid_dict)
|
||||
self._delete_from_pending(youtube_id)
|
||||
downloaded += 1
|
||||
|
||||
# post processing
|
||||
DownloadPostProcess(self.task).run()
|
||||
|
||||
return downloaded, failed
|
||||
|
||||
def _notify(self, video_data, message, progress=False):
|
||||
"""send progress notification to task"""
|
||||
if not self.task:
|
||||
return
|
||||
|
||||
typ = VideoTypeEnum(video_data["vid_type"]).value.rstrip("s").title()
|
||||
title = video_data.get("title")
|
||||
self.task.send_progress(
|
||||
[f"Processing {typ}: {title}", message], progress=progress
|
||||
)
|
||||
|
||||
def _get_next(self, auto_only):
|
||||
"""get next item in queue"""
|
||||
must_list = [{"term": {"status": {"value": "pending"}}}]
|
||||
must_not_list = [{"exists": {"field": "message"}}]
|
||||
if auto_only:
|
||||
must_list.append({"term": {"auto_start": {"value": True}}})
|
||||
|
||||
data = {
|
||||
"size": 1,
|
||||
"query": {"bool": {"must": must_list, "must_not": must_not_list}},
|
||||
"sort": [
|
||||
{"auto_start": {"order": "desc"}},
|
||||
{"timestamp": {"order": "asc"}},
|
||||
],
|
||||
}
|
||||
path = "ta_download/_search"
|
||||
response, _ = ElasticWrap(path).get(data=data)
|
||||
if not response["hits"]["hits"]:
|
||||
return False
|
||||
|
||||
return response["hits"]["hits"][0]["_source"]
|
||||
|
||||
def _progress_hook(self, response):
|
||||
"""process the progress_hooks from yt_dlp"""
|
||||
progress = False
|
||||
try:
|
||||
size = response.get("_total_bytes_str")
|
||||
if size.strip() == "N/A":
|
||||
size = response.get("_total_bytes_estimate_str", "N/A")
|
||||
|
||||
percent = response["_percent_str"]
|
||||
progress = float(percent.strip("%")) / 100
|
||||
speed = response["_speed_str"]
|
||||
eta = response["_eta_str"]
|
||||
message = f"{percent} of {size} at {speed} - time left: {eta}"
|
||||
except KeyError:
|
||||
message = "processing"
|
||||
|
||||
if self.task:
|
||||
title = response["info_dict"]["title"]
|
||||
self.task.send_progress([title, message], progress=progress)
|
||||
|
||||
def _build_obs(self):
|
||||
"""collection to build all obs passed to yt-dlp"""
|
||||
self._build_obs_basic()
|
||||
self._build_obs_user()
|
||||
self._build_obs_postprocessors()
|
||||
|
||||
def _build_obs_basic(self):
|
||||
"""initial obs"""
|
||||
self.obs = {
|
||||
"merge_output_format": "mp4",
|
||||
"outtmpl": (self.CACHE_DIR + "/download/%(id)s.mp4"),
|
||||
"progress_hooks": [self._progress_hook],
|
||||
"noprogress": True,
|
||||
"continuedl": True,
|
||||
"writethumbnail": False,
|
||||
"noplaylist": True,
|
||||
"color": "no_color",
|
||||
}
|
||||
|
||||
def _build_obs_user(self):
|
||||
"""build user customized options"""
|
||||
if self.config["downloads"]["format"]:
|
||||
self.obs["format"] = self.config["downloads"]["format"]
|
||||
if self.config["downloads"]["format_sort"]:
|
||||
format_sort = self.config["downloads"]["format_sort"]
|
||||
format_sort_list = [i.strip() for i in format_sort.split(",")]
|
||||
self.obs["format_sort"] = format_sort_list
|
||||
if self.config["downloads"]["limit_speed"]:
|
||||
self.obs["ratelimit"] = (
|
||||
self.config["downloads"]["limit_speed"] * 1024
|
||||
)
|
||||
|
||||
throttle = self.config["downloads"]["throttledratelimit"]
|
||||
if throttle:
|
||||
self.obs["throttledratelimit"] = throttle * 1024
|
||||
|
||||
def _build_obs_postprocessors(self):
|
||||
"""add postprocessor to obs"""
|
||||
postprocessors = []
|
||||
|
||||
if self.config["downloads"]["add_metadata"]:
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "FFmpegMetadata",
|
||||
"add_chapters": True,
|
||||
"add_metadata": True,
|
||||
}
|
||||
)
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "MetadataFromField",
|
||||
"formats": [
|
||||
"%(title)s:%(meta_title)s",
|
||||
"%(uploader)s:%(meta_artist)s",
|
||||
":(?P<album>)",
|
||||
],
|
||||
"when": "pre_process",
|
||||
}
|
||||
)
|
||||
|
||||
if self.config["downloads"]["add_thumbnail"]:
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "EmbedThumbnail",
|
||||
"already_have_thumbnail": True,
|
||||
}
|
||||
)
|
||||
self.obs["writethumbnail"] = True
|
||||
|
||||
self.obs["postprocessors"] = postprocessors
|
||||
|
||||
def _set_overwrites(self, obs: dict, channel_id: str) -> None:
|
||||
"""add overwrites to obs"""
|
||||
overwrites = self.channel_overwrites.get(channel_id)
|
||||
if overwrites and overwrites.get("download_format"):
|
||||
obs["format"] = overwrites.get("download_format")
|
||||
|
||||
def _dl_single_vid(self, youtube_id: str, channel_id: str) -> bool:
|
||||
"""download single video"""
|
||||
obs = self.obs.copy()
|
||||
self._set_overwrites(obs, channel_id)
|
||||
dl_cache = os.path.join(self.CACHE_DIR, "download")
|
||||
|
||||
success, message = YtWrap(obs, self.config).download(youtube_id)
|
||||
if not success:
|
||||
self._handle_error(youtube_id, message)
|
||||
|
||||
if self.obs["writethumbnail"]:
|
||||
# webp files don't get cleaned up automatically
|
||||
all_cached = ignore_filelist(os.listdir(dl_cache))
|
||||
to_clean = [i for i in all_cached if not i.endswith(".mp4")]
|
||||
for file_name in to_clean:
|
||||
file_path = os.path.join(dl_cache, file_name)
|
||||
os.remove(file_path)
|
||||
|
||||
return success
|
||||
|
||||
@staticmethod
|
||||
def _handle_error(youtube_id, message):
|
||||
"""store error message"""
|
||||
data = {"doc": {"message": message}}
|
||||
_, _ = ElasticWrap(f"ta_download/_update/{youtube_id}").post(data=data)
|
||||
|
||||
def move_to_archive(self, vid_dict):
|
||||
"""move downloaded video from cache to archive"""
|
||||
host_uid = EnvironmentSettings.HOST_UID
|
||||
host_gid = EnvironmentSettings.HOST_GID
|
||||
# make folder
|
||||
folder = os.path.join(
|
||||
self.MEDIA_DIR, vid_dict["channel"]["channel_id"]
|
||||
)
|
||||
if not os.path.exists(folder):
|
||||
os.makedirs(folder)
|
||||
if host_uid and host_gid:
|
||||
os.chown(folder, host_uid, host_gid)
|
||||
# move media file
|
||||
media_file = vid_dict["youtube_id"] + ".mp4"
|
||||
old_path = os.path.join(self.CACHE_DIR, "download", media_file)
|
||||
new_path = os.path.join(self.MEDIA_DIR, vid_dict["media_url"])
|
||||
# move media file and fix permission
|
||||
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
||||
if host_uid and host_gid:
|
||||
os.chown(new_path, host_uid, host_gid)
|
||||
|
||||
@staticmethod
|
||||
def _delete_from_pending(youtube_id):
|
||||
"""delete downloaded video from pending index if its there"""
|
||||
path = f"ta_download/_doc/{youtube_id}?refresh=true"
|
||||
_, _ = ElasticWrap(path).delete()
|
||||
|
||||
def _reset_auto(self):
|
||||
"""reset autostart to defaults after queue stop"""
|
||||
path = "ta_download/_update_by_query"
|
||||
data = {
|
||||
"query": {"term": {"auto_start": {"value": True}}},
|
||||
"script": {
|
||||
"source": "ctx._source.auto_start = false",
|
||||
"lang": "painless",
|
||||
},
|
||||
}
|
||||
response, _ = ElasticWrap(path).post(data=data)
|
||||
updated = response.get("updated")
|
||||
if updated:
|
||||
print(f"[download] reset auto start on {updated} videos.")
|
||||
|
||||
|
||||
class DownloadPostProcess(DownloaderBase):
|
||||
"""handle task to run after download queue finishes"""
|
||||
|
||||
def run(self):
|
||||
"""run all functions"""
|
||||
self.auto_delete_all()
|
||||
self.auto_delete_overwrites()
|
||||
self.refresh_playlist()
|
||||
self.match_videos()
|
||||
self.get_comments()
|
||||
|
||||
def auto_delete_all(self):
|
||||
"""handle auto delete"""
|
||||
autodelete_days = self.config["downloads"]["autodelete_days"]
|
||||
if not autodelete_days:
|
||||
return
|
||||
|
||||
print(f"auto delete older than {autodelete_days} days")
|
||||
now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
|
||||
data = {
|
||||
"query": {"range": {"player.watched_date": {"lte": now_lte}}},
|
||||
"sort": [{"player.watched_date": {"order": "asc"}}],
|
||||
}
|
||||
self._auto_delete_watched(data)
|
||||
|
||||
def auto_delete_overwrites(self):
|
||||
"""handle per channel auto delete from overwrites"""
|
||||
for channel_id, value in self.channel_overwrites.items():
|
||||
if "autodelete_days" in value:
|
||||
autodelete_days = value.get("autodelete_days")
|
||||
print(f"{channel_id}: delete older than {autodelete_days}d")
|
||||
now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
|
||||
must_list = [
|
||||
{"range": {"player.watched_date": {"lte": now_lte}}},
|
||||
{"term": {"channel.channel_id": {"value": channel_id}}},
|
||||
]
|
||||
data = {
|
||||
"query": {"bool": {"must": must_list}},
|
||||
"sort": [{"player.watched_date": {"order": "desc"}}],
|
||||
}
|
||||
self._auto_delete_watched(data)
|
||||
|
||||
@staticmethod
|
||||
def _auto_delete_watched(data):
|
||||
"""delete watched videos after x days"""
|
||||
to_delete = IndexPaginate("ta_video", data).get_results()
|
||||
if not to_delete:
|
||||
return
|
||||
|
||||
for video in to_delete:
|
||||
youtube_id = video["youtube_id"]
|
||||
print(f"{youtube_id}: auto delete video")
|
||||
YoutubeVideo(youtube_id).delete_media_file()
|
||||
|
||||
print("add deleted to ignore list")
|
||||
vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
|
||||
pending = PendingList(youtube_ids=vids)
|
||||
pending.parse_url_list()
|
||||
_ = pending.add_to_pending(status="ignore")
|
||||
|
||||
def refresh_playlist(self) -> None:
|
||||
"""match videos with playlists"""
|
||||
self.add_playlists_to_refresh()
|
||||
|
||||
queue = RedisQueue(self.PLAYLIST_QUEUE)
|
||||
while True:
|
||||
total = queue.max_score()
|
||||
playlist_id, idx = queue.get_next()
|
||||
if not playlist_id or not idx or not total:
|
||||
break
|
||||
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
playlist.update_playlist(skip_on_empty=True)
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
channel_name = playlist.json_data["playlist_channel"]
|
||||
playlist_title = playlist.json_data["playlist_name"]
|
||||
message = [
|
||||
f"Post Processing Playlists for: {channel_name}",
|
||||
f"{playlist_title} [{idx}/{total}]",
|
||||
]
|
||||
progress = idx / total
|
||||
self.task.send_progress(message, progress=progress)
|
||||
|
||||
def add_playlists_to_refresh(self) -> None:
|
||||
"""add playlists to refresh"""
|
||||
if self.task:
|
||||
message = ["Post Processing Playlists", "Scanning for Playlists"]
|
||||
self.task.send_progress(message)
|
||||
|
||||
self._add_playlist_sub()
|
||||
self._add_channel_playlists()
|
||||
self._add_video_playlists()
|
||||
|
||||
def _add_playlist_sub(self):
|
||||
"""add subscribed playlists to refresh"""
|
||||
subs = PlaylistSubscription().get_playlists()
|
||||
to_add = [i["playlist_id"] for i in subs]
|
||||
RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
|
||||
|
||||
def _add_channel_playlists(self):
|
||||
"""add playlists from channels to refresh"""
|
||||
queue = RedisQueue(self.CHANNEL_QUEUE)
|
||||
while True:
|
||||
channel_id, _ = queue.get_next()
|
||||
if not channel_id:
|
||||
break
|
||||
|
||||
channel = YoutubeChannel(channel_id)
|
||||
channel.get_from_es()
|
||||
overwrites = channel.get_overwrites()
|
||||
if "index_playlists" in overwrites:
|
||||
channel.get_all_playlists()
|
||||
to_add = [i[0] for i in channel.all_playlists]
|
||||
RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
|
||||
|
||||
def _add_video_playlists(self):
|
||||
"""add other playlists for quick sync"""
|
||||
all_playlists = RedisQueue(self.PLAYLIST_QUEUE).get_all()
|
||||
must_not = [{"terms": {"playlist_id": all_playlists}}]
|
||||
video_ids = RedisQueue(self.VIDEO_QUEUE).get_all()
|
||||
must = [{"terms": {"playlist_entries.youtube_id": video_ids}}]
|
||||
data = {
|
||||
"query": {"bool": {"must_not": must_not, "must": must}},
|
||||
"_source": ["playlist_id"],
|
||||
}
|
||||
playlists = IndexPaginate("ta_playlist", data).get_results()
|
||||
to_add = [i["playlist_id"] for i in playlists]
|
||||
RedisQueue(self.PLAYLIST_QUICK).add_list(to_add)
|
||||
|
||||
def match_videos(self) -> None:
|
||||
"""scan rest of indexed playlists to match videos"""
|
||||
queue = RedisQueue(self.PLAYLIST_QUICK)
|
||||
while True:
|
||||
total = queue.max_score()
|
||||
playlist_id, idx = queue.get_next()
|
||||
if not playlist_id or not idx or not total:
|
||||
break
|
||||
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
playlist.get_from_es()
|
||||
playlist.add_vids_to_playlist()
|
||||
playlist.remove_vids_from_playlist()
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
message = [
|
||||
"Post Processing Playlists.",
|
||||
f"Validate Playlists: - {idx}/{total}",
|
||||
]
|
||||
progress = idx / total
|
||||
self.task.send_progress(message, progress=progress)
|
||||
|
||||
def get_comments(self):
|
||||
"""get comments from youtube"""
|
||||
video_queue = RedisQueue(self.VIDEO_QUEUE)
|
||||
comment_list = CommentList(task=self.task)
|
||||
comment_list.add(video_ids=video_queue.get_all())
|
||||
|
||||
video_queue.clear()
|
||||
comment_list.index()
|
||||
18
backend/download/urls.py
Normal file
18
backend/download/urls.py
Normal file
@@ -0,0 +1,18 @@
|
||||
"""all download API urls"""
|
||||
|
||||
from django.urls import path
|
||||
from download import views
|
||||
|
||||
urlpatterns = [
|
||||
path("", views.DownloadApiListView.as_view(), name="api-download-list"),
|
||||
path(
|
||||
"aggs/",
|
||||
views.DownloadAggsApiView.as_view(),
|
||||
name="api-download-aggs",
|
||||
),
|
||||
path(
|
||||
"<slug:video_id>/",
|
||||
views.DownloadApiView.as_view(),
|
||||
name="api-download",
|
||||
),
|
||||
]
|
||||
170
backend/download/views.py
Normal file
170
backend/download/views.py
Normal file
@@ -0,0 +1,170 @@
|
||||
"""all download API views"""
|
||||
|
||||
from common.views_base import AdminOnly, ApiBaseView
|
||||
from download.src.queue import PendingInteract
|
||||
from rest_framework.response import Response
|
||||
from task.tasks import download_pending, extrac_dl
|
||||
|
||||
|
||||
class DownloadApiListView(ApiBaseView):
|
||||
"""resolves to /api/download/
|
||||
GET: returns latest videos in the download queue
|
||||
POST: add a list of videos to download queue
|
||||
DELETE: remove items based on query filter
|
||||
"""
|
||||
|
||||
search_base = "ta_download/_search/"
|
||||
valid_filter = ["pending", "ignore"]
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
def get(self, request):
|
||||
"""get request"""
|
||||
query_filter = request.GET.get("filter", False)
|
||||
self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
|
||||
|
||||
must_list = []
|
||||
if query_filter:
|
||||
if query_filter not in self.valid_filter:
|
||||
message = f"invalid url query filter: {query_filter}"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
must_list.append({"term": {"status": {"value": query_filter}}})
|
||||
|
||||
filter_channel = request.GET.get("channel", False)
|
||||
if filter_channel:
|
||||
must_list.append(
|
||||
{"term": {"channel_id": {"value": filter_channel}}}
|
||||
)
|
||||
|
||||
self.data["query"] = {"bool": {"must": must_list}}
|
||||
|
||||
self.get_document_list(request)
|
||||
return Response(self.response)
|
||||
|
||||
@staticmethod
|
||||
def post(request):
|
||||
"""add list of videos to download queue"""
|
||||
data = request.data
|
||||
auto_start = bool(request.GET.get("autostart"))
|
||||
try:
|
||||
to_add = data["data"]
|
||||
except KeyError:
|
||||
message = "missing expected data key"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
|
||||
url_str = " ".join(pending)
|
||||
extrac_dl.delay(url_str, auto_start=auto_start)
|
||||
|
||||
return Response(data)
|
||||
|
||||
def delete(self, request):
|
||||
"""delete download queue"""
|
||||
query_filter = request.GET.get("filter", False)
|
||||
if query_filter not in self.valid_filter:
|
||||
message = f"invalid url query filter: {query_filter}"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
message = f"delete queue by status: {query_filter}"
|
||||
print(message)
|
||||
PendingInteract(status=query_filter).delete_by_status()
|
||||
|
||||
return Response({"message": message})
|
||||
|
||||
|
||||
class DownloadAggsApiView(ApiBaseView):
|
||||
"""resolves to /api/download/aggs/
|
||||
GET: get download aggregations
|
||||
"""
|
||||
|
||||
search_base = "ta_download/_search"
|
||||
valid_filter_view = ["ignore", "pending"]
|
||||
|
||||
def get(self, request):
|
||||
"""get aggs"""
|
||||
filter_view = request.GET.get("filter")
|
||||
if filter_view:
|
||||
if filter_view not in self.valid_filter_view:
|
||||
message = f"invalid filter: {filter_view}"
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
self.data.update(
|
||||
{
|
||||
"query": {"term": {"status": {"value": filter_view}}},
|
||||
}
|
||||
)
|
||||
|
||||
self.data.update(
|
||||
{
|
||||
"aggs": {
|
||||
"channel_downloads": {
|
||||
"multi_terms": {
|
||||
"size": 30,
|
||||
"terms": [
|
||||
{"field": "channel_name.keyword"},
|
||||
{"field": "channel_id"},
|
||||
],
|
||||
"order": {"_count": "desc"},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
self.get_aggs()
|
||||
|
||||
return Response(self.response)
|
||||
|
||||
|
||||
class DownloadApiView(ApiBaseView):
|
||||
"""resolves to /api/download/<video_id>/
|
||||
GET: returns metadata dict of an item in the download queue
|
||||
POST: update status of item to pending or ignore
|
||||
DELETE: forget from download queue
|
||||
"""
|
||||
|
||||
search_base = "ta_download/_doc/"
|
||||
valid_status = ["pending", "ignore", "ignore-force", "priority"]
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
def get(self, request, video_id):
|
||||
# pylint: disable=unused-argument
|
||||
"""get request"""
|
||||
self.get_document(video_id)
|
||||
return Response(self.response, status=self.status_code)
|
||||
|
||||
def post(self, request, video_id):
|
||||
"""post to video to change status"""
|
||||
item_status = request.data.get("status")
|
||||
if item_status not in self.valid_status:
|
||||
message = f"{video_id}: invalid status {item_status}"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
if item_status == "ignore-force":
|
||||
extrac_dl.delay(video_id, status="ignore")
|
||||
message = f"{video_id}: set status to ignore"
|
||||
return Response(request.data)
|
||||
|
||||
_, status_code = PendingInteract(video_id).get_item()
|
||||
if status_code == 404:
|
||||
message = f"{video_id}: item not found {status_code}"
|
||||
return Response({"message": message}, status=404)
|
||||
|
||||
print(f"{video_id}: change status to {item_status}")
|
||||
PendingInteract(video_id, item_status).update_status()
|
||||
if item_status == "priority":
|
||||
download_pending.delay(auto_only=True)
|
||||
|
||||
return Response(request.data)
|
||||
|
||||
@staticmethod
|
||||
def delete(request, video_id):
|
||||
# pylint: disable=unused-argument
|
||||
"""delete single video from queue"""
|
||||
print(f"{video_id}: delete from queue")
|
||||
PendingInteract(video_id).delete_item()
|
||||
|
||||
return Response({"success": True})
|
||||
Reference in New Issue
Block a user