mirror of
https://git.vectorsigma.ru/public/tubearchivist.git
synced 2026-08-09 13:39:26 +00:00
renamed django app folder to backend
This commit is contained in:
Simon
14
backend/README.md
Normal file
14
backend/README.md
Normal file
@@ -0,0 +1,14 @@
|
||||
# Django Setup
|
||||
|
||||
## Apps
|
||||
|
||||
### appsettings
|
||||
### channel
|
||||
### common
|
||||
### config
|
||||
### download
|
||||
### playlist
|
||||
### stats
|
||||
### task
|
||||
### user
|
||||
### video
|
||||
0
backend/appsettings/__init__.py
Normal file
0
backend/appsettings/__init__.py
Normal file
687
backend/appsettings/index_mapping.json
Normal file
687
backend/appsettings/index_mapping.json
Normal file
@@ -0,0 +1,687 @@
|
||||
{
|
||||
"index_config": [{
|
||||
"index_name": "config",
|
||||
"expected_map": {
|
||||
"config": {
|
||||
"type": "object",
|
||||
"enabled": false
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
"number_of_replicas": "0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"index_name": "channel",
|
||||
"expected_map": {
|
||||
"channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"channel_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
},
|
||||
"search_as_you_type": {
|
||||
"type": "search_as_you_type",
|
||||
"doc_values": false,
|
||||
"max_shingle_size": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel_banner_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_tvart_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_thumb_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_description": {
|
||||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"channel_tags": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel_overwrites": {
|
||||
"properties": {
|
||||
"download_format": {
|
||||
"type": "text"
|
||||
},
|
||||
"autodelete_days": {
|
||||
"type": "long"
|
||||
},
|
||||
"index_playlists": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"integrate_sponsorblock": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"subscriptions_channel_size": {
|
||||
"type": "long"
|
||||
},
|
||||
"subscriptions_live_channel_size": {
|
||||
"type": "long"
|
||||
},
|
||||
"subscriptions_shorts_channel_size": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
"analysis": {
|
||||
"normalizer": {
|
||||
"to_lower": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"number_of_replicas": "0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"index_name": "video",
|
||||
"expected_map": {
|
||||
"vid_thumb_url": {
|
||||
"type": "text",
|
||||
"index": false
|
||||
},
|
||||
"vid_thumb_base64": {
|
||||
"type": "text",
|
||||
"index": false
|
||||
},
|
||||
"date_downloaded": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"channel": {
|
||||
"properties": {
|
||||
"channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"channel_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
},
|
||||
"search_as_you_type": {
|
||||
"type": "search_as_you_type",
|
||||
"doc_values": false,
|
||||
"max_shingle_size": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel_banner_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_tvart_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_thumb_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_description": {
|
||||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"channel_tags": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel_overwrites": {
|
||||
"properties": {
|
||||
"download_format": {
|
||||
"type": "text"
|
||||
},
|
||||
"autodelete_days": {
|
||||
"type": "long"
|
||||
},
|
||||
"index_playlists": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"integrate_sponsorblock": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"subscriptions_channel_size": {
|
||||
"type": "long"
|
||||
},
|
||||
"subscriptions_live_channel_size": {
|
||||
"type": "long"
|
||||
},
|
||||
"subscriptions_shorts_channel_size": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": {
|
||||
"type": "text"
|
||||
},
|
||||
"media_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"media_size": {
|
||||
"type": "long"
|
||||
},
|
||||
"tags": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
},
|
||||
"search_as_you_type": {
|
||||
"type": "search_as_you_type",
|
||||
"doc_values": false,
|
||||
"max_shingle_size": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"vid_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"vid_type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"published": {
|
||||
"type": "date"
|
||||
},
|
||||
"playlist": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"comment_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"stats": {
|
||||
"properties": {
|
||||
"average_rating": {
|
||||
"type": "float"
|
||||
},
|
||||
"dislike_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"like_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"view_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"player": {
|
||||
"properties": {
|
||||
"duration": {
|
||||
"type": "long"
|
||||
},
|
||||
"duration_str": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"watched": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"watched_date": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
}
|
||||
}
|
||||
},
|
||||
"subtitles": {
|
||||
"properties": {
|
||||
"ext": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"lang": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"media_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"source": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"streams": {
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"index": {
|
||||
"type": "short",
|
||||
"index": false
|
||||
},
|
||||
"codec": {
|
||||
"type": "text"
|
||||
},
|
||||
"width": {
|
||||
"type": "short"
|
||||
},
|
||||
"height": {
|
||||
"type": "short"
|
||||
},
|
||||
"bitrate": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"sponsorblock": {
|
||||
"properties": {
|
||||
"last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"has_unlocked": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"is_enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"segments": {
|
||||
"properties": {
|
||||
"UUID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"actionType": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"category": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"locked": {
|
||||
"type": "short"
|
||||
},
|
||||
"segment": {
|
||||
"type": "float"
|
||||
},
|
||||
"videoDuration": {
|
||||
"type": "float"
|
||||
},
|
||||
"votes": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
"analysis": {
|
||||
"normalizer": {
|
||||
"to_lower": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"number_of_replicas": "0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"index_name": "download",
|
||||
"expected_map": {
|
||||
"timestamp": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"channel_name": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"vid_thumb_url": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"vid_type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"auto_start": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"message": {
|
||||
"type": "text"
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
"analysis": {
|
||||
"normalizer": {
|
||||
"to_lower": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"number_of_replicas": "0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"index_name": "playlist",
|
||||
"expected_map": {
|
||||
"playlist_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_description": {
|
||||
"type": "text"
|
||||
},
|
||||
"playlist_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
},
|
||||
"search_as_you_type": {
|
||||
"type": "search_as_you_type",
|
||||
"doc_values": false,
|
||||
"max_shingle_size": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"playlist_channel": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"playlist_channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_thumbnail": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"playlist_entries": {
|
||||
"properties": {
|
||||
"downloaded": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"idx": {
|
||||
"type": "long"
|
||||
},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"uploader": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
"analysis": {
|
||||
"normalizer": {
|
||||
"to_lower": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"number_of_replicas": "0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"index_name": "subtitle",
|
||||
"expected_map": {
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"subtitle_fragment_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"subtitle_channel": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"subtitle_channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"subtitle_start": {
|
||||
"type": "text"
|
||||
},
|
||||
"subtitle_end": {
|
||||
"type": "text"
|
||||
},
|
||||
"subtitle_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"subtitle_index": {
|
||||
"type": "long"
|
||||
},
|
||||
"subtitle_lang": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"subtitle_source": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"subtitle_line": {
|
||||
"type": "text",
|
||||
"analyzer": "english"
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
"analysis": {
|
||||
"normalizer": {
|
||||
"to_lower": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"number_of_replicas": "0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"index_name": "comment",
|
||||
"expected_map": {
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"comment_channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_comments": {
|
||||
"properties": {
|
||||
"comment_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_timestamp": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"comment_time_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_likecount": {
|
||||
"type": "long"
|
||||
},
|
||||
"comment_is_favorited": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_author": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"comment_author_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_author_thumbnail": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_author_is_uploader": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_parent": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
"analysis": {
|
||||
"normalizer": {
|
||||
"to_lower": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"number_of_replicas": "0"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
0
backend/appsettings/migrations/__init__.py
Normal file
0
backend/appsettings/migrations/__init__.py
Normal file
0
backend/appsettings/src/__init__.py
Normal file
0
backend/appsettings/src/__init__.py
Normal file
273
backend/appsettings/src/backup.py
Normal file
273
backend/appsettings/src/backup.py
Normal file
@@ -0,0 +1,273 @@
|
||||
"""
|
||||
Functionality:
|
||||
- Handle json zip file based backup
|
||||
- create backup
|
||||
- restore backup
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import get_mapping, ignore_filelist
|
||||
from task.models import CustomPeriodicTask
|
||||
|
||||
|
||||
class ElasticBackup:
|
||||
"""dump index to nd-json files for later bulk import"""
|
||||
|
||||
INDEX_SPLIT = ["comment"]
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
BACKUP_DIR = os.path.join(CACHE_DIR, "backup")
|
||||
|
||||
def __init__(self, reason=False, task=False) -> None:
|
||||
self.timestamp = datetime.now().strftime("%Y%m%d")
|
||||
self.index_config = get_mapping()
|
||||
self.reason = reason
|
||||
self.task = task
|
||||
|
||||
def backup_all_indexes(self):
|
||||
"""backup all indexes, add reason to init"""
|
||||
print("backup all indexes")
|
||||
if not self.reason:
|
||||
raise ValueError("missing backup reason in ElasticBackup")
|
||||
|
||||
if self.task:
|
||||
self.task.send_progress(["Scanning your index."])
|
||||
for index in self.index_config:
|
||||
index_name = index["index_name"]
|
||||
print(f"backup: export in progress for {index_name}")
|
||||
if not self.index_exists(index_name):
|
||||
print(f"skip backup for not yet existing index {index_name}")
|
||||
continue
|
||||
|
||||
self.backup_index(index_name)
|
||||
|
||||
if self.task:
|
||||
self.task.send_progress(["Compress files to zip archive."])
|
||||
self.zip_it()
|
||||
if self.reason == "auto":
|
||||
self.rotate_backup()
|
||||
|
||||
def backup_index(self, index_name):
|
||||
"""export all documents of a single index"""
|
||||
paginate_kwargs = {
|
||||
"data": {"query": {"match_all": {}}},
|
||||
"keep_source": True,
|
||||
"callback": BackupCallback,
|
||||
"task": self.task,
|
||||
"total": self._get_total(index_name),
|
||||
}
|
||||
|
||||
if index_name in self.INDEX_SPLIT:
|
||||
paginate_kwargs.update({"size": 200})
|
||||
|
||||
paginate = IndexPaginate(f"ta_{index_name}", **paginate_kwargs)
|
||||
_ = paginate.get_results()
|
||||
|
||||
@staticmethod
|
||||
def _get_total(index_name):
|
||||
"""get total documents in index"""
|
||||
path = f"ta_{index_name}/_count"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
|
||||
return response.get("count")
|
||||
|
||||
def zip_it(self):
|
||||
"""pack it up into single zip file"""
|
||||
file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
|
||||
|
||||
to_backup = []
|
||||
for file in os.listdir(self.BACKUP_DIR):
|
||||
if file.endswith(".json"):
|
||||
to_backup.append(os.path.join(self.BACKUP_DIR, file))
|
||||
|
||||
backup_file = os.path.join(self.BACKUP_DIR, file_name)
|
||||
|
||||
comp = zipfile.ZIP_DEFLATED
|
||||
with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
|
||||
for backup_file in to_backup:
|
||||
zip_f.write(backup_file, os.path.basename(backup_file))
|
||||
|
||||
# cleanup
|
||||
for backup_file in to_backup:
|
||||
os.remove(backup_file)
|
||||
|
||||
def post_bulk_restore(self, file_name):
|
||||
"""send bulk to es"""
|
||||
file_path = os.path.join(self.CACHE_DIR, file_name)
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
data = f.read()
|
||||
|
||||
if not data.strip():
|
||||
return
|
||||
|
||||
_, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
|
||||
|
||||
def get_all_backup_files(self):
|
||||
"""build all available backup files for view"""
|
||||
all_backup_files = ignore_filelist(os.listdir(self.BACKUP_DIR))
|
||||
all_available_backups = [
|
||||
i
|
||||
for i in all_backup_files
|
||||
if i.startswith("ta_") and i.endswith(".zip")
|
||||
]
|
||||
all_available_backups.sort(reverse=True)
|
||||
|
||||
backup_dicts = []
|
||||
for filename in all_available_backups:
|
||||
data = self.build_backup_file_data(filename)
|
||||
backup_dicts.append(data)
|
||||
|
||||
return backup_dicts
|
||||
|
||||
def build_backup_file_data(self, filename):
|
||||
"""build metadata of single backup file"""
|
||||
file_path = os.path.join(self.BACKUP_DIR, filename)
|
||||
if not os.path.exists(file_path):
|
||||
return False
|
||||
|
||||
file_split = filename.split("-")
|
||||
if len(file_split) == 2:
|
||||
timestamp = file_split[1].strip(".zip")
|
||||
reason = False
|
||||
elif len(file_split) == 3:
|
||||
timestamp = file_split[1]
|
||||
reason = file_split[2].strip(".zip")
|
||||
else:
|
||||
raise ValueError
|
||||
|
||||
data = {
|
||||
"filename": filename,
|
||||
"file_path": file_path,
|
||||
"file_size": os.path.getsize(file_path),
|
||||
"timestamp": timestamp,
|
||||
"reason": reason,
|
||||
}
|
||||
|
||||
return data
|
||||
|
||||
def restore(self, filename):
|
||||
"""
|
||||
restore from backup zip file
|
||||
call reset from ElasitIndexWrap first to start blank
|
||||
"""
|
||||
zip_content = self._unpack_zip_backup(filename)
|
||||
self._restore_json_files(zip_content)
|
||||
|
||||
def _unpack_zip_backup(self, filename):
|
||||
"""extract backup zip and return filelist"""
|
||||
file_path = os.path.join(self.BACKUP_DIR, filename)
|
||||
|
||||
with zipfile.ZipFile(file_path, "r") as z:
|
||||
zip_content = z.namelist()
|
||||
z.extractall(self.BACKUP_DIR)
|
||||
|
||||
return zip_content
|
||||
|
||||
def _restore_json_files(self, zip_content):
|
||||
"""go through the unpacked files and restore"""
|
||||
for idx, json_f in enumerate(zip_content):
|
||||
self._notify_restore(idx, json_f, len(zip_content))
|
||||
file_name = os.path.join(self.BACKUP_DIR, json_f)
|
||||
|
||||
if not json_f.startswith("es_") or not json_f.endswith(".json"):
|
||||
os.remove(file_name)
|
||||
continue
|
||||
|
||||
print("restoring: " + json_f)
|
||||
self.post_bulk_restore(file_name)
|
||||
os.remove(file_name)
|
||||
|
||||
def _notify_restore(self, idx, json_f, total_files):
|
||||
"""notify restore progress"""
|
||||
message = [f"Restore index from json backup file {json_f}."]
|
||||
progress = (idx + 1) / total_files
|
||||
self.task.send_progress(message_lines=message, progress=progress)
|
||||
|
||||
@staticmethod
|
||||
def index_exists(index_name):
|
||||
"""check if index already exists to skip"""
|
||||
_, status_code = ElasticWrap(f"ta_{index_name}").get()
|
||||
exists = status_code == 200
|
||||
|
||||
return exists
|
||||
|
||||
def rotate_backup(self):
|
||||
"""delete old backups if needed"""
|
||||
try:
|
||||
task = CustomPeriodicTask.objects.get(name="run_backup")
|
||||
except CustomPeriodicTask.DoesNotExist:
|
||||
return
|
||||
|
||||
rotate = task.task_config.get("rotate")
|
||||
if not rotate:
|
||||
return
|
||||
|
||||
all_backup_files = self.get_all_backup_files()
|
||||
auto = [i for i in all_backup_files if i["reason"] == "auto"]
|
||||
|
||||
if len(auto) <= rotate:
|
||||
print("no backup files to rotate")
|
||||
return
|
||||
|
||||
all_to_delete = auto[rotate:]
|
||||
for to_delete in all_to_delete:
|
||||
self.delete_file(to_delete["filename"])
|
||||
|
||||
def delete_file(self, filename):
|
||||
"""delete backup file"""
|
||||
file_path = os.path.join(self.BACKUP_DIR, filename)
|
||||
if not os.path.exists(file_path):
|
||||
print(f"backup file not found: {filename}")
|
||||
return False
|
||||
|
||||
print(f"remove old backup file: {file_path}")
|
||||
os.remove(file_path)
|
||||
|
||||
return file_path
|
||||
|
||||
|
||||
class BackupCallback:
|
||||
"""handle backup ndjson writer as callback for IndexPaginate"""
|
||||
|
||||
def __init__(self, source, index_name, counter=0):
|
||||
self.source = source
|
||||
self.index_name = index_name
|
||||
self.counter = counter
|
||||
self.timestamp = datetime.now().strftime("%Y%m%d")
|
||||
self.cache_dir = EnvironmentSettings.CACHE_DIR
|
||||
|
||||
def run(self):
|
||||
"""run the junk task"""
|
||||
file_content = self._build_bulk()
|
||||
self._write_es_json(file_content)
|
||||
|
||||
def _build_bulk(self):
|
||||
"""build bulk query data from all_results"""
|
||||
bulk_list = []
|
||||
|
||||
for document in self.source:
|
||||
document_id = document["_id"]
|
||||
es_index = document["_index"]
|
||||
action = {"index": {"_index": es_index, "_id": document_id}}
|
||||
source = document["_source"]
|
||||
bulk_list.append(json.dumps(action))
|
||||
bulk_list.append(json.dumps(source))
|
||||
|
||||
# add last newline
|
||||
bulk_list.append("\n")
|
||||
file_content = "\n".join(bulk_list)
|
||||
|
||||
return file_content
|
||||
|
||||
def _write_es_json(self, file_content):
|
||||
"""write nd-json file for es _bulk API to disk"""
|
||||
index = self.index_name.lstrip("ta_")
|
||||
file_name = f"es_{index}-{self.timestamp}-{self.counter}.json"
|
||||
file_path = os.path.join(self.cache_dir, "backup", file_name)
|
||||
with open(file_path, "a+", encoding="utf-8") as f:
|
||||
f.write(file_content)
|
||||
245
backend/appsettings/src/config.py
Normal file
245
backend/appsettings/src/config.py
Normal file
@@ -0,0 +1,245 @@
|
||||
"""
|
||||
Functionality:
|
||||
- read and write config
|
||||
- load config variables into redis
|
||||
"""
|
||||
|
||||
from random import randint
|
||||
from time import sleep
|
||||
from typing import Literal, TypedDict
|
||||
|
||||
import requests
|
||||
from appsettings.src.snapshot import ElasticSnapshot
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
from django.conf import settings
|
||||
from download.src.yt_dlp_base import CookieHandler
|
||||
|
||||
|
||||
class SubscriptionsConfigType(TypedDict):
|
||||
"""describes subscriptions config"""
|
||||
|
||||
channel_size: int
|
||||
live_channel_size: int
|
||||
shorts_channel_size: int
|
||||
auto_start: bool
|
||||
|
||||
|
||||
class DownloadsConfigType(TypedDict):
|
||||
"""describes downloads config"""
|
||||
|
||||
limit_speed: int
|
||||
sleep_interval: int
|
||||
autodelete_days: int
|
||||
format: str | bool
|
||||
format_sort: str | bool
|
||||
add_metadata: bool
|
||||
add_thumbnail: bool
|
||||
subtitle: str | bool
|
||||
subtitle_source: Literal["user", "auto"] | bool
|
||||
subtitle_index: bool
|
||||
comment_max: str | bool
|
||||
comment_sort: Literal["top", "new"]
|
||||
cookie_import: bool
|
||||
throttledratelimit: int
|
||||
extractor_lang: str | bool
|
||||
integrate_ryd: bool
|
||||
integrate_sponsorblock: bool
|
||||
|
||||
|
||||
class ApplicationConfigType(TypedDict):
|
||||
"""describes application config"""
|
||||
|
||||
enable_snapshot: bool
|
||||
|
||||
|
||||
class AppConfigType(TypedDict):
|
||||
"""combined app config type"""
|
||||
|
||||
subscriptions: SubscriptionsConfigType
|
||||
downloads: DownloadsConfigType
|
||||
application: ApplicationConfigType
|
||||
|
||||
|
||||
class AppConfig:
|
||||
"""handle application variables"""
|
||||
|
||||
ES_PATH = "ta_config/_doc/appsettings"
|
||||
CONFIG_DEFAULTS: AppConfigType = {
|
||||
"subscriptions": {
|
||||
"channel_size": 50,
|
||||
"live_channel_size": 50,
|
||||
"shorts_channel_size": 50,
|
||||
"auto_start": False,
|
||||
},
|
||||
"downloads": {
|
||||
"limit_speed": False,
|
||||
"sleep_interval": 3,
|
||||
"autodelete_days": False,
|
||||
"format": False,
|
||||
"format_sort": False,
|
||||
"add_metadata": False,
|
||||
"add_thumbnail": False,
|
||||
"subtitle": False,
|
||||
"subtitle_source": False,
|
||||
"subtitle_index": False,
|
||||
"comment_max": False,
|
||||
"comment_sort": "top",
|
||||
"cookie_import": False,
|
||||
"throttledratelimit": False,
|
||||
"extractor_lang": False,
|
||||
"integrate_ryd": False,
|
||||
"integrate_sponsorblock": False,
|
||||
},
|
||||
"application": {"enable_snapshot": True},
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.config = self.get_config()
|
||||
|
||||
def get_config(self) -> AppConfigType:
|
||||
"""get config from ES"""
|
||||
response, status_code = ElasticWrap(self.ES_PATH).get()
|
||||
if not status_code == 200:
|
||||
return self.CONFIG_DEFAULTS
|
||||
|
||||
return response["_source"]
|
||||
|
||||
def update_config(self, data: dict) -> AppConfigType:
|
||||
"""update single config value"""
|
||||
for key, value in data.items():
|
||||
key_map = key.split(".")
|
||||
self._validate_key(key_map)
|
||||
self.config[key_map[0]][key_map[1]] = value
|
||||
|
||||
response, status_code = ElasticWrap(self.ES_PATH).post(self.config)
|
||||
if not status_code == 200:
|
||||
print(response)
|
||||
|
||||
return self.config
|
||||
|
||||
def _validate_key(self, key_map: list[str]) -> None:
|
||||
"""raise valueerror on invalid key"""
|
||||
exists = self.CONFIG_DEFAULTS.get(key_map[0], {}).get(key_map[1]) # type: ignore # noqa: E501
|
||||
if exists is None:
|
||||
raise ValueError(f"trying to access invalid config key: {key_map}")
|
||||
|
||||
def post_process_updated(self, data: dict) -> None:
|
||||
"""apply hooks for some config keys"""
|
||||
for config_value, updated_value in data:
|
||||
if config_value == "downloads.cookie_import":
|
||||
self.process_cookie(updated_value)
|
||||
if config_value == "application.enable_snapshot" and updated_value:
|
||||
ElasticSnapshot().setup()
|
||||
|
||||
def process_cookie(self, updated_value):
|
||||
"""import and validate cookie"""
|
||||
handler = CookieHandler(self.config)
|
||||
if updated_value:
|
||||
try:
|
||||
handler.import_cookie()
|
||||
except FileNotFoundError:
|
||||
print("cookie: import failed, file not found")
|
||||
handler.revoke()
|
||||
self._fail_message("Cookie file not found.")
|
||||
return
|
||||
|
||||
valid = handler.validate()
|
||||
if not valid:
|
||||
handler.revoke()
|
||||
self._fail_message("Failed to validate cookie file.")
|
||||
else:
|
||||
handler.revoke()
|
||||
|
||||
@staticmethod
|
||||
def _fail_message(message_line):
|
||||
"""notify our failure"""
|
||||
key = "message:setting"
|
||||
message = {
|
||||
"status": key,
|
||||
"group": "setting:application",
|
||||
"level": "error",
|
||||
"title": "Cookie import failed",
|
||||
"messages": [message_line],
|
||||
"id": "0000",
|
||||
}
|
||||
RedisArchivist().set_message(key, message=message, expire=True)
|
||||
|
||||
|
||||
class ReleaseVersion:
|
||||
"""compare local version with remote version"""
|
||||
|
||||
REMOTE_URL = "https://www.tubearchivist.com/api/release/latest/"
|
||||
NEW_KEY = "versioncheck:new"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.local_version: str = settings.TA_VERSION
|
||||
self.is_unstable: bool = settings.TA_VERSION.endswith("-unstable")
|
||||
self.remote_version: str = ""
|
||||
self.is_breaking: bool = False
|
||||
|
||||
def check(self) -> None:
|
||||
"""check version"""
|
||||
print(f"[{self.local_version}]: look for updates")
|
||||
self.get_remote_version()
|
||||
new_version = self._has_update()
|
||||
if new_version:
|
||||
message = {
|
||||
"status": True,
|
||||
"version": new_version,
|
||||
"is_breaking": self.is_breaking,
|
||||
}
|
||||
RedisArchivist().set_message(self.NEW_KEY, message)
|
||||
print(f"[{self.local_version}]: found new version {new_version}")
|
||||
|
||||
def get_local_version(self) -> str:
|
||||
"""read version from local"""
|
||||
return self.local_version
|
||||
|
||||
def get_remote_version(self) -> None:
|
||||
"""read version from remote"""
|
||||
sleep(randint(0, 60))
|
||||
response = requests.get(self.REMOTE_URL, timeout=20).json()
|
||||
self.remote_version = response["release_version"]
|
||||
self.is_breaking = response["breaking_changes"]
|
||||
|
||||
def _has_update(self) -> str | bool:
|
||||
"""check if there is an update"""
|
||||
remote_parsed = self._parse_version(self.remote_version)
|
||||
local_parsed = self._parse_version(self.local_version)
|
||||
if remote_parsed > local_parsed:
|
||||
return self.remote_version
|
||||
|
||||
if self.is_unstable and local_parsed == remote_parsed:
|
||||
return self.remote_version
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _parse_version(version) -> tuple[int, ...]:
|
||||
"""return version parts"""
|
||||
clean = version.rstrip("-unstable").lstrip("v")
|
||||
return tuple((int(i) for i in clean.split(".")))
|
||||
|
||||
def is_updated(self) -> str | bool:
|
||||
"""check if update happened in the mean time"""
|
||||
message = self.get_update()
|
||||
if not message:
|
||||
return False
|
||||
|
||||
local_parsed = self._parse_version(self.local_version)
|
||||
message_parsed = self._parse_version(message.get("version"))
|
||||
|
||||
if local_parsed >= message_parsed:
|
||||
RedisArchivist().del_message(self.NEW_KEY)
|
||||
return settings.TA_VERSION
|
||||
|
||||
return False
|
||||
|
||||
def get_update(self) -> dict:
|
||||
"""return new version dict if available"""
|
||||
message = RedisArchivist().get_message(self.NEW_KEY)
|
||||
if not message.get("status"):
|
||||
return {}
|
||||
|
||||
return message
|
||||
126
backend/appsettings/src/filesystem.py
Normal file
126
backend/appsettings/src/filesystem.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
Functionality:
|
||||
- scan the filesystem to delete or index
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import ignore_filelist
|
||||
from video.src.comments import CommentList
|
||||
from video.src.index import YoutubeVideo, index_new_video
|
||||
|
||||
|
||||
class Scanner:
|
||||
"""scan index and filesystem"""
|
||||
|
||||
VIDEOS: str = EnvironmentSettings.MEDIA_DIR
|
||||
|
||||
def __init__(self, task=False) -> None:
|
||||
self.task = task
|
||||
self.to_delete: set[str] = set()
|
||||
self.to_index: set[str] = set()
|
||||
|
||||
def scan(self) -> None:
|
||||
"""scan the filesystem"""
|
||||
downloaded: set[str] = self._get_downloaded()
|
||||
indexed: set[str] = self._get_indexed()
|
||||
self.to_index = downloaded - indexed
|
||||
self.to_delete = indexed - downloaded
|
||||
|
||||
def _get_downloaded(self) -> set[str]:
|
||||
"""get downloaded ids"""
|
||||
if self.task:
|
||||
self.task.send_progress(["Scan your filesystem for videos."])
|
||||
|
||||
downloaded: set = set()
|
||||
channels = ignore_filelist(os.listdir(self.VIDEOS))
|
||||
for channel in channels:
|
||||
folder = os.path.join(self.VIDEOS, channel)
|
||||
files = ignore_filelist(os.listdir(folder))
|
||||
downloaded.update({i.split(".")[0] for i in files})
|
||||
|
||||
return downloaded
|
||||
|
||||
def _get_indexed(self) -> set:
|
||||
"""get all indexed ids"""
|
||||
if self.task:
|
||||
self.task.send_progress(["Get all videos indexed."])
|
||||
|
||||
data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
|
||||
response = IndexPaginate("ta_video", data).get_results()
|
||||
return {i["youtube_id"] for i in response}
|
||||
|
||||
def apply(self) -> None:
|
||||
"""apply all changes"""
|
||||
self.delete()
|
||||
self.index()
|
||||
self.url_fix()
|
||||
|
||||
def delete(self) -> None:
|
||||
"""delete videos from index"""
|
||||
if not self.to_delete:
|
||||
print("nothing to delete")
|
||||
return
|
||||
|
||||
if self.task:
|
||||
self.task.send_progress(
|
||||
[f"Remove {len(self.to_delete)} videos from index."]
|
||||
)
|
||||
|
||||
for youtube_id in self.to_delete:
|
||||
YoutubeVideo(youtube_id).delete_media_file()
|
||||
|
||||
def index(self) -> None:
|
||||
"""index new"""
|
||||
if not self.to_index:
|
||||
print("nothing to index")
|
||||
return
|
||||
|
||||
total = len(self.to_index)
|
||||
for idx, youtube_id in enumerate(self.to_index):
|
||||
if self.task:
|
||||
self.task.send_progress(
|
||||
message_lines=[
|
||||
f"Index missing video {youtube_id}, {idx + 1}/{total}"
|
||||
],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
index_new_video(youtube_id)
|
||||
|
||||
comment_list = CommentList(task=self.task)
|
||||
comment_list.add(video_ids=list(self.to_index))
|
||||
comment_list.index()
|
||||
|
||||
def url_fix(self) -> None:
|
||||
"""
|
||||
update path v0.3.6 to v0.3.7
|
||||
fix url not matching channel-videoid pattern
|
||||
"""
|
||||
bool_must = (
|
||||
"doc['media_url'].value == "
|
||||
+ "(doc['channel.channel_id'].value + '/' + "
|
||||
+ "doc['youtube_id'].value) + '.mp4'"
|
||||
)
|
||||
to_update = (
|
||||
"ctx._source['media_url'] = "
|
||||
+ "ctx._source.channel['channel_id'] + '/' + "
|
||||
+ "ctx._source['youtube_id'] + '.mp4'"
|
||||
)
|
||||
data = {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must_not": [{"script": {"script": {"source": bool_must}}}]
|
||||
}
|
||||
},
|
||||
"script": {"source": to_update},
|
||||
}
|
||||
response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data)
|
||||
updated = response.get("updates")
|
||||
if updated:
|
||||
print(f"updated {updated} bad media_url")
|
||||
if self.task:
|
||||
self.task.send_progress(
|
||||
[f"Updated {updated} wrong media urls."]
|
||||
)
|
||||
212
backend/appsettings/src/index_setup.py
Normal file
212
backend/appsettings/src/index_setup.py
Normal file
@@ -0,0 +1,212 @@
|
||||
"""
|
||||
functionality:
|
||||
- setup elastic index at first start
|
||||
- verify and update index mapping and settings if needed
|
||||
- backup and restore metadata
|
||||
"""
|
||||
|
||||
from appsettings.src.backup import ElasticBackup
|
||||
from appsettings.src.config import AppConfig
|
||||
from appsettings.src.snapshot import ElasticSnapshot
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.helper import get_mapping
|
||||
|
||||
|
||||
class ElasticIndex:
|
||||
"""interact with a single index"""
|
||||
|
||||
def __init__(self, index_name, expected_map=False, expected_set=False):
|
||||
self.index_name = index_name
|
||||
self.expected_map = expected_map
|
||||
self.expected_set = expected_set
|
||||
self.exists, self.details = self.index_exists()
|
||||
|
||||
def index_exists(self):
|
||||
"""check if index already exists and return mapping if it does"""
|
||||
response, status_code = ElasticWrap(f"ta_{self.index_name}").get()
|
||||
exists = status_code == 200
|
||||
details = response.get(f"ta_{self.index_name}", False)
|
||||
|
||||
return exists, details
|
||||
|
||||
def validate(self):
|
||||
"""
|
||||
check if all expected mappings and settings match
|
||||
returns True when rebuild is needed
|
||||
"""
|
||||
|
||||
if self.expected_map:
|
||||
rebuild = self.validate_mappings()
|
||||
if rebuild:
|
||||
return rebuild
|
||||
|
||||
if self.expected_set:
|
||||
rebuild = self.validate_settings()
|
||||
if rebuild:
|
||||
return rebuild
|
||||
|
||||
return False
|
||||
|
||||
def validate_mappings(self):
|
||||
"""check if all mappings are as expected"""
|
||||
now_map = self.details["mappings"]["properties"]
|
||||
|
||||
for key, value in self.expected_map.items():
|
||||
# nested
|
||||
if list(value.keys()) == ["properties"]:
|
||||
for key_n, value_n in value["properties"].items():
|
||||
if key not in now_map:
|
||||
print(f"detected mapping change: {key_n}, {value_n}")
|
||||
return True
|
||||
if key_n not in now_map[key]["properties"].keys():
|
||||
print(f"detected mapping change: {key_n}, {value_n}")
|
||||
return True
|
||||
if not value_n == now_map[key]["properties"][key_n]:
|
||||
print(f"detected mapping change: {key_n}, {value_n}")
|
||||
return True
|
||||
|
||||
continue
|
||||
|
||||
# not nested
|
||||
if key not in now_map.keys():
|
||||
print(f"detected mapping change: {key}, {value}")
|
||||
return True
|
||||
if not value == now_map[key]:
|
||||
print(f"detected mapping change: {key}, {value}")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def validate_settings(self):
|
||||
"""check if all settings are as expected"""
|
||||
|
||||
now_set = self.details["settings"]["index"]
|
||||
|
||||
for key, value in self.expected_set.items():
|
||||
if key not in now_set.keys():
|
||||
print(key, value)
|
||||
return True
|
||||
|
||||
if not value == now_set[key]:
|
||||
print(key, value)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def rebuild_index(self):
|
||||
"""rebuild with new mapping"""
|
||||
print(f"applying new mappings to index ta_{self.index_name}...")
|
||||
self.create_blank(for_backup=True)
|
||||
self.reindex("backup")
|
||||
self.delete_index(backup=False)
|
||||
self.create_blank()
|
||||
self.reindex("restore")
|
||||
self.delete_index()
|
||||
|
||||
def reindex(self, method):
|
||||
"""create on elastic search"""
|
||||
if method == "backup":
|
||||
source = f"ta_{self.index_name}"
|
||||
destination = f"ta_{self.index_name}_backup"
|
||||
elif method == "restore":
|
||||
source = f"ta_{self.index_name}_backup"
|
||||
destination = f"ta_{self.index_name}"
|
||||
else:
|
||||
raise ValueError("invalid method, expected 'backup' or 'restore'")
|
||||
|
||||
data = {"source": {"index": source}, "dest": {"index": destination}}
|
||||
_, _ = ElasticWrap("_reindex?refresh=true").post(data=data)
|
||||
|
||||
def delete_index(self, backup=True):
|
||||
"""delete index passed as argument"""
|
||||
path = f"ta_{self.index_name}"
|
||||
if backup:
|
||||
path = path + "_backup"
|
||||
|
||||
_, _ = ElasticWrap(path).delete()
|
||||
|
||||
def create_blank(self, for_backup=False):
|
||||
"""apply new mapping and settings for blank new index"""
|
||||
print(f"create new blank index with name ta_{self.index_name}...")
|
||||
path = f"ta_{self.index_name}"
|
||||
if for_backup:
|
||||
path = f"{path}_backup"
|
||||
|
||||
data = {}
|
||||
if self.expected_set:
|
||||
data.update({"settings": self.expected_set})
|
||||
if self.expected_map:
|
||||
data.update({"mappings": {"properties": self.expected_map}})
|
||||
|
||||
_, _ = ElasticWrap(path).put(data)
|
||||
|
||||
|
||||
class ElasitIndexWrap:
|
||||
"""interact with all index mapping and setup"""
|
||||
|
||||
def __init__(self):
|
||||
self.index_config = get_mapping()
|
||||
self.backup_run = False
|
||||
|
||||
def setup(self):
|
||||
"""setup elastic index, run at startup"""
|
||||
for index in self.index_config:
|
||||
index_name, expected_map, expected_set = self._config_split(index)
|
||||
handler = ElasticIndex(index_name, expected_map, expected_set)
|
||||
if not handler.exists:
|
||||
handler.create_blank()
|
||||
continue
|
||||
|
||||
rebuild = handler.validate()
|
||||
if rebuild:
|
||||
self._check_backup()
|
||||
handler.rebuild_index()
|
||||
continue
|
||||
|
||||
# else all good
|
||||
print(f"ta_{index_name} index is created and up to date...")
|
||||
|
||||
def reset(self):
|
||||
"""reset all indexes to blank"""
|
||||
self.delete_all()
|
||||
self.create_all_blank()
|
||||
|
||||
def delete_all(self):
|
||||
"""delete all indexes"""
|
||||
print("reset elastic index")
|
||||
for index in self.index_config:
|
||||
index_name, _, _ = self._config_split(index)
|
||||
handler = ElasticIndex(index_name)
|
||||
handler.delete_index(backup=False)
|
||||
|
||||
def create_all_blank(self):
|
||||
"""create all blank indexes"""
|
||||
print("create all new indexes in elastic from template")
|
||||
for index in self.index_config:
|
||||
index_name, expected_map, expected_set = self._config_split(index)
|
||||
handler = ElasticIndex(index_name, expected_map, expected_set)
|
||||
handler.create_blank()
|
||||
|
||||
@staticmethod
|
||||
def _config_split(index):
|
||||
"""split index config keys"""
|
||||
index_name = index["index_name"]
|
||||
expected_map = index["expected_map"]
|
||||
expected_set = index["expected_set"]
|
||||
|
||||
return index_name, expected_map, expected_set
|
||||
|
||||
def _check_backup(self):
|
||||
"""create backup if needed"""
|
||||
if self.backup_run:
|
||||
return
|
||||
|
||||
config = AppConfig().config
|
||||
if config["application"]["enable_snapshot"]:
|
||||
# take snapshot if enabled
|
||||
ElasticSnapshot().take_snapshot_now(wait=True)
|
||||
else:
|
||||
# fallback to json backup
|
||||
ElasticBackup(reason="update").backup_all_indexes()
|
||||
|
||||
self.backup_run = True
|
||||
483
backend/appsettings/src/manual.py
Normal file
483
backend/appsettings/src/manual.py
Normal file
@@ -0,0 +1,483 @@
|
||||
"""
|
||||
Functionality:
|
||||
- Handle manual import task
|
||||
- Scan and identify media files in import folder
|
||||
- Process import media files
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.helper import ignore_filelist
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from PIL import Image
|
||||
from video.src.comments import CommentList
|
||||
from video.src.index import YoutubeVideo
|
||||
from yt_dlp.utils import ISO639Utils
|
||||
|
||||
|
||||
class ImportFolderScanner:
|
||||
"""import and indexing existing video files
|
||||
- identify all media files belonging to a video
|
||||
- identify youtube id
|
||||
- convert if needed
|
||||
"""
|
||||
|
||||
CONFIG = AppConfig().config
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
IMPORT_DIR = os.path.join(CACHE_DIR, "import")
|
||||
|
||||
"""All extensions should be in lowercase until better handling is in place.
|
||||
Described in Issue #502.
|
||||
"""
|
||||
EXT_MAP = {
|
||||
"media": [".mp4", ".mkv", ".webm"],
|
||||
"metadata": [".json"],
|
||||
"thumb": [".jpg", ".png", ".webp"],
|
||||
"subtitle": [".vtt"],
|
||||
}
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.task = task
|
||||
self.to_import = False
|
||||
|
||||
def scan(self):
|
||||
"""scan and match media files"""
|
||||
if self.task:
|
||||
self.task.send_progress(["Scanning your import folder."])
|
||||
|
||||
all_files = self.get_all_files()
|
||||
self.match_files(all_files)
|
||||
self.process_videos()
|
||||
|
||||
return self.to_import
|
||||
|
||||
def get_all_files(self):
|
||||
"""get all files in /import"""
|
||||
rel_paths = ignore_filelist(os.listdir(self.IMPORT_DIR))
|
||||
all_files = [os.path.join(self.IMPORT_DIR, i) for i in rel_paths]
|
||||
all_files.sort()
|
||||
|
||||
return all_files
|
||||
|
||||
@staticmethod
|
||||
def _get_template():
|
||||
"""base dict for video"""
|
||||
return {
|
||||
"media": False,
|
||||
"video_id": False,
|
||||
"metadata": False,
|
||||
"thumb": False,
|
||||
"subtitle": [],
|
||||
}
|
||||
|
||||
def match_files(self, all_files):
|
||||
"""loop through all files, join what matches"""
|
||||
self.to_import = []
|
||||
|
||||
current_video = self._get_template()
|
||||
last_base = False
|
||||
|
||||
for file_path in all_files:
|
||||
base_name, ext = self._detect_base_name(file_path)
|
||||
key, file_path = self._detect_type(file_path, ext)
|
||||
if not key or not file_path:
|
||||
continue
|
||||
|
||||
if base_name != last_base:
|
||||
if last_base:
|
||||
print(f"manual import: {current_video}")
|
||||
self.to_import.append(current_video)
|
||||
|
||||
current_video = self._get_template()
|
||||
last_base = base_name
|
||||
|
||||
if key == "subtitle":
|
||||
current_video["subtitle"].append(file_path)
|
||||
else:
|
||||
current_video[key] = file_path
|
||||
|
||||
if current_video.get("media"):
|
||||
print(f"manual import: {current_video}")
|
||||
self.to_import.append(current_video)
|
||||
|
||||
def _detect_base_name(self, file_path):
|
||||
"""extract base_name and ext for matching"""
|
||||
base_name_raw, ext = os.path.splitext(file_path)
|
||||
base_name, ext2 = os.path.splitext(base_name_raw)
|
||||
|
||||
if ext2:
|
||||
if ISO639Utils.short2long(ext2.strip(".")) or ext2 == ".info":
|
||||
# valid secondary extension
|
||||
return base_name, ext
|
||||
|
||||
return base_name_raw, ext
|
||||
|
||||
def _detect_type(self, file_path, ext):
|
||||
"""detect metadata type for file"""
|
||||
|
||||
for key, value in self.EXT_MAP.items():
|
||||
if ext.lower() in value:
|
||||
return key, file_path
|
||||
|
||||
return False, False
|
||||
|
||||
def process_videos(self):
|
||||
"""loop through all videos"""
|
||||
for idx, current_video in enumerate(self.to_import):
|
||||
if not current_video["media"]:
|
||||
print(f"{current_video}: no matching media file found.")
|
||||
raise ValueError
|
||||
|
||||
if self.task:
|
||||
self._notify(idx, current_video)
|
||||
|
||||
self._detect_youtube_id(current_video)
|
||||
self._dump_thumb(current_video)
|
||||
self._convert_thumb(current_video)
|
||||
self._get_subtitles(current_video)
|
||||
self._convert_video(current_video)
|
||||
print(f"manual import: {current_video}")
|
||||
|
||||
ManualImport(current_video, self.CONFIG).run()
|
||||
|
||||
video_ids = [i["video_id"] for i in self.to_import]
|
||||
comment_list = CommentList(task=self.task)
|
||||
comment_list.add(video_ids=video_ids)
|
||||
comment_list.index()
|
||||
|
||||
def _notify(self, idx, current_video):
|
||||
"""send notification back to task"""
|
||||
filename = os.path.split(current_video["media"])[-1]
|
||||
if len(filename) > 50:
|
||||
filename = filename[:50] + "..."
|
||||
|
||||
message = [
|
||||
f"Import queue processing video {idx + 1}/{len(self.to_import)}",
|
||||
filename,
|
||||
]
|
||||
progress = (idx + 1) / len(self.to_import)
|
||||
self.task.send_progress(message, progress=progress)
|
||||
|
||||
def _detect_youtube_id(self, current_video):
|
||||
"""find video id from filename or json"""
|
||||
youtube_id = self._extract_id_from_filename(current_video["media"])
|
||||
if youtube_id:
|
||||
current_video["video_id"] = youtube_id
|
||||
return
|
||||
|
||||
youtube_id = self._extract_id_from_json(current_video["metadata"])
|
||||
if youtube_id:
|
||||
current_video["video_id"] = youtube_id
|
||||
return
|
||||
|
||||
raise ValueError("failed to find video id")
|
||||
|
||||
@staticmethod
|
||||
def _extract_id_from_filename(file_name):
|
||||
"""
|
||||
look at the file name for the youtube id
|
||||
expects filename ending in [<youtube_id>].<ext>
|
||||
"""
|
||||
base_name, _ = os.path.splitext(file_name)
|
||||
id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
|
||||
if id_search:
|
||||
youtube_id = id_search.group(1)
|
||||
return youtube_id
|
||||
|
||||
print(f"id extraction failed from filename: {file_name}")
|
||||
|
||||
return False
|
||||
|
||||
def _extract_id_from_json(self, json_file):
|
||||
"""open json file and extract id"""
|
||||
json_path = os.path.join(self.CACHE_DIR, "import", json_file)
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
json_content = f.read()
|
||||
|
||||
youtube_id = json.loads(json_content)["id"]
|
||||
|
||||
return youtube_id
|
||||
|
||||
def _dump_thumb(self, current_video):
|
||||
"""extract embedded thumb before converting"""
|
||||
if current_video["thumb"]:
|
||||
return
|
||||
|
||||
media_path = current_video["media"]
|
||||
_, ext = os.path.splitext(media_path)
|
||||
|
||||
new_path = False
|
||||
if ext == ".mkv":
|
||||
idx, thumb_type = self._get_mkv_thumb_stream(media_path)
|
||||
if idx is not None:
|
||||
new_path = self.dump_mpv_thumb(media_path, idx, thumb_type)
|
||||
|
||||
elif ext == ".mp4":
|
||||
thumb_type = self.get_mp4_thumb_type(media_path)
|
||||
if thumb_type:
|
||||
new_path = self.dump_mp4_thumb(media_path, thumb_type)
|
||||
|
||||
if new_path:
|
||||
current_video["thumb"] = new_path
|
||||
|
||||
def _get_mkv_thumb_stream(self, media_path):
|
||||
"""get stream idx of thumbnail for mkv files"""
|
||||
streams = self._get_streams(media_path)
|
||||
attachments = [
|
||||
i for i in streams["streams"] if i["codec_type"] == "attachment"
|
||||
]
|
||||
|
||||
for idx, stream in enumerate(attachments):
|
||||
tags = stream["tags"]
|
||||
if "mimetype" in tags and tags["filename"].startswith("cover"):
|
||||
_, ext = os.path.splitext(tags["filename"])
|
||||
return idx, ext
|
||||
|
||||
return None, None
|
||||
|
||||
@staticmethod
|
||||
def dump_mpv_thumb(media_path, idx, thumb_type):
|
||||
"""write cover to disk for mkv"""
|
||||
_, media_ext = os.path.splitext(media_path)
|
||||
new_path = f"{media_path.rstrip(media_ext)}{thumb_type}"
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-v",
|
||||
"quiet",
|
||||
f"-dump_attachment:t:{idx}",
|
||||
new_path,
|
||||
"-i",
|
||||
media_path,
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
|
||||
return new_path
|
||||
|
||||
def get_mp4_thumb_type(self, media_path):
|
||||
"""detect filetype of embedded thumbnail"""
|
||||
streams = self._get_streams(media_path)
|
||||
|
||||
for stream in streams["streams"]:
|
||||
if stream["codec_name"] in ["png", "jpg"]:
|
||||
return stream["codec_name"]
|
||||
|
||||
return False
|
||||
|
||||
def _convert_thumb(self, current_video):
|
||||
"""convert all thumbnails to jpg"""
|
||||
if not current_video["thumb"]:
|
||||
return
|
||||
|
||||
thumb_path = current_video["thumb"]
|
||||
|
||||
base_path, ext = os.path.splitext(thumb_path)
|
||||
if ext == ".jpg":
|
||||
return
|
||||
|
||||
new_path = f"{base_path}.jpg"
|
||||
img_raw = Image.open(thumb_path)
|
||||
img_raw.convert("RGB").save(new_path)
|
||||
|
||||
os.remove(thumb_path)
|
||||
current_video["thumb"] = new_path
|
||||
|
||||
def _get_subtitles(self, current_video):
|
||||
"""find all subtitles in media file"""
|
||||
if current_video["subtitle"]:
|
||||
return
|
||||
|
||||
media_path = current_video["media"]
|
||||
streams = self._get_streams(media_path)
|
||||
base_path, ext = os.path.splitext(media_path)
|
||||
|
||||
if ext == ".webm":
|
||||
print(f"{media_path}: subtitle extract from webm not supported")
|
||||
return
|
||||
|
||||
for idx, stream in enumerate(streams["streams"]):
|
||||
if stream["codec_type"] == "subtitle":
|
||||
lang = ISO639Utils.long2short(stream["tags"]["language"])
|
||||
sub_path = f"{base_path}.{lang}.vtt"
|
||||
self._dump_subtitle(idx, media_path, sub_path)
|
||||
current_video["subtitle"].append(sub_path)
|
||||
|
||||
@staticmethod
|
||||
def _dump_subtitle(idx, media_path, sub_path):
|
||||
"""extract subtitle from media file"""
|
||||
subprocess.run(
|
||||
["ffmpeg", "-i", media_path, "-map", f"0:{idx}", sub_path],
|
||||
check=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_streams(media_path):
|
||||
"""return all streams from media_path"""
|
||||
streams_raw = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_streams",
|
||||
"-print_format",
|
||||
"json",
|
||||
media_path,
|
||||
],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
streams = json.loads(streams_raw.stdout.decode())
|
||||
|
||||
return streams
|
||||
|
||||
@staticmethod
|
||||
def dump_mp4_thumb(media_path, thumb_type):
|
||||
"""save cover to disk"""
|
||||
_, ext = os.path.splitext(media_path)
|
||||
new_path = f"{media_path.rstrip(ext)}.{thumb_type}"
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-i",
|
||||
media_path,
|
||||
"-map",
|
||||
"0:v",
|
||||
"-map",
|
||||
"-0:V",
|
||||
"-c",
|
||||
"copy",
|
||||
new_path,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
return new_path
|
||||
|
||||
def _convert_video(self, current_video):
|
||||
"""convert if needed"""
|
||||
current_path = current_video["media"]
|
||||
base_path, ext = os.path.splitext(current_path)
|
||||
if ext == ".mp4":
|
||||
return
|
||||
|
||||
new_path = base_path + ".mp4"
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-i",
|
||||
current_path,
|
||||
new_path,
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-stats",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
current_video["media"] = new_path
|
||||
os.remove(current_path)
|
||||
|
||||
|
||||
class ManualImport:
|
||||
"""import single identified video"""
|
||||
|
||||
def __init__(self, current_video, config):
|
||||
self.current_video = current_video
|
||||
self.config = config
|
||||
|
||||
def run(self):
|
||||
"""run all"""
|
||||
json_data = self.index_metadata()
|
||||
self._move_to_archive(json_data)
|
||||
self._cleanup(json_data)
|
||||
|
||||
def index_metadata(self):
|
||||
"""get metadata from yt or json"""
|
||||
video_id = self.current_video["video_id"]
|
||||
video = YoutubeVideo(video_id)
|
||||
video.build_json(
|
||||
youtube_meta_overwrite=self._get_info_json(),
|
||||
media_path=self.current_video["media"],
|
||||
)
|
||||
if not video.json_data:
|
||||
print(f"{video_id}: manual import failed, and no metadata found.")
|
||||
raise ValueError
|
||||
|
||||
video.check_subtitles(subtitle_files=self.current_video["subtitle"])
|
||||
video.upload_to_es()
|
||||
|
||||
if video.offline_import and self.current_video["thumb"]:
|
||||
old_path = self.current_video["thumb"]
|
||||
thumbs = ThumbManager(video_id)
|
||||
new_path = thumbs.vid_thumb_path(absolute=True, create_folder=True)
|
||||
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
||||
else:
|
||||
url = video.json_data["vid_thumb_url"]
|
||||
ThumbManager(video_id).download_video_thumb(url)
|
||||
|
||||
return video.json_data
|
||||
|
||||
def _get_info_json(self):
|
||||
"""read info_json from file"""
|
||||
if not self.current_video["metadata"]:
|
||||
return False
|
||||
|
||||
with open(self.current_video["metadata"], "r", encoding="utf-8") as f:
|
||||
info_json = json.loads(f.read())
|
||||
|
||||
return info_json
|
||||
|
||||
def _move_to_archive(self, json_data):
|
||||
"""move identified media file to archive"""
|
||||
videos = EnvironmentSettings.MEDIA_DIR
|
||||
host_uid = EnvironmentSettings.HOST_UID
|
||||
host_gid = EnvironmentSettings.HOST_GID
|
||||
|
||||
channel, file = os.path.split(json_data["media_url"])
|
||||
channel_folder = os.path.join(videos, channel)
|
||||
if not os.path.exists(channel_folder):
|
||||
os.makedirs(channel_folder)
|
||||
|
||||
if host_uid and host_gid:
|
||||
os.chown(channel_folder, host_uid, host_gid)
|
||||
|
||||
old_path = self.current_video["media"]
|
||||
new_path = os.path.join(channel_folder, file)
|
||||
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
||||
if host_uid and host_gid:
|
||||
os.chown(new_path, host_uid, host_gid)
|
||||
|
||||
base_name, _ = os.path.splitext(new_path)
|
||||
for old_path in self.current_video["subtitle"]:
|
||||
lang = old_path.split(".")[-2]
|
||||
new_path = f"{base_name}.{lang}.vtt"
|
||||
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
||||
|
||||
def _cleanup(self, json_data):
|
||||
"""cleanup leftover files"""
|
||||
meta_data = self.current_video["metadata"]
|
||||
if meta_data and os.path.exists(meta_data):
|
||||
os.remove(meta_data)
|
||||
|
||||
thumb = self.current_video["thumb"]
|
||||
if thumb and os.path.exists(thumb):
|
||||
os.remove(thumb)
|
||||
|
||||
for subtitle_file in self.current_video["subtitle"]:
|
||||
if os.path.exists(subtitle_file):
|
||||
os.remove(subtitle_file)
|
||||
|
||||
channel_info = os.path.join(
|
||||
EnvironmentSettings.CACHE_DIR,
|
||||
"import",
|
||||
f"{json_data['channel']['channel_id']}.info.json",
|
||||
)
|
||||
if os.path.exists(channel_info):
|
||||
os.remove(channel_info)
|
||||
563
backend/appsettings/src/reindex.py
Normal file
563
backend/appsettings/src/reindex.py
Normal file
@@ -0,0 +1,563 @@
|
||||
"""
|
||||
functionality:
|
||||
- periodically refresh documents
|
||||
- index and update in es
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
from typing import Callable, TypedDict
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from channel.src.index import YoutubeChannel
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.ta_redis import RedisQueue
|
||||
from download.src.subscriptions import ChannelSubscription
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from download.src.yt_dlp_base import CookieHandler
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
from task.models import CustomPeriodicTask
|
||||
from video.src.comments import Comments
|
||||
from video.src.index import YoutubeVideo
|
||||
|
||||
|
||||
class ReindexConfigType(TypedDict):
|
||||
"""represents config type"""
|
||||
|
||||
index_name: str
|
||||
queue_name: str
|
||||
active_key: str
|
||||
refresh_key: str
|
||||
|
||||
|
||||
class ReindexBase:
|
||||
"""base config class for reindex task"""
|
||||
|
||||
REINDEX_CONFIG: dict[str, ReindexConfigType] = {
|
||||
"video": {
|
||||
"index_name": "ta_video",
|
||||
"queue_name": "reindex:ta_video",
|
||||
"active_key": "active",
|
||||
"refresh_key": "vid_last_refresh",
|
||||
},
|
||||
"channel": {
|
||||
"index_name": "ta_channel",
|
||||
"queue_name": "reindex:ta_channel",
|
||||
"active_key": "channel_active",
|
||||
"refresh_key": "channel_last_refresh",
|
||||
},
|
||||
"playlist": {
|
||||
"index_name": "ta_playlist",
|
||||
"queue_name": "reindex:ta_playlist",
|
||||
"active_key": "playlist_active",
|
||||
"refresh_key": "playlist_last_refresh",
|
||||
},
|
||||
}
|
||||
|
||||
MULTIPLY = 1.2
|
||||
DAYS3 = 60 * 60 * 24 * 3
|
||||
|
||||
def __init__(self):
|
||||
self.config = AppConfig().config
|
||||
self.now = int(datetime.now().timestamp())
|
||||
|
||||
def populate(self, all_ids, reindex_config: ReindexConfigType):
|
||||
"""add all to reindex ids to redis queue"""
|
||||
if not all_ids:
|
||||
return
|
||||
|
||||
RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
|
||||
|
||||
|
||||
class ReindexPopulate(ReindexBase):
|
||||
"""add outdated and recent documents to reindex queue"""
|
||||
|
||||
INTERVAL_DEFAIULT: int = 90
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.interval = self.INTERVAL_DEFAIULT
|
||||
|
||||
def get_interval(self) -> None:
|
||||
"""get reindex days interval from task"""
|
||||
try:
|
||||
task = CustomPeriodicTask.objects.get(name="check_reindex")
|
||||
except CustomPeriodicTask.DoesNotExist:
|
||||
return
|
||||
|
||||
task_config = task.task_config
|
||||
if task_config.get("days"):
|
||||
self.interval = task_config.get("days")
|
||||
|
||||
def add_recent(self) -> None:
|
||||
"""add recent videos to refresh"""
|
||||
gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
|
||||
must_list = [
|
||||
{"term": {"active": {"value": True}}},
|
||||
{"range": {"published": {"gte": gte}}},
|
||||
]
|
||||
data = {
|
||||
"size": 10000,
|
||||
"query": {"bool": {"must": must_list}},
|
||||
"sort": [{"published": {"order": "desc"}}],
|
||||
}
|
||||
response, _ = ElasticWrap("ta_video/_search").get(data=data)
|
||||
hits = response["hits"]["hits"]
|
||||
if not hits:
|
||||
return
|
||||
|
||||
all_ids = [i["_source"]["youtube_id"] for i in hits]
|
||||
reindex_config: ReindexConfigType = self.REINDEX_CONFIG["video"]
|
||||
self.populate(all_ids, reindex_config)
|
||||
|
||||
def add_outdated(self) -> None:
|
||||
"""add outdated documents"""
|
||||
for reindex_config in self.REINDEX_CONFIG.values():
|
||||
total_hits = self._get_total_hits(reindex_config)
|
||||
daily_should = self._get_daily_should(total_hits)
|
||||
all_ids = self._get_outdated_ids(reindex_config, daily_should)
|
||||
self.populate(all_ids, reindex_config)
|
||||
|
||||
@staticmethod
|
||||
def _get_total_hits(reindex_config: ReindexConfigType) -> int:
|
||||
"""get total hits from index"""
|
||||
index_name = reindex_config["index_name"]
|
||||
active_key = reindex_config["active_key"]
|
||||
data = {
|
||||
"query": {"term": {active_key: {"value": True}}},
|
||||
"_source": False,
|
||||
}
|
||||
total = IndexPaginate(index_name, data, keep_source=True).get_results()
|
||||
|
||||
return len(total)
|
||||
|
||||
def _get_daily_should(self, total_hits: int) -> int:
|
||||
"""calc how many should reindex daily"""
|
||||
daily_should = int((total_hits // self.interval + 1) * self.MULTIPLY)
|
||||
if daily_should >= 10000:
|
||||
daily_should = 9999
|
||||
|
||||
return daily_should
|
||||
|
||||
def _get_outdated_ids(
|
||||
self, reindex_config: ReindexConfigType, daily_should: int
|
||||
) -> list[str]:
|
||||
"""get outdated from index_name"""
|
||||
index_name = reindex_config["index_name"]
|
||||
refresh_key = reindex_config["refresh_key"]
|
||||
now_lte = str(self.now - self.interval * 24 * 60 * 60)
|
||||
must_list = [
|
||||
{"match": {reindex_config["active_key"]: True}},
|
||||
{"range": {refresh_key: {"lte": now_lte}}},
|
||||
]
|
||||
data = {
|
||||
"size": daily_should,
|
||||
"query": {"bool": {"must": must_list}},
|
||||
"sort": [{refresh_key: {"order": "asc"}}],
|
||||
"_source": False,
|
||||
}
|
||||
response, _ = ElasticWrap(f"{index_name}/_search").get(data=data)
|
||||
|
||||
all_ids = [i["_id"] for i in response["hits"]["hits"]]
|
||||
return all_ids
|
||||
|
||||
|
||||
class ReindexManual(ReindexBase):
|
||||
"""
|
||||
manually add ids to reindex queue from API
|
||||
data_example = {
|
||||
"video": ["video1", "video2", "video3"],
|
||||
"channel": ["channel1", "channel2", "channel3"],
|
||||
"playlist": ["playlist1", "playlist2"],
|
||||
}
|
||||
extract_videos to also reindex all videos of channel/playlist
|
||||
"""
|
||||
|
||||
def __init__(self, extract_videos=False):
|
||||
super().__init__()
|
||||
self.extract_videos = extract_videos
|
||||
self.data = False
|
||||
|
||||
def extract_data(self, data) -> None:
|
||||
"""process data"""
|
||||
self.data = data
|
||||
for key, values in self.data.items():
|
||||
reindex_config = self.REINDEX_CONFIG.get(key)
|
||||
if not reindex_config:
|
||||
print(f"reindex type {key} not valid")
|
||||
raise ValueError
|
||||
|
||||
self.process_index(reindex_config, values)
|
||||
|
||||
def process_index(
|
||||
self, index_config: ReindexConfigType, values: list[str]
|
||||
) -> None:
|
||||
"""process values per index"""
|
||||
index_name = index_config["index_name"]
|
||||
if index_name == "ta_video":
|
||||
self._add_videos(values)
|
||||
elif index_name == "ta_channel":
|
||||
self._add_channels(values)
|
||||
elif index_name == "ta_playlist":
|
||||
self._add_playlists(values)
|
||||
|
||||
def _add_videos(self, values: list[str]) -> None:
|
||||
"""add list of videos to reindex queue"""
|
||||
if not values:
|
||||
return
|
||||
|
||||
queue_name = self.REINDEX_CONFIG["video"]["queue_name"]
|
||||
RedisQueue(queue_name).add_list(values)
|
||||
|
||||
def _add_channels(self, values: list[str]) -> None:
|
||||
"""add list of channels to reindex queue"""
|
||||
queue_name = self.REINDEX_CONFIG["channel"]["queue_name"]
|
||||
RedisQueue(queue_name).add_list(values)
|
||||
|
||||
if self.extract_videos:
|
||||
for channel_id in values:
|
||||
all_videos = self._get_channel_videos(channel_id)
|
||||
self._add_videos(all_videos)
|
||||
|
||||
def _add_playlists(self, values: list[str]) -> None:
|
||||
"""add list of playlists to reindex queue"""
|
||||
queue_name = self.REINDEX_CONFIG["playlist"]["queue_name"]
|
||||
RedisQueue(queue_name).add_list(values)
|
||||
|
||||
if self.extract_videos:
|
||||
for playlist_id in values:
|
||||
all_videos = self._get_playlist_videos(playlist_id)
|
||||
self._add_videos(all_videos)
|
||||
|
||||
def _get_channel_videos(self, channel_id: str) -> list[str]:
|
||||
"""get all videos from channel"""
|
||||
data = {
|
||||
"query": {"term": {"channel.channel_id": {"value": channel_id}}},
|
||||
"_source": ["youtube_id"],
|
||||
}
|
||||
all_results = IndexPaginate("ta_video", data).get_results()
|
||||
return [i["youtube_id"] for i in all_results]
|
||||
|
||||
def _get_playlist_videos(self, playlist_id: str) -> list[str]:
|
||||
"""get all videos from playlist"""
|
||||
data = {
|
||||
"query": {"term": {"playlist.keyword": {"value": playlist_id}}},
|
||||
"_source": ["youtube_id"],
|
||||
}
|
||||
all_results = IndexPaginate("ta_video", data).get_results()
|
||||
return [i["youtube_id"] for i in all_results]
|
||||
|
||||
|
||||
class Reindex(ReindexBase):
|
||||
"""reindex all documents from redis queue"""
|
||||
|
||||
def __init__(self, task=False):
|
||||
super().__init__()
|
||||
self.task = task
|
||||
self.processed = {
|
||||
"videos": 0,
|
||||
"channels": 0,
|
||||
"playlists": 0,
|
||||
}
|
||||
|
||||
def reindex_all(self) -> None:
|
||||
"""reindex all in queue"""
|
||||
if not self.cookie_is_valid():
|
||||
print("[reindex] cookie invalid, exiting...")
|
||||
return
|
||||
|
||||
for name, index_config in self.REINDEX_CONFIG.items():
|
||||
if not RedisQueue(index_config["queue_name"]).length():
|
||||
continue
|
||||
|
||||
self.reindex_type(name, index_config)
|
||||
|
||||
def reindex_type(self, name: str, index_config: ReindexConfigType) -> None:
|
||||
"""reindex all of a single index"""
|
||||
reindex = self._get_reindex_map(index_config["index_name"])
|
||||
queue = RedisQueue(index_config["queue_name"])
|
||||
while True:
|
||||
total = queue.max_score()
|
||||
youtube_id, idx = queue.get_next()
|
||||
if not youtube_id or not idx or not total:
|
||||
break
|
||||
|
||||
if self.task:
|
||||
self._notify(name, total, idx)
|
||||
|
||||
reindex(youtube_id)
|
||||
sleep_interval = self.config["downloads"].get("sleep_interval", 0)
|
||||
sleep(sleep_interval)
|
||||
|
||||
def _get_reindex_map(self, index_name: str) -> Callable:
|
||||
"""return def to run for index"""
|
||||
def_map = {
|
||||
"ta_video": self._reindex_single_video,
|
||||
"ta_channel": self._reindex_single_channel,
|
||||
"ta_playlist": self._reindex_single_playlist,
|
||||
}
|
||||
|
||||
return def_map[index_name]
|
||||
|
||||
def _notify(self, name: str, total: int, idx: int) -> None:
|
||||
"""send notification back to task"""
|
||||
message = [f"Reindexing {name.title()}s {idx}/{total}"]
|
||||
progress = idx / total
|
||||
self.task.send_progress(message, progress=progress)
|
||||
|
||||
def _reindex_single_video(self, youtube_id: str) -> None:
|
||||
"""refresh data for single video"""
|
||||
video = YoutubeVideo(youtube_id)
|
||||
|
||||
# read current state
|
||||
video.get_from_es()
|
||||
if not video.json_data:
|
||||
return
|
||||
|
||||
es_meta = video.json_data.copy()
|
||||
|
||||
# get new
|
||||
media_url = os.path.join(
|
||||
EnvironmentSettings.MEDIA_DIR, es_meta["media_url"]
|
||||
)
|
||||
video.build_json(media_path=media_url)
|
||||
if not video.youtube_meta:
|
||||
video.deactivate()
|
||||
return
|
||||
|
||||
video.delete_subtitles(subtitles=es_meta.get("subtitles"))
|
||||
video.check_subtitles()
|
||||
|
||||
# add back
|
||||
video.json_data["player"] = es_meta.get("player")
|
||||
video.json_data["date_downloaded"] = es_meta.get("date_downloaded")
|
||||
video.json_data["channel"] = es_meta.get("channel")
|
||||
if es_meta.get("playlist"):
|
||||
video.json_data["playlist"] = es_meta.get("playlist")
|
||||
|
||||
video.upload_to_es()
|
||||
|
||||
thumb_handler = ThumbManager(youtube_id)
|
||||
thumb_handler.delete_video_thumb()
|
||||
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
|
||||
|
||||
Comments(youtube_id, config=self.config).reindex_comments()
|
||||
self.processed["videos"] += 1
|
||||
|
||||
def _reindex_single_channel(self, channel_id: str) -> None:
|
||||
"""refresh channel data and sync to videos"""
|
||||
# read current state
|
||||
channel = YoutubeChannel(channel_id)
|
||||
channel.get_from_es()
|
||||
if not channel.json_data:
|
||||
return
|
||||
|
||||
es_meta = channel.json_data.copy()
|
||||
|
||||
# get new
|
||||
channel.get_from_youtube()
|
||||
if not channel.youtube_meta:
|
||||
channel.deactivate()
|
||||
channel.get_from_es()
|
||||
channel.sync_to_videos()
|
||||
return
|
||||
|
||||
channel.process_youtube_meta()
|
||||
channel.get_channel_art()
|
||||
|
||||
# add back
|
||||
channel.json_data["channel_subscribed"] = es_meta["channel_subscribed"]
|
||||
overwrites = es_meta.get("channel_overwrites")
|
||||
if overwrites:
|
||||
channel.json_data["channel_overwrites"] = overwrites
|
||||
|
||||
channel.upload_to_es()
|
||||
channel.sync_to_videos()
|
||||
ChannelFullScan(channel_id).scan()
|
||||
self.processed["channels"] += 1
|
||||
|
||||
def _reindex_single_playlist(self, playlist_id: str) -> None:
|
||||
"""refresh playlist data"""
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
playlist.get_from_es()
|
||||
if (
|
||||
not playlist.json_data
|
||||
or playlist.json_data["playlist_type"] == "custom"
|
||||
):
|
||||
return
|
||||
|
||||
is_active = playlist.update_playlist()
|
||||
if not is_active:
|
||||
playlist.deactivate()
|
||||
return
|
||||
|
||||
self.processed["playlists"] += 1
|
||||
|
||||
def cookie_is_valid(self) -> bool:
|
||||
"""return true if cookie is enabled and valid"""
|
||||
if not self.config["downloads"]["cookie_import"]:
|
||||
# is not activated, continue reindex
|
||||
return True
|
||||
|
||||
valid = CookieHandler(self.config).validate()
|
||||
return valid
|
||||
|
||||
def build_message(self) -> str:
|
||||
"""build progress message"""
|
||||
message = ""
|
||||
for key, value in self.processed.items():
|
||||
if value:
|
||||
message = message + f"{value} {key}, "
|
||||
|
||||
if message:
|
||||
message = f"reindexed {message.rstrip(', ')}"
|
||||
|
||||
return message
|
||||
|
||||
|
||||
class ReindexProgress(ReindexBase):
|
||||
"""
|
||||
get progress of reindex task
|
||||
request_type: key of self.REINDEX_CONFIG
|
||||
request_id: id of request_type
|
||||
return = {
|
||||
"state": "running" | "queued" | False
|
||||
"total_queued": int
|
||||
"in_queue_name": "queue_name"
|
||||
}
|
||||
"""
|
||||
|
||||
def __init__(self, request_type=False, request_id=False):
|
||||
super().__init__()
|
||||
self.request_type = request_type
|
||||
self.request_id = request_id
|
||||
|
||||
def get_progress(self) -> dict:
|
||||
"""get progress from task"""
|
||||
queue_name, request_type = self._get_queue_name()
|
||||
total = self._get_total_in_queue(queue_name)
|
||||
|
||||
progress = {
|
||||
"total_queued": total,
|
||||
"type": request_type,
|
||||
}
|
||||
state = self._get_state(total, queue_name)
|
||||
progress.update(state)
|
||||
|
||||
return progress
|
||||
|
||||
def _get_queue_name(self):
|
||||
"""return queue_name, queue_type, raise exception on error"""
|
||||
if not self.request_type:
|
||||
return "all", "all"
|
||||
|
||||
reindex_config = self.REINDEX_CONFIG.get(self.request_type)
|
||||
if not reindex_config:
|
||||
print(f"reindex_config not found: {self.request_type}")
|
||||
raise ValueError
|
||||
|
||||
return reindex_config["queue_name"], self.request_type
|
||||
|
||||
def _get_total_in_queue(self, queue_name):
|
||||
"""get all items in queue"""
|
||||
total = 0
|
||||
if queue_name == "all":
|
||||
queues = [i["queue_name"] for i in self.REINDEX_CONFIG.values()]
|
||||
for queue in queues:
|
||||
total += len(RedisQueue(queue).get_all())
|
||||
else:
|
||||
total += len(RedisQueue(queue_name).get_all())
|
||||
|
||||
return total
|
||||
|
||||
def _get_state(self, total, queue_name):
|
||||
"""get state based on request_id"""
|
||||
state_dict = {}
|
||||
if self.request_id:
|
||||
state = RedisQueue(queue_name).in_queue(self.request_id)
|
||||
state_dict.update({"id": self.request_id, "state": state})
|
||||
|
||||
return state_dict
|
||||
|
||||
if total:
|
||||
state = "running"
|
||||
else:
|
||||
state = "empty"
|
||||
|
||||
state_dict.update({"state": state})
|
||||
|
||||
return state_dict
|
||||
|
||||
|
||||
class ChannelFullScan:
|
||||
"""
|
||||
update from v0.3.0 to v0.3.1
|
||||
full scan of channel to fix vid_type mismatch
|
||||
"""
|
||||
|
||||
def __init__(self, channel_id):
|
||||
self.channel_id = channel_id
|
||||
self.to_update = False
|
||||
|
||||
def scan(self):
|
||||
"""match local with remote"""
|
||||
print(f"{self.channel_id}: start full scan")
|
||||
all_local_videos = self._get_all_local()
|
||||
all_remote_videos = self._get_all_remote()
|
||||
self.to_update = []
|
||||
for video in all_local_videos:
|
||||
video_id = video["youtube_id"]
|
||||
remote_match = [i for i in all_remote_videos if i[0] == video_id]
|
||||
if not remote_match:
|
||||
print(f"{video_id}: no remote match found")
|
||||
continue
|
||||
|
||||
expected_type = remote_match[0][-1]
|
||||
if video["vid_type"] != expected_type:
|
||||
self.to_update.append(
|
||||
{
|
||||
"video_id": video_id,
|
||||
"vid_type": expected_type,
|
||||
}
|
||||
)
|
||||
|
||||
self.update()
|
||||
|
||||
def _get_all_remote(self):
|
||||
"""get all channel videos"""
|
||||
sub = ChannelSubscription()
|
||||
all_remote_videos = sub.get_last_youtube_videos(
|
||||
self.channel_id, limit=False
|
||||
)
|
||||
|
||||
return all_remote_videos
|
||||
|
||||
def _get_all_local(self):
|
||||
"""get all local indexed channel_videos"""
|
||||
channel = YoutubeChannel(self.channel_id)
|
||||
all_local_videos = channel.get_channel_videos()
|
||||
|
||||
return all_local_videos
|
||||
|
||||
def update(self):
|
||||
"""build bulk query for updates"""
|
||||
if not self.to_update:
|
||||
print(f"{self.channel_id}: nothing to update")
|
||||
return
|
||||
|
||||
print(f"{self.channel_id}: fixing {len(self.to_update)} videos")
|
||||
bulk_list = []
|
||||
for video in self.to_update:
|
||||
action = {
|
||||
"update": {"_id": video.get("video_id"), "_index": "ta_video"}
|
||||
}
|
||||
source = {"doc": {"vid_type": video.get("vid_type")}}
|
||||
bulk_list.append(json.dumps(action))
|
||||
bulk_list.append(json.dumps(source))
|
||||
# add last newline
|
||||
bulk_list.append("\n")
|
||||
data = "\n".join(bulk_list)
|
||||
_, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
|
||||
285
backend/appsettings/src/snapshot.py
Normal file
285
backend/appsettings/src/snapshot.py
Normal file
@@ -0,0 +1,285 @@
|
||||
"""
|
||||
functionality:
|
||||
- handle snapshots in ES
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.helper import get_mapping
|
||||
|
||||
|
||||
class ElasticSnapshot:
|
||||
"""interact with snapshots on ES"""
|
||||
|
||||
REPO = "ta_snapshot"
|
||||
REPO_SETTINGS = {
|
||||
"compress": "true",
|
||||
"chunk_size": "1g",
|
||||
"location": EnvironmentSettings.ES_SNAPSHOT_DIR,
|
||||
}
|
||||
POLICY = "ta_daily"
|
||||
|
||||
def __init__(self):
|
||||
self.all_indices = self._get_all_indices()
|
||||
|
||||
def _get_all_indices(self):
|
||||
"""return all indices names managed by TA"""
|
||||
mapping = get_mapping()
|
||||
all_indices = [f"ta_{i['index_name']}" for i in mapping]
|
||||
|
||||
return all_indices
|
||||
|
||||
def setup(self):
|
||||
"""setup the snapshot in ES, create or update if needed"""
|
||||
print("snapshot: run setup")
|
||||
repo_exists = self._check_repo_exists()
|
||||
if not repo_exists:
|
||||
self.create_repo()
|
||||
|
||||
policy_exists = self._check_policy_exists()
|
||||
if not policy_exists:
|
||||
self.create_policy()
|
||||
|
||||
is_outdated = self._needs_startup_snapshot()
|
||||
if is_outdated:
|
||||
_ = self.take_snapshot_now()
|
||||
|
||||
def _check_repo_exists(self):
|
||||
"""check if expected repo already exists"""
|
||||
path = f"_snapshot/{self.REPO}"
|
||||
response, statuscode = ElasticWrap(path).get()
|
||||
if statuscode == 200:
|
||||
print(f"snapshot: repo {self.REPO} already created")
|
||||
matching = response[self.REPO]["settings"] == self.REPO_SETTINGS
|
||||
if not matching:
|
||||
print(f"snapshot: update repo settings {self.REPO_SETTINGS}")
|
||||
|
||||
return matching
|
||||
|
||||
print(f"snapshot: setup repo {self.REPO} config {self.REPO_SETTINGS}")
|
||||
return False
|
||||
|
||||
def create_repo(self):
|
||||
"""create filesystem repo"""
|
||||
path = f"_snapshot/{self.REPO}"
|
||||
data = {
|
||||
"type": "fs",
|
||||
"settings": self.REPO_SETTINGS,
|
||||
}
|
||||
response, statuscode = ElasticWrap(path).post(data=data)
|
||||
if statuscode == 200:
|
||||
print(f"snapshot: repo setup correctly: {response}")
|
||||
|
||||
def _check_policy_exists(self):
|
||||
"""check if snapshot policy is set correctly"""
|
||||
policy = self._get_policy()
|
||||
expected_policy = self._build_policy_data()
|
||||
if not policy:
|
||||
print(f"snapshot: create policy {self.POLICY} {expected_policy}")
|
||||
return False
|
||||
|
||||
if policy["policy"] != expected_policy:
|
||||
print(f"snapshot: update policy settings {expected_policy}")
|
||||
return False
|
||||
|
||||
print("snapshot: policy is set.")
|
||||
return True
|
||||
|
||||
def _get_policy(self):
|
||||
"""get policy from es"""
|
||||
path = f"_slm/policy/{self.POLICY}"
|
||||
response, statuscode = ElasticWrap(path).get()
|
||||
if statuscode != 200:
|
||||
return False
|
||||
|
||||
return response[self.POLICY]
|
||||
|
||||
def create_policy(self):
|
||||
"""create snapshot lifetime policy"""
|
||||
path = f"_slm/policy/{self.POLICY}"
|
||||
data = self._build_policy_data()
|
||||
response, statuscode = ElasticWrap(path).put(data)
|
||||
if statuscode == 200:
|
||||
print(f"snapshot: policy setup correctly: {response}")
|
||||
|
||||
def _build_policy_data(self):
|
||||
"""build policy dict from config"""
|
||||
at_12 = datetime.now().replace(hour=12, minute=0, second=0)
|
||||
hour = at_12.astimezone(ZoneInfo("UTC")).hour
|
||||
|
||||
return {
|
||||
"schedule": f"0 0 {hour} * * ?",
|
||||
"name": f"<{self.POLICY}_>",
|
||||
"repository": self.REPO,
|
||||
"config": {
|
||||
"indices": self.all_indices,
|
||||
"ignore_unavailable": True,
|
||||
"include_global_state": True,
|
||||
},
|
||||
"retention": {
|
||||
"expire_after": "30d",
|
||||
"min_count": 5,
|
||||
"max_count": 50,
|
||||
},
|
||||
}
|
||||
|
||||
def _needs_startup_snapshot(self):
|
||||
"""check if last snapshot is expired"""
|
||||
snap_dicts = self._get_all_snapshots()
|
||||
if not snap_dicts:
|
||||
print("snapshot: create initial snapshot")
|
||||
return True
|
||||
|
||||
last_stamp = snap_dicts[0]["end_stamp"]
|
||||
now = int(datetime.now().timestamp())
|
||||
outdated = (now - last_stamp) / 60 / 60 > 24
|
||||
if outdated:
|
||||
print("snapshot: is outdated, create new now")
|
||||
|
||||
print("snapshot: last snapshot is up-to-date")
|
||||
return outdated
|
||||
|
||||
def take_snapshot_now(self, wait=False):
|
||||
"""execute daily snapshot now"""
|
||||
path = f"_slm/policy/{self.POLICY}/_execute"
|
||||
response, statuscode = ElasticWrap(path).post()
|
||||
if statuscode == 200:
|
||||
print(f"snapshot: executing now: {response}")
|
||||
|
||||
if wait:
|
||||
self._wait_for_snapshot(response["snapshot_name"])
|
||||
|
||||
return response
|
||||
|
||||
def _wait_for_snapshot(self, snapshot_name):
|
||||
"""return after snapshot_name completes"""
|
||||
path = f"_snapshot/{self.REPO}/{snapshot_name}"
|
||||
|
||||
while True:
|
||||
# wait for task to be created
|
||||
sleep(1)
|
||||
_, statuscode = ElasticWrap(path).get()
|
||||
if statuscode == 200:
|
||||
break
|
||||
|
||||
while True:
|
||||
# wait for snapshot success
|
||||
response, statuscode = ElasticWrap(path).get()
|
||||
snapshot_state = response["snapshots"][0]["state"]
|
||||
if snapshot_state == "SUCCESS":
|
||||
break
|
||||
|
||||
print(f"snapshot: {snapshot_name} in state {snapshot_state}")
|
||||
print("snapshot: wait to complete")
|
||||
sleep(5)
|
||||
|
||||
print(f"snapshot: completed - {response}")
|
||||
|
||||
def get_snapshot_stats(self):
|
||||
"""get snapshot info for frontend"""
|
||||
snapshot_info = self._build_policy_details()
|
||||
if snapshot_info:
|
||||
snapshot_info.update({"snapshots": self._get_all_snapshots()})
|
||||
|
||||
return snapshot_info
|
||||
|
||||
def get_single_snapshot(self, snapshot_id):
|
||||
"""get single snapshot metadata"""
|
||||
path = f"_snapshot/{self.REPO}/{snapshot_id}"
|
||||
response, statuscode = ElasticWrap(path).get()
|
||||
if statuscode == 404:
|
||||
print(f"snapshots: not found: {snapshot_id}")
|
||||
return False
|
||||
|
||||
snapshot = response["snapshots"][0]
|
||||
return self._parse_single_snapshot(snapshot)
|
||||
|
||||
def _get_all_snapshots(self):
|
||||
"""get a list of all registered snapshots"""
|
||||
path = f"_snapshot/{self.REPO}/*?sort=start_time&order=desc"
|
||||
response, statuscode = ElasticWrap(path).get()
|
||||
if statuscode == 404:
|
||||
print("snapshots: not configured")
|
||||
return False
|
||||
|
||||
all_snapshots = response["snapshots"]
|
||||
if not all_snapshots:
|
||||
print("snapshots: no snapshots found")
|
||||
return False
|
||||
|
||||
snap_dicts = []
|
||||
for snapshot in all_snapshots:
|
||||
snap_dict = self._parse_single_snapshot(snapshot)
|
||||
snap_dicts.append(snap_dict)
|
||||
|
||||
return snap_dicts
|
||||
|
||||
def _parse_single_snapshot(self, snapshot):
|
||||
"""extract relevant metadata from single snapshot"""
|
||||
snap_dict = {
|
||||
"id": snapshot["snapshot"],
|
||||
"state": snapshot["state"],
|
||||
"es_version": snapshot["version"],
|
||||
"start_date": self._date_converter(snapshot["start_time"]),
|
||||
"end_date": self._date_converter(snapshot["end_time"]),
|
||||
"end_stamp": snapshot["end_time_in_millis"] // 1000,
|
||||
"duration_s": snapshot["duration_in_millis"] // 1000,
|
||||
}
|
||||
return snap_dict
|
||||
|
||||
def _build_policy_details(self):
|
||||
"""get additional policy details"""
|
||||
policy = self._get_policy()
|
||||
if not policy:
|
||||
return False
|
||||
|
||||
next_exec = policy["next_execution_millis"] // 1000
|
||||
next_exec_date = datetime.fromtimestamp(next_exec)
|
||||
next_exec_str = next_exec_date.strftime("%Y-%m-%d %H:%M")
|
||||
expire_after = policy["policy"]["retention"]["expire_after"]
|
||||
policy_metadata = {
|
||||
"next_exec": next_exec,
|
||||
"next_exec_str": next_exec_str,
|
||||
"expire_after": expire_after,
|
||||
}
|
||||
return policy_metadata
|
||||
|
||||
@staticmethod
|
||||
def _date_converter(date_utc):
|
||||
"""convert datetime string"""
|
||||
date = datetime.strptime(date_utc, "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
utc_date = date.replace(tzinfo=ZoneInfo("UTC"))
|
||||
converted = utc_date.astimezone(ZoneInfo(EnvironmentSettings.TZ))
|
||||
converted_str = converted.strftime("%Y-%m-%d %H:%M")
|
||||
|
||||
return converted_str
|
||||
|
||||
def restore_all(self, snapshot_name):
|
||||
"""restore snapshot by name"""
|
||||
for index in self.all_indices:
|
||||
_, _ = ElasticWrap(index).delete()
|
||||
|
||||
path = f"_snapshot/{self.REPO}/{snapshot_name}/_restore"
|
||||
data = {"indices": "*"}
|
||||
response, statuscode = ElasticWrap(path).post(data=data)
|
||||
if statuscode == 200:
|
||||
print(f"snapshot: executing now: {response}")
|
||||
return response
|
||||
|
||||
print(f"snapshot: failed to restore, {statuscode} {response}")
|
||||
return False
|
||||
|
||||
def delete_single_snapshot(self, snapshot_id):
|
||||
"""delete single snapshot from index"""
|
||||
path = f"_snapshot/{self.REPO}/{snapshot_id}"
|
||||
response, statuscode = ElasticWrap(path).delete()
|
||||
if statuscode == 200:
|
||||
print(f"snapshot: deleting {snapshot_id} {response}")
|
||||
return response
|
||||
|
||||
print(f"snapshot: failed to delete, {statuscode} {response}")
|
||||
return False
|
||||
42
backend/appsettings/urls.py
Normal file
42
backend/appsettings/urls.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""all app settings API urls"""
|
||||
|
||||
from appsettings import views
|
||||
from django.urls import path
|
||||
|
||||
urlpatterns = [
|
||||
path(
|
||||
"config/",
|
||||
views.AppConfigApiView.as_view(),
|
||||
name="api-config",
|
||||
),
|
||||
path(
|
||||
"snapshot/",
|
||||
views.SnapshotApiListView.as_view(),
|
||||
name="api-snapshot-list",
|
||||
),
|
||||
path(
|
||||
"snapshot/<slug:snapshot_id>/",
|
||||
views.SnapshotApiView.as_view(),
|
||||
name="api-snapshot",
|
||||
),
|
||||
path(
|
||||
"backup/",
|
||||
views.BackupApiListView.as_view(),
|
||||
name="api-backup-list",
|
||||
),
|
||||
path(
|
||||
"backup/<str:filename>/",
|
||||
views.BackupApiView.as_view(),
|
||||
name="api-backup",
|
||||
),
|
||||
path(
|
||||
"cookie/",
|
||||
views.CookieView.as_view(),
|
||||
name="api-cookie",
|
||||
),
|
||||
path(
|
||||
"token/",
|
||||
views.TokenView.as_view(),
|
||||
name="api-token",
|
||||
),
|
||||
]
|
||||
252
backend/appsettings/views.py
Normal file
252
backend/appsettings/views.py
Normal file
@@ -0,0 +1,252 @@
|
||||
"""all app settings API views"""
|
||||
|
||||
from appsettings.src.backup import ElasticBackup
|
||||
from appsettings.src.config import AppConfig
|
||||
from appsettings.src.snapshot import ElasticSnapshot
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
from common.views_base import AdminOnly, ApiBaseView
|
||||
from download.src.yt_dlp_base import CookieHandler
|
||||
from rest_framework.response import Response
|
||||
from task.src.task_manager import TaskCommand
|
||||
from task.tasks import run_restore_backup
|
||||
|
||||
|
||||
class AppConfigApiView(ApiBaseView):
|
||||
"""resolves to /api/appsettings/config/
|
||||
GET: return app settings
|
||||
POST: update app settings
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
@staticmethod
|
||||
def get(request):
|
||||
"""get config"""
|
||||
response = AppConfig().config
|
||||
return Response(response)
|
||||
|
||||
@staticmethod
|
||||
def post(request):
|
||||
"""
|
||||
update config values
|
||||
data object where key is flatted CONFIG_DEFAULTS separated by '.', e.g.
|
||||
{"subscriptions.channel_size": 5, "subscriptions.live_channel_size": 5}
|
||||
"""
|
||||
data = request.data
|
||||
try:
|
||||
config = AppConfig().update_config(data)
|
||||
except ValueError as err:
|
||||
return Response({"error": str(err)}, status=400)
|
||||
|
||||
return Response(config)
|
||||
|
||||
|
||||
class SnapshotApiListView(ApiBaseView):
|
||||
"""resolves to /api/appsettings/snapshot/
|
||||
GET: returns snapshot config plus list of existing snapshots
|
||||
POST: take snapshot now
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
@staticmethod
|
||||
def get(request):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
snapshots = ElasticSnapshot().get_snapshot_stats()
|
||||
|
||||
return Response(snapshots)
|
||||
|
||||
@staticmethod
|
||||
def post(request):
|
||||
"""take snapshot now with post request"""
|
||||
# pylint: disable=unused-argument
|
||||
response = ElasticSnapshot().take_snapshot_now()
|
||||
|
||||
return Response(response)
|
||||
|
||||
|
||||
class SnapshotApiView(ApiBaseView):
|
||||
"""resolves to /api/appsettings/snapshot/<snapshot-id>/
|
||||
GET: return a single snapshot
|
||||
POST: restore snapshot
|
||||
DELETE: delete a snapshot
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
@staticmethod
|
||||
def get(request, snapshot_id):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
|
||||
|
||||
if not snapshot:
|
||||
return Response({"message": "snapshot not found"}, status=404)
|
||||
|
||||
return Response(snapshot)
|
||||
|
||||
@staticmethod
|
||||
def post(request, snapshot_id):
|
||||
"""restore snapshot with post request"""
|
||||
# pylint: disable=unused-argument
|
||||
response = ElasticSnapshot().restore_all(snapshot_id)
|
||||
if not response:
|
||||
message = {"message": "failed to restore snapshot"}
|
||||
return Response(message, status=400)
|
||||
|
||||
return Response(response)
|
||||
|
||||
@staticmethod
|
||||
def delete(request, snapshot_id):
|
||||
"""delete snapshot from index"""
|
||||
# pylint: disable=unused-argument
|
||||
response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
|
||||
if not response:
|
||||
message = {"message": "failed to delete snapshot"}
|
||||
return Response(message, status=400)
|
||||
|
||||
return Response(response)
|
||||
|
||||
|
||||
class BackupApiListView(ApiBaseView):
|
||||
"""resolves to /api/appsettings/backup/
|
||||
GET: returns list of available zip backups
|
||||
POST: take zip backup now
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
task_name = "run_backup"
|
||||
|
||||
@staticmethod
|
||||
def get(request):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
backup_files = ElasticBackup().get_all_backup_files()
|
||||
return Response(backup_files)
|
||||
|
||||
def post(self, request):
|
||||
"""handle post request"""
|
||||
# pylint: disable=unused-argument
|
||||
response = TaskCommand().start(self.task_name)
|
||||
message = {
|
||||
"message": "backup task started",
|
||||
"task_id": response["task_id"],
|
||||
}
|
||||
|
||||
return Response(message)
|
||||
|
||||
|
||||
class BackupApiView(ApiBaseView):
|
||||
"""resolves to /api/appsettings/backup/<filename>/
|
||||
GET: return a single backup
|
||||
POST: restore backup
|
||||
DELETE: delete backup
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
task_name = "restore_backup"
|
||||
|
||||
@staticmethod
|
||||
def get(request, filename):
|
||||
"""get single backup"""
|
||||
# pylint: disable=unused-argument
|
||||
backup_file = ElasticBackup().build_backup_file_data(filename)
|
||||
if not backup_file:
|
||||
message = {"message": "file not found"}
|
||||
return Response(message, status=404)
|
||||
|
||||
return Response(backup_file)
|
||||
|
||||
def post(self, request, filename):
|
||||
"""restore backup file"""
|
||||
# pylint: disable=unused-argument
|
||||
task = run_restore_backup.delay(filename)
|
||||
message = {
|
||||
"message": "backup restore task started",
|
||||
"filename": filename,
|
||||
"task_id": task.id,
|
||||
}
|
||||
return Response(message)
|
||||
|
||||
@staticmethod
|
||||
def delete(request, filename):
|
||||
"""delete backup file"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
backup_file = ElasticBackup().delete_file(filename)
|
||||
if not backup_file:
|
||||
message = {"message": "file not found"}
|
||||
return Response(message, status=404)
|
||||
|
||||
message = {"message": f"file {filename} deleted"}
|
||||
return Response(message)
|
||||
|
||||
|
||||
class CookieView(ApiBaseView):
|
||||
"""resolves to /api/appsettings/cookie/
|
||||
GET: check if cookie is enabled
|
||||
POST: verify validity of cookie
|
||||
PUT: import cookie
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
@staticmethod
|
||||
def get(request):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
config = AppConfig().config
|
||||
valid = RedisArchivist().get_message("cookie:valid")
|
||||
response = {"cookie_enabled": config["downloads"]["cookie_import"]}
|
||||
response.update(valid)
|
||||
|
||||
return Response(response)
|
||||
|
||||
@staticmethod
|
||||
def post(request):
|
||||
"""handle post request"""
|
||||
# pylint: disable=unused-argument
|
||||
config = AppConfig().config
|
||||
validated = CookieHandler(config).validate()
|
||||
|
||||
return Response({"cookie_validated": validated})
|
||||
|
||||
@staticmethod
|
||||
def put(request):
|
||||
"""handle put request"""
|
||||
# pylint: disable=unused-argument
|
||||
config = AppConfig().config
|
||||
cookie = request.data.get("cookie")
|
||||
if not cookie:
|
||||
message = "missing cookie key in request data"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
print(f"cookie preview:\n\n{cookie[:300]}")
|
||||
handler = CookieHandler(config)
|
||||
handler.set_cookie(cookie)
|
||||
validated = handler.validate()
|
||||
if not validated:
|
||||
handler.revoke()
|
||||
message = {"cookie_import": "fail", "cookie_validated": validated}
|
||||
print(f"cookie: {message}")
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
message = {"cookie_import": "done", "cookie_validated": validated}
|
||||
return Response(message)
|
||||
|
||||
|
||||
class TokenView(ApiBaseView):
|
||||
"""resolves to /api/appsettings/token/
|
||||
DELETE: revoke the token
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
@staticmethod
|
||||
def delete(request):
|
||||
"""delete the token, new will get created automatically"""
|
||||
print("revoke API token")
|
||||
request.user.auth_token.delete()
|
||||
return Response({"success": True})
|
||||
0
backend/channel/__init__.py
Normal file
0
backend/channel/__init__.py
Normal file
0
backend/channel/migrations/__init__.py
Normal file
0
backend/channel/migrations/__init__.py
Normal file
0
backend/channel/src/__init__.py
Normal file
0
backend/channel/src/__init__.py
Normal file
355
backend/channel/src/index.py
Normal file
355
backend/channel/src/index.py
Normal file
@@ -0,0 +1,355 @@
|
||||
"""
|
||||
functionality:
|
||||
- get metadata from youtube for a channel
|
||||
- index and update in es
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.index_generic import YouTubeItem
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
|
||||
|
||||
class YoutubeChannel(YouTubeItem):
|
||||
"""represents a single youtube channel"""
|
||||
|
||||
es_path = False
|
||||
index_name = "ta_channel"
|
||||
yt_base = "https://www.youtube.com/channel/"
|
||||
yt_obs = {
|
||||
"playlist_items": "1,0",
|
||||
"skip_download": True,
|
||||
}
|
||||
|
||||
def __init__(self, youtube_id, task=False):
|
||||
super().__init__(youtube_id)
|
||||
self.all_playlists = False
|
||||
self.task = task
|
||||
|
||||
def build_json(self, upload=False, fallback=False):
|
||||
"""get from es or from youtube"""
|
||||
self.get_from_es()
|
||||
if self.json_data:
|
||||
return
|
||||
|
||||
self.get_from_youtube()
|
||||
if not self.youtube_meta and fallback:
|
||||
self._video_fallback(fallback)
|
||||
else:
|
||||
self.process_youtube_meta()
|
||||
self.get_channel_art()
|
||||
|
||||
if upload:
|
||||
self.upload_to_es()
|
||||
|
||||
def process_youtube_meta(self):
|
||||
"""extract relevant fields"""
|
||||
self.youtube_meta["thumbnails"].reverse()
|
||||
self.json_data = {
|
||||
"channel_active": True,
|
||||
"channel_description": self.youtube_meta.get("description", False),
|
||||
"channel_id": self.youtube_id,
|
||||
"channel_last_refresh": int(datetime.now().timestamp()),
|
||||
"channel_name": self.youtube_meta["uploader"],
|
||||
"channel_subs": self.youtube_meta.get("channel_follower_count", 0),
|
||||
"channel_subscribed": False,
|
||||
"channel_tags": self._parse_tags(self.youtube_meta.get("tags")),
|
||||
"channel_banner_url": self._get_banner_art(),
|
||||
"channel_thumb_url": self._get_thumb_art(),
|
||||
"channel_tvart_url": self._get_tv_art(),
|
||||
"channel_views": self.youtube_meta.get("view_count") or 0,
|
||||
}
|
||||
|
||||
def _parse_tags(self, tags):
|
||||
"""parse channel tags"""
|
||||
if not tags:
|
||||
return False
|
||||
|
||||
joined = " ".join(tags)
|
||||
return [i.strip() for i in joined.split('"') if i and not i == " "]
|
||||
|
||||
def _get_thumb_art(self):
|
||||
"""extract thumb art"""
|
||||
for i in self.youtube_meta["thumbnails"]:
|
||||
if not i.get("width"):
|
||||
continue
|
||||
if i.get("width") == i.get("height"):
|
||||
return i["url"]
|
||||
|
||||
return False
|
||||
|
||||
def _get_tv_art(self):
|
||||
"""extract tv artwork"""
|
||||
for i in self.youtube_meta["thumbnails"]:
|
||||
if i.get("id") == "banner_uncropped":
|
||||
return i["url"]
|
||||
for i in self.youtube_meta["thumbnails"]:
|
||||
if not i.get("width"):
|
||||
continue
|
||||
if i["width"] // i["height"] < 2 and not i["width"] == i["height"]:
|
||||
return i["url"]
|
||||
|
||||
return False
|
||||
|
||||
def _get_banner_art(self):
|
||||
"""extract banner artwork"""
|
||||
for i in self.youtube_meta["thumbnails"]:
|
||||
if not i.get("width"):
|
||||
continue
|
||||
if i["width"] // i["height"] > 5:
|
||||
return i["url"]
|
||||
|
||||
return False
|
||||
|
||||
def _video_fallback(self, fallback):
|
||||
"""use video metadata as fallback"""
|
||||
print(f"{self.youtube_id}: fallback to video metadata")
|
||||
self.json_data = {
|
||||
"channel_active": False,
|
||||
"channel_last_refresh": int(datetime.now().timestamp()),
|
||||
"channel_subs": fallback.get("channel_follower_count", 0),
|
||||
"channel_name": fallback["uploader"],
|
||||
"channel_banner_url": False,
|
||||
"channel_tvart_url": False,
|
||||
"channel_id": self.youtube_id,
|
||||
"channel_subscribed": False,
|
||||
"channel_tags": False,
|
||||
"channel_description": False,
|
||||
"channel_thumb_url": False,
|
||||
"channel_views": 0,
|
||||
}
|
||||
self._info_json_fallback()
|
||||
|
||||
def _info_json_fallback(self):
|
||||
"""read channel info.json for additional metadata"""
|
||||
info_json = os.path.join(
|
||||
EnvironmentSettings.CACHE_DIR,
|
||||
"import",
|
||||
f"{self.youtube_id}.info.json",
|
||||
)
|
||||
if os.path.exists(info_json):
|
||||
print(f"{self.youtube_id}: read info.json file")
|
||||
with open(info_json, "r", encoding="utf-8") as f:
|
||||
content = json.loads(f.read())
|
||||
|
||||
self.json_data.update(
|
||||
{
|
||||
"channel_subs": content.get("channel_follower_count", 0),
|
||||
"channel_description": content.get("description", False),
|
||||
}
|
||||
)
|
||||
os.remove(info_json)
|
||||
|
||||
def get_channel_art(self):
|
||||
"""download channel art for new channels"""
|
||||
urls = (
|
||||
self.json_data["channel_thumb_url"],
|
||||
self.json_data["channel_banner_url"],
|
||||
self.json_data["channel_tvart_url"],
|
||||
)
|
||||
ThumbManager(self.youtube_id, item_type="channel").download(urls)
|
||||
|
||||
def sync_to_videos(self):
|
||||
"""sync new channel_dict to all videos of channel"""
|
||||
# add ingest pipeline
|
||||
processors = []
|
||||
for field, value in self.json_data.items():
|
||||
line = {"set": {"field": "channel." + field, "value": value}}
|
||||
processors.append(line)
|
||||
data = {"description": self.youtube_id, "processors": processors}
|
||||
ingest_path = f"_ingest/pipeline/{self.youtube_id}"
|
||||
_, _ = ElasticWrap(ingest_path).put(data)
|
||||
# apply pipeline
|
||||
data = {"query": {"match": {"channel.channel_id": self.youtube_id}}}
|
||||
update_path = f"ta_video/_update_by_query?pipeline={self.youtube_id}"
|
||||
_, _ = ElasticWrap(update_path).post(data)
|
||||
|
||||
def get_folder_path(self):
|
||||
"""get folder where media files get stored"""
|
||||
folder_path = os.path.join(
|
||||
EnvironmentSettings.MEDIA_DIR,
|
||||
self.json_data["channel_id"],
|
||||
)
|
||||
return folder_path
|
||||
|
||||
def delete_es_videos(self):
|
||||
"""delete all channel documents from elasticsearch"""
|
||||
data = {
|
||||
"query": {
|
||||
"term": {"channel.channel_id": {"value": self.youtube_id}}
|
||||
}
|
||||
}
|
||||
_, _ = ElasticWrap("ta_video/_delete_by_query").post(data)
|
||||
|
||||
def delete_es_comments(self):
|
||||
"""delete all comments from this channel"""
|
||||
data = {
|
||||
"query": {
|
||||
"term": {"comment_channel_id": {"value": self.youtube_id}}
|
||||
}
|
||||
}
|
||||
_, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
|
||||
|
||||
def delete_es_subtitles(self):
|
||||
"""delete all subtitles from this channel"""
|
||||
data = {
|
||||
"query": {
|
||||
"term": {"subtitle_channel_id": {"value": self.youtube_id}}
|
||||
}
|
||||
}
|
||||
_, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data)
|
||||
|
||||
def delete_playlists(self):
|
||||
"""delete all indexed playlist from es"""
|
||||
all_playlists = self.get_indexed_playlists()
|
||||
for playlist in all_playlists:
|
||||
YoutubePlaylist(playlist["playlist_id"]).delete_metadata()
|
||||
|
||||
def delete_channel(self):
|
||||
"""delete channel and all videos"""
|
||||
print(f"{self.youtube_id}: delete channel")
|
||||
self.get_from_es()
|
||||
if not self.json_data:
|
||||
raise FileNotFoundError
|
||||
|
||||
folder_path = self.get_folder_path()
|
||||
print(f"{self.youtube_id}: delete all media files")
|
||||
try:
|
||||
all_videos = os.listdir(folder_path)
|
||||
for video in all_videos:
|
||||
video_path = os.path.join(folder_path, video)
|
||||
os.remove(video_path)
|
||||
os.rmdir(folder_path)
|
||||
except FileNotFoundError:
|
||||
print(f"no videos found for {folder_path}")
|
||||
|
||||
print(f"{self.youtube_id}: delete indexed playlists")
|
||||
self.delete_playlists()
|
||||
print(f"{self.youtube_id}: delete indexed videos")
|
||||
self.delete_es_videos()
|
||||
self.delete_es_comments()
|
||||
self.delete_es_subtitles()
|
||||
self.del_in_es()
|
||||
|
||||
def index_channel_playlists(self):
|
||||
"""add all playlists of channel to index"""
|
||||
print(f"{self.youtube_id}: index all playlists")
|
||||
self.get_from_es()
|
||||
channel_name = self.json_data["channel_name"]
|
||||
self.task.send_progress([f"{channel_name}: Looking for Playlists"])
|
||||
self.get_all_playlists()
|
||||
if not self.all_playlists:
|
||||
print(f"{self.youtube_id}: no playlists found.")
|
||||
return
|
||||
|
||||
total = len(self.all_playlists)
|
||||
for idx, playlist in enumerate(self.all_playlists):
|
||||
if self.task:
|
||||
self._notify_single_playlist(idx, total)
|
||||
|
||||
self._index_single_playlist(playlist)
|
||||
print("add playlist: " + playlist[1])
|
||||
|
||||
def _notify_single_playlist(self, idx, total):
|
||||
"""send notification"""
|
||||
channel_name = self.json_data["channel_name"]
|
||||
message = [
|
||||
f"{channel_name}: Scanning channel for playlists",
|
||||
f"Progress: {idx + 1}/{total}",
|
||||
]
|
||||
self.task.send_progress(message, progress=(idx + 1) / total)
|
||||
|
||||
@staticmethod
|
||||
def _index_single_playlist(playlist):
|
||||
"""add single playlist if needed"""
|
||||
playlist = YoutubePlaylist(playlist[0])
|
||||
playlist.update_playlist(skip_on_empty=True)
|
||||
|
||||
def get_channel_videos(self):
|
||||
"""get all videos from channel"""
|
||||
data = {
|
||||
"query": {
|
||||
"term": {"channel.channel_id": {"value": self.youtube_id}}
|
||||
},
|
||||
"_source": ["youtube_id", "vid_type"],
|
||||
}
|
||||
all_videos = IndexPaginate("ta_video", data).get_results()
|
||||
return all_videos
|
||||
|
||||
def get_all_playlists(self):
|
||||
"""get all playlists owned by this channel"""
|
||||
url = (
|
||||
f"https://www.youtube.com/channel/{self.youtube_id}"
|
||||
+ "/playlists?view=1&sort=dd&shelf_id=0"
|
||||
)
|
||||
obs = {"skip_download": True, "extract_flat": True}
|
||||
playlists = YtWrap(obs, self.config).extract(url)
|
||||
all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
|
||||
self.all_playlists = all_entries
|
||||
|
||||
def get_indexed_playlists(self, active_only=False):
|
||||
"""get all indexed playlists from channel"""
|
||||
must_list = [
|
||||
{"term": {"playlist_channel_id": {"value": self.youtube_id}}}
|
||||
]
|
||||
if active_only:
|
||||
must_list.append({"term": {"playlist_active": {"value": True}}})
|
||||
|
||||
data = {"query": {"bool": {"must": must_list}}}
|
||||
|
||||
all_playlists = IndexPaginate("ta_playlist", data).get_results()
|
||||
return all_playlists
|
||||
|
||||
def get_overwrites(self) -> dict:
|
||||
"""get all per channel overwrites"""
|
||||
return self.json_data.get("channel_overwrites", {})
|
||||
|
||||
def set_overwrites(self, overwrites):
|
||||
"""set per channel overwrites"""
|
||||
valid_keys = [
|
||||
"download_format",
|
||||
"autodelete_days",
|
||||
"index_playlists",
|
||||
"integrate_sponsorblock",
|
||||
"subscriptions_channel_size",
|
||||
"subscriptions_live_channel_size",
|
||||
"subscriptions_shorts_channel_size",
|
||||
]
|
||||
|
||||
to_write = self.json_data.get("channel_overwrites", {})
|
||||
for key, value in overwrites.items():
|
||||
if key not in valid_keys:
|
||||
raise ValueError(f"invalid overwrite key: {key}")
|
||||
elif value == "disable":
|
||||
to_write[key] = False
|
||||
continue
|
||||
elif value == "0":
|
||||
if key in to_write:
|
||||
del to_write[key]
|
||||
continue
|
||||
elif value == "1":
|
||||
to_write[key] = True
|
||||
continue
|
||||
elif isinstance(value, int) and int(value) < 0:
|
||||
if key in to_write:
|
||||
del to_write[key]
|
||||
continue
|
||||
elif value is not None and value != "":
|
||||
to_write.update({key: value})
|
||||
|
||||
self.json_data["channel_overwrites"] = to_write
|
||||
|
||||
|
||||
def channel_overwrites(channel_id, overwrites):
|
||||
"""collection to overwrite settings per channel"""
|
||||
channel = YoutubeChannel(channel_id)
|
||||
channel.build_json()
|
||||
channel.set_overwrites(overwrites)
|
||||
channel.upload_to_es()
|
||||
channel.sync_to_videos()
|
||||
78
backend/channel/src/nav.py
Normal file
78
backend/channel/src/nav.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""build channel nav"""
|
||||
|
||||
from common.src.es_connect import ElasticWrap
|
||||
|
||||
|
||||
class ChannelNav:
|
||||
"""get all nav items"""
|
||||
|
||||
def __init__(self, channel_id):
|
||||
self.channel_id = channel_id
|
||||
|
||||
def get_nav(self):
|
||||
"""build nav items"""
|
||||
nav = {
|
||||
"has_pending": self._get_has_pending(),
|
||||
"has_playlists": self._get_has_playlists(),
|
||||
}
|
||||
nav.update(self._get_vid_types())
|
||||
return nav
|
||||
|
||||
def _get_vid_types(self):
|
||||
"""get available vid_types in given channel"""
|
||||
data = {
|
||||
"size": 0,
|
||||
"query": {
|
||||
"term": {"channel.channel_id": {"value": self.channel_id}}
|
||||
},
|
||||
"aggs": {"unique_values": {"terms": {"field": "vid_type"}}},
|
||||
}
|
||||
response, _ = ElasticWrap("ta_video/_search").get(data)
|
||||
buckets = response["aggregations"]["unique_values"]["buckets"]
|
||||
|
||||
type_nav = {
|
||||
"has_videos": False,
|
||||
"has_streams": False,
|
||||
"has_shorts": False,
|
||||
}
|
||||
for bucket in buckets:
|
||||
if bucket["key"] == "videos":
|
||||
type_nav["has_videos"] = True
|
||||
if bucket["key"] == "streams":
|
||||
type_nav["has_streams"] = True
|
||||
if bucket["key"] == "shorts":
|
||||
type_nav["has_shorts"] = True
|
||||
|
||||
return type_nav
|
||||
|
||||
def _get_has_pending(self):
|
||||
"""check if has pending videos in download queue"""
|
||||
data = {
|
||||
"size": 1,
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"term": {"status": {"value": "pending"}}},
|
||||
{"term": {"channel_id": {"value": self.channel_id}}},
|
||||
]
|
||||
}
|
||||
},
|
||||
"_source": False,
|
||||
}
|
||||
response, _ = ElasticWrap("ta_download/_search").get(data=data)
|
||||
|
||||
return bool(response["hits"]["hits"])
|
||||
|
||||
def _get_has_playlists(self):
|
||||
"""check if channel has playlists"""
|
||||
path = "ta_playlist/_search"
|
||||
data = {
|
||||
"size": 1,
|
||||
"query": {
|
||||
"term": {"playlist_channel_id": {"value": self.channel_id}}
|
||||
},
|
||||
"_source": False,
|
||||
}
|
||||
response, _ = ElasticWrap(path).get(data=data)
|
||||
|
||||
return bool(response["hits"]["hits"])
|
||||
32
backend/channel/urls.py
Normal file
32
backend/channel/urls.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""all channel API urls"""
|
||||
|
||||
from channel import views
|
||||
from django.urls import path
|
||||
|
||||
urlpatterns = [
|
||||
path(
|
||||
"",
|
||||
views.ChannelApiListView.as_view(),
|
||||
name="api-channel-list",
|
||||
),
|
||||
path(
|
||||
"search/",
|
||||
views.ChannelApiSearchView.as_view(),
|
||||
name="api-channel-search",
|
||||
),
|
||||
path(
|
||||
"<slug:channel_id>/",
|
||||
views.ChannelApiView.as_view(),
|
||||
name="api-channel",
|
||||
),
|
||||
path(
|
||||
"<slug:channel_id>/aggs/",
|
||||
views.ChannelAggsApiView.as_view(),
|
||||
name="api-channel-aggs",
|
||||
),
|
||||
path(
|
||||
"<slug:channel_id>/nav/",
|
||||
views.ChannelNavApiView.as_view(),
|
||||
name="api-channel-nav",
|
||||
),
|
||||
]
|
||||
170
backend/channel/views.py
Normal file
170
backend/channel/views.py
Normal file
@@ -0,0 +1,170 @@
|
||||
"""all channel API views"""
|
||||
|
||||
from channel.src.index import YoutubeChannel
|
||||
from channel.src.nav import ChannelNav
|
||||
from common.src.urlparser import Parser
|
||||
from common.views_base import AdminWriteOnly, ApiBaseView
|
||||
from download.src.subscriptions import ChannelSubscription
|
||||
from rest_framework.response import Response
|
||||
from task.tasks import subscribe_to
|
||||
|
||||
|
||||
class ChannelApiListView(ApiBaseView):
|
||||
"""resolves to /api/channel/
|
||||
GET: returns list of channels
|
||||
POST: edit a list of channels
|
||||
"""
|
||||
|
||||
search_base = "ta_channel/_search/"
|
||||
valid_filter = ["subscribed"]
|
||||
permission_classes = [AdminWriteOnly]
|
||||
|
||||
def get(self, request):
|
||||
"""get request"""
|
||||
self.data.update(
|
||||
{"sort": [{"channel_name.keyword": {"order": "asc"}}]}
|
||||
)
|
||||
|
||||
query_filter = request.GET.get("filter", False)
|
||||
must_list = []
|
||||
if query_filter:
|
||||
if query_filter not in self.valid_filter:
|
||||
message = f"invalid url query filter: {query_filter}"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
must_list.append({"term": {"channel_subscribed": {"value": True}}})
|
||||
|
||||
self.data["query"] = {"bool": {"must": must_list}}
|
||||
self.get_document_list(request)
|
||||
|
||||
return Response(self.response)
|
||||
|
||||
def post(self, request):
|
||||
"""subscribe/unsubscribe to list of channels"""
|
||||
data = request.data
|
||||
try:
|
||||
to_add = data["data"]
|
||||
except KeyError:
|
||||
message = "missing expected data key"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
pending = []
|
||||
for channel_item in to_add:
|
||||
channel_id = channel_item["channel_id"]
|
||||
if channel_item["channel_subscribed"]:
|
||||
pending.append(channel_id)
|
||||
else:
|
||||
self._unsubscribe(channel_id)
|
||||
|
||||
if pending:
|
||||
url_str = " ".join(pending)
|
||||
subscribe_to.delay(url_str, expected_type="channel")
|
||||
|
||||
return Response(data)
|
||||
|
||||
@staticmethod
|
||||
def _unsubscribe(channel_id: str):
|
||||
"""unsubscribe"""
|
||||
print(f"[{channel_id}] unsubscribe from channel")
|
||||
ChannelSubscription().change_subscribe(
|
||||
channel_id, channel_subscribed=False
|
||||
)
|
||||
|
||||
|
||||
class ChannelApiView(ApiBaseView):
|
||||
"""resolves to /api/channel/<channel_id>/
|
||||
GET: returns metadata dict of channel
|
||||
"""
|
||||
|
||||
search_base = "ta_channel/_doc/"
|
||||
permission_classes = [AdminWriteOnly]
|
||||
|
||||
def get(self, request, channel_id):
|
||||
# pylint: disable=unused-argument
|
||||
"""get request"""
|
||||
self.get_document(channel_id)
|
||||
return Response(self.response, status=self.status_code)
|
||||
|
||||
def delete(self, request, channel_id):
|
||||
# pylint: disable=unused-argument
|
||||
"""delete channel"""
|
||||
message = {"channel": channel_id}
|
||||
try:
|
||||
YoutubeChannel(channel_id).delete_channel()
|
||||
status_code = 200
|
||||
message.update({"state": "delete"})
|
||||
except FileNotFoundError:
|
||||
status_code = 404
|
||||
message.update({"state": "not found"})
|
||||
|
||||
return Response(message, status=status_code)
|
||||
|
||||
|
||||
class ChannelAggsApiView(ApiBaseView):
|
||||
"""resolves to /api/channel/<channel_id>/aggs/
|
||||
GET: get channel aggregations
|
||||
"""
|
||||
|
||||
search_base = "ta_video/_search"
|
||||
|
||||
def get(self, request, channel_id):
|
||||
"""get aggs"""
|
||||
self.data.update(
|
||||
{
|
||||
"query": {
|
||||
"term": {"channel.channel_id": {"value": channel_id}}
|
||||
},
|
||||
"aggs": {
|
||||
"total_items": {"value_count": {"field": "youtube_id"}},
|
||||
"total_size": {"sum": {"field": "media_size"}},
|
||||
"total_duration": {"sum": {"field": "player.duration"}},
|
||||
},
|
||||
}
|
||||
)
|
||||
self.get_aggs()
|
||||
|
||||
return Response(self.response)
|
||||
|
||||
|
||||
class ChannelNavApiView(ApiBaseView):
|
||||
"""resolves to /api/channel/<channel_id>/nav/
|
||||
GET: get channel nav
|
||||
"""
|
||||
|
||||
def get(self, request, channel_id):
|
||||
"""get nav"""
|
||||
|
||||
nav = ChannelNav(channel_id).get_nav()
|
||||
return Response(nav)
|
||||
|
||||
|
||||
class ChannelApiSearchView(ApiBaseView):
|
||||
"""resolves to /api/channel/search/
|
||||
search for channel
|
||||
"""
|
||||
|
||||
search_base = "ta_channel/_doc/"
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request, search with s parameter"""
|
||||
|
||||
query = request.GET.get("q")
|
||||
if not query:
|
||||
message = "missing expected q parameter"
|
||||
return Response({"message": message, "data": False}, status=400)
|
||||
|
||||
try:
|
||||
parsed = Parser(query).parse()[0]
|
||||
except (ValueError, IndexError, AttributeError):
|
||||
message = f"channel not found: {query}"
|
||||
return Response({"message": message, "data": False}, status=404)
|
||||
|
||||
if not parsed["type"] == "channel":
|
||||
message = "expected type channel"
|
||||
return Response({"message": message, "data": False}, status=400)
|
||||
|
||||
self.get_document(parsed["url"])
|
||||
|
||||
return Response(self.response, status=self.status_code)
|
||||
0
backend/common/__init__.py
Normal file
0
backend/common/__init__.py
Normal file
0
backend/common/migrations/__init__.py
Normal file
0
backend/common/migrations/__init__.py
Normal file
0
backend/common/src/__init__.py
Normal file
0
backend/common/src/__init__.py
Normal file
117
backend/common/src/env_settings.py
Normal file
117
backend/common/src/env_settings.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
Functionality:
|
||||
- read and write application config backed by ES
|
||||
- encapsulate persistence of application properties
|
||||
"""
|
||||
|
||||
from os import environ
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
print("loading local dotenv")
|
||||
load_dotenv(".env")
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
class EnvironmentSettings:
|
||||
"""
|
||||
Handle settings for the application that are driven from the environment.
|
||||
These will not change when the user is using the application.
|
||||
These settings are only provided only on startup.
|
||||
"""
|
||||
|
||||
HOST_UID: int = int(environ.get("HOST_UID", False))
|
||||
HOST_GID: int = int(environ.get("HOST_GID", False))
|
||||
ENABLE_CAST: bool = bool(environ.get("ENABLE_CAST"))
|
||||
TZ: str = str(environ.get("TZ", "UTC"))
|
||||
TA_PORT: int = int(environ.get("TA_PORT", False))
|
||||
TA_UWSGI_PORT: int = int(environ.get("TA_UWSGI_PORT", False))
|
||||
TA_USERNAME: str = str(environ.get("TA_USERNAME"))
|
||||
TA_PASSWORD: str = str(environ.get("TA_PASSWORD"))
|
||||
|
||||
# Application Paths
|
||||
MEDIA_DIR: str = str(environ.get("TA_MEDIA_DIR", "/youtube"))
|
||||
APP_DIR: str = str(environ.get("TA_APP_DIR", "/app"))
|
||||
CACHE_DIR: str = str(environ.get("TA_CACHE_DIR", "/cache"))
|
||||
|
||||
# Redis
|
||||
REDIS_HOST: str = str(environ.get("REDIS_HOST"))
|
||||
REDIS_PORT: int = int(environ.get("REDIS_PORT", 6379))
|
||||
REDIS_NAME_SPACE: str = str(environ.get("REDIS_NAME_SPACE", "ta:"))
|
||||
|
||||
# ElasticSearch
|
||||
ES_URL: str = str(environ.get("ES_URL"))
|
||||
ES_PASS: str = str(environ.get("ELASTIC_PASSWORD"))
|
||||
ES_USER: str = str(environ.get("ELASTIC_USER", "elastic"))
|
||||
ES_SNAPSHOT_DIR: str = str(
|
||||
environ.get(
|
||||
"ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
|
||||
)
|
||||
)
|
||||
ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL"))
|
||||
|
||||
def get_cache_root(self):
|
||||
"""get root for web server"""
|
||||
if self.CACHE_DIR.startswith("/"):
|
||||
return self.CACHE_DIR
|
||||
|
||||
return f"/{self.CACHE_DIR}"
|
||||
|
||||
def get_media_root(self):
|
||||
"""get root for media folder"""
|
||||
if self.MEDIA_DIR.startswith("/"):
|
||||
return self.MEDIA_DIR
|
||||
|
||||
return f"/{self.MEDIA_DIR}"
|
||||
|
||||
def print_generic(self):
|
||||
"""print generic env vars"""
|
||||
print(
|
||||
f"""
|
||||
HOST_UID: {self.HOST_UID}
|
||||
HOST_GID: {self.HOST_GID}
|
||||
TZ: {self.TZ}
|
||||
ENABLE_CAST: {self.ENABLE_CAST}
|
||||
TA_PORT: {self.TA_PORT}
|
||||
TA_UWSGI_PORT: {self.TA_UWSGI_PORT}
|
||||
TA_USERNAME: {self.TA_USERNAME}
|
||||
TA_PASSWORD: *****"""
|
||||
)
|
||||
|
||||
def print_paths(self):
|
||||
"""debug paths set"""
|
||||
print(
|
||||
f"""
|
||||
MEDIA_DIR: {self.MEDIA_DIR}
|
||||
APP_DIR: {self.APP_DIR}
|
||||
CACHE_DIR: {self.CACHE_DIR}"""
|
||||
)
|
||||
|
||||
def print_redis_conf(self):
|
||||
"""debug redis conf paths"""
|
||||
print(
|
||||
f"""
|
||||
REDIS_HOST: {self.REDIS_HOST}
|
||||
REDIS_PORT: {self.REDIS_PORT}
|
||||
REDIS_NAME_SPACE: {self.REDIS_NAME_SPACE}"""
|
||||
)
|
||||
|
||||
def print_es_paths(self):
|
||||
"""debug es conf"""
|
||||
print(
|
||||
f"""
|
||||
ES_URL: {self.ES_URL}
|
||||
ES_PASS: *****
|
||||
ES_USER: {self.ES_USER}
|
||||
ES_SNAPSHOT_DIR: {self.ES_SNAPSHOT_DIR}
|
||||
ES_DISABLE_VERIFY_SSL: {self.ES_DISABLE_VERIFY_SSL}"""
|
||||
)
|
||||
|
||||
def print_all(self):
|
||||
"""print all"""
|
||||
self.print_generic()
|
||||
self.print_paths()
|
||||
self.print_redis_conf()
|
||||
self.print_es_paths()
|
||||
231
backend/common/src/es_connect.py
Normal file
231
backend/common/src/es_connect.py
Normal file
@@ -0,0 +1,231 @@
|
||||
"""
|
||||
functionality:
|
||||
- wrapper around requests to call elastic search
|
||||
- reusable search_after to extract total index
|
||||
"""
|
||||
|
||||
# pylint: disable=missing-timeout
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
import urllib3
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
|
||||
|
||||
class ElasticWrap:
|
||||
"""makes all calls to elastic search
|
||||
returns response json and status code tuple
|
||||
"""
|
||||
|
||||
def __init__(self, path: str):
|
||||
self.url: str = f"{EnvironmentSettings.ES_URL}/{path}"
|
||||
self.auth: tuple[str, str] = (
|
||||
EnvironmentSettings.ES_USER,
|
||||
EnvironmentSettings.ES_PASS,
|
||||
)
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
def get(
|
||||
self,
|
||||
data: bool | dict = False,
|
||||
timeout: int = 10,
|
||||
print_error: bool = True,
|
||||
) -> tuple[dict, int]:
|
||||
"""get data from es"""
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"auth": self.auth,
|
||||
"timeout": timeout,
|
||||
}
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
kwargs["verify"] = False
|
||||
|
||||
if data:
|
||||
kwargs["json"] = data
|
||||
|
||||
response = requests.get(self.url, **kwargs)
|
||||
|
||||
if print_error and not response.ok:
|
||||
print(response.text)
|
||||
|
||||
return response.json(), response.status_code
|
||||
|
||||
def post(
|
||||
self, data: bool | dict = False, ndjson: bool = False
|
||||
) -> tuple[dict, int]:
|
||||
"""post data to es"""
|
||||
|
||||
kwargs: dict[str, Any] = {"auth": self.auth}
|
||||
|
||||
if ndjson and data:
|
||||
kwargs.update(
|
||||
{
|
||||
"headers": {"Content-type": "application/x-ndjson"},
|
||||
"data": data,
|
||||
}
|
||||
)
|
||||
elif data:
|
||||
kwargs.update(
|
||||
{
|
||||
"headers": {"Content-type": "application/json"},
|
||||
"data": json.dumps(data),
|
||||
}
|
||||
)
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
kwargs["verify"] = False
|
||||
|
||||
response = requests.post(self.url, **kwargs)
|
||||
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
|
||||
return response.json(), response.status_code
|
||||
|
||||
def put(
|
||||
self,
|
||||
data: bool | dict = False,
|
||||
refresh: bool = False,
|
||||
) -> tuple[dict, Any]:
|
||||
"""put data to es"""
|
||||
|
||||
if refresh:
|
||||
self.url = f"{self.url}/?refresh=true"
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"json": data,
|
||||
"auth": self.auth,
|
||||
}
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
kwargs["verify"] = False
|
||||
|
||||
response = requests.put(self.url, **kwargs)
|
||||
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
print(data)
|
||||
raise ValueError("failed to add item to index")
|
||||
|
||||
return response.json(), response.status_code
|
||||
|
||||
def delete(
|
||||
self,
|
||||
data: bool | dict = False,
|
||||
refresh: bool = False,
|
||||
) -> tuple[dict, Any]:
|
||||
"""delete document from es"""
|
||||
|
||||
if refresh:
|
||||
self.url = f"{self.url}/?refresh=true"
|
||||
|
||||
kwargs: dict[str, Any] = {"auth": self.auth}
|
||||
|
||||
if data:
|
||||
kwargs["json"] = data
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
kwargs["verify"] = False
|
||||
|
||||
response = requests.delete(self.url, **kwargs)
|
||||
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
|
||||
return response.json(), response.status_code
|
||||
|
||||
|
||||
class IndexPaginate:
|
||||
"""use search_after to go through whole index
|
||||
kwargs:
|
||||
- size: int, overwrite DEFAULT_SIZE
|
||||
- keep_source: bool, keep _source key from es results
|
||||
- callback: obj, Class implementing run method callback for every loop
|
||||
- task: task object to send notification
|
||||
- total: int, total items in index for progress message
|
||||
"""
|
||||
|
||||
DEFAULT_SIZE = 500
|
||||
|
||||
def __init__(self, index_name, data, **kwargs):
|
||||
self.index_name = index_name
|
||||
self.data = data
|
||||
self.pit_id = False
|
||||
self.kwargs = kwargs
|
||||
|
||||
def get_results(self):
|
||||
"""get all results, add task and total for notifications"""
|
||||
self.get_pit()
|
||||
self.validate_data()
|
||||
all_results = self.run_loop()
|
||||
self.clean_pit()
|
||||
return all_results
|
||||
|
||||
def get_pit(self):
|
||||
"""get pit for index"""
|
||||
path = f"{self.index_name}/_pit?keep_alive=10m"
|
||||
response, _ = ElasticWrap(path).post()
|
||||
self.pit_id = response["id"]
|
||||
|
||||
def validate_data(self):
|
||||
"""add pit and size to data"""
|
||||
if not self.data:
|
||||
self.data = {}
|
||||
|
||||
if "query" not in self.data.keys():
|
||||
self.data.update({"query": {"match_all": {}}})
|
||||
|
||||
if "sort" not in self.data.keys():
|
||||
self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
|
||||
|
||||
self.data["size"] = self.kwargs.get("size") or self.DEFAULT_SIZE
|
||||
self.data["pit"] = {"id": self.pit_id, "keep_alive": "10m"}
|
||||
|
||||
def run_loop(self):
|
||||
"""loop through results until last hit"""
|
||||
all_results = []
|
||||
counter = 0
|
||||
while True:
|
||||
response, _ = ElasticWrap("_search").get(data=self.data)
|
||||
all_hits = response["hits"]["hits"]
|
||||
if not all_hits:
|
||||
break
|
||||
|
||||
for hit in all_hits:
|
||||
if self.kwargs.get("keep_source"):
|
||||
all_results.append(hit)
|
||||
else:
|
||||
all_results.append(hit["_source"])
|
||||
|
||||
if self.kwargs.get("callback"):
|
||||
self.kwargs.get("callback")(
|
||||
all_hits, self.index_name, counter=counter
|
||||
).run()
|
||||
|
||||
if self.kwargs.get("task"):
|
||||
print(f"{self.index_name}: processing page {counter}")
|
||||
self._notify(len(all_results))
|
||||
|
||||
counter += 1
|
||||
|
||||
# update search_after with last hit data
|
||||
self.data["search_after"] = all_hits[-1]["sort"]
|
||||
|
||||
return all_results
|
||||
|
||||
def _notify(self, processed):
|
||||
"""send notification on task"""
|
||||
total = self.kwargs.get("total")
|
||||
progress = processed / total
|
||||
index_clean = self.index_name.lstrip("ta_").title()
|
||||
message = [f"Processing {index_clean}s {processed}/{total}"]
|
||||
self.kwargs.get("task").send_progress(message, progress=progress)
|
||||
|
||||
def clean_pit(self):
|
||||
"""delete pit from elastic search"""
|
||||
ElasticWrap("_pit").delete(data={"id": self.pit_id})
|
||||
11
backend/common/src/health.py
Normal file
11
backend/common/src/health.py
Normal file
@@ -0,0 +1,11 @@
|
||||
from django.http import HttpResponse
|
||||
|
||||
|
||||
class HealthCheckMiddleware:
|
||||
def __init__(self, get_response):
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request):
|
||||
if request.path == "/health":
|
||||
return HttpResponse("ok")
|
||||
return self.get_response(request)
|
||||
269
backend/common/src/helper.py
Normal file
269
backend/common/src/helper.py
Normal file
@@ -0,0 +1,269 @@
|
||||
"""
|
||||
Loose collection of helper functions
|
||||
- don't import AppConfig class here to avoid circular imports
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import string
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import IndexPaginate
|
||||
|
||||
|
||||
def ignore_filelist(filelist: list[str]) -> list[str]:
|
||||
"""ignore temp files for os.listdir sanitizer"""
|
||||
to_ignore = [
|
||||
"@eaDir",
|
||||
"Icon\r\r",
|
||||
"Network Trash Folder",
|
||||
"Temporary Items",
|
||||
]
|
||||
cleaned: list[str] = []
|
||||
for file_name in filelist:
|
||||
if file_name.startswith(".") or file_name in to_ignore:
|
||||
continue
|
||||
|
||||
cleaned.append(file_name)
|
||||
|
||||
return cleaned
|
||||
|
||||
|
||||
def randomizor(length: int) -> str:
|
||||
"""generate random alpha numeric string"""
|
||||
pool: str = string.digits + string.ascii_letters
|
||||
return "".join(random.choice(pool) for i in range(length))
|
||||
|
||||
|
||||
def requests_headers() -> dict[str, str]:
|
||||
"""build header with random user agent for requests outside of yt-dlp"""
|
||||
|
||||
chrome_versions = (
|
||||
"90.0.4430.212",
|
||||
"90.0.4430.24",
|
||||
"90.0.4430.70",
|
||||
"90.0.4430.72",
|
||||
"90.0.4430.85",
|
||||
"90.0.4430.93",
|
||||
"91.0.4472.101",
|
||||
"91.0.4472.106",
|
||||
"91.0.4472.114",
|
||||
"91.0.4472.124",
|
||||
"91.0.4472.164",
|
||||
"91.0.4472.19",
|
||||
"91.0.4472.77",
|
||||
"92.0.4515.107",
|
||||
"92.0.4515.115",
|
||||
"92.0.4515.131",
|
||||
"92.0.4515.159",
|
||||
"92.0.4515.43",
|
||||
"93.0.4556.0",
|
||||
"93.0.4577.15",
|
||||
"93.0.4577.63",
|
||||
"93.0.4577.82",
|
||||
"94.0.4606.41",
|
||||
"94.0.4606.54",
|
||||
"94.0.4606.61",
|
||||
"94.0.4606.71",
|
||||
"94.0.4606.81",
|
||||
"94.0.4606.85",
|
||||
"95.0.4638.17",
|
||||
"95.0.4638.50",
|
||||
"95.0.4638.54",
|
||||
"95.0.4638.69",
|
||||
"95.0.4638.74",
|
||||
"96.0.4664.18",
|
||||
"96.0.4664.45",
|
||||
"96.0.4664.55",
|
||||
"96.0.4664.93",
|
||||
"97.0.4692.20",
|
||||
)
|
||||
template = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
+ f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
|
||||
)
|
||||
|
||||
return {"User-Agent": template}
|
||||
|
||||
|
||||
def date_parser(timestamp: int | str) -> str:
|
||||
"""return formatted date string"""
|
||||
if isinstance(timestamp, int):
|
||||
date_obj = datetime.fromtimestamp(timestamp)
|
||||
elif isinstance(timestamp, str):
|
||||
date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
|
||||
else:
|
||||
raise TypeError(f"invalid timestamp: {timestamp}")
|
||||
|
||||
return date_obj.date().isoformat()
|
||||
|
||||
|
||||
def time_parser(timestamp: str) -> float:
|
||||
"""return seconds from timestamp, false on empty"""
|
||||
if not timestamp:
|
||||
return False
|
||||
|
||||
if timestamp.isnumeric():
|
||||
return int(timestamp)
|
||||
|
||||
hours, minutes, seconds = timestamp.split(":", maxsplit=3)
|
||||
return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
|
||||
|
||||
|
||||
def clear_dl_cache(cache_dir: str) -> int:
|
||||
"""clear leftover files from dl cache"""
|
||||
print("clear download cache")
|
||||
download_cache_dir = os.path.join(cache_dir, "download")
|
||||
leftover_files = ignore_filelist(os.listdir(download_cache_dir))
|
||||
for cached in leftover_files:
|
||||
to_delete = os.path.join(download_cache_dir, cached)
|
||||
os.remove(to_delete)
|
||||
|
||||
return len(leftover_files)
|
||||
|
||||
|
||||
def get_mapping() -> dict:
|
||||
"""read index_mapping.json and get expected mapping and settings"""
|
||||
with open("appsettings/index_mapping.json", "r", encoding="utf-8") as f:
|
||||
index_config: dict = json.load(f).get("index_config")
|
||||
|
||||
return index_config
|
||||
|
||||
|
||||
def is_shorts(youtube_id: str) -> bool:
|
||||
"""check if youtube_id is a shorts video, bot not it it's not a shorts"""
|
||||
shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
|
||||
cookies = {"SOCS": "CAI"}
|
||||
response = requests.head(
|
||||
shorts_url, cookies=cookies, headers=requests_headers(), timeout=10
|
||||
)
|
||||
|
||||
return response.status_code == 200
|
||||
|
||||
|
||||
def get_duration_sec(file_path: str) -> int:
|
||||
"""get duration of media file from file path"""
|
||||
|
||||
duration = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
"default=noprint_wrappers=1:nokey=1",
|
||||
file_path,
|
||||
],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
duration_raw = duration.stdout.decode().strip()
|
||||
if duration_raw == "N/A":
|
||||
return 0
|
||||
|
||||
duration_sec = int(float(duration_raw))
|
||||
return duration_sec
|
||||
|
||||
|
||||
def get_duration_str(seconds: int) -> str:
|
||||
"""Return a human-readable duration string from seconds."""
|
||||
if not seconds:
|
||||
return "NA"
|
||||
|
||||
units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
|
||||
duration_parts = []
|
||||
|
||||
for unit_label, unit_seconds in units:
|
||||
if seconds >= unit_seconds:
|
||||
unit_count, seconds = divmod(seconds, unit_seconds)
|
||||
duration_parts.append(f"{unit_count:02}{unit_label}")
|
||||
|
||||
duration_parts[0] = duration_parts[0].lstrip("0")
|
||||
|
||||
return " ".join(duration_parts)
|
||||
|
||||
|
||||
def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
|
||||
"""parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
|
||||
allowed_hosts: list[str] = [
|
||||
"localhost",
|
||||
"tubearchivist",
|
||||
]
|
||||
csrf_trusted_origins: list[str] = [
|
||||
"http://localhost",
|
||||
"http://tubearchivist",
|
||||
]
|
||||
for host in ta_host.split():
|
||||
host_clean = host.strip()
|
||||
if not host_clean.startswith("http"):
|
||||
host_clean = f"http://{host_clean}"
|
||||
|
||||
parsed = urlparse(host_clean)
|
||||
allowed_hosts.append(f"{parsed.hostname}")
|
||||
csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}")
|
||||
|
||||
return allowed_hosts, csrf_trusted_origins
|
||||
|
||||
|
||||
def get_stylesheets() -> list:
|
||||
"""Get all valid stylesheets from /static/css"""
|
||||
app_root = EnvironmentSettings.APP_DIR
|
||||
try:
|
||||
stylesheets = os.listdir(os.path.join(app_root, "static/css"))
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
|
||||
stylesheets.remove("style.css")
|
||||
stylesheets.sort()
|
||||
stylesheets = list(filter(lambda x: x.endswith(".css"), stylesheets))
|
||||
return stylesheets
|
||||
|
||||
|
||||
def check_stylesheet(stylesheet: str):
|
||||
"""Check if a stylesheet exists. Return dark.css as a fallback"""
|
||||
if stylesheet in get_stylesheets():
|
||||
return stylesheet
|
||||
|
||||
return "dark.css"
|
||||
|
||||
|
||||
def is_missing(
|
||||
to_check: str | list[str],
|
||||
index_name: str = "ta_video,ta_download",
|
||||
on_key: str = "youtube_id",
|
||||
) -> list[str]:
|
||||
"""id or list of ids that are missing from index_name"""
|
||||
if isinstance(to_check, str):
|
||||
to_check = [to_check]
|
||||
|
||||
data = {
|
||||
"query": {"terms": {on_key: to_check}},
|
||||
"_source": [on_key],
|
||||
}
|
||||
result = IndexPaginate(index_name, data=data).get_results()
|
||||
existing_ids = [i[on_key] for i in result]
|
||||
dl = [i for i in to_check if i not in existing_ids]
|
||||
|
||||
return dl
|
||||
|
||||
|
||||
def get_channel_overwrites() -> dict[str, dict[str, Any]]:
|
||||
"""get overwrites indexed my channel_id"""
|
||||
data = {
|
||||
"query": {
|
||||
"bool": {"must": [{"exists": {"field": "channel_overwrites"}}]}
|
||||
},
|
||||
"_source": ["channel_id", "channel_overwrites"],
|
||||
}
|
||||
result = IndexPaginate("ta_channel", data).get_results()
|
||||
overwrites = {i["channel_id"]: i["channel_overwrites"] for i in result}
|
||||
|
||||
return overwrites
|
||||
145
backend/common/src/index_generic.py
Normal file
145
backend/common/src/index_generic.py
Normal file
@@ -0,0 +1,145 @@
|
||||
"""
|
||||
functionality:
|
||||
- generic base class to inherit from for video, channel and playlist
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from user.src.user_config import UserConfig
|
||||
|
||||
|
||||
class YouTubeItem:
|
||||
"""base class for youtube"""
|
||||
|
||||
es_path = False
|
||||
index_name = ""
|
||||
yt_base = ""
|
||||
yt_obs: dict[str, bool | str] = {
|
||||
"skip_download": True,
|
||||
"noplaylist": True,
|
||||
}
|
||||
|
||||
def __init__(self, youtube_id):
|
||||
self.youtube_id = youtube_id
|
||||
self.es_path = f"{self.index_name}/_doc/{youtube_id}"
|
||||
self.config = AppConfig().config
|
||||
self.youtube_meta = False
|
||||
self.json_data = False
|
||||
|
||||
def build_yt_url(self):
|
||||
"""build youtube url"""
|
||||
return self.yt_base + self.youtube_id
|
||||
|
||||
def get_from_youtube(self):
|
||||
"""use yt-dlp to get meta data from youtube"""
|
||||
print(f"{self.youtube_id}: get metadata from youtube")
|
||||
obs_request = self.yt_obs.copy()
|
||||
if self.config["downloads"]["extractor_lang"]:
|
||||
langs = self.config["downloads"]["extractor_lang"]
|
||||
langs_list = [i.strip() for i in langs.split(",")]
|
||||
obs_request["extractor_args"] = {"youtube": {"lang": langs_list}}
|
||||
|
||||
url = self.build_yt_url()
|
||||
self.youtube_meta = YtWrap(obs_request, self.config).extract(url)
|
||||
|
||||
def get_from_es(self):
|
||||
"""get indexed data from elastic search"""
|
||||
print(f"{self.youtube_id}: get metadata from es")
|
||||
response, _ = ElasticWrap(f"{self.es_path}").get()
|
||||
source = response.get("_source")
|
||||
self.json_data = source
|
||||
|
||||
def upload_to_es(self):
|
||||
"""add json_data to elastic"""
|
||||
_, _ = ElasticWrap(self.es_path).put(self.json_data, refresh=True)
|
||||
|
||||
def deactivate(self):
|
||||
"""deactivate document in es"""
|
||||
print(f"{self.youtube_id}: deactivate document")
|
||||
key_match = {
|
||||
"ta_video": "active",
|
||||
"ta_channel": "channel_active",
|
||||
"ta_playlist": "playlist_active",
|
||||
}
|
||||
path = f"{self.index_name}/_update/{self.youtube_id}?refresh=true"
|
||||
data = {
|
||||
"script": f"ctx._source.{key_match.get(self.index_name)} = false"
|
||||
}
|
||||
_, _ = ElasticWrap(path).post(data)
|
||||
|
||||
def del_in_es(self):
|
||||
"""delete item from elastic search"""
|
||||
print(f"{self.youtube_id}: delete from es")
|
||||
_, _ = ElasticWrap(self.es_path).delete(refresh=True)
|
||||
|
||||
|
||||
class Pagination:
|
||||
"""
|
||||
figure out the pagination based on page size and total_hits
|
||||
"""
|
||||
|
||||
def __init__(self, request):
|
||||
self.request = request
|
||||
self.page_get = False
|
||||
self.params = False
|
||||
self.get_params()
|
||||
self.page_size = self.get_page_size()
|
||||
self.pagination = self.first_guess()
|
||||
|
||||
def get_params(self):
|
||||
"""process url query parameters"""
|
||||
query_dict = self.request.GET.copy()
|
||||
self.page_get = int(query_dict.get("page", 0))
|
||||
|
||||
_ = query_dict.pop("page", False)
|
||||
self.params = query_dict.urlencode()
|
||||
|
||||
def get_page_size(self):
|
||||
"""get default or user modified page_size"""
|
||||
return UserConfig(self.request.user.id).get_value("page_size")
|
||||
|
||||
def first_guess(self):
|
||||
"""build first guess before api call"""
|
||||
page_get = self.page_get
|
||||
page_from = 0
|
||||
if page_get in [0, 1]:
|
||||
prev_pages = False
|
||||
elif page_get > 1:
|
||||
page_from = (page_get - 1) * self.page_size
|
||||
prev_pages = [
|
||||
i for i in range(page_get - 1, page_get - 6, -1) if i > 1
|
||||
]
|
||||
prev_pages.reverse()
|
||||
pagination = {
|
||||
"page_size": self.page_size,
|
||||
"page_from": page_from,
|
||||
"prev_pages": prev_pages,
|
||||
"current_page": page_get,
|
||||
"max_hits": False,
|
||||
"params": self.params,
|
||||
}
|
||||
|
||||
return pagination
|
||||
|
||||
def validate(self, total_hits):
|
||||
"""validate pagination with total_hits after making api call"""
|
||||
page_get = self.page_get
|
||||
max_pages = math.ceil(total_hits / self.page_size)
|
||||
if total_hits >= 10000:
|
||||
# es returns maximal 10000 results
|
||||
self.pagination["max_hits"] = True
|
||||
max_pages = max_pages - 1
|
||||
|
||||
if page_get < max_pages and max_pages > 1:
|
||||
self.pagination["last_page"] = max_pages
|
||||
else:
|
||||
self.pagination["last_page"] = False
|
||||
next_pages = [
|
||||
i for i in range(page_get + 1, page_get + 6) if 1 < i < max_pages
|
||||
]
|
||||
|
||||
self.pagination["next_pages"] = next_pages
|
||||
self.pagination["total_hits"] = total_hits
|
||||
182
backend/common/src/search_processor.py
Normal file
182
backend/common/src/search_processor.py
Normal file
@@ -0,0 +1,182 @@
|
||||
"""
|
||||
Functionality:
|
||||
- processing search results for frontend
|
||||
- this is duplicated code from home.src.frontend.searching.SearchHandler
|
||||
"""
|
||||
|
||||
import urllib.parse
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.helper import date_parser, get_duration_str
|
||||
from download.src.thumbnails import ThumbManager
|
||||
|
||||
|
||||
class SearchProcess:
|
||||
"""process search results"""
|
||||
|
||||
def __init__(self, response):
|
||||
self.response = response
|
||||
self.processed = False
|
||||
|
||||
def process(self):
|
||||
"""detect type and process"""
|
||||
print(self.response)
|
||||
if "_source" in self.response.keys():
|
||||
# single
|
||||
self.processed = self._process_result(self.response)
|
||||
|
||||
elif "hits" in self.response.keys():
|
||||
# multiple
|
||||
self.processed = []
|
||||
all_sources = self.response["hits"]["hits"]
|
||||
for result in all_sources:
|
||||
self.processed.append(self._process_result(result))
|
||||
|
||||
return self.processed
|
||||
|
||||
def _process_result(self, result):
|
||||
"""detect which type of data to process"""
|
||||
index = result["_index"]
|
||||
processed = False
|
||||
if index == "ta_video":
|
||||
processed = self._process_video(result["_source"])
|
||||
if index == "ta_channel":
|
||||
processed = self._process_channel(result["_source"])
|
||||
if index == "ta_playlist":
|
||||
processed = self._process_playlist(result["_source"])
|
||||
if index == "ta_download":
|
||||
processed = self._process_download(result["_source"])
|
||||
if index == "ta_comment":
|
||||
processed = self._process_comment(result["_source"])
|
||||
if index == "ta_subtitle":
|
||||
processed = self._process_subtitle(result)
|
||||
|
||||
if isinstance(processed, dict):
|
||||
processed.update(
|
||||
{
|
||||
"_index": index,
|
||||
"_score": round(result.get("_score") or 0, 2),
|
||||
}
|
||||
)
|
||||
|
||||
return processed
|
||||
|
||||
@staticmethod
|
||||
def _process_channel(channel_dict):
|
||||
"""run on single channel"""
|
||||
channel_id = channel_dict["channel_id"]
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
art_base = f"{cache_root}/channels/{channel_id}"
|
||||
date_str = date_parser(channel_dict["channel_last_refresh"])
|
||||
channel_dict.update(
|
||||
{
|
||||
"channel_last_refresh": date_str,
|
||||
"channel_banner_url": f"{art_base}_banner.jpg",
|
||||
"channel_thumb_url": f"{art_base}_thumb.jpg",
|
||||
"channel_tvart_url": f"{art_base}_tvart.jpg",
|
||||
}
|
||||
)
|
||||
|
||||
return dict(sorted(channel_dict.items()))
|
||||
|
||||
def _process_video(self, video_dict):
|
||||
"""run on single video dict"""
|
||||
video_id = video_dict["youtube_id"]
|
||||
media_url = urllib.parse.quote(video_dict["media_url"])
|
||||
vid_last_refresh = date_parser(video_dict["vid_last_refresh"])
|
||||
published = date_parser(video_dict["published"])
|
||||
vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
|
||||
channel = self._process_channel(video_dict["channel"])
|
||||
|
||||
if "subtitles" in video_dict:
|
||||
for idx, _ in enumerate(video_dict["subtitles"]):
|
||||
url = video_dict["subtitles"][idx]["media_url"]
|
||||
video_dict["subtitles"][idx]["media_url"] = f"/media/{url}"
|
||||
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
media_root = EnvironmentSettings().get_media_root()
|
||||
|
||||
video_dict.update(
|
||||
{
|
||||
"channel": channel,
|
||||
"media_url": f"{media_root}/{media_url}",
|
||||
"vid_last_refresh": vid_last_refresh,
|
||||
"published": published,
|
||||
"vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
|
||||
}
|
||||
)
|
||||
|
||||
return dict(sorted(video_dict.items()))
|
||||
|
||||
@staticmethod
|
||||
def _process_playlist(playlist_dict):
|
||||
"""run on single playlist dict"""
|
||||
playlist_id = playlist_dict["playlist_id"]
|
||||
playlist_last_refresh = date_parser(
|
||||
playlist_dict["playlist_last_refresh"]
|
||||
)
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
playlist_thumbnail = f"{cache_root}/playlists/{playlist_id}.jpg"
|
||||
playlist_dict.update(
|
||||
{
|
||||
"playlist_thumbnail": playlist_thumbnail,
|
||||
"playlist_last_refresh": playlist_last_refresh,
|
||||
}
|
||||
)
|
||||
|
||||
return dict(sorted(playlist_dict.items()))
|
||||
|
||||
def _process_download(self, download_dict):
|
||||
"""run on single download item"""
|
||||
video_id = download_dict["youtube_id"]
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
|
||||
published = date_parser(download_dict["published"])
|
||||
|
||||
download_dict.update(
|
||||
{
|
||||
"vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
|
||||
"published": published,
|
||||
}
|
||||
)
|
||||
return dict(sorted(download_dict.items()))
|
||||
|
||||
def _process_comment(self, comment_dict):
|
||||
"""run on all comments, create reply thread"""
|
||||
all_comments = comment_dict["comment_comments"]
|
||||
processed_comments = []
|
||||
|
||||
for comment in all_comments:
|
||||
if comment["comment_parent"] == "root":
|
||||
comment.update({"comment_replies": []})
|
||||
processed_comments.append(comment)
|
||||
else:
|
||||
processed_comments[-1]["comment_replies"].append(comment)
|
||||
|
||||
return processed_comments
|
||||
|
||||
def _process_subtitle(self, result):
|
||||
"""take complete result dict to extract highlight"""
|
||||
subtitle_dict = result["_source"]
|
||||
highlight = result.get("highlight")
|
||||
if highlight:
|
||||
# replace lines with the highlighted markdown
|
||||
subtitle_line = highlight.get("subtitle_line")[0]
|
||||
subtitle_dict.update({"subtitle_line": subtitle_line})
|
||||
|
||||
thumb_path = ThumbManager(subtitle_dict["youtube_id"]).vid_thumb_path()
|
||||
subtitle_dict.update({"vid_thumb_url": f"/cache/{thumb_path}"})
|
||||
|
||||
return subtitle_dict
|
||||
|
||||
|
||||
def process_aggs(response):
|
||||
"""convert aggs duration to str"""
|
||||
|
||||
if response.get("aggregations"):
|
||||
aggs = response["aggregations"]
|
||||
if "total_duration" in aggs:
|
||||
duration_sec = int(aggs["total_duration"]["value"])
|
||||
aggs["total_duration"].update(
|
||||
{"value_str": get_duration_str(duration_sec)}
|
||||
)
|
||||
382
backend/common/src/searching.py
Normal file
382
backend/common/src/searching.py
Normal file
@@ -0,0 +1,382 @@
|
||||
"""
|
||||
Functionality:
|
||||
- handle search to populate results to view
|
||||
- cache youtube video thumbnails and channel artwork
|
||||
- parse values in hit_cleanup for frontend
|
||||
- calculate pagination values
|
||||
"""
|
||||
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.search_processor import SearchProcess
|
||||
|
||||
|
||||
class SearchForm:
|
||||
"""build query from search form data"""
|
||||
|
||||
def multi_search(self, search_query):
|
||||
"""searching through index"""
|
||||
path, query, query_type = SearchParser(search_query).run()
|
||||
response, _ = ElasticWrap(path).get(data=query)
|
||||
search_results = SearchProcess(response).process()
|
||||
all_results = self.build_results(search_results)
|
||||
|
||||
return {"results": all_results, "queryType": query_type}
|
||||
|
||||
@staticmethod
|
||||
def build_results(search_results):
|
||||
"""build the all_results dict"""
|
||||
video_results = []
|
||||
channel_results = []
|
||||
playlist_results = []
|
||||
fulltext_results = []
|
||||
if search_results:
|
||||
for result in search_results:
|
||||
if result["_index"] == "ta_video":
|
||||
video_results.append(result)
|
||||
elif result["_index"] == "ta_channel":
|
||||
channel_results.append(result)
|
||||
elif result["_index"] == "ta_playlist":
|
||||
playlist_results.append(result)
|
||||
elif result["_index"] == "ta_subtitle":
|
||||
fulltext_results.append(result)
|
||||
|
||||
all_results = {
|
||||
"video_results": video_results,
|
||||
"channel_results": channel_results,
|
||||
"playlist_results": playlist_results,
|
||||
"fulltext_results": fulltext_results,
|
||||
}
|
||||
|
||||
return all_results
|
||||
|
||||
|
||||
class SearchParser:
|
||||
"""handle structured searches"""
|
||||
|
||||
def __init__(self, search_query):
|
||||
self.query_words = search_query.lower().split()
|
||||
self.query_map = {"term": [], "fuzzy": []}
|
||||
self.append_to = "term"
|
||||
|
||||
def run(self):
|
||||
"""collection, return path and query dict for es"""
|
||||
print(f"query words: {self.query_words}")
|
||||
query_type = self._find_map()
|
||||
self._run_words()
|
||||
self._delete_unset()
|
||||
self._match_data_types()
|
||||
|
||||
path, query = QueryBuilder(self.query_map, query_type).run()
|
||||
|
||||
return path, query, query_type
|
||||
|
||||
def _find_map(self):
|
||||
"""find query in keyword map"""
|
||||
first_word = self.query_words[0]
|
||||
key_word_map = self._get_map()
|
||||
|
||||
if ":" in first_word:
|
||||
index_match, query_string = first_word.split(":")
|
||||
if index_match in key_word_map:
|
||||
self.query_map.update(key_word_map.get(index_match))
|
||||
self.query_words[0] = query_string
|
||||
return index_match
|
||||
|
||||
self.query_map.update(key_word_map.get("simple"))
|
||||
print(f"query_map: {self.query_map}")
|
||||
|
||||
return "simple"
|
||||
|
||||
@staticmethod
|
||||
def _get_map():
|
||||
"""return map to build on"""
|
||||
return {
|
||||
"simple": {
|
||||
"index": "ta_video,ta_channel,ta_playlist",
|
||||
},
|
||||
"video": {
|
||||
"index": "ta_video",
|
||||
"channel": [],
|
||||
"active": [],
|
||||
},
|
||||
"channel": {
|
||||
"index": "ta_channel",
|
||||
"active": [],
|
||||
"subscribed": [],
|
||||
},
|
||||
"playlist": {
|
||||
"index": "ta_playlist",
|
||||
"active": [],
|
||||
"subscribed": [],
|
||||
},
|
||||
"full": {
|
||||
"index": "ta_subtitle",
|
||||
"lang": [],
|
||||
"source": [],
|
||||
},
|
||||
}
|
||||
|
||||
def _run_words(self):
|
||||
"""append word by word"""
|
||||
for word in self.query_words:
|
||||
if ":" in word:
|
||||
keyword, search_string = word.split(":")
|
||||
if keyword in self.query_map:
|
||||
self.append_to = keyword
|
||||
word = search_string
|
||||
|
||||
if word:
|
||||
self.query_map[self.append_to].append(word)
|
||||
|
||||
def _delete_unset(self):
|
||||
"""delete unset keys"""
|
||||
new_query_map = {}
|
||||
for key, value in self.query_map.items():
|
||||
if value:
|
||||
new_query_map.update({key: value})
|
||||
self.query_map = new_query_map
|
||||
|
||||
def _match_data_types(self):
|
||||
"""match values with data types"""
|
||||
for key, value in self.query_map.items():
|
||||
if key in ["term", "channel"]:
|
||||
self.query_map[key] = " ".join(self.query_map[key])
|
||||
if key in ["active", "subscribed"]:
|
||||
self.query_map[key] = "yes" in value
|
||||
|
||||
|
||||
class QueryBuilder:
|
||||
"""build query for ES from form data"""
|
||||
|
||||
def __init__(self, query_map, query_type):
|
||||
self.query_map = query_map
|
||||
self.query_type = query_type
|
||||
|
||||
def run(self):
|
||||
"""build query"""
|
||||
path = self._build_path()
|
||||
query = self.build_query()
|
||||
print(f"es path: {path}")
|
||||
print(f"query: {query}")
|
||||
|
||||
return path, query
|
||||
|
||||
def _build_path(self):
|
||||
"""build es index search path"""
|
||||
return f"{self.query_map.get('index')}/_search"
|
||||
|
||||
def build_query(self):
|
||||
"""build query based on query_type"""
|
||||
|
||||
exec_map = {
|
||||
"simple": self._build_simple,
|
||||
"video": self._build_video,
|
||||
"channel": self._build_channel,
|
||||
"playlist": self._build_playlist,
|
||||
"full": self._build_fulltext,
|
||||
}
|
||||
|
||||
build_must_list = exec_map[self.query_type]
|
||||
|
||||
if self.query_type == "full":
|
||||
query = build_must_list()
|
||||
else:
|
||||
query = {
|
||||
"size": 30,
|
||||
"query": {"bool": {"must": build_must_list()}},
|
||||
}
|
||||
|
||||
return query
|
||||
|
||||
def _get_fuzzy(self):
|
||||
"""return fuziness valuee"""
|
||||
fuzzy_value = self.query_map.get("fuzzy", ["auto"])[0]
|
||||
if fuzzy_value == "no":
|
||||
return 0
|
||||
|
||||
if not fuzzy_value.isdigit():
|
||||
return "auto"
|
||||
|
||||
if int(fuzzy_value) > 2:
|
||||
return "2"
|
||||
|
||||
return fuzzy_value
|
||||
|
||||
def _build_simple(self):
|
||||
"""build simple cross index query"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": term,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"channel_name._2gram",
|
||||
"channel_name._3gram",
|
||||
"channel_name.search_as_you_type",
|
||||
"playlist_name._2gram",
|
||||
"playlist_name._3gram",
|
||||
"playlist_name.search_as_you_type",
|
||||
"title._2gram",
|
||||
"title._3gram",
|
||||
"title.search_as_you_type",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return must_list
|
||||
|
||||
def _build_video(self):
|
||||
"""build video query"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": term,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"title._2gram^2",
|
||||
"title._3gram^2",
|
||||
"title.search_as_you_type^2",
|
||||
"tags",
|
||||
"category",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (active := self.query_map.get("active")) is not None:
|
||||
must_list.append({"term": {"active": {"value": active}}})
|
||||
|
||||
if (channel := self.query_map.get("channel")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": channel,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"channel.channel_name._2gram",
|
||||
"channel.channel_name._3gram",
|
||||
"channel.channel_name.search_as_you_type",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return must_list
|
||||
|
||||
def _build_channel(self):
|
||||
"""build query for channel"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": term,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"channel_description",
|
||||
"channel_name._2gram^2",
|
||||
"channel_name._3gram^2",
|
||||
"channel_name.search_as_you_type^2",
|
||||
"channel_tags",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (active := self.query_map.get("active")) is not None:
|
||||
must_list.append({"term": {"channel_active": {"value": active}}})
|
||||
|
||||
if (subscribed := self.query_map.get("subscribed")) is not None:
|
||||
must_list.append(
|
||||
{"term": {"channel_subscribed": {"value": subscribed}}}
|
||||
)
|
||||
|
||||
return must_list
|
||||
|
||||
def _build_playlist(self):
|
||||
"""build query for playlist"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": term,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"playlist_description",
|
||||
"playlist_name._2gram^2",
|
||||
"playlist_name._3gram^2",
|
||||
"playlist_name.search_as_you_type^2",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (active := self.query_map.get("active")) is not None:
|
||||
must_list.append({"term": {"playlist_active": {"value": active}}})
|
||||
|
||||
if (subscribed := self.query_map.get("subscribed")) is not None:
|
||||
must_list.append(
|
||||
{"term": {"playlist_subscribed": {"value": subscribed}}}
|
||||
)
|
||||
|
||||
return must_list
|
||||
|
||||
def _build_fulltext(self):
|
||||
"""build query for fulltext search"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"match": {
|
||||
"subtitle_line": {
|
||||
"query": term,
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (lang := self.query_map.get("lang")) is not None:
|
||||
must_list.append({"term": {"subtitle_lang": {"value": lang[0]}}})
|
||||
|
||||
if (source := self.query_map.get("source")) is not None:
|
||||
must_list.append(
|
||||
{"term": {"subtitle_source": {"value": source[0]}}}
|
||||
)
|
||||
|
||||
query = {
|
||||
"size": 30,
|
||||
"query": {"bool": {"must": must_list}},
|
||||
"highlight": {
|
||||
"fields": {
|
||||
"subtitle_line": {
|
||||
"number_of_fragments": 0,
|
||||
"pre_tags": ['<span class="settings-current">'],
|
||||
"post_tags": ["</span>"],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
return query
|
||||
241
backend/common/src/ta_redis.py
Normal file
241
backend/common/src/ta_redis.py
Normal file
@@ -0,0 +1,241 @@
|
||||
"""
|
||||
functionality:
|
||||
- interact with redis
|
||||
- hold temporary download queue in redis
|
||||
- interact with celery tasks results
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import redis
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
|
||||
|
||||
class RedisBase:
|
||||
"""connection base for redis"""
|
||||
|
||||
NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE
|
||||
|
||||
def __init__(self):
|
||||
self.conn = redis.Redis(
|
||||
host=EnvironmentSettings.REDIS_HOST,
|
||||
port=EnvironmentSettings.REDIS_PORT,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
|
||||
class RedisArchivist(RedisBase):
|
||||
"""collection of methods to interact with redis"""
|
||||
|
||||
CHANNELS: list[str] = [
|
||||
"download",
|
||||
"add",
|
||||
"rescan",
|
||||
"subchannel",
|
||||
"subplaylist",
|
||||
"playlistscan",
|
||||
"setting",
|
||||
]
|
||||
|
||||
def set_message(
|
||||
self,
|
||||
key: str,
|
||||
message: dict,
|
||||
path: str = ".",
|
||||
expire: bool | int = False,
|
||||
save: bool = False,
|
||||
) -> None:
|
||||
"""write new message to redis"""
|
||||
self.conn.execute_command(
|
||||
"JSON.SET", self.NAME_SPACE + key, path, json.dumps(message)
|
||||
)
|
||||
|
||||
if expire:
|
||||
if isinstance(expire, bool):
|
||||
secs: int = 20
|
||||
else:
|
||||
secs = expire
|
||||
self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
|
||||
|
||||
if save:
|
||||
self.bg_save()
|
||||
|
||||
def bg_save(self) -> None:
|
||||
"""save to aof"""
|
||||
try:
|
||||
self.conn.bgsave()
|
||||
except redis.exceptions.ResponseError:
|
||||
pass
|
||||
|
||||
def get_message(self, key: str) -> dict:
|
||||
"""get message dict from redis"""
|
||||
reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
|
||||
if reply:
|
||||
return json.loads(reply)
|
||||
|
||||
return {"status": False}
|
||||
|
||||
def list_keys(self, query: str) -> list:
|
||||
"""return all key matches"""
|
||||
reply = self.conn.execute_command(
|
||||
"KEYS", self.NAME_SPACE + query + "*"
|
||||
)
|
||||
if not reply:
|
||||
return []
|
||||
|
||||
return [i.lstrip(self.NAME_SPACE) for i in reply]
|
||||
|
||||
def list_items(self, query: str) -> list:
|
||||
"""list all matches"""
|
||||
all_matches = self.list_keys(query)
|
||||
if not all_matches:
|
||||
return []
|
||||
|
||||
return [self.get_message(i) for i in all_matches]
|
||||
|
||||
def del_message(self, key: str) -> bool:
|
||||
"""delete key from redis"""
|
||||
response = self.conn.execute_command("DEL", self.NAME_SPACE + key)
|
||||
return response
|
||||
|
||||
|
||||
class RedisQueue(RedisBase):
|
||||
"""
|
||||
dynamically interact with queues in redis using sorted set
|
||||
- low score number is first in queue
|
||||
- add new items with high score number
|
||||
|
||||
queue names in use:
|
||||
download:channel channels during download
|
||||
download:playlist:full playlists during dl for full refresh
|
||||
download:playlist:quick playlists during dl for quick refresh
|
||||
download:video videos during downloads
|
||||
index:comment videos needing comment indexing
|
||||
reindex:ta_video reindex videos
|
||||
reindex:ta_channel reindex channels
|
||||
reindex:ta_playlist reindex playlists
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, queue_name: str):
|
||||
super().__init__()
|
||||
self.key = f"{self.NAME_SPACE}{queue_name}"
|
||||
|
||||
def get_all(self) -> list[str]:
|
||||
"""return all elements in list"""
|
||||
result = self.conn.zrange(self.key, 0, -1)
|
||||
return result
|
||||
|
||||
def length(self) -> int:
|
||||
"""return total elements in list"""
|
||||
return self.conn.zcard(self.key)
|
||||
|
||||
def in_queue(self, element) -> str | bool:
|
||||
"""check if element is in list"""
|
||||
result = self.conn.zrank(self.key, element)
|
||||
if result is not None:
|
||||
return "in_queue"
|
||||
|
||||
return False
|
||||
|
||||
def add(self, to_add: str) -> None:
|
||||
"""add single item to queue"""
|
||||
if not to_add:
|
||||
return
|
||||
|
||||
next_score = self._get_next_score()
|
||||
self.conn.zadd(self.key, {to_add: next_score})
|
||||
|
||||
def add_list(self, to_add: list) -> None:
|
||||
"""add list to queue"""
|
||||
if not to_add:
|
||||
return
|
||||
|
||||
next_score = self._get_next_score()
|
||||
mapping = {i[1]: next_score + i[0] for i in enumerate(to_add)}
|
||||
self.conn.zadd(self.key, mapping)
|
||||
|
||||
def max_score(self) -> int | None:
|
||||
"""get max score"""
|
||||
last = self.conn.zrange(self.key, -1, -1, withscores=True)
|
||||
if not last:
|
||||
return None
|
||||
|
||||
return int(last[0][1])
|
||||
|
||||
def _get_next_score(self) -> float:
|
||||
"""get next score in queue to append"""
|
||||
last = self.conn.zrange(self.key, -1, -1, withscores=True)
|
||||
if not last:
|
||||
return 1.0
|
||||
|
||||
return last[0][1] + 1
|
||||
|
||||
def get_next(self) -> tuple[str | None, int | None]:
|
||||
"""return next element in the queue, if available"""
|
||||
result = self.conn.zpopmin(self.key)
|
||||
if not result:
|
||||
return None, None
|
||||
|
||||
item, idx = result[0][0], int(result[0][1])
|
||||
|
||||
return item, idx
|
||||
|
||||
def clear(self) -> None:
|
||||
"""delete list from redis"""
|
||||
self.conn.delete(self.key)
|
||||
|
||||
|
||||
class TaskRedis(RedisBase):
|
||||
"""interact with redis tasks"""
|
||||
|
||||
BASE: str = "celery-task-meta-"
|
||||
EXPIRE: int = 60 * 60 * 24
|
||||
COMMANDS: list[str] = ["STOP", "KILL"]
|
||||
|
||||
def get_all(self) -> list:
|
||||
"""return all tasks"""
|
||||
all_keys = self.conn.execute_command("KEYS", f"{self.BASE}*")
|
||||
return [i.replace(self.BASE, "") for i in all_keys]
|
||||
|
||||
def get_single(self, task_id: str) -> dict:
|
||||
"""return content of single task"""
|
||||
result = self.conn.execute_command("GET", self.BASE + task_id)
|
||||
if not result:
|
||||
return {}
|
||||
|
||||
return json.loads(result)
|
||||
|
||||
def set_key(
|
||||
self, task_id: str, message: dict, expire: bool | int = False
|
||||
) -> None:
|
||||
"""set value for lock, initial or update"""
|
||||
key: str = f"{self.BASE}{task_id}"
|
||||
self.conn.execute_command("SET", key, json.dumps(message))
|
||||
|
||||
if expire:
|
||||
self.conn.execute_command("EXPIRE", key, self.EXPIRE)
|
||||
|
||||
def set_command(self, task_id: str, command: str) -> None:
|
||||
"""set task command"""
|
||||
if command not in self.COMMANDS:
|
||||
print(f"{command} not in valid commands {self.COMMANDS}")
|
||||
raise ValueError
|
||||
|
||||
message = self.get_single(task_id)
|
||||
if not message:
|
||||
print(f"{task_id} not found")
|
||||
raise KeyError
|
||||
|
||||
message.update({"command": command})
|
||||
self.set_key(task_id, message)
|
||||
|
||||
def del_task(self, task_id: str) -> None:
|
||||
"""delete task result by id"""
|
||||
self.conn.execute_command("DEL", f"{self.BASE}{task_id}")
|
||||
|
||||
def del_all(self) -> None:
|
||||
"""delete all task results"""
|
||||
all_tasks = self.get_all()
|
||||
for task_id in all_tasks:
|
||||
self.del_task(task_id)
|
||||
138
backend/common/src/urlparser.py
Normal file
138
backend/common/src/urlparser.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""
|
||||
Functionality:
|
||||
- detect valid youtube ids and links from multi line string
|
||||
- identify vid_type if possible
|
||||
"""
|
||||
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from video.src.constants import VideoTypeEnum
|
||||
|
||||
|
||||
class Parser:
|
||||
"""take a multi line string and detect valid youtube ids"""
|
||||
|
||||
def __init__(self, url_str):
|
||||
self.url_list = [i.strip() for i in url_str.split()]
|
||||
|
||||
def parse(self):
|
||||
"""parse the list"""
|
||||
ids = []
|
||||
for url in self.url_list:
|
||||
parsed = urlparse(url)
|
||||
if parsed.netloc:
|
||||
# is url
|
||||
identified = self.process_url(parsed)
|
||||
else:
|
||||
# is not url
|
||||
identified = self._find_valid_id(url)
|
||||
|
||||
if "vid_type" not in identified:
|
||||
identified.update(self._detect_vid_type(parsed.path))
|
||||
|
||||
ids.append(identified)
|
||||
|
||||
return ids
|
||||
|
||||
def process_url(self, parsed):
|
||||
"""process as url"""
|
||||
if parsed.netloc == "youtu.be":
|
||||
# shortened
|
||||
youtube_id = parsed.path.strip("/")
|
||||
return self._validate_expected(youtube_id, "video")
|
||||
|
||||
if "youtube.com" not in parsed.netloc:
|
||||
message = f"invalid domain: {parsed.netloc}"
|
||||
raise ValueError(message)
|
||||
|
||||
query_parsed = parse_qs(parsed.query)
|
||||
if "v" in query_parsed:
|
||||
# video from v query str
|
||||
youtube_id = query_parsed["v"][0]
|
||||
return self._validate_expected(youtube_id, "video")
|
||||
|
||||
if "list" in query_parsed:
|
||||
# playlist from list query str
|
||||
youtube_id = query_parsed["list"][0]
|
||||
return self._validate_expected(youtube_id, "playlist")
|
||||
|
||||
all_paths = parsed.path.strip("/").split("/")
|
||||
if all_paths[0] == "shorts":
|
||||
# is shorts video
|
||||
item = self._validate_expected(all_paths[1], "video")
|
||||
item.update({"vid_type": VideoTypeEnum.SHORTS.value})
|
||||
return item
|
||||
|
||||
if all_paths[0] == "channel":
|
||||
return self._validate_expected(all_paths[1], "channel")
|
||||
|
||||
# detect channel
|
||||
channel_id = self._extract_channel_name(parsed.geturl())
|
||||
return {"type": "channel", "url": channel_id}
|
||||
|
||||
def _validate_expected(self, youtube_id, expected_type):
|
||||
"""raise value error if not matching"""
|
||||
matched = self._find_valid_id(youtube_id)
|
||||
if matched["type"] != expected_type:
|
||||
raise ValueError(
|
||||
f"{youtube_id} not of expected type {expected_type}"
|
||||
)
|
||||
|
||||
return {"type": expected_type, "url": youtube_id}
|
||||
|
||||
def _find_valid_id(self, id_str):
|
||||
"""detect valid id from length of string"""
|
||||
if id_str in ("LL", "WL"):
|
||||
return {"type": "playlist", "url": id_str}
|
||||
|
||||
if id_str.startswith("@"):
|
||||
url = f"https://www.youtube.com/{id_str}"
|
||||
channel_id = self._extract_channel_name(url)
|
||||
return {"type": "channel", "url": channel_id}
|
||||
|
||||
len_id_str = len(id_str)
|
||||
if len_id_str == 11:
|
||||
item_type = "video"
|
||||
elif len_id_str == 24:
|
||||
item_type = "channel"
|
||||
elif len_id_str in (34, 26, 18) or id_str.startswith("TA_playlist_"):
|
||||
item_type = "playlist"
|
||||
else:
|
||||
raise ValueError(f"not a valid id_str: {id_str}")
|
||||
|
||||
return {"type": item_type, "url": id_str}
|
||||
|
||||
@staticmethod
|
||||
def _extract_channel_name(url):
|
||||
"""find channel id from channel name with yt-dlp help"""
|
||||
obs_request = {
|
||||
"check_formats": None,
|
||||
"skip_download": True,
|
||||
"extract_flat": True,
|
||||
"playlistend": 0,
|
||||
}
|
||||
url_info = YtWrap(obs_request).extract(url)
|
||||
channel_id = url_info.get("channel_id", False)
|
||||
if channel_id:
|
||||
return channel_id
|
||||
|
||||
url = url_info.get("url", False)
|
||||
if url:
|
||||
# handle old channel name redirect with url path split
|
||||
channel_id = urlparse(url).path.strip("/").split("/")[1]
|
||||
|
||||
return channel_id
|
||||
|
||||
print(f"failed to extract channel id from {url}")
|
||||
raise ValueError
|
||||
|
||||
def _detect_vid_type(self, path):
|
||||
"""try to match enum from path, needs to be serializable"""
|
||||
last = path.strip("/").split("/")[-1]
|
||||
try:
|
||||
vid_type = VideoTypeEnum(last).value
|
||||
except ValueError:
|
||||
vid_type = VideoTypeEnum.UNKNOWN.value
|
||||
|
||||
return {"vid_type": vid_type}
|
||||
105
backend/common/src/watched.py
Normal file
105
backend/common/src/watched.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
functionality:
|
||||
- handle watched state for videos, channels and playlists
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.urlparser import Parser
|
||||
|
||||
|
||||
class WatchState:
|
||||
"""handle watched checkbox for videos and channels"""
|
||||
|
||||
def __init__(self, youtube_id, is_watched):
|
||||
self.youtube_id = youtube_id
|
||||
self.is_watched = is_watched
|
||||
self.stamp = int(datetime.now().timestamp())
|
||||
self.pipeline = f"_ingest/pipeline/watch_{youtube_id}"
|
||||
|
||||
def change(self):
|
||||
"""change watched state of item(s)"""
|
||||
print(f"{self.youtube_id}: change watched state to {self.is_watched}")
|
||||
url_type = self._dedect_type()
|
||||
if url_type == "video":
|
||||
self.change_vid_state()
|
||||
return
|
||||
|
||||
self._add_pipeline()
|
||||
path = f"ta_video/_update_by_query?pipeline=watch_{self.youtube_id}"
|
||||
data = self._build_update_data(url_type)
|
||||
_, _ = ElasticWrap(path).post(data)
|
||||
self._delete_pipeline()
|
||||
|
||||
def _dedect_type(self):
|
||||
"""find youtube id type"""
|
||||
url_process = Parser(self.youtube_id).parse()
|
||||
url_type = url_process[0]["type"]
|
||||
return url_type
|
||||
|
||||
def change_vid_state(self):
|
||||
"""change watched state of video"""
|
||||
path = f"ta_video/_update/{self.youtube_id}"
|
||||
data = {
|
||||
"doc": {
|
||||
"player": {
|
||||
"watched": self.is_watched,
|
||||
"watched_date": self.stamp,
|
||||
}
|
||||
}
|
||||
}
|
||||
response, status_code = ElasticWrap(path).post(data=data)
|
||||
if status_code != 200:
|
||||
print(response)
|
||||
raise ValueError("failed to mark video as watched")
|
||||
|
||||
def _build_update_data(self, url_type):
|
||||
"""build update by query data based on url_type"""
|
||||
term_key_map = {
|
||||
"channel": "channel.channel_id",
|
||||
"playlist": "playlist.keyword",
|
||||
}
|
||||
term_key = term_key_map.get(url_type)
|
||||
|
||||
return {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"term": {term_key: {"value": self.youtube_id}}},
|
||||
{
|
||||
"term": {
|
||||
"player.watched": {
|
||||
"value": not self.is_watched
|
||||
}
|
||||
}
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def _add_pipeline(self):
|
||||
"""add ingest pipeline"""
|
||||
data = {
|
||||
"description": f"{self.youtube_id}: watched {self.is_watched}",
|
||||
"processors": [
|
||||
{
|
||||
"set": {
|
||||
"field": "player.watched",
|
||||
"value": self.is_watched,
|
||||
}
|
||||
},
|
||||
{
|
||||
"set": {
|
||||
"field": "player.watched_date",
|
||||
"value": self.stamp,
|
||||
}
|
||||
},
|
||||
],
|
||||
}
|
||||
_, _ = ElasticWrap(self.pipeline).put(data)
|
||||
|
||||
def _delete_pipeline(self):
|
||||
"""delete pipeline"""
|
||||
ElasticWrap(self.pipeline).delete()
|
||||
0
backend/common/tests/__init__.py
Normal file
0
backend/common/tests/__init__.py
Normal file
11
backend/common/tests/conftest.py
Normal file
11
backend/common/tests/conftest.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""test configs"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def change_test_dir(request):
|
||||
"""change directory to project folder"""
|
||||
os.chdir(request.config.rootdir / "tubearchivist")
|
||||
0
backend/common/tests/test_src/__init__.py
Normal file
0
backend/common/tests/test_src/__init__.py
Normal file
113
backend/common/tests/test_src/test_helper.py
Normal file
113
backend/common/tests/test_src/test_helper.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""tests for helper functions"""
|
||||
|
||||
import pytest
|
||||
from common.src.helper import (
|
||||
date_parser,
|
||||
get_duration_str,
|
||||
get_mapping,
|
||||
is_shorts,
|
||||
randomizor,
|
||||
time_parser,
|
||||
)
|
||||
|
||||
|
||||
def test_randomizor_with_positive_length():
|
||||
"""test randomizer"""
|
||||
length = 10
|
||||
result = randomizor(length)
|
||||
assert len(result) == length
|
||||
assert result.isalnum()
|
||||
|
||||
|
||||
def test_date_parser_with_int():
|
||||
"""unix timestamp"""
|
||||
timestamp = 1621539600
|
||||
expected_date = "2021-05-20"
|
||||
assert date_parser(timestamp) == expected_date
|
||||
|
||||
|
||||
def test_date_parser_with_str():
|
||||
"""iso timestamp"""
|
||||
date_str = "2021-05-21"
|
||||
expected_date = "2021-05-21"
|
||||
assert date_parser(date_str) == expected_date
|
||||
|
||||
|
||||
def test_date_parser_with_invalid_input():
|
||||
"""invalid type"""
|
||||
invalid_input = [1621539600]
|
||||
with pytest.raises(TypeError):
|
||||
date_parser(invalid_input)
|
||||
|
||||
|
||||
def test_date_parser_with_invalid_string_format():
|
||||
"""invalid date string"""
|
||||
invalid_date_str = "21/05/2021"
|
||||
with pytest.raises(ValueError):
|
||||
date_parser(invalid_date_str)
|
||||
|
||||
|
||||
def test_time_parser_with_numeric_string():
|
||||
"""as number"""
|
||||
timestamp = "100"
|
||||
expected_seconds = 100
|
||||
assert time_parser(timestamp) == expected_seconds
|
||||
|
||||
|
||||
def test_time_parser_with_hh_mm_ss_format():
|
||||
"""to seconds"""
|
||||
timestamp = "01:00:00"
|
||||
expected_seconds = 3600.0
|
||||
assert time_parser(timestamp) == expected_seconds
|
||||
|
||||
|
||||
def test_time_parser_with_empty_string():
|
||||
"""handle empty"""
|
||||
timestamp = ""
|
||||
assert time_parser(timestamp) is False
|
||||
|
||||
|
||||
def test_time_parser_with_invalid_format():
|
||||
"""not enough to unpack"""
|
||||
timestamp = "01:00"
|
||||
with pytest.raises(ValueError):
|
||||
time_parser(timestamp)
|
||||
|
||||
|
||||
def test_time_parser_with_non_numeric_input():
|
||||
"""non numeric"""
|
||||
timestamp = "1a:00:00"
|
||||
with pytest.raises(ValueError):
|
||||
time_parser(timestamp)
|
||||
|
||||
|
||||
def test_get_mapping():
|
||||
"""test mappint"""
|
||||
index_config = get_mapping()
|
||||
assert isinstance(index_config, list)
|
||||
assert all(isinstance(i, dict) for i in index_config)
|
||||
|
||||
|
||||
def test_is_shorts():
|
||||
"""is shorts id"""
|
||||
youtube_id = "YG3-Pw3rixU"
|
||||
assert is_shorts(youtube_id)
|
||||
|
||||
|
||||
def test_is_not_shorts():
|
||||
"""is not shorts id"""
|
||||
youtube_id = "Ogr9kbypSNg"
|
||||
assert is_shorts(youtube_id) is False
|
||||
|
||||
|
||||
def test_get_duration_str():
|
||||
"""only seconds"""
|
||||
assert get_duration_str(None) == "NA"
|
||||
assert get_duration_str(5) == "5s"
|
||||
assert get_duration_str(10) == "10s"
|
||||
assert get_duration_str(500) == "8m 20s"
|
||||
assert get_duration_str(1000) == "16m 40s"
|
||||
assert get_duration_str(5000) == "1h 23m 20s"
|
||||
assert get_duration_str(500000) == "5d 18h 53m 20s"
|
||||
assert get_duration_str(5000000) == "57d 20h 53m 20s"
|
||||
assert get_duration_str(50000000) == "1y 213d 16h 53m 20s"
|
||||
144
backend/common/tests/test_src/test_urlparser.py
Normal file
144
backend/common/tests/test_src/test_urlparser.py
Normal file
@@ -0,0 +1,144 @@
|
||||
"""tests for url parser"""
|
||||
|
||||
import pytest
|
||||
from common.src.urlparser import Parser
|
||||
|
||||
# video id parsing
|
||||
VIDEO_URL_IN = [
|
||||
"7DKv5H5Frt0",
|
||||
"https://www.youtube.com/watch?v=7DKv5H5Frt0",
|
||||
"https://www.youtube.com/watch?v=7DKv5H5Frt0&t=113&feature=shared",
|
||||
"https://www.youtube.com/watch?v=7DKv5H5Frt0&list=PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5&index=1&pp=iAQB" # noqa: E501
|
||||
"https://youtu.be/7DKv5H5Frt0",
|
||||
]
|
||||
VIDEO_OUT = [{"type": "video", "url": "7DKv5H5Frt0", "vid_type": "unknown"}]
|
||||
VIDEO_TEST_CASES = [(i, VIDEO_OUT) for i in VIDEO_URL_IN]
|
||||
|
||||
# shorts id parsing
|
||||
SHORTS_URL_IN = [
|
||||
"https://www.youtube.com/shorts/YG3-Pw3rixU",
|
||||
"https://youtube.com/shorts/YG3-Pw3rixU?feature=shared",
|
||||
]
|
||||
SHORTS_OUT = [{"type": "video", "url": "YG3-Pw3rixU", "vid_type": "shorts"}]
|
||||
SHORTS_TEST_CASES = [(i, SHORTS_OUT) for i in SHORTS_URL_IN]
|
||||
|
||||
# channel id parsing
|
||||
CHANNEL_URL_IN = [
|
||||
"UCBa659QWEk1AI4Tg--mrJ2A",
|
||||
"@TomScottGo",
|
||||
"https://www.youtube.com/channel/UCBa659QWEk1AI4Tg--mrJ2A",
|
||||
"https://www.youtube.com/@TomScottGo",
|
||||
]
|
||||
CHANNEL_OUT = [
|
||||
{
|
||||
"type": "channel",
|
||||
"url": "UCBa659QWEk1AI4Tg--mrJ2A",
|
||||
"vid_type": "unknown",
|
||||
}
|
||||
]
|
||||
CHANNEL_TEST_CASES = [(i, CHANNEL_OUT) for i in CHANNEL_URL_IN]
|
||||
|
||||
# channel vid type parsing
|
||||
CHANNEL_VID_TYPES = [
|
||||
(
|
||||
"https://www.youtube.com/@IBRACORP/videos",
|
||||
[
|
||||
{
|
||||
"type": "channel",
|
||||
"url": "UC7aW7chIafJG6ECYAd3N5uQ",
|
||||
"vid_type": "videos",
|
||||
}
|
||||
],
|
||||
),
|
||||
(
|
||||
"https://www.youtube.com/@IBRACORP/shorts",
|
||||
[
|
||||
{
|
||||
"type": "channel",
|
||||
"url": "UC7aW7chIafJG6ECYAd3N5uQ",
|
||||
"vid_type": "shorts",
|
||||
}
|
||||
],
|
||||
),
|
||||
(
|
||||
"https://www.youtube.com/@IBRACORP/streams",
|
||||
[
|
||||
{
|
||||
"type": "channel",
|
||||
"url": "UC7aW7chIafJG6ECYAd3N5uQ",
|
||||
"vid_type": "streams",
|
||||
}
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
# playlist id parsing
|
||||
PLAYLIST_URL_IN = [
|
||||
"PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5",
|
||||
"https://www.youtube.com/playlist?list=PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5",
|
||||
]
|
||||
PLAYLIST_OUT = [
|
||||
{
|
||||
"type": "playlist",
|
||||
"url": "PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5",
|
||||
"vid_type": "unknown",
|
||||
}
|
||||
]
|
||||
PLAYLIST_TEST_CASES = [(i, PLAYLIST_OUT) for i in PLAYLIST_URL_IN]
|
||||
|
||||
# personal playlists
|
||||
EXPECTED_WL = [{"type": "playlist", "url": "WL", "vid_type": "unknown"}]
|
||||
EXPECTED_LL = [{"type": "playlist", "url": "LL", "vid_type": "unknown"}]
|
||||
PERSONAL_PLAYLISTS_TEST_CASES = [
|
||||
("WL", EXPECTED_WL),
|
||||
("https://www.youtube.com/playlist?list=WL", EXPECTED_WL),
|
||||
("LL", EXPECTED_LL),
|
||||
("https://www.youtube.com/playlist?list=LL", EXPECTED_LL),
|
||||
]
|
||||
|
||||
# collect tests expected to pass
|
||||
PASSTING_TESTS = []
|
||||
PASSTING_TESTS.extend(VIDEO_TEST_CASES)
|
||||
PASSTING_TESTS.extend(SHORTS_TEST_CASES)
|
||||
PASSTING_TESTS.extend(CHANNEL_TEST_CASES)
|
||||
PASSTING_TESTS.extend(CHANNEL_VID_TYPES)
|
||||
PASSTING_TESTS.extend(PLAYLIST_TEST_CASES)
|
||||
PASSTING_TESTS.extend(PERSONAL_PLAYLISTS_TEST_CASES)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("url_str, expected_result", PASSTING_TESTS)
|
||||
def test_passing_parse(url_str, expected_result):
|
||||
"""test parser"""
|
||||
parser = Parser(url_str)
|
||||
parsed = parser.parse()
|
||||
assert parsed == expected_result
|
||||
|
||||
|
||||
INVALID_IDS_ERRORS = [
|
||||
"aaaaa",
|
||||
"https://www.youtube.com/playlist?list=AAAA",
|
||||
"https://www.youtube.com/channel/UC9-y-6csu5WGm29I7Jiwpn",
|
||||
"https://www.youtube.com/watch?v=CK3_zarXkw",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_value", INVALID_IDS_ERRORS)
|
||||
def test_invalid_ids(invalid_value):
|
||||
"""test for invalid IDs"""
|
||||
with pytest.raises(ValueError, match="not a valid id_str"):
|
||||
parser = Parser(invalid_value)
|
||||
parser.parse()
|
||||
|
||||
|
||||
INVALID_DOMAINS = [
|
||||
"https://vimeo.com/32001208",
|
||||
"https://peertube.tv/w/8RiJE2j2nw569FVgPNjDt7",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_value", INVALID_DOMAINS)
|
||||
def test_invalid_domains(invalid_value):
|
||||
"""raise error on none YT domains"""
|
||||
parser = Parser(invalid_value)
|
||||
with pytest.raises(ValueError, match="invalid domain"):
|
||||
parser.parse()
|
||||
28
backend/common/urls.py
Normal file
28
backend/common/urls.py
Normal file
@@ -0,0 +1,28 @@
|
||||
"""all api urls"""
|
||||
|
||||
from common import views
|
||||
from django.urls import path
|
||||
|
||||
urlpatterns = [
|
||||
path("ping/", views.PingView.as_view(), name="ping"),
|
||||
path(
|
||||
"refresh/",
|
||||
views.RefreshView.as_view(),
|
||||
name="api-refresh",
|
||||
),
|
||||
path(
|
||||
"watched/",
|
||||
views.WatchedView.as_view(),
|
||||
name="api-watched",
|
||||
),
|
||||
path(
|
||||
"search/",
|
||||
views.SearchView.as_view(),
|
||||
name="api-search",
|
||||
),
|
||||
path(
|
||||
"notification/",
|
||||
views.NotificationView.as_view(),
|
||||
name="api-notification",
|
||||
),
|
||||
]
|
||||
115
backend/common/views.py
Normal file
115
backend/common/views.py
Normal file
@@ -0,0 +1,115 @@
|
||||
"""all API views"""
|
||||
|
||||
from appsettings.src.config import ReleaseVersion
|
||||
from appsettings.src.reindex import ReindexProgress
|
||||
from common.src.searching import SearchForm
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
from common.src.watched import WatchState
|
||||
from common.views_base import AdminOnly, ApiBaseView
|
||||
from rest_framework.response import Response
|
||||
from task.tasks import check_reindex
|
||||
|
||||
|
||||
class PingView(ApiBaseView):
|
||||
"""resolves to /api/ping/
|
||||
GET: test your connection
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get(request):
|
||||
"""get pong"""
|
||||
data = {
|
||||
"response": "pong",
|
||||
"user": request.user.id,
|
||||
"version": ReleaseVersion().get_local_version(),
|
||||
}
|
||||
return Response(data)
|
||||
|
||||
|
||||
class RefreshView(ApiBaseView):
|
||||
"""resolves to /api/refresh/
|
||||
GET: get refresh progress
|
||||
POST: start a manual refresh task
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
request_type = request.GET.get("type")
|
||||
request_id = request.GET.get("id")
|
||||
|
||||
if request_id and not request_type:
|
||||
return Response({"status": "Bad Request"}, status=400)
|
||||
|
||||
try:
|
||||
progress = ReindexProgress(
|
||||
request_type=request_type, request_id=request_id
|
||||
).get_progress()
|
||||
except ValueError:
|
||||
return Response({"status": "Bad Request"}, status=400)
|
||||
|
||||
return Response(progress)
|
||||
|
||||
def post(self, request):
|
||||
"""handle post request"""
|
||||
data = request.data
|
||||
extract_videos = bool(request.GET.get("extract_videos", False))
|
||||
check_reindex.delay(data=data, extract_videos=extract_videos)
|
||||
|
||||
return Response(data)
|
||||
|
||||
|
||||
class WatchedView(ApiBaseView):
|
||||
"""resolves to /api/watched/
|
||||
POST: change watched state of video, channel or playlist
|
||||
"""
|
||||
|
||||
def post(self, request):
|
||||
"""change watched state"""
|
||||
youtube_id = request.data.get("id")
|
||||
is_watched = request.data.get("is_watched")
|
||||
|
||||
if not youtube_id or is_watched is None:
|
||||
message = {"message": "missing id or is_watched"}
|
||||
return Response(message, status=400)
|
||||
|
||||
WatchState(youtube_id, is_watched).change()
|
||||
return Response({"message": "success"}, status=200)
|
||||
|
||||
|
||||
class SearchView(ApiBaseView):
|
||||
"""resolves to /api/search/
|
||||
GET: run a search with the string in the ?query parameter
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get(request):
|
||||
"""handle get request
|
||||
search through all indexes"""
|
||||
search_query = request.GET.get("query", None)
|
||||
if search_query is None:
|
||||
return Response(
|
||||
{"message": "no search query specified"}, status=400
|
||||
)
|
||||
|
||||
search_results = SearchForm().multi_search(search_query)
|
||||
return Response(search_results)
|
||||
|
||||
|
||||
class NotificationView(ApiBaseView):
|
||||
"""resolves to /api/notification/
|
||||
GET: returns a list of notifications
|
||||
filter query to filter messages by group
|
||||
"""
|
||||
|
||||
valid_filters = ["download", "settings", "channel"]
|
||||
|
||||
def get(self, request):
|
||||
"""get all notifications"""
|
||||
query = "message"
|
||||
filter_by = request.GET.get("filter", None)
|
||||
if filter_by in self.valid_filters:
|
||||
query = f"{query}:{filter_by}"
|
||||
|
||||
return Response(RedisArchivist().list_items(query))
|
||||
106
backend/common/views_base.py
Normal file
106
backend/common/views_base.py
Normal file
@@ -0,0 +1,106 @@
|
||||
"""base classes to inherit from"""
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.index_generic import Pagination
|
||||
from common.src.search_processor import SearchProcess, process_aggs
|
||||
from rest_framework import permissions
|
||||
from rest_framework.authentication import (
|
||||
SessionAuthentication,
|
||||
TokenAuthentication,
|
||||
)
|
||||
from rest_framework.views import APIView
|
||||
|
||||
|
||||
def check_admin(user):
|
||||
"""check for admin permission for restricted views"""
|
||||
return user.is_staff or user.groups.filter(name="admin").exists()
|
||||
|
||||
|
||||
class AdminOnly(permissions.BasePermission):
|
||||
"""allow only admin"""
|
||||
|
||||
def has_permission(self, request, view):
|
||||
return check_admin(request.user)
|
||||
|
||||
|
||||
class AdminWriteOnly(permissions.BasePermission):
|
||||
"""allow only admin writes"""
|
||||
|
||||
def has_permission(self, request, view):
|
||||
if request.method in permissions.SAFE_METHODS:
|
||||
return permissions.IsAuthenticated().has_permission(request, view)
|
||||
|
||||
return check_admin(request.user)
|
||||
|
||||
|
||||
class ApiBaseView(APIView):
|
||||
"""base view to inherit from"""
|
||||
|
||||
authentication_classes = [SessionAuthentication, TokenAuthentication]
|
||||
permission_classes = [permissions.IsAuthenticated]
|
||||
search_base = ""
|
||||
data = ""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.response = {
|
||||
"data": False,
|
||||
"config": {
|
||||
"enable_cast": EnvironmentSettings.ENABLE_CAST,
|
||||
"downloads": AppConfig().config["downloads"],
|
||||
},
|
||||
}
|
||||
self.data = {"query": {"match_all": {}}}
|
||||
self.status_code = False
|
||||
self.context = False
|
||||
self.pagination_handler = False
|
||||
|
||||
def get_document(self, document_id):
|
||||
"""get single document from es"""
|
||||
path = f"{self.search_base}{document_id}"
|
||||
response, status_code = ElasticWrap(path).get()
|
||||
try:
|
||||
self.response["data"] = SearchProcess(response).process()
|
||||
except KeyError:
|
||||
print(f"item not found: {document_id}")
|
||||
self.response["data"] = False
|
||||
self.status_code = status_code
|
||||
|
||||
def initiate_pagination(self, request):
|
||||
"""set initial pagination values"""
|
||||
self.pagination_handler = Pagination(request)
|
||||
self.data.update(
|
||||
{
|
||||
"size": self.pagination_handler.pagination["page_size"],
|
||||
"from": self.pagination_handler.pagination["page_from"],
|
||||
}
|
||||
)
|
||||
|
||||
def get_document_list(self, request, pagination=True):
|
||||
"""get a list of results"""
|
||||
if pagination:
|
||||
self.initiate_pagination(request)
|
||||
|
||||
es_handler = ElasticWrap(self.search_base)
|
||||
response, status_code = es_handler.get(data=self.data)
|
||||
self.response["data"] = SearchProcess(response).process()
|
||||
if self.response["data"]:
|
||||
self.status_code = status_code
|
||||
else:
|
||||
self.status_code = 404
|
||||
|
||||
if pagination:
|
||||
self.pagination_handler.validate(
|
||||
response["hits"]["total"]["value"]
|
||||
)
|
||||
self.response["paginate"] = self.pagination_handler.pagination
|
||||
|
||||
def get_aggs(self):
|
||||
"""get aggs alone"""
|
||||
self.data["size"] = 0
|
||||
response, _ = ElasticWrap(self.search_base).get(data=self.data)
|
||||
process_aggs(response)
|
||||
|
||||
self.response = response.get("aggregations")
|
||||
0
backend/config/__init__.py
Normal file
0
backend/config/__init__.py
Normal file
16
backend/config/asgi.py
Normal file
16
backend/config/asgi.py
Normal file
@@ -0,0 +1,16 @@
|
||||
"""
|
||||
ASGI config for config project.
|
||||
|
||||
It exposes the ASGI callable as a module-level variable named ``application``.
|
||||
|
||||
For more information on this file, see
|
||||
https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from django.core.asgi import get_asgi_application
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
||||
|
||||
application = get_asgi_application()
|
||||
0
backend/config/management/__init__.py
Normal file
0
backend/config/management/__init__.py
Normal file
0
backend/config/management/commands/__init__.py
Normal file
0
backend/config/management/commands/__init__.py
Normal file
163
backend/config/management/commands/ta_connection.py
Normal file
163
backend/config/management/commands/ta_connection.py
Normal file
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
Functionality:
|
||||
- check that all connections are working
|
||||
"""
|
||||
|
||||
from time import sleep
|
||||
|
||||
import requests
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
TOPIC = """
|
||||
|
||||
#######################
|
||||
# Connection check #
|
||||
#######################
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""command framework"""
|
||||
|
||||
TIMEOUT = 120
|
||||
MIN_MAJOR, MAX_MAJOR = 8, 8
|
||||
MIN_MINOR = 0
|
||||
|
||||
# pylint: disable=no-member
|
||||
help = "Check connections"
|
||||
|
||||
def handle(self, *args, **options):
|
||||
"""run all commands"""
|
||||
self.stdout.write(TOPIC)
|
||||
self._redis_connection_check()
|
||||
self._redis_config_set()
|
||||
self._es_connection_check()
|
||||
self._es_version_check()
|
||||
self._es_path_check()
|
||||
|
||||
def _redis_connection_check(self):
|
||||
"""check ir redis connection is established"""
|
||||
self.stdout.write("[1] connect to Redis")
|
||||
redis_conn = RedisArchivist().conn
|
||||
for _ in range(5):
|
||||
try:
|
||||
pong = redis_conn.execute_command("PING")
|
||||
if pong:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" ✓ Redis connection verified")
|
||||
)
|
||||
return
|
||||
|
||||
except Exception: # pylint: disable=broad-except
|
||||
self.stdout.write(" ... retry Redis connection")
|
||||
sleep(2)
|
||||
|
||||
message = " 🗙 Redis connection failed"
|
||||
self.stdout.write(self.style.ERROR(f"{message}"))
|
||||
RedisArchivist().exec("PING")
|
||||
sleep(60)
|
||||
raise CommandError(message)
|
||||
|
||||
def _redis_config_set(self):
|
||||
"""set config for redis if not set already"""
|
||||
self.stdout.write("[2] set Redis config")
|
||||
redis_conn = RedisArchivist().conn
|
||||
timeout_is = int(redis_conn.config_get("timeout").get("timeout"))
|
||||
if not timeout_is:
|
||||
redis_conn.config_set("timeout", 3600)
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(" ✓ Redis config set"))
|
||||
|
||||
def _es_connection_check(self):
|
||||
"""wait for elasticsearch connection"""
|
||||
self.stdout.write("[3] connect to Elastic Search")
|
||||
total = self.TIMEOUT // 5
|
||||
for i in range(total):
|
||||
self.stdout.write(f" ... waiting for ES [{i}/{total}]")
|
||||
try:
|
||||
_, status_code = ElasticWrap("/").get(
|
||||
timeout=1, print_error=False
|
||||
)
|
||||
except (
|
||||
requests.exceptions.ConnectionError,
|
||||
requests.exceptions.Timeout,
|
||||
):
|
||||
sleep(5)
|
||||
continue
|
||||
|
||||
if status_code and status_code == 200:
|
||||
path = "_cluster/health?wait_for_status=yellow&timeout=60s"
|
||||
_, _ = ElasticWrap(path).get(timeout=60)
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" ✓ ES connection established")
|
||||
)
|
||||
return
|
||||
|
||||
response, status_code = ElasticWrap("/").get(
|
||||
timeout=1, print_error=False
|
||||
)
|
||||
|
||||
message = " 🗙 ES connection failed"
|
||||
self.stdout.write(self.style.ERROR(f"{message}"))
|
||||
self.stdout.write(f" error message: {response}")
|
||||
self.stdout.write(f" status code: {status_code}")
|
||||
sleep(60)
|
||||
raise CommandError(message)
|
||||
|
||||
def _es_version_check(self):
|
||||
"""check for minimal elasticsearch version"""
|
||||
self.stdout.write("[4] Elastic Search version check")
|
||||
response, _ = ElasticWrap("/").get()
|
||||
version = response["version"]["number"]
|
||||
major = int(version.split(".")[0])
|
||||
|
||||
if self.MIN_MAJOR <= major <= self.MAX_MAJOR:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" ✓ ES version check passed")
|
||||
)
|
||||
return
|
||||
|
||||
message = (
|
||||
" 🗙 ES version check failed. "
|
||||
+ f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}"
|
||||
)
|
||||
self.stdout.write(self.style.ERROR(f"{message}"))
|
||||
sleep(60)
|
||||
raise CommandError(message)
|
||||
|
||||
def _es_path_check(self):
|
||||
"""check that path.repo var is set"""
|
||||
self.stdout.write("[5] check ES path.repo env var")
|
||||
response, _ = ElasticWrap("_nodes/_all/settings").get()
|
||||
snaphost_roles = [
|
||||
"data",
|
||||
"data_cold",
|
||||
"data_content",
|
||||
"data_frozen",
|
||||
"data_hot",
|
||||
"data_warm",
|
||||
"master",
|
||||
]
|
||||
for node in response["nodes"].values():
|
||||
if not (set(node["roles"]) & set(snaphost_roles)):
|
||||
continue
|
||||
|
||||
if node["settings"]["path"].get("repo"):
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" ✓ path.repo env var is set")
|
||||
)
|
||||
return
|
||||
|
||||
message = (
|
||||
" 🗙 path.repo env var not found. "
|
||||
+ "set the following env var to the ES container:\n"
|
||||
+ " path.repo="
|
||||
+ EnvironmentSettings.ES_SNAPSHOT_DIR
|
||||
)
|
||||
self.stdout.write(self.style.ERROR(message))
|
||||
sleep(60)
|
||||
raise CommandError(message)
|
||||
193
backend/config/management/commands/ta_envcheck.py
Normal file
193
backend/config/management/commands/ta_envcheck.py
Normal file
@@ -0,0 +1,193 @@
|
||||
"""
|
||||
Functionality:
|
||||
- Check environment at startup
|
||||
- Process config file overwrites from env var
|
||||
- Stop startup on error
|
||||
- python management.py ta_envcheck
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from user.models import Account
|
||||
|
||||
LOGO = """
|
||||
|
||||
.... .....
|
||||
...'',;:cc,. .;::;;,'...
|
||||
..,;:cccllclc, .:ccllllcc;,..
|
||||
..,:cllcc:;,'.',. ....'',;ccllc:,..
|
||||
..;cllc:,'.. ...,:cccc:'.
|
||||
.;cccc;.. ..,:ccc:'.
|
||||
.ckkkOkxollllllllllllc. .,:::;. .,cclc;
|
||||
.:0MMMMMMMMMMMMMMMMMMMX: .cNMMMWx. .;clc:
|
||||
.;lOXK0000KNMMMMX00000KO; ;KMMMMMNl. .;ccl:,.
|
||||
.;:c:'.....kMMMNo........ 'OMMMWMMMK: '::;;'.
|
||||
....... .xMMMNl .dWMMXdOMMMO' ........
|
||||
.:cc:;. .xMMMNc .lNMMNo.:XMMWx. .:cl:.
|
||||
.:llc,. .:xxxd, ;KMMMk. .oWMMNl. .:llc'
|
||||
.cll:. .;:;;:::,. 'OMMMK:';''kWMMK: .;llc,
|
||||
.cll:. .,;;;;;;,. .,xWMMNl.:l:.;KMMMO' .;llc'
|
||||
.:llc. .cOOOk; .lKNMMWx..:l:..lNMMWx. .:llc'
|
||||
.;lcc,. .xMMMNc :KMMMM0, .:lc. .xWMMNl.'ccl:.
|
||||
.cllc. .xMMMNc 'OMMMMXc...:lc...,0MMMKl:lcc,.
|
||||
.,ccl:. .xMMMNc .xWMMMWo.,;;:lc;;;.cXMMMXdcc;.
|
||||
.,clc:. .xMMMNc .lNMMMWk. .':clc:,. .dWMMW0o;.
|
||||
.,clcc,. .ckkkx; .okkkOx, .';,. 'kKKK0l.
|
||||
.':lcc:'..... . .. ..,;cllc,.
|
||||
.,cclc,.... ....;clc;..
|
||||
..,:,..,c:'.. ...';:,..,:,.
|
||||
....:lcccc:;,'''.....'',;;:clllc,....
|
||||
.'',;:cllllllccccclllllcc:,'..
|
||||
...'',,;;;;;;;;;,''...
|
||||
.....
|
||||
|
||||
"""
|
||||
|
||||
TOPIC = """
|
||||
#######################
|
||||
# Environment Setup #
|
||||
#######################
|
||||
|
||||
"""
|
||||
|
||||
EXPECTED_ENV_VARS = [
|
||||
"TA_USERNAME",
|
||||
"TA_PASSWORD",
|
||||
"ELASTIC_PASSWORD",
|
||||
"ES_URL",
|
||||
"TA_HOST",
|
||||
]
|
||||
INST = "https://github.com/tubearchivist/tubearchivist#installing-and-updating"
|
||||
NGINX = "/etc/nginx/sites-available/default"
|
||||
UWSGI = "/app/uwsgi.ini"
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""command framework"""
|
||||
|
||||
# pylint: disable=no-member
|
||||
help = "Check environment before startup"
|
||||
|
||||
def handle(self, *args, **options):
|
||||
"""run all commands"""
|
||||
self.stdout.write(LOGO)
|
||||
self.stdout.write(TOPIC)
|
||||
self._expected_vars()
|
||||
self._elastic_user_overwrite()
|
||||
self._ta_port_overwrite()
|
||||
self._ta_uwsgi_overwrite()
|
||||
self._enable_cast_overwrite()
|
||||
self._create_superuser()
|
||||
|
||||
def _expected_vars(self):
|
||||
"""check if expected env vars are set"""
|
||||
self.stdout.write("[1] checking expected env vars")
|
||||
env = os.environ
|
||||
for var in EXPECTED_ENV_VARS:
|
||||
if not env.get(var):
|
||||
message = f" 🗙 expected env var {var} not set\n {INST}"
|
||||
self.stdout.write(self.style.ERROR(message))
|
||||
raise CommandError(message)
|
||||
|
||||
message = " ✓ all expected env vars are set"
|
||||
self.stdout.write(self.style.SUCCESS(message))
|
||||
|
||||
def _elastic_user_overwrite(self):
|
||||
"""check for ELASTIC_USER overwrite"""
|
||||
self.stdout.write("[2] check ES user overwrite")
|
||||
env = EnvironmentSettings.ES_USER
|
||||
self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}"))
|
||||
|
||||
def _ta_port_overwrite(self):
|
||||
"""set TA_PORT overwrite for nginx"""
|
||||
self.stdout.write("[3] check TA_PORT overwrite")
|
||||
overwrite = EnvironmentSettings.TA_PORT
|
||||
if not overwrite:
|
||||
self.stdout.write(self.style.SUCCESS(" TA_PORT is not set"))
|
||||
return
|
||||
|
||||
regex = re.compile(r"listen [0-9]{1,5}")
|
||||
to_overwrite = f"listen {overwrite}"
|
||||
changed = file_overwrite(NGINX, regex, to_overwrite)
|
||||
if changed:
|
||||
message = f" ✓ TA_PORT changed to {overwrite}"
|
||||
else:
|
||||
message = f" ✓ TA_PORT already set to {overwrite}"
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(message))
|
||||
|
||||
def _ta_uwsgi_overwrite(self):
|
||||
"""set TA_UWSGI_PORT overwrite"""
|
||||
self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
|
||||
overwrite = EnvironmentSettings.TA_UWSGI_PORT
|
||||
if not overwrite:
|
||||
message = " TA_UWSGI_PORT is not set"
|
||||
self.stdout.write(self.style.SUCCESS(message))
|
||||
return
|
||||
|
||||
# nginx
|
||||
regex = re.compile(r"uwsgi_pass localhost:[0-9]{1,5}")
|
||||
to_overwrite = f"uwsgi_pass localhost:{overwrite}"
|
||||
changed = file_overwrite(NGINX, regex, to_overwrite)
|
||||
|
||||
# uwsgi
|
||||
regex = re.compile(r"socket = :[0-9]{1,5}")
|
||||
to_overwrite = f"socket = :{overwrite}"
|
||||
changed = file_overwrite(UWSGI, regex, to_overwrite)
|
||||
|
||||
if changed:
|
||||
message = f" ✓ TA_UWSGI_PORT changed to {overwrite}"
|
||||
else:
|
||||
message = f" ✓ TA_UWSGI_PORT already set to {overwrite}"
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(message))
|
||||
|
||||
def _enable_cast_overwrite(self):
|
||||
"""cast workaround, remove auth for static files in nginx"""
|
||||
self.stdout.write("[5] check ENABLE_CAST overwrite")
|
||||
overwrite = EnvironmentSettings.ENABLE_CAST
|
||||
if not overwrite:
|
||||
self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set"))
|
||||
return
|
||||
|
||||
regex = re.compile(r"[^\S\r\n]*auth_request /api/ping/;\n")
|
||||
changed = file_overwrite(NGINX, regex, "")
|
||||
if changed:
|
||||
message = " ✓ process nginx to enable Cast"
|
||||
else:
|
||||
message = " ✓ Cast is already enabled in nginx"
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(message))
|
||||
|
||||
def _create_superuser(self):
|
||||
"""create superuser if not exist"""
|
||||
self.stdout.write("[6] create superuser")
|
||||
is_created = Account.objects.filter(is_superuser=True)
|
||||
if is_created:
|
||||
message = " superuser already created"
|
||||
self.stdout.write(self.style.SUCCESS(message))
|
||||
return
|
||||
|
||||
name = EnvironmentSettings.TA_USERNAME
|
||||
password = EnvironmentSettings.TA_PASSWORD
|
||||
Account.objects.create_superuser(name, password)
|
||||
message = f" ✓ new superuser with name {name} created"
|
||||
self.stdout.write(self.style.SUCCESS(message))
|
||||
|
||||
|
||||
def file_overwrite(file_path, regex, overwrite):
|
||||
"""change file content from old to overwrite, return true when changed"""
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
file_content = f.read()
|
||||
|
||||
changed = re.sub(regex, overwrite, file_content)
|
||||
if changed == file_content:
|
||||
return False
|
||||
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(changed)
|
||||
|
||||
return True
|
||||
251
backend/config/management/commands/ta_startup.py
Normal file
251
backend/config/management/commands/ta_startup.py
Normal file
@@ -0,0 +1,251 @@
|
||||
"""
|
||||
Functionality:
|
||||
- Application startup
|
||||
- Apply migrations
|
||||
"""
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
from random import randint
|
||||
from time import sleep
|
||||
|
||||
from appsettings.src.config import ReleaseVersion
|
||||
from appsettings.src.index_setup import ElasitIndexWrap
|
||||
from appsettings.src.snapshot import ElasticSnapshot
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.helper import clear_dl_cache
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.utils import dateformat
|
||||
from django_celery_beat.models import CrontabSchedule, PeriodicTasks
|
||||
from task.models import CustomPeriodicTask
|
||||
from task.src.config_schedule import ScheduleBuilder
|
||||
from task.src.task_manager import TaskManager
|
||||
from task.tasks import version_check
|
||||
|
||||
TOPIC = """
|
||||
|
||||
#######################
|
||||
# Application Start #
|
||||
#######################
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""command framework"""
|
||||
|
||||
# pylint: disable=no-member
|
||||
|
||||
def handle(self, *args, **options):
|
||||
"""run all commands"""
|
||||
self.stdout.write(TOPIC)
|
||||
self._make_folders()
|
||||
self._clear_redis_keys()
|
||||
self._clear_tasks()
|
||||
self._clear_dl_cache()
|
||||
self._version_check()
|
||||
self._index_setup()
|
||||
self._snapshot_check()
|
||||
self._create_default_schedules()
|
||||
self._update_schedule_tz()
|
||||
|
||||
def _mig_app_settings(self) -> None:
|
||||
"""update from v0.4.10 to v0.5.0, migrate application settings"""
|
||||
self.stdout.write("[MIGRATION] move appconfig to ES")
|
||||
config = RedisArchivist().get_message("config")
|
||||
if not config:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" no config values to migrate")
|
||||
)
|
||||
return
|
||||
|
||||
path = "ta_config/_doc/appsettings"
|
||||
response, status_code = ElasticWrap(path).post(config)
|
||||
|
||||
if status_code in [200, 201]:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" ✓ migrated appconfig to ES")
|
||||
)
|
||||
RedisArchivist().del_message("config")
|
||||
return
|
||||
|
||||
message = " 🗙 failed to migrate app config"
|
||||
self.stdout.write(self.style.ERROR(message))
|
||||
self.stdout.write(response)
|
||||
sleep(60)
|
||||
raise CommandError(message)
|
||||
|
||||
def _make_folders(self):
|
||||
"""make expected cache folders"""
|
||||
self.stdout.write("[2] create expected cache folders")
|
||||
folders = [
|
||||
"backup",
|
||||
"channels",
|
||||
"download",
|
||||
"import",
|
||||
"playlists",
|
||||
"videos",
|
||||
]
|
||||
cache_dir = EnvironmentSettings.CACHE_DIR
|
||||
for folder in folders:
|
||||
folder_path = os.path.join(cache_dir, folder)
|
||||
os.makedirs(folder_path, exist_ok=True)
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(" ✓ expected folders created"))
|
||||
|
||||
def _clear_redis_keys(self):
|
||||
"""make sure there are no leftover locks or keys set in redis"""
|
||||
self.stdout.write("[3] clear leftover keys in redis")
|
||||
all_keys = [
|
||||
"dl_queue_id",
|
||||
"dl_queue",
|
||||
"downloading",
|
||||
"manual_import",
|
||||
"reindex",
|
||||
"rescan",
|
||||
"run_backup",
|
||||
"startup_check",
|
||||
"reindex:ta_video",
|
||||
"reindex:ta_channel",
|
||||
"reindex:ta_playlist",
|
||||
]
|
||||
|
||||
redis_con = RedisArchivist()
|
||||
has_changed = False
|
||||
for key in all_keys:
|
||||
if redis_con.del_message(key):
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ cleared key {key}")
|
||||
)
|
||||
has_changed = True
|
||||
|
||||
if not has_changed:
|
||||
self.stdout.write(self.style.SUCCESS(" no keys found"))
|
||||
|
||||
def _clear_tasks(self):
|
||||
"""clear tasks and messages"""
|
||||
self.stdout.write("[4] clear task leftovers")
|
||||
TaskManager().fail_pending()
|
||||
redis_con = RedisArchivist()
|
||||
to_delete = redis_con.list_keys("message:")
|
||||
if to_delete:
|
||||
for key in to_delete:
|
||||
redis_con.del_message(key)
|
||||
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ cleared {len(to_delete)} messages")
|
||||
)
|
||||
|
||||
def _clear_dl_cache(self):
|
||||
"""clear leftover files from dl cache"""
|
||||
self.stdout.write("[5] clear leftover files from dl cache")
|
||||
leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
|
||||
if leftover_files:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ cleared {leftover_files} files")
|
||||
)
|
||||
else:
|
||||
self.stdout.write(self.style.SUCCESS(" no files found"))
|
||||
|
||||
def _version_check(self):
|
||||
"""remove new release key if updated now"""
|
||||
self.stdout.write("[6] check for first run after update")
|
||||
new_version = ReleaseVersion().is_updated()
|
||||
if new_version:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ update to {new_version} completed")
|
||||
)
|
||||
else:
|
||||
self.stdout.write(self.style.SUCCESS(" no new update found"))
|
||||
|
||||
version_task = CustomPeriodicTask.objects.filter(name="version_check")
|
||||
if not version_task.exists():
|
||||
return
|
||||
|
||||
if not version_task.first().last_run_at:
|
||||
self.style.SUCCESS(" ✓ send initial version check task")
|
||||
version_check.delay()
|
||||
|
||||
def _index_setup(self):
|
||||
"""migration: validate index mappings"""
|
||||
self.stdout.write("[7] validate index mappings")
|
||||
ElasitIndexWrap().setup()
|
||||
|
||||
def _snapshot_check(self):
|
||||
"""migration setup snapshots"""
|
||||
self.stdout.write("[8] setup snapshots")
|
||||
ElasticSnapshot().setup()
|
||||
|
||||
def _create_default_schedules(self) -> None:
|
||||
"""
|
||||
create default schedules for new installations
|
||||
needs to be called after _mig_schedule_store
|
||||
"""
|
||||
self.stdout.write("[9] create initial schedules")
|
||||
init_has_run = CustomPeriodicTask.objects.filter(
|
||||
name="version_check"
|
||||
).exists()
|
||||
|
||||
if init_has_run:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
" schedule init already done, skipping..."
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
builder = ScheduleBuilder()
|
||||
check_reindex = builder.get_set_task(
|
||||
"check_reindex", schedule=builder.SCHEDULES["check_reindex"]
|
||||
)
|
||||
check_reindex.task_config.update({"days": 90})
|
||||
check_reindex.last_run_at = dateformat.make_aware(datetime.now())
|
||||
check_reindex.save()
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
f" ✓ created new default schedule: {check_reindex}"
|
||||
)
|
||||
)
|
||||
|
||||
thumbnail_check = builder.get_set_task(
|
||||
"thumbnail_check", schedule=builder.SCHEDULES["thumbnail_check"]
|
||||
)
|
||||
thumbnail_check.last_run_at = dateformat.make_aware(datetime.now())
|
||||
thumbnail_check.save()
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
f" ✓ created new default schedule: {thumbnail_check}"
|
||||
)
|
||||
)
|
||||
daily_random = f"{randint(0, 59)} {randint(0, 23)} *"
|
||||
version_check_task = builder.get_set_task(
|
||||
"version_check", schedule=daily_random
|
||||
)
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
f" ✓ created new default schedule: {version_check_task}"
|
||||
)
|
||||
)
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" ✓ all default schedules created")
|
||||
)
|
||||
|
||||
def _update_schedule_tz(self) -> None:
|
||||
"""update timezone for Schedule instances"""
|
||||
self.stdout.write("[9] validate schedules TZ")
|
||||
tz = EnvironmentSettings.TZ
|
||||
to_update = CrontabSchedule.objects.exclude(timezone=tz)
|
||||
|
||||
if not to_update.exists():
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" all schedules have correct TZ")
|
||||
)
|
||||
return
|
||||
|
||||
updated = to_update.update(timezone=tz)
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ updated {updated} schedules to {tz}.")
|
||||
)
|
||||
PeriodicTasks.update_changed()
|
||||
294
backend/config/settings.py
Normal file
294
backend/config/settings.py
Normal file
@@ -0,0 +1,294 @@
|
||||
"""
|
||||
Django settings for config project.
|
||||
|
||||
Generated by 'django-admin startproject' using Django 3.2.5.
|
||||
|
||||
For more information on this file, see
|
||||
https://docs.djangoproject.com/en/3.2/topics/settings/
|
||||
|
||||
For the full list of settings and their values, see
|
||||
https://docs.djangoproject.com/en/3.2/ref/settings/
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from os import environ, path
|
||||
from pathlib import Path
|
||||
|
||||
import ldap
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.helper import ta_host_parser
|
||||
from corsheaders.defaults import default_headers
|
||||
from django_auth_ldap.config import LDAPSearch
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(".env")
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
# Build paths inside the project like this: BASE_DIR / 'subdir'.
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
# Quick-start development settings - unsuitable for production
|
||||
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
|
||||
|
||||
PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode())
|
||||
SECRET_KEY = PW_HASH.hexdigest()
|
||||
|
||||
# SECURITY WARNING: don't run with debug turned on in production!
|
||||
DEBUG = bool(environ.get("DJANGO_DEBUG"))
|
||||
|
||||
ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(
|
||||
environ.get("TA_HOST", "localhost")
|
||||
)
|
||||
|
||||
# Application definition
|
||||
|
||||
INSTALLED_APPS = [
|
||||
"django_celery_beat",
|
||||
"django.contrib.admin",
|
||||
"django.contrib.auth",
|
||||
"django.contrib.contenttypes",
|
||||
"django.contrib.sessions",
|
||||
"django.contrib.messages",
|
||||
"corsheaders",
|
||||
"whitenoise.runserver_nostatic",
|
||||
"django.contrib.staticfiles",
|
||||
"django.contrib.humanize",
|
||||
"rest_framework",
|
||||
"rest_framework.authtoken",
|
||||
"common",
|
||||
"video",
|
||||
"channel",
|
||||
"playlist",
|
||||
"download",
|
||||
"task",
|
||||
"appsettings",
|
||||
"stats",
|
||||
"user",
|
||||
"config",
|
||||
]
|
||||
|
||||
MIDDLEWARE = [
|
||||
"django.middleware.security.SecurityMiddleware",
|
||||
"django.contrib.sessions.middleware.SessionMiddleware",
|
||||
"corsheaders.middleware.CorsMiddleware",
|
||||
"whitenoise.middleware.WhiteNoiseMiddleware",
|
||||
"django.middleware.common.CommonMiddleware",
|
||||
"django.middleware.csrf.CsrfViewMiddleware",
|
||||
"django.contrib.auth.middleware.AuthenticationMiddleware",
|
||||
"django.contrib.messages.middleware.MessageMiddleware",
|
||||
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
||||
"common.src.health.HealthCheckMiddleware",
|
||||
]
|
||||
|
||||
ROOT_URLCONF = "config.urls"
|
||||
|
||||
TEMPLATES = [
|
||||
{
|
||||
"BACKEND": "django.template.backends.django.DjangoTemplates",
|
||||
"DIRS": [],
|
||||
"APP_DIRS": True,
|
||||
"OPTIONS": {
|
||||
"context_processors": [
|
||||
"django.template.context_processors.debug",
|
||||
"django.template.context_processors.request",
|
||||
"django.contrib.auth.context_processors.auth",
|
||||
"django.contrib.messages.context_processors.messages",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
WSGI_APPLICATION = "config.wsgi.application"
|
||||
|
||||
if bool(environ.get("TA_LDAP")):
|
||||
# pylint: disable=global-at-module-level
|
||||
global AUTH_LDAP_SERVER_URI
|
||||
AUTH_LDAP_SERVER_URI = environ.get("TA_LDAP_SERVER_URI")
|
||||
|
||||
global AUTH_LDAP_BIND_DN
|
||||
AUTH_LDAP_BIND_DN = environ.get("TA_LDAP_BIND_DN")
|
||||
|
||||
global AUTH_LDAP_BIND_PASSWORD
|
||||
AUTH_LDAP_BIND_PASSWORD = environ.get("TA_LDAP_BIND_PASSWORD")
|
||||
|
||||
"""
|
||||
Since these are new environment variables, taking the opporunity to use
|
||||
more accurate env names.
|
||||
Given Names are *_technically_* different from Personal names, as people
|
||||
who change their names have different given names and personal names,
|
||||
and they go by personal names. Additionally, "LastName" is actually
|
||||
incorrect for many cultures, such as Korea, where the
|
||||
family name comes first, and the personal name comes last.
|
||||
|
||||
But we all know people are going to try to guess at these, so still want
|
||||
to include names that people will guess, hence using first/last as well.
|
||||
"""
|
||||
|
||||
# Attribute mapping options
|
||||
|
||||
global AUTH_LDAP_USER_ATTR_MAP_USERNAME
|
||||
AUTH_LDAP_USER_ATTR_MAP_USERNAME = (
|
||||
environ.get("TA_LDAP_USER_ATTR_MAP_USERNAME")
|
||||
or environ.get("TA_LDAP_USER_ATTR_MAP_UID")
|
||||
or "uid"
|
||||
)
|
||||
|
||||
global AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME
|
||||
AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME = (
|
||||
environ.get("TA_LDAP_USER_ATTR_MAP_PERSONALNAME")
|
||||
or environ.get("TA_LDAP_USER_ATTR_MAP_FIRSTNAME")
|
||||
or environ.get("TA_LDAP_USER_ATTR_MAP_GIVENNAME")
|
||||
or "givenName"
|
||||
)
|
||||
|
||||
global AUTH_LDAP_USER_ATTR_MAP_SURNAME
|
||||
AUTH_LDAP_USER_ATTR_MAP_SURNAME = (
|
||||
environ.get("TA_LDAP_USER_ATTR_MAP_SURNAME")
|
||||
or environ.get("TA_LDAP_USER_ATTR_MAP_LASTNAME")
|
||||
or environ.get("TA_LDAP_USER_ATTR_MAP_FAMILYNAME")
|
||||
or "sn"
|
||||
)
|
||||
|
||||
global AUTH_LDAP_USER_ATTR_MAP_EMAIL
|
||||
AUTH_LDAP_USER_ATTR_MAP_EMAIL = (
|
||||
environ.get("TA_LDAP_USER_ATTR_MAP_EMAIL")
|
||||
or environ.get("TA_LDAP_USER_ATTR_MAP_MAIL")
|
||||
or "mail"
|
||||
)
|
||||
|
||||
global AUTH_LDAP_USER_BASE
|
||||
AUTH_LDAP_USER_BASE = environ.get("TA_LDAP_USER_BASE")
|
||||
|
||||
global AUTH_LDAP_USER_FILTER
|
||||
AUTH_LDAP_USER_FILTER = environ.get("TA_LDAP_USER_FILTER")
|
||||
|
||||
global AUTH_LDAP_USER_SEARCH
|
||||
# pylint: disable=no-member
|
||||
AUTH_LDAP_USER_SEARCH = LDAPSearch(
|
||||
AUTH_LDAP_USER_BASE,
|
||||
ldap.SCOPE_SUBTREE,
|
||||
"(&("
|
||||
+ AUTH_LDAP_USER_ATTR_MAP_USERNAME
|
||||
+ "=%(user)s)"
|
||||
+ AUTH_LDAP_USER_FILTER
|
||||
+ ")",
|
||||
)
|
||||
|
||||
global AUTH_LDAP_USER_ATTR_MAP
|
||||
AUTH_LDAP_USER_ATTR_MAP = {
|
||||
"username": AUTH_LDAP_USER_ATTR_MAP_USERNAME,
|
||||
"first_name": AUTH_LDAP_USER_ATTR_MAP_PERSONALNAME,
|
||||
"last_name": AUTH_LDAP_USER_ATTR_MAP_SURNAME,
|
||||
"email": AUTH_LDAP_USER_ATTR_MAP_EMAIL,
|
||||
}
|
||||
|
||||
if bool(environ.get("TA_LDAP_DISABLE_CERT_CHECK")):
|
||||
global AUTH_LDAP_GLOBAL_OPTIONS
|
||||
AUTH_LDAP_GLOBAL_OPTIONS = {
|
||||
ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
|
||||
}
|
||||
|
||||
AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
|
||||
|
||||
# Database
|
||||
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
|
||||
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
DB_PATH = path.join(CACHE_DIR, "db.sqlite3")
|
||||
DATABASES = {
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.sqlite3",
|
||||
"NAME": DB_PATH,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Password validation
|
||||
# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
|
||||
|
||||
AUTH_PASSWORD_VALIDATORS = [
|
||||
{
|
||||
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa: E501
|
||||
},
|
||||
{
|
||||
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", # noqa: E501
|
||||
},
|
||||
{
|
||||
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", # noqa: E501
|
||||
},
|
||||
{
|
||||
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", # noqa: E501
|
||||
},
|
||||
]
|
||||
|
||||
AUTH_USER_MODEL = "user.Account"
|
||||
|
||||
# Forward-auth authentication
|
||||
if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
|
||||
TA_AUTH_PROXY_USERNAME_HEADER = (
|
||||
environ.get("TA_AUTH_PROXY_USERNAME_HEADER") or "HTTP_REMOTE_USER"
|
||||
)
|
||||
TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")
|
||||
|
||||
MIDDLEWARE.append("user.src.remote_user_auth.HttpRemoteUserMiddleware")
|
||||
|
||||
AUTHENTICATION_BACKENDS = (
|
||||
"django.contrib.auth.backends.RemoteUserBackend",
|
||||
)
|
||||
|
||||
|
||||
# Internationalization
|
||||
# https://docs.djangoproject.com/en/3.2/topics/i18n/
|
||||
|
||||
LANGUAGE_CODE = "en-us"
|
||||
TIME_ZONE = EnvironmentSettings.TZ
|
||||
USE_I18N = True
|
||||
USE_L10N = True
|
||||
USE_TZ = True
|
||||
|
||||
|
||||
# Static files (CSS, JavaScript, Images)
|
||||
# https://docs.djangoproject.com/en/3.2/howto/static-files/
|
||||
|
||||
STATIC_URL = "/static/"
|
||||
STATICFILES_DIRS = (str(BASE_DIR.joinpath("static")),)
|
||||
STATIC_ROOT = str(BASE_DIR.joinpath("staticfiles"))
|
||||
STORAGES = {
|
||||
"staticfiles": {
|
||||
"BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage",
|
||||
},
|
||||
}
|
||||
|
||||
# Default primary key field type
|
||||
# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
|
||||
|
||||
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
|
||||
|
||||
LOGIN_URL = "/login/"
|
||||
LOGOUT_REDIRECT_URL = "/login/"
|
||||
|
||||
# Cors needed for browser extension
|
||||
# background.js makes the request so HTTP_ORIGIN will be from extension
|
||||
if environ.get("DISABLE_CORS"):
|
||||
# disable cors
|
||||
CORS_ORIGIN_ALLOW_ALL = True
|
||||
else:
|
||||
CORS_ALLOWED_ORIGIN_REGEXES = [
|
||||
r"moz-extension://*",
|
||||
r"chrome-extension://*",
|
||||
]
|
||||
CORS_ALLOWED_ORIGINS = ["http://localhost:3000"]
|
||||
|
||||
|
||||
CORS_ALLOW_HEADERS = list(default_headers) + [
|
||||
"mode",
|
||||
]
|
||||
|
||||
# TA application settings
|
||||
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
|
||||
TA_VERSION = "v0.4.10-unstable"
|
||||
31
backend/config/urls.py
Normal file
31
backend/config/urls.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""config URL Configuration
|
||||
|
||||
The `urlpatterns` list routes URLs to views. For more information please see:
|
||||
https://docs.djangoproject.com/en/3.2/topics/http/urls/
|
||||
Examples:
|
||||
Function views
|
||||
1. Add an import: from my_app import views
|
||||
2. Add a URL to urlpatterns: path('', views.home, name='home')
|
||||
Class-based views
|
||||
1. Add an import: from other_app.views import Home
|
||||
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
|
||||
Including another URLconf
|
||||
1. Import the include() function: from django.urls import include, path
|
||||
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
|
||||
"""
|
||||
|
||||
from django.contrib import admin
|
||||
from django.urls import include, path
|
||||
|
||||
urlpatterns = [
|
||||
path("api/", include("common.urls")),
|
||||
path("api/video/", include("video.urls")),
|
||||
path("api/channel/", include("channel.urls")),
|
||||
path("api/playlist/", include("playlist.urls")),
|
||||
path("api/download/", include("download.urls")),
|
||||
path("api/task/", include("task.urls")),
|
||||
path("api/appsettings/", include("appsettings.urls")),
|
||||
path("api/stats/", include("stats.urls")),
|
||||
path("api/user/", include("user.urls")),
|
||||
path("admin/", admin.site.urls),
|
||||
]
|
||||
16
backend/config/wsgi.py
Normal file
16
backend/config/wsgi.py
Normal file
@@ -0,0 +1,16 @@
|
||||
"""
|
||||
WSGI config for config project.
|
||||
|
||||
It exposes the WSGI callable as a module-level variable named ``application``.
|
||||
|
||||
For more information on this file, see
|
||||
https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from django.core.wsgi import get_wsgi_application
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
||||
|
||||
application = get_wsgi_application()
|
||||
0
backend/download/__init__.py
Normal file
0
backend/download/__init__.py
Normal file
0
backend/download/migrations/__init__.py
Normal file
0
backend/download/migrations/__init__.py
Normal file
0
backend/download/src/__init__.py
Normal file
0
backend/download/src/__init__.py
Normal file
370
backend/download/src/queue.py
Normal file
370
backend/download/src/queue.py
Normal file
@@ -0,0 +1,370 @@
|
||||
"""
|
||||
Functionality:
|
||||
- handle download queue
|
||||
- linked with ta_dowload index
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import get_duration_str, is_shorts
|
||||
from download.src.subscriptions import ChannelSubscription
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
from video.src.constants import VideoTypeEnum
|
||||
|
||||
|
||||
class PendingIndex:
|
||||
"""base class holding all export methods"""
|
||||
|
||||
def __init__(self):
|
||||
self.all_pending = False
|
||||
self.all_ignored = False
|
||||
self.all_videos = False
|
||||
self.all_channels = False
|
||||
self.channel_overwrites = False
|
||||
self.video_overwrites = False
|
||||
self.to_skip = False
|
||||
|
||||
def get_download(self):
|
||||
"""get a list of all pending videos in ta_download"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"sort": [{"timestamp": {"order": "asc"}}],
|
||||
}
|
||||
all_results = IndexPaginate("ta_download", data).get_results()
|
||||
|
||||
self.all_pending = []
|
||||
self.all_ignored = []
|
||||
self.to_skip = []
|
||||
|
||||
for result in all_results:
|
||||
self.to_skip.append(result["youtube_id"])
|
||||
if result["status"] == "pending":
|
||||
self.all_pending.append(result)
|
||||
elif result["status"] == "ignore":
|
||||
self.all_ignored.append(result)
|
||||
|
||||
def get_indexed(self):
|
||||
"""get a list of all videos indexed"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"sort": [{"published": {"order": "desc"}}],
|
||||
}
|
||||
self.all_videos = IndexPaginate("ta_video", data).get_results()
|
||||
for video in self.all_videos:
|
||||
self.to_skip.append(video["youtube_id"])
|
||||
|
||||
def get_channels(self):
|
||||
"""get a list of all channels indexed"""
|
||||
self.all_channels = []
|
||||
self.channel_overwrites = {}
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"sort": [{"channel_id": {"order": "asc"}}],
|
||||
}
|
||||
channels = IndexPaginate("ta_channel", data).get_results()
|
||||
|
||||
for channel in channels:
|
||||
channel_id = channel["channel_id"]
|
||||
self.all_channels.append(channel_id)
|
||||
if channel.get("channel_overwrites"):
|
||||
self.channel_overwrites.update(
|
||||
{channel_id: channel.get("channel_overwrites")}
|
||||
)
|
||||
|
||||
self._map_overwrites()
|
||||
|
||||
def _map_overwrites(self):
|
||||
"""map video ids to channel ids overwrites"""
|
||||
self.video_overwrites = {}
|
||||
for video in self.all_pending:
|
||||
video_id = video["youtube_id"]
|
||||
channel_id = video["channel_id"]
|
||||
overwrites = self.channel_overwrites.get(channel_id, False)
|
||||
if overwrites:
|
||||
self.video_overwrites.update({video_id: overwrites})
|
||||
|
||||
|
||||
class PendingInteract:
|
||||
"""interact with items in download queue"""
|
||||
|
||||
def __init__(self, youtube_id=False, status=False):
|
||||
self.youtube_id = youtube_id
|
||||
self.status = status
|
||||
|
||||
def delete_item(self):
|
||||
"""delete single item from pending"""
|
||||
path = f"ta_download/_doc/{self.youtube_id}"
|
||||
_, _ = ElasticWrap(path).delete(refresh=True)
|
||||
|
||||
def delete_by_status(self):
|
||||
"""delete all matching item by status"""
|
||||
data = {"query": {"term": {"status": {"value": self.status}}}}
|
||||
path = "ta_download/_delete_by_query"
|
||||
_, _ = ElasticWrap(path).post(data=data)
|
||||
|
||||
def update_status(self):
|
||||
"""update status of pending item"""
|
||||
if self.status == "priority":
|
||||
data = {
|
||||
"doc": {
|
||||
"status": "pending",
|
||||
"auto_start": True,
|
||||
"message": None,
|
||||
}
|
||||
}
|
||||
else:
|
||||
data = {"doc": {"status": self.status}}
|
||||
|
||||
path = f"ta_download/_update/{self.youtube_id}/?refresh=true"
|
||||
_, _ = ElasticWrap(path).post(data=data)
|
||||
|
||||
def get_item(self):
|
||||
"""return pending item dict"""
|
||||
path = f"ta_download/_doc/{self.youtube_id}"
|
||||
response, status_code = ElasticWrap(path).get()
|
||||
return response["_source"], status_code
|
||||
|
||||
def get_channel(self):
|
||||
"""
|
||||
get channel metadata from queue to not depend on channel to be indexed
|
||||
"""
|
||||
data = {
|
||||
"size": 1,
|
||||
"query": {"term": {"channel_id": {"value": self.youtube_id}}},
|
||||
}
|
||||
response, _ = ElasticWrap("ta_download/_search").get(data=data)
|
||||
hits = response["hits"]["hits"]
|
||||
if not hits:
|
||||
channel_name = "NA"
|
||||
else:
|
||||
channel_name = hits[0]["_source"].get("channel_name", "NA")
|
||||
|
||||
return {
|
||||
"channel_id": self.youtube_id,
|
||||
"channel_name": channel_name,
|
||||
}
|
||||
|
||||
|
||||
class PendingList(PendingIndex):
|
||||
"""manage the pending videos list"""
|
||||
|
||||
yt_obs = {
|
||||
"noplaylist": True,
|
||||
"writethumbnail": True,
|
||||
"simulate": True,
|
||||
"check_formats": None,
|
||||
}
|
||||
|
||||
def __init__(self, youtube_ids=False, task=False):
|
||||
super().__init__()
|
||||
self.config = AppConfig().config
|
||||
self.youtube_ids = youtube_ids
|
||||
self.task = task
|
||||
self.to_skip = False
|
||||
self.missing_videos = False
|
||||
|
||||
def parse_url_list(self):
|
||||
"""extract youtube ids from list"""
|
||||
self.missing_videos = []
|
||||
self.get_download()
|
||||
self.get_indexed()
|
||||
total = len(self.youtube_ids)
|
||||
for idx, entry in enumerate(self.youtube_ids):
|
||||
self._process_entry(entry)
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
self.task.send_progress(
|
||||
message_lines=[f"Extracting items {idx + 1}/{total}"],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
|
||||
def _process_entry(self, entry):
|
||||
"""process single entry from url list"""
|
||||
vid_type = self._get_vid_type(entry)
|
||||
if entry["type"] == "video":
|
||||
self._add_video(entry["url"], vid_type)
|
||||
elif entry["type"] == "channel":
|
||||
self._parse_channel(entry["url"], vid_type)
|
||||
elif entry["type"] == "playlist":
|
||||
self._parse_playlist(entry["url"])
|
||||
else:
|
||||
raise ValueError(f"invalid url_type: {entry}")
|
||||
|
||||
@staticmethod
|
||||
def _get_vid_type(entry):
|
||||
"""add vid type enum if available"""
|
||||
vid_type_str = entry.get("vid_type")
|
||||
if not vid_type_str:
|
||||
return VideoTypeEnum.UNKNOWN
|
||||
|
||||
return VideoTypeEnum(vid_type_str)
|
||||
|
||||
def _add_video(self, url, vid_type):
|
||||
"""add video to list"""
|
||||
if url not in self.missing_videos and url not in self.to_skip:
|
||||
self.missing_videos.append((url, vid_type))
|
||||
else:
|
||||
print(f"{url}: skipped adding already indexed video to download.")
|
||||
|
||||
def _parse_channel(self, url, vid_type):
|
||||
"""add all videos of channel to list"""
|
||||
video_results = ChannelSubscription().get_last_youtube_videos(
|
||||
url, limit=False, query_filter=vid_type
|
||||
)
|
||||
for video_id, _, vid_type in video_results:
|
||||
self._add_video(video_id, vid_type)
|
||||
|
||||
def _parse_playlist(self, url):
|
||||
"""add all videos of playlist to list"""
|
||||
playlist = YoutubePlaylist(url)
|
||||
is_active = playlist.update_playlist()
|
||||
if not is_active:
|
||||
message = f"{playlist.youtube_id}: failed to extract metadata"
|
||||
print(message)
|
||||
raise ValueError(message)
|
||||
|
||||
entries = playlist.json_data["playlist_entries"]
|
||||
to_add = [i["youtube_id"] for i in entries if not i["downloaded"]]
|
||||
if not to_add:
|
||||
return
|
||||
|
||||
for video_id in to_add:
|
||||
# match vid_type later
|
||||
self._add_video(video_id, VideoTypeEnum.UNKNOWN)
|
||||
|
||||
def add_to_pending(self, status="pending", auto_start=False):
|
||||
"""add missing videos to pending list"""
|
||||
self.get_channels()
|
||||
bulk_list = []
|
||||
|
||||
total = len(self.missing_videos)
|
||||
videos_added = []
|
||||
for idx, (youtube_id, vid_type) in enumerate(self.missing_videos):
|
||||
if self.task and self.task.is_stopped():
|
||||
break
|
||||
|
||||
print(f"{youtube_id}: [{idx + 1}/{total}]: add to queue")
|
||||
self._notify_add(idx, total)
|
||||
video_details = self.get_youtube_details(youtube_id, vid_type)
|
||||
if not video_details:
|
||||
continue
|
||||
|
||||
video_details.update(
|
||||
{
|
||||
"status": status,
|
||||
"auto_start": auto_start,
|
||||
}
|
||||
)
|
||||
|
||||
action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
|
||||
bulk_list.append(json.dumps(action))
|
||||
bulk_list.append(json.dumps(video_details))
|
||||
|
||||
url = video_details["vid_thumb_url"]
|
||||
ThumbManager(youtube_id).download_video_thumb(url)
|
||||
videos_added.append(youtube_id)
|
||||
|
||||
if len(bulk_list) >= 20:
|
||||
self._ingest_bulk(bulk_list)
|
||||
bulk_list = []
|
||||
|
||||
self._ingest_bulk(bulk_list)
|
||||
|
||||
return videos_added
|
||||
|
||||
def _ingest_bulk(self, bulk_list):
|
||||
"""add items to queue in bulk"""
|
||||
if not bulk_list:
|
||||
return
|
||||
|
||||
# add last newline
|
||||
bulk_list.append("\n")
|
||||
query_str = "\n".join(bulk_list)
|
||||
_, _ = ElasticWrap("_bulk?refresh=true").post(query_str, ndjson=True)
|
||||
|
||||
def _notify_add(self, idx, total):
|
||||
"""send notification for adding videos to download queue"""
|
||||
if not self.task:
|
||||
return
|
||||
|
||||
self.task.send_progress(
|
||||
message_lines=[
|
||||
"Adding new videos to download queue.",
|
||||
f"Extracting items {idx + 1}/{total}",
|
||||
],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
|
||||
def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEOS):
|
||||
"""get details from youtubedl for single pending video"""
|
||||
vid = YtWrap(self.yt_obs, self.config).extract(youtube_id)
|
||||
if not vid:
|
||||
return False
|
||||
|
||||
if vid.get("id") != youtube_id:
|
||||
# skip premium videos with different id
|
||||
print(f"{youtube_id}: skipping premium video, id not matching")
|
||||
return False
|
||||
# stop if video is streaming live now
|
||||
if vid["live_status"] in ["is_upcoming", "is_live"]:
|
||||
print(f"{youtube_id}: skip is_upcoming or is_live")
|
||||
return False
|
||||
|
||||
if vid["live_status"] == "was_live":
|
||||
vid_type = VideoTypeEnum.STREAMS
|
||||
else:
|
||||
if self._check_shorts(vid):
|
||||
vid_type = VideoTypeEnum.SHORTS
|
||||
else:
|
||||
vid_type = VideoTypeEnum.VIDEOS
|
||||
|
||||
if not vid.get("channel"):
|
||||
print(f"{youtube_id}: skip video not part of channel")
|
||||
return False
|
||||
|
||||
return self._parse_youtube_details(vid, vid_type)
|
||||
|
||||
@staticmethod
|
||||
def _check_shorts(vid):
|
||||
"""check if vid is shorts video"""
|
||||
if vid["width"] > vid["height"]:
|
||||
return False
|
||||
|
||||
duration = vid.get("duration")
|
||||
if duration and isinstance(duration, int):
|
||||
if duration > 60:
|
||||
return False
|
||||
|
||||
return is_shorts(vid["id"])
|
||||
|
||||
def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS):
|
||||
"""parse response"""
|
||||
vid_id = vid.get("id")
|
||||
published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
|
||||
"%Y-%m-%d"
|
||||
)
|
||||
|
||||
# build dict
|
||||
youtube_details = {
|
||||
"youtube_id": vid_id,
|
||||
"channel_name": vid["channel"],
|
||||
"vid_thumb_url": vid["thumbnail"],
|
||||
"title": vid["title"],
|
||||
"channel_id": vid["channel_id"],
|
||||
"duration": get_duration_str(vid["duration"]),
|
||||
"published": published,
|
||||
"timestamp": int(datetime.now().timestamp()),
|
||||
# Pulling enum value out so it is serializable
|
||||
"vid_type": vid_type.value,
|
||||
}
|
||||
if self.all_channels:
|
||||
youtube_details.update(
|
||||
{"channel_indexed": vid["channel_id"] in self.all_channels}
|
||||
)
|
||||
return youtube_details
|
||||
436
backend/download/src/subscriptions.py
Normal file
436
backend/download/src/subscriptions.py
Normal file
@@ -0,0 +1,436 @@
|
||||
"""
|
||||
Functionality:
|
||||
- handle channel subscriptions
|
||||
- handle playlist subscriptions
|
||||
"""
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from channel.src.index import YoutubeChannel
|
||||
from common.src.es_connect import IndexPaginate
|
||||
from common.src.helper import is_missing
|
||||
from common.src.urlparser import Parser
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
from video.src.constants import VideoTypeEnum
|
||||
from video.src.index import YoutubeVideo
|
||||
|
||||
|
||||
class ChannelSubscription:
|
||||
"""manage the list of channels subscribed"""
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.config = AppConfig().config
|
||||
self.task = task
|
||||
|
||||
@staticmethod
|
||||
def get_channels(subscribed_only=True):
|
||||
"""get a list of all channels subscribed to"""
|
||||
data = {
|
||||
"sort": [{"channel_name.keyword": {"order": "asc"}}],
|
||||
}
|
||||
if subscribed_only:
|
||||
data["query"] = {"term": {"channel_subscribed": {"value": True}}}
|
||||
else:
|
||||
data["query"] = {"match_all": {}}
|
||||
|
||||
all_channels = IndexPaginate("ta_channel", data).get_results()
|
||||
|
||||
return all_channels
|
||||
|
||||
def get_last_youtube_videos(
|
||||
self,
|
||||
channel_id,
|
||||
limit=True,
|
||||
query_filter=None,
|
||||
channel_overwrites=None,
|
||||
):
|
||||
"""get a list of last videos from channel"""
|
||||
query_handler = VideoQueryBuilder(self.config, channel_overwrites)
|
||||
queries = query_handler.build_queries(query_filter)
|
||||
last_videos = []
|
||||
|
||||
for vid_type_enum, limit_amount in queries:
|
||||
obs = {
|
||||
"skip_download": True,
|
||||
"extract_flat": True,
|
||||
}
|
||||
vid_type = vid_type_enum.value
|
||||
|
||||
if limit:
|
||||
obs["playlistend"] = limit_amount
|
||||
|
||||
url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
|
||||
channel_query = YtWrap(obs, self.config).extract(url)
|
||||
if not channel_query:
|
||||
continue
|
||||
|
||||
last_videos.extend(
|
||||
[
|
||||
(i["id"], i["title"], vid_type)
|
||||
for i in channel_query["entries"]
|
||||
]
|
||||
)
|
||||
|
||||
return last_videos
|
||||
|
||||
def find_missing(self):
|
||||
"""add missing videos from subscribed channels to pending"""
|
||||
all_channels = self.get_channels()
|
||||
if not all_channels:
|
||||
return False
|
||||
|
||||
missing_videos = []
|
||||
|
||||
total = len(all_channels)
|
||||
for idx, channel in enumerate(all_channels):
|
||||
channel_id = channel["channel_id"]
|
||||
print(f"{channel_id}: find missing videos.")
|
||||
last_videos = self.get_last_youtube_videos(
|
||||
channel_id,
|
||||
channel_overwrites=channel.get("channel_overwrites"),
|
||||
)
|
||||
|
||||
if last_videos:
|
||||
ids_to_add = is_missing([i[0] for i in last_videos])
|
||||
for video_id, _, vid_type in last_videos:
|
||||
if video_id in ids_to_add:
|
||||
missing_videos.append((video_id, vid_type))
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
if self.task.is_stopped():
|
||||
self.task.send_progress(["Received Stop signal."])
|
||||
break
|
||||
|
||||
self.task.send_progress(
|
||||
message_lines=[f"Scanning Channel {idx + 1}/{total}"],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
|
||||
return missing_videos
|
||||
|
||||
@staticmethod
|
||||
def change_subscribe(channel_id, channel_subscribed):
|
||||
"""subscribe or unsubscribe from channel and update"""
|
||||
channel = YoutubeChannel(channel_id)
|
||||
channel.build_json()
|
||||
channel.json_data["channel_subscribed"] = channel_subscribed
|
||||
channel.upload_to_es()
|
||||
channel.sync_to_videos()
|
||||
|
||||
|
||||
class VideoQueryBuilder:
|
||||
"""Build queries for yt-dlp."""
|
||||
|
||||
def __init__(self, config: dict, channel_overwrites: dict | None = None):
|
||||
self.config = config
|
||||
self.channel_overwrites = channel_overwrites or {}
|
||||
|
||||
def build_queries(
|
||||
self, video_type: VideoTypeEnum | None, limit: bool = True
|
||||
) -> list[tuple[VideoTypeEnum, int | None]]:
|
||||
"""Build queries for all or specific video type."""
|
||||
query_methods = {
|
||||
VideoTypeEnum.VIDEOS: self.videos_query,
|
||||
VideoTypeEnum.STREAMS: self.streams_query,
|
||||
VideoTypeEnum.SHORTS: self.shorts_query,
|
||||
}
|
||||
|
||||
if video_type:
|
||||
# build query for specific type
|
||||
query_method = query_methods.get(video_type)
|
||||
if query_method:
|
||||
query = query_method(limit)
|
||||
if query[1] != 0:
|
||||
return [query]
|
||||
return []
|
||||
|
||||
# Build and return queries for all video types
|
||||
queries = []
|
||||
for build_query in query_methods.values():
|
||||
query = build_query(limit)
|
||||
if query[1] != 0:
|
||||
queries.append(query)
|
||||
|
||||
return queries
|
||||
|
||||
def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
|
||||
"""Build query for videos."""
|
||||
return self._build_generic_query(
|
||||
video_type=VideoTypeEnum.VIDEOS,
|
||||
overwrite_key="subscriptions_channel_size",
|
||||
config_key="channel_size",
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
|
||||
"""Build query for streams."""
|
||||
return self._build_generic_query(
|
||||
video_type=VideoTypeEnum.STREAMS,
|
||||
overwrite_key="subscriptions_live_channel_size",
|
||||
config_key="live_channel_size",
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
|
||||
"""Build query for shorts."""
|
||||
return self._build_generic_query(
|
||||
video_type=VideoTypeEnum.SHORTS,
|
||||
overwrite_key="subscriptions_shorts_channel_size",
|
||||
config_key="shorts_channel_size",
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def _build_generic_query(
|
||||
self,
|
||||
video_type: VideoTypeEnum,
|
||||
overwrite_key: str,
|
||||
config_key: str,
|
||||
limit: bool,
|
||||
) -> tuple[VideoTypeEnum, int | None]:
|
||||
"""Generic query for video page scraping."""
|
||||
if not limit:
|
||||
return (video_type, None)
|
||||
|
||||
if (
|
||||
overwrite_key in self.channel_overwrites
|
||||
and self.channel_overwrites[overwrite_key] is not None
|
||||
):
|
||||
overwrite = self.channel_overwrites[overwrite_key]
|
||||
return (video_type, overwrite)
|
||||
|
||||
if overwrite := self.config["subscriptions"].get(config_key):
|
||||
return (video_type, overwrite)
|
||||
|
||||
return (video_type, 0)
|
||||
|
||||
|
||||
class PlaylistSubscription:
|
||||
"""manage the playlist download functionality"""
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.config = AppConfig().config
|
||||
self.task = task
|
||||
|
||||
@staticmethod
|
||||
def get_playlists(subscribed_only=True):
|
||||
"""get a list of all active playlists"""
|
||||
data = {
|
||||
"sort": [{"playlist_channel.keyword": {"order": "desc"}}],
|
||||
}
|
||||
data["query"] = {
|
||||
"bool": {"must": [{"term": {"playlist_active": {"value": True}}}]}
|
||||
}
|
||||
if subscribed_only:
|
||||
data["query"]["bool"]["must"].append(
|
||||
{"term": {"playlist_subscribed": {"value": True}}}
|
||||
)
|
||||
|
||||
all_playlists = IndexPaginate("ta_playlist", data).get_results()
|
||||
|
||||
return all_playlists
|
||||
|
||||
def process_url_str(self, new_playlists, subscribed=True):
|
||||
"""process playlist subscribe form url_str"""
|
||||
for idx, playlist in enumerate(new_playlists):
|
||||
playlist_id = playlist["url"]
|
||||
if not playlist["type"] == "playlist":
|
||||
print(f"{playlist_id} not a playlist, skipping...")
|
||||
continue
|
||||
|
||||
playlist_h = YoutubePlaylist(playlist_id)
|
||||
playlist_h.build_json()
|
||||
if not playlist_h.json_data:
|
||||
message = f"{playlist_h.youtube_id}: failed to extract data"
|
||||
print(message)
|
||||
raise ValueError(message)
|
||||
|
||||
playlist_h.json_data["playlist_subscribed"] = subscribed
|
||||
playlist_h.upload_to_es()
|
||||
playlist_h.add_vids_to_playlist()
|
||||
self.channel_validate(playlist_h.json_data["playlist_channel_id"])
|
||||
|
||||
url = playlist_h.json_data["playlist_thumbnail"]
|
||||
thumb = ThumbManager(playlist_id, item_type="playlist")
|
||||
thumb.download_playlist_thumb(url)
|
||||
|
||||
if self.task:
|
||||
self.task.send_progress(
|
||||
message_lines=[
|
||||
f"Processing {idx + 1} of {len(new_playlists)}"
|
||||
],
|
||||
progress=(idx + 1) / len(new_playlists),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def channel_validate(channel_id):
|
||||
"""make sure channel of playlist is there"""
|
||||
channel = YoutubeChannel(channel_id)
|
||||
channel.build_json(upload=True)
|
||||
|
||||
@staticmethod
|
||||
def change_subscribe(playlist_id, subscribe_status):
|
||||
"""change the subscribe status of a playlist"""
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
playlist.build_json()
|
||||
playlist.json_data["playlist_subscribed"] = subscribe_status
|
||||
playlist.upload_to_es()
|
||||
|
||||
def find_missing(self):
|
||||
"""find videos in subscribed playlists not downloaded yet"""
|
||||
all_playlists = [i["playlist_id"] for i in self.get_playlists()]
|
||||
if not all_playlists:
|
||||
return False
|
||||
|
||||
missing_videos = []
|
||||
total = len(all_playlists)
|
||||
for idx, playlist_id in enumerate(all_playlists):
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
is_active = playlist.update_playlist()
|
||||
if not is_active:
|
||||
playlist.deactivate()
|
||||
continue
|
||||
|
||||
playlist_entries = playlist.json_data["playlist_entries"]
|
||||
size_limit = self.config["subscriptions"]["channel_size"]
|
||||
if size_limit:
|
||||
del playlist_entries[size_limit:]
|
||||
|
||||
to_check = [
|
||||
i["youtube_id"]
|
||||
for i in playlist_entries
|
||||
if i["downloaded"] is False
|
||||
]
|
||||
needs_downloading = is_missing(to_check)
|
||||
missing_videos.extend(needs_downloading)
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
if self.task.is_stopped():
|
||||
self.task.send_progress(["Received Stop signal."])
|
||||
break
|
||||
|
||||
self.task.send_progress(
|
||||
message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
|
||||
return missing_videos
|
||||
|
||||
|
||||
class SubscriptionScanner:
|
||||
"""add missing videos to queue"""
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.task = task
|
||||
self.missing_videos = False
|
||||
self.auto_start = AppConfig().config["subscriptions"].get("auto_start")
|
||||
|
||||
def scan(self):
|
||||
"""scan channels and playlists"""
|
||||
if self.task:
|
||||
self.task.send_progress(["Rescanning channels and playlists."])
|
||||
|
||||
self.missing_videos = []
|
||||
self.scan_channels()
|
||||
if self.task and not self.task.is_stopped():
|
||||
self.scan_playlists()
|
||||
|
||||
return self.missing_videos
|
||||
|
||||
def scan_channels(self):
|
||||
"""get missing from channels"""
|
||||
channel_handler = ChannelSubscription(task=self.task)
|
||||
missing = channel_handler.find_missing()
|
||||
if not missing:
|
||||
return
|
||||
|
||||
for vid_id, vid_type in missing:
|
||||
self.missing_videos.append(
|
||||
{"type": "video", "vid_type": vid_type, "url": vid_id}
|
||||
)
|
||||
|
||||
def scan_playlists(self):
|
||||
"""get missing from playlists"""
|
||||
playlist_handler = PlaylistSubscription(task=self.task)
|
||||
missing = playlist_handler.find_missing()
|
||||
if not missing:
|
||||
return
|
||||
|
||||
for i in missing:
|
||||
self.missing_videos.append(
|
||||
{
|
||||
"type": "video",
|
||||
"vid_type": VideoTypeEnum.VIDEOS.value,
|
||||
"url": i,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class SubscriptionHandler:
|
||||
"""subscribe to channels and playlists from url_str"""
|
||||
|
||||
def __init__(self, url_str, task=False):
|
||||
self.url_str = url_str
|
||||
self.task = task
|
||||
self.to_subscribe = False
|
||||
|
||||
def subscribe(self, expected_type=False):
|
||||
"""subscribe to url_str items"""
|
||||
if self.task:
|
||||
self.task.send_progress(["Processing form content."])
|
||||
self.to_subscribe = Parser(self.url_str).parse()
|
||||
|
||||
total = len(self.to_subscribe)
|
||||
for idx, item in enumerate(self.to_subscribe):
|
||||
if self.task:
|
||||
self._notify(idx, item, total)
|
||||
|
||||
self.subscribe_type(item, expected_type=expected_type)
|
||||
|
||||
def subscribe_type(self, item, expected_type):
|
||||
"""process single item"""
|
||||
if item["type"] == "playlist":
|
||||
if expected_type and expected_type != "playlist":
|
||||
raise TypeError(
|
||||
f"expected {expected_type} url but got {item.get('type')}"
|
||||
)
|
||||
|
||||
PlaylistSubscription().process_url_str([item])
|
||||
return
|
||||
|
||||
if item["type"] == "video":
|
||||
# extract channel id from video
|
||||
video = YoutubeVideo(item["url"])
|
||||
video.get_from_youtube()
|
||||
video.process_youtube_meta()
|
||||
channel_id = video.channel_id
|
||||
elif item["type"] == "channel":
|
||||
channel_id = item["url"]
|
||||
else:
|
||||
raise ValueError("failed to subscribe to: " + item["url"])
|
||||
|
||||
if expected_type and expected_type != "channel":
|
||||
raise TypeError(
|
||||
f"expected {expected_type} url but got {item.get('type')}"
|
||||
)
|
||||
|
||||
self._subscribe(channel_id)
|
||||
|
||||
def _subscribe(self, channel_id):
|
||||
"""subscribe to channel"""
|
||||
ChannelSubscription().change_subscribe(
|
||||
channel_id, channel_subscribed=True
|
||||
)
|
||||
|
||||
def _notify(self, idx, item, total):
|
||||
"""send notification message to redis"""
|
||||
subscribe_type = item["type"].title()
|
||||
message_lines = [
|
||||
f"Subscribe to {subscribe_type}",
|
||||
f"Progress: {idx + 1}/{total}",
|
||||
]
|
||||
self.task.send_progress(message_lines, progress=(idx + 1) / total)
|
||||
505
backend/download/src/thumbnails.py
Normal file
505
backend/download/src/thumbnails.py
Normal file
@@ -0,0 +1,505 @@
|
||||
"""
|
||||
functionality:
|
||||
- handle download and caching for thumbnails
|
||||
- check for missing thumbnails
|
||||
"""
|
||||
|
||||
import base64
|
||||
import os
|
||||
from io import BytesIO
|
||||
from time import sleep
|
||||
|
||||
import requests
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import is_missing
|
||||
from mutagen.mp4 import MP4, MP4Cover
|
||||
from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
|
||||
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = True
|
||||
|
||||
|
||||
class ThumbManagerBase:
|
||||
"""base class for thumbnail management"""
|
||||
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
|
||||
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
|
||||
PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
|
||||
|
||||
def __init__(self, item_id, item_type, fallback=False):
|
||||
self.item_id = item_id
|
||||
self.item_type = item_type
|
||||
self.fallback = fallback
|
||||
|
||||
def download_raw(self, url):
|
||||
"""download thumbnail for video"""
|
||||
if not url:
|
||||
return self.get_fallback()
|
||||
|
||||
for i in range(3):
|
||||
try:
|
||||
response = requests.get(url, stream=True, timeout=5)
|
||||
if response.ok:
|
||||
try:
|
||||
img = Image.open(response.raw)
|
||||
if isinstance(img, Image.Image):
|
||||
return img
|
||||
return self.get_fallback()
|
||||
|
||||
except (UnidentifiedImageError, OSError):
|
||||
print(f"failed to open thumbnail: {url}")
|
||||
return self.get_fallback()
|
||||
|
||||
if response.status_code == 404:
|
||||
return self.get_fallback()
|
||||
|
||||
except (
|
||||
requests.exceptions.RequestException,
|
||||
requests.exceptions.ReadTimeout,
|
||||
):
|
||||
print(f"{self.item_id}: retry thumbnail download {url}")
|
||||
sleep((i + 1) ** i)
|
||||
|
||||
return self.get_fallback()
|
||||
|
||||
def get_fallback(self):
|
||||
"""get fallback thumbnail if not available"""
|
||||
print(f"{self.item_id}: failed to extract thumbnail, use fallback")
|
||||
if self.fallback:
|
||||
img_raw = Image.open(self.fallback)
|
||||
return img_raw
|
||||
|
||||
app_root = EnvironmentSettings.APP_DIR
|
||||
default_map = {
|
||||
"video": os.path.join(
|
||||
app_root, "static/img/default-video-thumb.jpg"
|
||||
),
|
||||
"playlist": os.path.join(
|
||||
app_root, "static/img/default-playlist-thumb.jpg"
|
||||
),
|
||||
"icon": os.path.join(
|
||||
app_root, "static/img/default-channel-icon.jpg"
|
||||
),
|
||||
"banner": os.path.join(
|
||||
app_root, "static/img/default-channel-banner.jpg"
|
||||
),
|
||||
"tvart": os.path.join(
|
||||
app_root, "static/img/default-channel-art.jpg"
|
||||
),
|
||||
}
|
||||
|
||||
img_raw = Image.open(default_map[self.item_type])
|
||||
|
||||
return img_raw
|
||||
|
||||
|
||||
class ThumbManager(ThumbManagerBase):
|
||||
"""handle thumbnails related functions"""
|
||||
|
||||
def __init__(self, item_id, item_type="video", fallback=False):
|
||||
super().__init__(item_id, item_type, fallback=fallback)
|
||||
|
||||
def download(self, url):
|
||||
"""download thumbnail"""
|
||||
print(f"{self.item_id}: download {self.item_type} thumbnail")
|
||||
if self.item_type == "video":
|
||||
self.download_video_thumb(url)
|
||||
elif self.item_type == "channel":
|
||||
self.download_channel_art(url)
|
||||
elif self.item_type == "playlist":
|
||||
self.download_playlist_thumb(url)
|
||||
|
||||
def delete(self):
|
||||
"""delete thumbnail file"""
|
||||
print(f"{self.item_id}: delete {self.item_type} thumbnail")
|
||||
if self.item_type == "video":
|
||||
self.delete_video_thumb()
|
||||
elif self.item_type == "channel":
|
||||
self.delete_channel_thumb()
|
||||
elif self.item_type == "playlist":
|
||||
self.delete_playlist_thumb()
|
||||
|
||||
def download_video_thumb(self, url, skip_existing=False):
|
||||
"""pass url for video thumbnail"""
|
||||
folder_path = os.path.join(self.VIDEO_DIR, self.item_id[0].lower())
|
||||
thumb_path = self.vid_thumb_path(absolute=True)
|
||||
|
||||
if skip_existing and os.path.exists(thumb_path):
|
||||
return
|
||||
|
||||
os.makedirs(folder_path, exist_ok=True)
|
||||
img_raw = self.download_raw(url)
|
||||
width, height = img_raw.size
|
||||
|
||||
if not width / height == 16 / 9:
|
||||
new_height = width / 16 * 9
|
||||
offset = (height - new_height) / 2
|
||||
img_raw = img_raw.crop((0, offset, width, height - offset))
|
||||
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
def vid_thumb_path(self, absolute=False, create_folder=False):
|
||||
"""build expected path for video thumbnail from youtube_id"""
|
||||
folder_name = self.item_id[0].lower()
|
||||
folder_path = os.path.join("videos", folder_name)
|
||||
thumb_path = os.path.join(folder_path, f"{self.item_id}.jpg")
|
||||
if absolute:
|
||||
thumb_path = os.path.join(self.CACHE_DIR, thumb_path)
|
||||
|
||||
if create_folder:
|
||||
folder_path = os.path.join(self.CACHE_DIR, folder_path)
|
||||
os.makedirs(folder_path, exist_ok=True)
|
||||
|
||||
return thumb_path
|
||||
|
||||
def download_channel_art(self, urls, skip_existing=False):
|
||||
"""pass tuple of channel thumbnails"""
|
||||
channel_thumb, channel_banner, channel_tv = urls
|
||||
self._download_channel_thumb(channel_thumb, skip_existing)
|
||||
self._download_channel_banner(channel_banner, skip_existing)
|
||||
self._download_channel_tv(channel_tv, skip_existing)
|
||||
|
||||
def _download_channel_thumb(self, channel_thumb, skip_existing):
|
||||
"""download channel thumbnail"""
|
||||
|
||||
thumb_path = os.path.join(
|
||||
self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg"
|
||||
)
|
||||
self.item_type = "icon"
|
||||
|
||||
if skip_existing and os.path.exists(thumb_path):
|
||||
return
|
||||
|
||||
img_raw = self.download_raw(channel_thumb)
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
def _download_channel_banner(self, channel_banner, skip_existing):
|
||||
"""download channel banner"""
|
||||
|
||||
banner_path = os.path.join(
|
||||
self.CHANNEL_DIR, self.item_id + "_banner.jpg"
|
||||
)
|
||||
self.item_type = "banner"
|
||||
if skip_existing and os.path.exists(banner_path):
|
||||
return
|
||||
|
||||
img_raw = self.download_raw(channel_banner)
|
||||
img_raw.convert("RGB").save(banner_path)
|
||||
|
||||
def _download_channel_tv(self, channel_tv, skip_existing):
|
||||
"""download channel tv art"""
|
||||
art_path = os.path.join(self.CHANNEL_DIR, self.item_id + "_tvart.jpg")
|
||||
self.item_type = "tvart"
|
||||
if skip_existing and os.path.exists(art_path):
|
||||
return
|
||||
|
||||
img_raw = self.download_raw(channel_tv)
|
||||
img_raw.convert("RGB").save(art_path)
|
||||
|
||||
def download_playlist_thumb(self, url, skip_existing=False):
|
||||
"""pass thumbnail url"""
|
||||
thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
|
||||
if skip_existing and os.path.exists(thumb_path):
|
||||
return
|
||||
|
||||
img_raw = (
|
||||
self.download_raw(url)
|
||||
if not isinstance(url, str) or url.startswith("http")
|
||||
else Image.open(os.path.join(self.CACHE_DIR, url))
|
||||
)
|
||||
width, height = img_raw.size
|
||||
|
||||
if not width / height == 16 / 9:
|
||||
new_height = width / 16 * 9
|
||||
offset = (height - new_height) / 2
|
||||
img_raw = img_raw.crop((0, offset, width, height - offset))
|
||||
img_raw = img_raw.resize((336, 189))
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
def delete_video_thumb(self):
|
||||
"""delete video thumbnail if exists"""
|
||||
thumb_path = self.vid_thumb_path()
|
||||
to_delete = os.path.join(self.CACHE_DIR, thumb_path)
|
||||
if os.path.exists(to_delete):
|
||||
os.remove(to_delete)
|
||||
|
||||
def delete_channel_thumb(self):
|
||||
"""delete all artwork of channel"""
|
||||
thumb = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg")
|
||||
banner = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_banner.jpg")
|
||||
if os.path.exists(thumb):
|
||||
os.remove(thumb)
|
||||
if os.path.exists(banner):
|
||||
os.remove(banner)
|
||||
|
||||
def delete_playlist_thumb(self):
|
||||
"""delete playlist thumbnail"""
|
||||
thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
|
||||
if os.path.exists(thumb_path):
|
||||
os.remove(thumb_path)
|
||||
|
||||
def get_vid_base64_blur(self):
|
||||
"""return base64 encoded placeholder"""
|
||||
file_path = os.path.join(self.CACHE_DIR, self.vid_thumb_path())
|
||||
img_raw = Image.open(file_path)
|
||||
img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20))
|
||||
img_blur = img_raw.filter(ImageFilter.BLUR)
|
||||
buffer = BytesIO()
|
||||
img_blur.save(buffer, format="JPEG")
|
||||
img_data = buffer.getvalue()
|
||||
img_base64 = base64.b64encode(img_data).decode()
|
||||
data_url = f"data:image/jpg;base64,{img_base64}"
|
||||
|
||||
return data_url
|
||||
|
||||
|
||||
class ValidatorCallback:
|
||||
"""handle callback validate thumbnails page by page"""
|
||||
|
||||
def __init__(self, source, index_name, counter=0):
|
||||
self.source = source
|
||||
self.index_name = index_name
|
||||
self.counter = counter
|
||||
|
||||
def run(self):
|
||||
"""run the task for page"""
|
||||
print(f"{self.index_name}: validate artwork")
|
||||
if self.index_name == "ta_video":
|
||||
self._validate_videos()
|
||||
elif self.index_name == "ta_channel":
|
||||
self._validate_channels()
|
||||
elif self.index_name == "ta_playlist":
|
||||
self._validate_playlists()
|
||||
|
||||
def _validate_videos(self):
|
||||
"""check if video thumbnails are correct"""
|
||||
for video in self.source:
|
||||
url = video["_source"]["vid_thumb_url"]
|
||||
handler = ThumbManager(video["_source"]["youtube_id"])
|
||||
handler.download_video_thumb(url, skip_existing=True)
|
||||
|
||||
def _validate_channels(self):
|
||||
"""check if all channel artwork is there"""
|
||||
for channel in self.source:
|
||||
urls = (
|
||||
channel["_source"]["channel_thumb_url"],
|
||||
channel["_source"]["channel_banner_url"],
|
||||
channel["_source"].get("channel_tvart_url", False),
|
||||
)
|
||||
handler = ThumbManager(channel["_source"]["channel_id"])
|
||||
handler.download_channel_art(urls, skip_existing=True)
|
||||
|
||||
def _validate_playlists(self):
|
||||
"""check if all playlist artwork is there"""
|
||||
for playlist in self.source:
|
||||
url = playlist["_source"]["playlist_thumbnail"]
|
||||
handler = ThumbManager(playlist["_source"]["playlist_id"])
|
||||
handler.download_playlist_thumb(url, skip_existing=True)
|
||||
|
||||
|
||||
class ThumbValidator:
|
||||
"""validate thumbnails"""
|
||||
|
||||
INDEX = [
|
||||
{
|
||||
"data": {
|
||||
"query": {"term": {"active": {"value": True}}},
|
||||
"_source": ["vid_thumb_url", "youtube_id"],
|
||||
},
|
||||
"name": "ta_video",
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"query": {"term": {"channel_active": {"value": True}}},
|
||||
"_source": {
|
||||
"excludes": ["channel_description", "channel_overwrites"]
|
||||
},
|
||||
},
|
||||
"name": "ta_channel",
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"query": {"term": {"playlist_active": {"value": True}}},
|
||||
"_source": ["playlist_id", "playlist_thumbnail"],
|
||||
},
|
||||
"name": "ta_playlist",
|
||||
},
|
||||
]
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.task = task
|
||||
|
||||
def validate(self):
|
||||
"""validate all indexes"""
|
||||
for index in self.INDEX:
|
||||
total = self._get_total(index["name"])
|
||||
if not total:
|
||||
continue
|
||||
|
||||
paginate = IndexPaginate(
|
||||
index_name=index["name"],
|
||||
data=index["data"],
|
||||
size=1000,
|
||||
callback=ValidatorCallback,
|
||||
task=self.task,
|
||||
total=total,
|
||||
)
|
||||
_ = paginate.get_results()
|
||||
|
||||
def clean_up(self):
|
||||
"""clean up all thumbs"""
|
||||
self._clean_up_vids()
|
||||
self._clean_up_channels()
|
||||
self._clean_up_playlists()
|
||||
|
||||
def _clean_up_vids(self):
|
||||
"""clean unneeded vid thumbs"""
|
||||
video_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "videos")
|
||||
video_folders = os.listdir(video_dir)
|
||||
for video_folder in video_folders:
|
||||
folder_path = os.path.join(video_dir, video_folder)
|
||||
thumbs_is = {i.split(".")[0] for i in os.listdir(folder_path)}
|
||||
thumbs_should = self._get_vid_thumbs_should(video_folder)
|
||||
to_delete = thumbs_is - thumbs_should
|
||||
for thumb in to_delete:
|
||||
delete_path = os.path.join(folder_path, f"{thumb}.jpg")
|
||||
os.remove(delete_path)
|
||||
|
||||
if to_delete:
|
||||
message = (
|
||||
f"[thumbs][video][{video_folder}] "
|
||||
+ f"delete {len(to_delete)} unused thumbnails"
|
||||
)
|
||||
print(message)
|
||||
if self.task:
|
||||
self.task.send_progress([message])
|
||||
|
||||
@staticmethod
|
||||
def _get_vid_thumbs_should(video_folder: str) -> set[str]:
|
||||
"""get indexed"""
|
||||
should_list = [
|
||||
{"prefix": {"youtube_id": {"value": video_folder.lower()}}},
|
||||
{"prefix": {"youtube_id": {"value": video_folder.upper()}}},
|
||||
]
|
||||
data = {
|
||||
"query": {"bool": {"should": should_list}},
|
||||
"_source": ["youtube_id"],
|
||||
}
|
||||
result = IndexPaginate("ta_video,ta_download", data).get_results()
|
||||
thumbs_should = {i["youtube_id"] for i in result}
|
||||
|
||||
return thumbs_should
|
||||
|
||||
def _clean_up_channels(self):
|
||||
"""clean unneeded channel thumbs"""
|
||||
channel_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "channels")
|
||||
channel_art = os.listdir(channel_dir)
|
||||
thumbs_is = {"_".join(i.split("_")[:-1]) for i in channel_art}
|
||||
to_delete = is_missing(list(thumbs_is), "ta_channel", "channel_id")
|
||||
for channel_thumb in channel_art:
|
||||
if channel_thumb[:24] in to_delete:
|
||||
delete_path = os.path.join(channel_dir, channel_thumb)
|
||||
os.remove(delete_path)
|
||||
|
||||
if to_delete:
|
||||
message = (
|
||||
"[thumbs][channel] "
|
||||
+ f"delete {len(to_delete)} unused channel art"
|
||||
)
|
||||
print(message)
|
||||
if self.task:
|
||||
self.task.send_progress([message])
|
||||
|
||||
def _clean_up_playlists(self):
|
||||
"""clean up unneeded playlist thumbs"""
|
||||
playlist_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "playlists")
|
||||
playlist_art = os.listdir(playlist_dir)
|
||||
thumbs_is = {i.split(".")[0] for i in playlist_art}
|
||||
to_delete = is_missing(list(thumbs_is), "ta_playlist", "playlist_id")
|
||||
for playlist_id in to_delete:
|
||||
delete_path = os.path.join(playlist_dir, f"{playlist_id}.jpg")
|
||||
os.remove(delete_path)
|
||||
|
||||
if to_delete:
|
||||
message = (
|
||||
"[thumbs][playlist] "
|
||||
+ f"delete {len(to_delete)} unused playlist art"
|
||||
)
|
||||
print(message)
|
||||
if self.task:
|
||||
self.task.send_progress([message])
|
||||
|
||||
@staticmethod
|
||||
def _get_total(index_name):
|
||||
"""get total documents in index"""
|
||||
path = f"{index_name}/_count"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
|
||||
return response.get("count")
|
||||
|
||||
|
||||
class ThumbFilesystem:
|
||||
"""sync thumbnail files to media files"""
|
||||
|
||||
INDEX_NAME = "ta_video"
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.task = task
|
||||
|
||||
def embed(self):
|
||||
"""entry point"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"_source": ["media_url", "youtube_id"],
|
||||
}
|
||||
paginate = IndexPaginate(
|
||||
index_name=self.INDEX_NAME,
|
||||
data=data,
|
||||
size=200,
|
||||
callback=EmbedCallback,
|
||||
task=self.task,
|
||||
total=self._get_total(),
|
||||
)
|
||||
_ = paginate.get_results()
|
||||
|
||||
def _get_total(self):
|
||||
"""get total documents in index"""
|
||||
path = f"{self.INDEX_NAME}/_count"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
|
||||
return response.get("count")
|
||||
|
||||
|
||||
class EmbedCallback:
|
||||
"""callback class to embed thumbnails"""
|
||||
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
|
||||
FORMAT = MP4Cover.FORMAT_JPEG
|
||||
|
||||
def __init__(self, source, index_name, counter=0):
|
||||
self.source = source
|
||||
self.index_name = index_name
|
||||
self.counter = counter
|
||||
|
||||
def run(self):
|
||||
"""run embed"""
|
||||
for video in self.source:
|
||||
video_id = video["_source"]["youtube_id"]
|
||||
media_url = os.path.join(
|
||||
self.MEDIA_DIR, video["_source"]["media_url"]
|
||||
)
|
||||
thumb_path = os.path.join(
|
||||
self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path()
|
||||
)
|
||||
if os.path.exists(thumb_path):
|
||||
self.embed(media_url, thumb_path)
|
||||
|
||||
def embed(self, media_url, thumb_path):
|
||||
"""embed thumb in single media file"""
|
||||
video = MP4(media_url)
|
||||
with open(thumb_path, "rb") as f:
|
||||
video["covr"] = [MP4Cover(f.read(), imageformat=self.FORMAT)]
|
||||
|
||||
video.save()
|
||||
173
backend/download/src/yt_dlp_base.py
Normal file
173
backend/download/src/yt_dlp_base.py
Normal file
@@ -0,0 +1,173 @@
|
||||
"""
|
||||
functionality:
|
||||
- base class to make all calls to yt-dlp
|
||||
- handle yt-dlp errors
|
||||
"""
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
from http import cookiejar
|
||||
from io import StringIO
|
||||
|
||||
import yt_dlp
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
|
||||
|
||||
class YtWrap:
|
||||
"""wrap calls to yt"""
|
||||
|
||||
OBS_BASE = {
|
||||
"default_search": "ytsearch",
|
||||
"quiet": True,
|
||||
"check_formats": "selected",
|
||||
"socket_timeout": 10,
|
||||
"extractor_retries": 3,
|
||||
"retries": 10,
|
||||
}
|
||||
|
||||
def __init__(self, obs_request, config=False):
|
||||
self.obs_request = obs_request
|
||||
self.config = config
|
||||
self.build_obs()
|
||||
|
||||
def build_obs(self):
|
||||
"""build yt-dlp obs"""
|
||||
self.obs = self.OBS_BASE.copy()
|
||||
self.obs.update(self.obs_request)
|
||||
if self.config:
|
||||
self.add_cookie()
|
||||
|
||||
def add_cookie(self):
|
||||
"""add cookie if enabled"""
|
||||
if self.config["downloads"]["cookie_import"]:
|
||||
cookie_io = CookieHandler(self.config).get()
|
||||
self.obs["cookiefile"] = cookie_io
|
||||
|
||||
def download(self, url):
|
||||
"""make download request"""
|
||||
with yt_dlp.YoutubeDL(self.obs) as ydl:
|
||||
try:
|
||||
ydl.download([url])
|
||||
except yt_dlp.utils.DownloadError as err:
|
||||
print(f"{url}: failed to download with message {err}")
|
||||
if "Temporary failure in name resolution" in str(err):
|
||||
raise ConnectionError("lost the internet, abort!") from err
|
||||
|
||||
return False, str(err)
|
||||
|
||||
return True, True
|
||||
|
||||
def extract(self, url):
|
||||
"""make extract request"""
|
||||
try:
|
||||
response = yt_dlp.YoutubeDL(self.obs).extract_info(url)
|
||||
except cookiejar.LoadError as err:
|
||||
print(f"cookie file is invalid: {err}")
|
||||
return False
|
||||
except yt_dlp.utils.ExtractorError as err:
|
||||
print(f"{url}: failed to extract with message: {err}, continue...")
|
||||
return False
|
||||
except yt_dlp.utils.DownloadError as err:
|
||||
if "This channel does not have a" in str(err):
|
||||
return False
|
||||
|
||||
print(f"{url}: failed to get info from youtube with message {err}")
|
||||
if "Temporary failure in name resolution" in str(err):
|
||||
raise ConnectionError("lost the internet, abort!") from err
|
||||
|
||||
return False
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class CookieHandler:
|
||||
"""handle youtube cookie for yt-dlp"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.cookie_io = False
|
||||
self.config = config
|
||||
self.cache_dir = EnvironmentSettings.CACHE_DIR
|
||||
|
||||
def get(self):
|
||||
"""get cookie io stream"""
|
||||
cookie = RedisArchivist().get_message("cookie")
|
||||
self.cookie_io = StringIO(cookie)
|
||||
return self.cookie_io
|
||||
|
||||
def import_cookie(self):
|
||||
"""import cookie from file"""
|
||||
import_path = os.path.join(
|
||||
self.cache_dir, "import", "cookies.google.txt"
|
||||
)
|
||||
|
||||
try:
|
||||
with open(import_path, encoding="utf-8") as cookie_file:
|
||||
cookie = cookie_file.read()
|
||||
except FileNotFoundError as err:
|
||||
print(f"cookie: {import_path} file not found")
|
||||
raise err
|
||||
|
||||
self.set_cookie(cookie)
|
||||
|
||||
os.remove(import_path)
|
||||
print("cookie: import successful")
|
||||
|
||||
def set_cookie(self, cookie):
|
||||
"""set cookie str and activate in config"""
|
||||
RedisArchivist().set_message("cookie", cookie, save=True)
|
||||
path = ".downloads.cookie_import"
|
||||
RedisArchivist().set_message("config", True, path=path, save=True)
|
||||
self.config["downloads"]["cookie_import"] = True
|
||||
print("cookie: activated and stored in Redis")
|
||||
|
||||
@staticmethod
|
||||
def revoke():
|
||||
"""revoke cookie"""
|
||||
RedisArchivist().del_message("cookie")
|
||||
RedisArchivist().del_message("cookie:valid")
|
||||
RedisArchivist().set_message(
|
||||
"config", False, path=".downloads.cookie_import"
|
||||
)
|
||||
print("cookie: revoked")
|
||||
|
||||
def validate(self):
|
||||
"""validate cookie using the liked videos playlist"""
|
||||
print("validating cookie")
|
||||
obs_request = {
|
||||
"skip_download": True,
|
||||
"extract_flat": True,
|
||||
}
|
||||
validator = YtWrap(obs_request, self.config)
|
||||
response = bool(validator.extract("LL"))
|
||||
self.store_validation(response)
|
||||
|
||||
# update in redis to avoid expiring
|
||||
modified = validator.obs["cookiefile"].getvalue()
|
||||
if modified:
|
||||
RedisArchivist().set_message("cookie", modified)
|
||||
|
||||
if not response:
|
||||
mess_dict = {
|
||||
"status": "message:download",
|
||||
"level": "error",
|
||||
"title": "Cookie validation failed, exiting...",
|
||||
"message": "",
|
||||
}
|
||||
RedisArchivist().set_message(
|
||||
"message:download", mess_dict, expire=4
|
||||
)
|
||||
print("cookie validation failed, exiting...")
|
||||
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def store_validation(response):
|
||||
"""remember last validation"""
|
||||
now = datetime.now()
|
||||
message = {
|
||||
"status": response,
|
||||
"validated": int(now.timestamp()),
|
||||
"validated_str": now.strftime("%Y-%m-%d %H:%M"),
|
||||
}
|
||||
RedisArchivist().set_message("cookie:valid", message)
|
||||
451
backend/download/src/yt_dlp_handler.py
Normal file
451
backend/download/src/yt_dlp_handler.py
Normal file
@@ -0,0 +1,451 @@
|
||||
"""
|
||||
functionality:
|
||||
- handle yt_dlp
|
||||
- build options and post processor
|
||||
- download video files
|
||||
- move to archive
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from channel.src.index import YoutubeChannel
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import get_channel_overwrites, ignore_filelist
|
||||
from common.src.ta_redis import RedisQueue
|
||||
from download.src.queue import PendingList
|
||||
from download.src.subscriptions import PlaylistSubscription
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
from video.src.comments import CommentList
|
||||
from video.src.constants import VideoTypeEnum
|
||||
from video.src.index import YoutubeVideo, index_new_video
|
||||
|
||||
|
||||
class DownloaderBase:
|
||||
"""base class for shared config"""
|
||||
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
|
||||
CHANNEL_QUEUE = "download:channel"
|
||||
PLAYLIST_QUEUE = "download:playlist:full"
|
||||
PLAYLIST_QUICK = "download:playlist:quick"
|
||||
VIDEO_QUEUE = "download:video"
|
||||
|
||||
def __init__(self, task):
|
||||
self.task = task
|
||||
self.config = AppConfig().config
|
||||
self.channel_overwrites = get_channel_overwrites()
|
||||
self.now = int(datetime.now().timestamp())
|
||||
|
||||
|
||||
class VideoDownloader(DownloaderBase):
|
||||
"""handle the video download functionality"""
|
||||
|
||||
def __init__(self, task=False):
|
||||
super().__init__(task)
|
||||
self.obs = False
|
||||
self._build_obs()
|
||||
|
||||
def run_queue(self, auto_only=False) -> tuple[int, int]:
|
||||
"""setup download queue in redis loop until no more items"""
|
||||
downloaded = 0
|
||||
failed = 0
|
||||
while True:
|
||||
video_data = self._get_next(auto_only)
|
||||
if self.task.is_stopped() or not video_data:
|
||||
self._reset_auto()
|
||||
break
|
||||
|
||||
youtube_id = video_data["youtube_id"]
|
||||
channel_id = video_data["channel_id"]
|
||||
print(f"{youtube_id}: Downloading video")
|
||||
self._notify(video_data, "Validate download format")
|
||||
|
||||
success = self._dl_single_vid(youtube_id, channel_id)
|
||||
if not success:
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
self._notify(video_data, "Add video metadata to index", progress=1)
|
||||
video_type = VideoTypeEnum(video_data["vid_type"])
|
||||
vid_dict = index_new_video(youtube_id, video_type=video_type)
|
||||
RedisQueue(self.CHANNEL_QUEUE).add(channel_id)
|
||||
RedisQueue(self.VIDEO_QUEUE).add(youtube_id)
|
||||
|
||||
self._notify(video_data, "Move downloaded file to archive")
|
||||
self.move_to_archive(vid_dict)
|
||||
self._delete_from_pending(youtube_id)
|
||||
downloaded += 1
|
||||
|
||||
# post processing
|
||||
DownloadPostProcess(self.task).run()
|
||||
|
||||
return downloaded, failed
|
||||
|
||||
def _notify(self, video_data, message, progress=False):
|
||||
"""send progress notification to task"""
|
||||
if not self.task:
|
||||
return
|
||||
|
||||
typ = VideoTypeEnum(video_data["vid_type"]).value.rstrip("s").title()
|
||||
title = video_data.get("title")
|
||||
self.task.send_progress(
|
||||
[f"Processing {typ}: {title}", message], progress=progress
|
||||
)
|
||||
|
||||
def _get_next(self, auto_only):
|
||||
"""get next item in queue"""
|
||||
must_list = [{"term": {"status": {"value": "pending"}}}]
|
||||
must_not_list = [{"exists": {"field": "message"}}]
|
||||
if auto_only:
|
||||
must_list.append({"term": {"auto_start": {"value": True}}})
|
||||
|
||||
data = {
|
||||
"size": 1,
|
||||
"query": {"bool": {"must": must_list, "must_not": must_not_list}},
|
||||
"sort": [
|
||||
{"auto_start": {"order": "desc"}},
|
||||
{"timestamp": {"order": "asc"}},
|
||||
],
|
||||
}
|
||||
path = "ta_download/_search"
|
||||
response, _ = ElasticWrap(path).get(data=data)
|
||||
if not response["hits"]["hits"]:
|
||||
return False
|
||||
|
||||
return response["hits"]["hits"][0]["_source"]
|
||||
|
||||
def _progress_hook(self, response):
|
||||
"""process the progress_hooks from yt_dlp"""
|
||||
progress = False
|
||||
try:
|
||||
size = response.get("_total_bytes_str")
|
||||
if size.strip() == "N/A":
|
||||
size = response.get("_total_bytes_estimate_str", "N/A")
|
||||
|
||||
percent = response["_percent_str"]
|
||||
progress = float(percent.strip("%")) / 100
|
||||
speed = response["_speed_str"]
|
||||
eta = response["_eta_str"]
|
||||
message = f"{percent} of {size} at {speed} - time left: {eta}"
|
||||
except KeyError:
|
||||
message = "processing"
|
||||
|
||||
if self.task:
|
||||
title = response["info_dict"]["title"]
|
||||
self.task.send_progress([title, message], progress=progress)
|
||||
|
||||
def _build_obs(self):
|
||||
"""collection to build all obs passed to yt-dlp"""
|
||||
self._build_obs_basic()
|
||||
self._build_obs_user()
|
||||
self._build_obs_postprocessors()
|
||||
|
||||
def _build_obs_basic(self):
|
||||
"""initial obs"""
|
||||
self.obs = {
|
||||
"merge_output_format": "mp4",
|
||||
"outtmpl": (self.CACHE_DIR + "/download/%(id)s.mp4"),
|
||||
"progress_hooks": [self._progress_hook],
|
||||
"noprogress": True,
|
||||
"continuedl": True,
|
||||
"writethumbnail": False,
|
||||
"noplaylist": True,
|
||||
"color": "no_color",
|
||||
}
|
||||
|
||||
def _build_obs_user(self):
|
||||
"""build user customized options"""
|
||||
if self.config["downloads"]["format"]:
|
||||
self.obs["format"] = self.config["downloads"]["format"]
|
||||
if self.config["downloads"]["format_sort"]:
|
||||
format_sort = self.config["downloads"]["format_sort"]
|
||||
format_sort_list = [i.strip() for i in format_sort.split(",")]
|
||||
self.obs["format_sort"] = format_sort_list
|
||||
if self.config["downloads"]["limit_speed"]:
|
||||
self.obs["ratelimit"] = (
|
||||
self.config["downloads"]["limit_speed"] * 1024
|
||||
)
|
||||
|
||||
throttle = self.config["downloads"]["throttledratelimit"]
|
||||
if throttle:
|
||||
self.obs["throttledratelimit"] = throttle * 1024
|
||||
|
||||
def _build_obs_postprocessors(self):
|
||||
"""add postprocessor to obs"""
|
||||
postprocessors = []
|
||||
|
||||
if self.config["downloads"]["add_metadata"]:
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "FFmpegMetadata",
|
||||
"add_chapters": True,
|
||||
"add_metadata": True,
|
||||
}
|
||||
)
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "MetadataFromField",
|
||||
"formats": [
|
||||
"%(title)s:%(meta_title)s",
|
||||
"%(uploader)s:%(meta_artist)s",
|
||||
":(?P<album>)",
|
||||
],
|
||||
"when": "pre_process",
|
||||
}
|
||||
)
|
||||
|
||||
if self.config["downloads"]["add_thumbnail"]:
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "EmbedThumbnail",
|
||||
"already_have_thumbnail": True,
|
||||
}
|
||||
)
|
||||
self.obs["writethumbnail"] = True
|
||||
|
||||
self.obs["postprocessors"] = postprocessors
|
||||
|
||||
def _set_overwrites(self, obs: dict, channel_id: str) -> None:
|
||||
"""add overwrites to obs"""
|
||||
overwrites = self.channel_overwrites.get(channel_id)
|
||||
if overwrites and overwrites.get("download_format"):
|
||||
obs["format"] = overwrites.get("download_format")
|
||||
|
||||
def _dl_single_vid(self, youtube_id: str, channel_id: str) -> bool:
|
||||
"""download single video"""
|
||||
obs = self.obs.copy()
|
||||
self._set_overwrites(obs, channel_id)
|
||||
dl_cache = os.path.join(self.CACHE_DIR, "download")
|
||||
|
||||
success, message = YtWrap(obs, self.config).download(youtube_id)
|
||||
if not success:
|
||||
self._handle_error(youtube_id, message)
|
||||
|
||||
if self.obs["writethumbnail"]:
|
||||
# webp files don't get cleaned up automatically
|
||||
all_cached = ignore_filelist(os.listdir(dl_cache))
|
||||
to_clean = [i for i in all_cached if not i.endswith(".mp4")]
|
||||
for file_name in to_clean:
|
||||
file_path = os.path.join(dl_cache, file_name)
|
||||
os.remove(file_path)
|
||||
|
||||
return success
|
||||
|
||||
@staticmethod
|
||||
def _handle_error(youtube_id, message):
|
||||
"""store error message"""
|
||||
data = {"doc": {"message": message}}
|
||||
_, _ = ElasticWrap(f"ta_download/_update/{youtube_id}").post(data=data)
|
||||
|
||||
def move_to_archive(self, vid_dict):
|
||||
"""move downloaded video from cache to archive"""
|
||||
host_uid = EnvironmentSettings.HOST_UID
|
||||
host_gid = EnvironmentSettings.HOST_GID
|
||||
# make folder
|
||||
folder = os.path.join(
|
||||
self.MEDIA_DIR, vid_dict["channel"]["channel_id"]
|
||||
)
|
||||
if not os.path.exists(folder):
|
||||
os.makedirs(folder)
|
||||
if host_uid and host_gid:
|
||||
os.chown(folder, host_uid, host_gid)
|
||||
# move media file
|
||||
media_file = vid_dict["youtube_id"] + ".mp4"
|
||||
old_path = os.path.join(self.CACHE_DIR, "download", media_file)
|
||||
new_path = os.path.join(self.MEDIA_DIR, vid_dict["media_url"])
|
||||
# move media file and fix permission
|
||||
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
||||
if host_uid and host_gid:
|
||||
os.chown(new_path, host_uid, host_gid)
|
||||
|
||||
@staticmethod
|
||||
def _delete_from_pending(youtube_id):
|
||||
"""delete downloaded video from pending index if its there"""
|
||||
path = f"ta_download/_doc/{youtube_id}?refresh=true"
|
||||
_, _ = ElasticWrap(path).delete()
|
||||
|
||||
def _reset_auto(self):
|
||||
"""reset autostart to defaults after queue stop"""
|
||||
path = "ta_download/_update_by_query"
|
||||
data = {
|
||||
"query": {"term": {"auto_start": {"value": True}}},
|
||||
"script": {
|
||||
"source": "ctx._source.auto_start = false",
|
||||
"lang": "painless",
|
||||
},
|
||||
}
|
||||
response, _ = ElasticWrap(path).post(data=data)
|
||||
updated = response.get("updated")
|
||||
if updated:
|
||||
print(f"[download] reset auto start on {updated} videos.")
|
||||
|
||||
|
||||
class DownloadPostProcess(DownloaderBase):
|
||||
"""handle task to run after download queue finishes"""
|
||||
|
||||
def run(self):
|
||||
"""run all functions"""
|
||||
self.auto_delete_all()
|
||||
self.auto_delete_overwrites()
|
||||
self.refresh_playlist()
|
||||
self.match_videos()
|
||||
self.get_comments()
|
||||
|
||||
def auto_delete_all(self):
|
||||
"""handle auto delete"""
|
||||
autodelete_days = self.config["downloads"]["autodelete_days"]
|
||||
if not autodelete_days:
|
||||
return
|
||||
|
||||
print(f"auto delete older than {autodelete_days} days")
|
||||
now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
|
||||
data = {
|
||||
"query": {"range": {"player.watched_date": {"lte": now_lte}}},
|
||||
"sort": [{"player.watched_date": {"order": "asc"}}],
|
||||
}
|
||||
self._auto_delete_watched(data)
|
||||
|
||||
def auto_delete_overwrites(self):
|
||||
"""handle per channel auto delete from overwrites"""
|
||||
for channel_id, value in self.channel_overwrites.items():
|
||||
if "autodelete_days" in value:
|
||||
autodelete_days = value.get("autodelete_days")
|
||||
print(f"{channel_id}: delete older than {autodelete_days}d")
|
||||
now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
|
||||
must_list = [
|
||||
{"range": {"player.watched_date": {"lte": now_lte}}},
|
||||
{"term": {"channel.channel_id": {"value": channel_id}}},
|
||||
]
|
||||
data = {
|
||||
"query": {"bool": {"must": must_list}},
|
||||
"sort": [{"player.watched_date": {"order": "desc"}}],
|
||||
}
|
||||
self._auto_delete_watched(data)
|
||||
|
||||
@staticmethod
|
||||
def _auto_delete_watched(data):
|
||||
"""delete watched videos after x days"""
|
||||
to_delete = IndexPaginate("ta_video", data).get_results()
|
||||
if not to_delete:
|
||||
return
|
||||
|
||||
for video in to_delete:
|
||||
youtube_id = video["youtube_id"]
|
||||
print(f"{youtube_id}: auto delete video")
|
||||
YoutubeVideo(youtube_id).delete_media_file()
|
||||
|
||||
print("add deleted to ignore list")
|
||||
vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
|
||||
pending = PendingList(youtube_ids=vids)
|
||||
pending.parse_url_list()
|
||||
_ = pending.add_to_pending(status="ignore")
|
||||
|
||||
def refresh_playlist(self) -> None:
|
||||
"""match videos with playlists"""
|
||||
self.add_playlists_to_refresh()
|
||||
|
||||
queue = RedisQueue(self.PLAYLIST_QUEUE)
|
||||
while True:
|
||||
total = queue.max_score()
|
||||
playlist_id, idx = queue.get_next()
|
||||
if not playlist_id or not idx or not total:
|
||||
break
|
||||
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
playlist.update_playlist(skip_on_empty=True)
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
channel_name = playlist.json_data["playlist_channel"]
|
||||
playlist_title = playlist.json_data["playlist_name"]
|
||||
message = [
|
||||
f"Post Processing Playlists for: {channel_name}",
|
||||
f"{playlist_title} [{idx}/{total}]",
|
||||
]
|
||||
progress = idx / total
|
||||
self.task.send_progress(message, progress=progress)
|
||||
|
||||
def add_playlists_to_refresh(self) -> None:
|
||||
"""add playlists to refresh"""
|
||||
if self.task:
|
||||
message = ["Post Processing Playlists", "Scanning for Playlists"]
|
||||
self.task.send_progress(message)
|
||||
|
||||
self._add_playlist_sub()
|
||||
self._add_channel_playlists()
|
||||
self._add_video_playlists()
|
||||
|
||||
def _add_playlist_sub(self):
|
||||
"""add subscribed playlists to refresh"""
|
||||
subs = PlaylistSubscription().get_playlists()
|
||||
to_add = [i["playlist_id"] for i in subs]
|
||||
RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
|
||||
|
||||
def _add_channel_playlists(self):
|
||||
"""add playlists from channels to refresh"""
|
||||
queue = RedisQueue(self.CHANNEL_QUEUE)
|
||||
while True:
|
||||
channel_id, _ = queue.get_next()
|
||||
if not channel_id:
|
||||
break
|
||||
|
||||
channel = YoutubeChannel(channel_id)
|
||||
channel.get_from_es()
|
||||
overwrites = channel.get_overwrites()
|
||||
if "index_playlists" in overwrites:
|
||||
channel.get_all_playlists()
|
||||
to_add = [i[0] for i in channel.all_playlists]
|
||||
RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
|
||||
|
||||
def _add_video_playlists(self):
|
||||
"""add other playlists for quick sync"""
|
||||
all_playlists = RedisQueue(self.PLAYLIST_QUEUE).get_all()
|
||||
must_not = [{"terms": {"playlist_id": all_playlists}}]
|
||||
video_ids = RedisQueue(self.VIDEO_QUEUE).get_all()
|
||||
must = [{"terms": {"playlist_entries.youtube_id": video_ids}}]
|
||||
data = {
|
||||
"query": {"bool": {"must_not": must_not, "must": must}},
|
||||
"_source": ["playlist_id"],
|
||||
}
|
||||
playlists = IndexPaginate("ta_playlist", data).get_results()
|
||||
to_add = [i["playlist_id"] for i in playlists]
|
||||
RedisQueue(self.PLAYLIST_QUICK).add_list(to_add)
|
||||
|
||||
def match_videos(self) -> None:
|
||||
"""scan rest of indexed playlists to match videos"""
|
||||
queue = RedisQueue(self.PLAYLIST_QUICK)
|
||||
while True:
|
||||
total = queue.max_score()
|
||||
playlist_id, idx = queue.get_next()
|
||||
if not playlist_id or not idx or not total:
|
||||
break
|
||||
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
playlist.get_from_es()
|
||||
playlist.add_vids_to_playlist()
|
||||
playlist.remove_vids_from_playlist()
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
message = [
|
||||
"Post Processing Playlists.",
|
||||
f"Validate Playlists: - {idx}/{total}",
|
||||
]
|
||||
progress = idx / total
|
||||
self.task.send_progress(message, progress=progress)
|
||||
|
||||
def get_comments(self):
|
||||
"""get comments from youtube"""
|
||||
video_queue = RedisQueue(self.VIDEO_QUEUE)
|
||||
comment_list = CommentList(task=self.task)
|
||||
comment_list.add(video_ids=video_queue.get_all())
|
||||
|
||||
video_queue.clear()
|
||||
comment_list.index()
|
||||
18
backend/download/urls.py
Normal file
18
backend/download/urls.py
Normal file
@@ -0,0 +1,18 @@
|
||||
"""all download API urls"""
|
||||
|
||||
from django.urls import path
|
||||
from download import views
|
||||
|
||||
urlpatterns = [
|
||||
path("", views.DownloadApiListView.as_view(), name="api-download-list"),
|
||||
path(
|
||||
"aggs/",
|
||||
views.DownloadAggsApiView.as_view(),
|
||||
name="api-download-aggs",
|
||||
),
|
||||
path(
|
||||
"<slug:video_id>/",
|
||||
views.DownloadApiView.as_view(),
|
||||
name="api-download",
|
||||
),
|
||||
]
|
||||
170
backend/download/views.py
Normal file
170
backend/download/views.py
Normal file
@@ -0,0 +1,170 @@
|
||||
"""all download API views"""
|
||||
|
||||
from common.views_base import AdminOnly, ApiBaseView
|
||||
from download.src.queue import PendingInteract
|
||||
from rest_framework.response import Response
|
||||
from task.tasks import download_pending, extrac_dl
|
||||
|
||||
|
||||
class DownloadApiListView(ApiBaseView):
|
||||
"""resolves to /api/download/
|
||||
GET: returns latest videos in the download queue
|
||||
POST: add a list of videos to download queue
|
||||
DELETE: remove items based on query filter
|
||||
"""
|
||||
|
||||
search_base = "ta_download/_search/"
|
||||
valid_filter = ["pending", "ignore"]
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
def get(self, request):
|
||||
"""get request"""
|
||||
query_filter = request.GET.get("filter", False)
|
||||
self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
|
||||
|
||||
must_list = []
|
||||
if query_filter:
|
||||
if query_filter not in self.valid_filter:
|
||||
message = f"invalid url query filter: {query_filter}"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
must_list.append({"term": {"status": {"value": query_filter}}})
|
||||
|
||||
filter_channel = request.GET.get("channel", False)
|
||||
if filter_channel:
|
||||
must_list.append(
|
||||
{"term": {"channel_id": {"value": filter_channel}}}
|
||||
)
|
||||
|
||||
self.data["query"] = {"bool": {"must": must_list}}
|
||||
|
||||
self.get_document_list(request)
|
||||
return Response(self.response)
|
||||
|
||||
@staticmethod
|
||||
def post(request):
|
||||
"""add list of videos to download queue"""
|
||||
data = request.data
|
||||
auto_start = bool(request.GET.get("autostart"))
|
||||
try:
|
||||
to_add = data["data"]
|
||||
except KeyError:
|
||||
message = "missing expected data key"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
|
||||
url_str = " ".join(pending)
|
||||
extrac_dl.delay(url_str, auto_start=auto_start)
|
||||
|
||||
return Response(data)
|
||||
|
||||
def delete(self, request):
|
||||
"""delete download queue"""
|
||||
query_filter = request.GET.get("filter", False)
|
||||
if query_filter not in self.valid_filter:
|
||||
message = f"invalid url query filter: {query_filter}"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
message = f"delete queue by status: {query_filter}"
|
||||
print(message)
|
||||
PendingInteract(status=query_filter).delete_by_status()
|
||||
|
||||
return Response({"message": message})
|
||||
|
||||
|
||||
class DownloadAggsApiView(ApiBaseView):
|
||||
"""resolves to /api/download/aggs/
|
||||
GET: get download aggregations
|
||||
"""
|
||||
|
||||
search_base = "ta_download/_search"
|
||||
valid_filter_view = ["ignore", "pending"]
|
||||
|
||||
def get(self, request):
|
||||
"""get aggs"""
|
||||
filter_view = request.GET.get("filter")
|
||||
if filter_view:
|
||||
if filter_view not in self.valid_filter_view:
|
||||
message = f"invalid filter: {filter_view}"
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
self.data.update(
|
||||
{
|
||||
"query": {"term": {"status": {"value": filter_view}}},
|
||||
}
|
||||
)
|
||||
|
||||
self.data.update(
|
||||
{
|
||||
"aggs": {
|
||||
"channel_downloads": {
|
||||
"multi_terms": {
|
||||
"size": 30,
|
||||
"terms": [
|
||||
{"field": "channel_name.keyword"},
|
||||
{"field": "channel_id"},
|
||||
],
|
||||
"order": {"_count": "desc"},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
self.get_aggs()
|
||||
|
||||
return Response(self.response)
|
||||
|
||||
|
||||
class DownloadApiView(ApiBaseView):
|
||||
"""resolves to /api/download/<video_id>/
|
||||
GET: returns metadata dict of an item in the download queue
|
||||
POST: update status of item to pending or ignore
|
||||
DELETE: forget from download queue
|
||||
"""
|
||||
|
||||
search_base = "ta_download/_doc/"
|
||||
valid_status = ["pending", "ignore", "ignore-force", "priority"]
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
def get(self, request, video_id):
|
||||
# pylint: disable=unused-argument
|
||||
"""get request"""
|
||||
self.get_document(video_id)
|
||||
return Response(self.response, status=self.status_code)
|
||||
|
||||
def post(self, request, video_id):
|
||||
"""post to video to change status"""
|
||||
item_status = request.data.get("status")
|
||||
if item_status not in self.valid_status:
|
||||
message = f"{video_id}: invalid status {item_status}"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
if item_status == "ignore-force":
|
||||
extrac_dl.delay(video_id, status="ignore")
|
||||
message = f"{video_id}: set status to ignore"
|
||||
return Response(request.data)
|
||||
|
||||
_, status_code = PendingInteract(video_id).get_item()
|
||||
if status_code == 404:
|
||||
message = f"{video_id}: item not found {status_code}"
|
||||
return Response({"message": message}, status=404)
|
||||
|
||||
print(f"{video_id}: change status to {item_status}")
|
||||
PendingInteract(video_id, item_status).update_status()
|
||||
if item_status == "priority":
|
||||
download_pending.delay(auto_only=True)
|
||||
|
||||
return Response(request.data)
|
||||
|
||||
@staticmethod
|
||||
def delete(request, video_id):
|
||||
# pylint: disable=unused-argument
|
||||
"""delete single video from queue"""
|
||||
print(f"{video_id}: delete from queue")
|
||||
PendingInteract(video_id).delete_item()
|
||||
|
||||
return Response({"success": True})
|
||||
23
backend/manage.py
Executable file
23
backend/manage.py
Executable file
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env python
|
||||
"""Django's command-line utility for administrative tasks."""
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
# pylint: disable=import-outside-toplevel
|
||||
"""Run administrative tasks."""
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
||||
try:
|
||||
from django.core.management import execute_from_command_line
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Couldn't import Django. Are you sure it's installed and "
|
||||
"available on your PYTHONPATH environment variable? Did you "
|
||||
"forget to activate a virtual environment?"
|
||||
) from exc
|
||||
execute_from_command_line(sys.argv)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
0
backend/playlist/__init__.py
Normal file
0
backend/playlist/__init__.py
Normal file
0
backend/playlist/migrations/__init__.py
Normal file
0
backend/playlist/migrations/__init__.py
Normal file
0
backend/playlist/src/__init__.py
Normal file
0
backend/playlist/src/__init__.py
Normal file
10
backend/playlist/src/constants.py
Normal file
10
backend/playlist/src/constants.py
Normal file
@@ -0,0 +1,10 @@
|
||||
"""playlist constants"""
|
||||
|
||||
import enum
|
||||
|
||||
|
||||
class PlaylistTypesEnum(enum.Enum):
|
||||
"""all playlist_type options"""
|
||||
|
||||
REGULAR = "regular"
|
||||
CUSTOM = "custom"
|
||||
443
backend/playlist/src/index.py
Normal file
443
backend/playlist/src/index.py
Normal file
@@ -0,0 +1,443 @@
|
||||
"""
|
||||
functionality:
|
||||
- get metadata from youtube for a playlist
|
||||
- index and update in es
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from channel.src import index as channel
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.index_generic import YouTubeItem
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from video.src import index as ta_video
|
||||
|
||||
|
||||
class YoutubePlaylist(YouTubeItem):
|
||||
"""represents a single youtube playlist"""
|
||||
|
||||
es_path = False
|
||||
index_name = "ta_playlist"
|
||||
yt_obs = {
|
||||
"extract_flat": True,
|
||||
"allow_playlist_files": True,
|
||||
}
|
||||
yt_base = "https://www.youtube.com/playlist?list="
|
||||
|
||||
def __init__(self, youtube_id):
|
||||
super().__init__(youtube_id)
|
||||
self.all_members = False
|
||||
self.nav = False
|
||||
|
||||
def build_json(self, scrape=False):
|
||||
"""collection to create json_data"""
|
||||
self.get_from_es()
|
||||
if self.json_data:
|
||||
subscribed = self.json_data.get("playlist_subscribed")
|
||||
else:
|
||||
subscribed = False
|
||||
|
||||
if scrape or not self.json_data:
|
||||
self.get_from_youtube()
|
||||
if not self.youtube_meta:
|
||||
self.json_data = False
|
||||
return
|
||||
|
||||
self.process_youtube_meta()
|
||||
self._ensure_channel()
|
||||
ids_found = self.get_local_vids()
|
||||
self.get_entries(ids_found)
|
||||
self.json_data["playlist_entries"] = self.all_members
|
||||
self.json_data["playlist_subscribed"] = subscribed
|
||||
|
||||
def process_youtube_meta(self):
|
||||
"""extract relevant fields from youtube"""
|
||||
try:
|
||||
playlist_thumbnail = self.youtube_meta["thumbnails"][-1]["url"]
|
||||
except IndexError:
|
||||
print(f"{self.youtube_id}: thumbnail extraction failed")
|
||||
playlist_thumbnail = False
|
||||
|
||||
self.json_data = {
|
||||
"playlist_id": self.youtube_id,
|
||||
"playlist_active": True,
|
||||
"playlist_name": self.youtube_meta["title"],
|
||||
"playlist_channel": self.youtube_meta["channel"],
|
||||
"playlist_channel_id": self.youtube_meta["channel_id"],
|
||||
"playlist_thumbnail": playlist_thumbnail,
|
||||
"playlist_description": self.youtube_meta["description"] or False,
|
||||
"playlist_last_refresh": int(datetime.now().timestamp()),
|
||||
"playlist_type": "regular",
|
||||
}
|
||||
|
||||
def _ensure_channel(self):
|
||||
"""make sure channel is indexed"""
|
||||
channel_id = self.json_data["playlist_channel_id"]
|
||||
channel_handler = channel.YoutubeChannel(channel_id)
|
||||
channel_handler.build_json(upload=True)
|
||||
|
||||
def get_local_vids(self) -> list[str]:
|
||||
"""get local video ids from youtube entries"""
|
||||
entries = self.youtube_meta["entries"]
|
||||
data = {
|
||||
"query": {"terms": {"youtube_id": [i["id"] for i in entries]}},
|
||||
"_source": ["youtube_id"],
|
||||
}
|
||||
indexed_vids = IndexPaginate("ta_video", data).get_results()
|
||||
ids_found = [i["youtube_id"] for i in indexed_vids]
|
||||
|
||||
return ids_found
|
||||
|
||||
def get_entries(self, ids_found) -> None:
|
||||
"""get all videos in playlist, match downloaded with ids_found"""
|
||||
all_members = []
|
||||
for idx, entry in enumerate(self.youtube_meta["entries"]):
|
||||
if not entry["channel"]:
|
||||
continue
|
||||
|
||||
to_append = {
|
||||
"youtube_id": entry["id"],
|
||||
"title": entry["title"],
|
||||
"uploader": entry["channel"],
|
||||
"idx": idx,
|
||||
"downloaded": entry["id"] in ids_found,
|
||||
}
|
||||
all_members.append(to_append)
|
||||
|
||||
self.all_members = all_members
|
||||
|
||||
def get_playlist_art(self):
|
||||
"""download artwork of playlist"""
|
||||
url = self.json_data["playlist_thumbnail"]
|
||||
ThumbManager(self.youtube_id, item_type="playlist").download(url)
|
||||
|
||||
def add_vids_to_playlist(self):
|
||||
"""sync the playlist id to videos"""
|
||||
script = (
|
||||
'if (!ctx._source.containsKey("playlist")) '
|
||||
+ "{ctx._source.playlist = [params.playlist]} "
|
||||
+ "else if (!ctx._source.playlist.contains(params.playlist)) "
|
||||
+ "{ctx._source.playlist.add(params.playlist)} "
|
||||
+ "else {ctx.op = 'none'}"
|
||||
)
|
||||
|
||||
bulk_list = []
|
||||
for entry in self.json_data["playlist_entries"]:
|
||||
video_id = entry["youtube_id"]
|
||||
action = {"update": {"_id": video_id, "_index": "ta_video"}}
|
||||
source = {
|
||||
"script": {
|
||||
"source": script,
|
||||
"lang": "painless",
|
||||
"params": {"playlist": self.youtube_id},
|
||||
}
|
||||
}
|
||||
bulk_list.append(json.dumps(action))
|
||||
bulk_list.append(json.dumps(source))
|
||||
|
||||
# add last newline
|
||||
bulk_list.append("\n")
|
||||
query_str = "\n".join(bulk_list)
|
||||
|
||||
ElasticWrap("_bulk").post(query_str, ndjson=True)
|
||||
|
||||
def remove_vids_from_playlist(self):
|
||||
"""remove playlist ids from videos if needed"""
|
||||
needed = [i["youtube_id"] for i in self.json_data["playlist_entries"]]
|
||||
data = {
|
||||
"query": {"match": {"playlist": self.youtube_id}},
|
||||
"_source": ["youtube_id"],
|
||||
}
|
||||
result = IndexPaginate("ta_video", data).get_results()
|
||||
to_remove = [
|
||||
i["youtube_id"] for i in result if i["youtube_id"] not in needed
|
||||
]
|
||||
s = "ctx._source.playlist.removeAll(Collections.singleton(params.rm))"
|
||||
for video_id in to_remove:
|
||||
query = {
|
||||
"script": {
|
||||
"source": s,
|
||||
"lang": "painless",
|
||||
"params": {"rm": self.youtube_id},
|
||||
},
|
||||
"query": {"match": {"youtube_id": video_id}},
|
||||
}
|
||||
path = "ta_video/_update_by_query"
|
||||
_, status_code = ElasticWrap(path).post(query)
|
||||
if status_code == 200:
|
||||
print(f"{self.youtube_id}: removed {video_id} from playlist")
|
||||
|
||||
def update_playlist(self, skip_on_empty=False):
|
||||
"""update metadata for playlist with data from YouTube"""
|
||||
self.build_json(scrape=True)
|
||||
if not self.json_data:
|
||||
# return false to deactivate
|
||||
return False
|
||||
|
||||
if skip_on_empty:
|
||||
has_item_downloaded = any(
|
||||
i["downloaded"] for i in self.json_data["playlist_entries"]
|
||||
)
|
||||
if not has_item_downloaded:
|
||||
return True
|
||||
|
||||
self.upload_to_es()
|
||||
self.add_vids_to_playlist()
|
||||
self.remove_vids_from_playlist()
|
||||
self.get_playlist_art()
|
||||
return True
|
||||
|
||||
def build_nav(self, youtube_id):
|
||||
"""find next and previous in playlist of a given youtube_id"""
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
all_entries_available = self.json_data["playlist_entries"]
|
||||
all_entries = [i for i in all_entries_available if i["downloaded"]]
|
||||
current = [i for i in all_entries if i["youtube_id"] == youtube_id]
|
||||
# stop if not found or playlist of 1
|
||||
if not current or not len(all_entries) > 1:
|
||||
return
|
||||
|
||||
current_idx = all_entries.index(current[0])
|
||||
if current_idx == 0:
|
||||
previous_item = False
|
||||
else:
|
||||
previous_item = all_entries[current_idx - 1]
|
||||
prev_id = previous_item["youtube_id"]
|
||||
prev_thumb_path = ThumbManager(prev_id).vid_thumb_path()
|
||||
previous_item["vid_thumb"] = f"{cache_root}/{prev_thumb_path}"
|
||||
|
||||
if current_idx == len(all_entries) - 1:
|
||||
next_item = False
|
||||
else:
|
||||
next_item = all_entries[current_idx + 1]
|
||||
next_id = next_item["youtube_id"]
|
||||
next_thumb_path = ThumbManager(next_id).vid_thumb_path()
|
||||
next_item["vid_thumb"] = f"{cache_root}/{next_thumb_path}"
|
||||
|
||||
self.nav = {
|
||||
"playlist_meta": {
|
||||
"current_idx": current[0]["idx"],
|
||||
"playlist_id": self.youtube_id,
|
||||
"playlist_name": self.json_data["playlist_name"],
|
||||
"playlist_channel": self.json_data["playlist_channel"],
|
||||
},
|
||||
"playlist_previous": previous_item,
|
||||
"playlist_next": next_item,
|
||||
}
|
||||
return
|
||||
|
||||
def delete_metadata(self):
|
||||
"""delete metadata for playlist"""
|
||||
self.delete_videos_metadata()
|
||||
script = (
|
||||
"ctx._source.playlist.removeAll("
|
||||
+ "Collections.singleton(params.playlist)) "
|
||||
)
|
||||
data = {
|
||||
"query": {
|
||||
"term": {"playlist.keyword": {"value": self.youtube_id}}
|
||||
},
|
||||
"script": {
|
||||
"source": script,
|
||||
"lang": "painless",
|
||||
"params": {"playlist": self.youtube_id},
|
||||
},
|
||||
}
|
||||
_, _ = ElasticWrap("ta_video/_update_by_query").post(data)
|
||||
self.del_in_es()
|
||||
|
||||
def is_custom_playlist(self):
|
||||
self.get_from_es()
|
||||
return self.json_data["playlist_type"] == "custom"
|
||||
|
||||
def delete_videos_metadata(self, channel_id=None):
|
||||
"""delete video metadata for a specific channel"""
|
||||
self.get_from_es()
|
||||
playlist = self.json_data["playlist_entries"]
|
||||
i = 0
|
||||
while i < len(playlist):
|
||||
video_id = playlist[i]["youtube_id"]
|
||||
video = ta_video.YoutubeVideo(video_id)
|
||||
video.get_from_es()
|
||||
if (
|
||||
channel_id is None
|
||||
or video.json_data["channel"]["channel_id"] == channel_id
|
||||
):
|
||||
playlist.pop(i)
|
||||
self.remove_playlist_from_video(video_id)
|
||||
i -= 1
|
||||
i += 1
|
||||
self.set_playlist_thumbnail()
|
||||
self.upload_to_es()
|
||||
|
||||
def delete_videos_playlist(self):
|
||||
"""delete playlist with all videos"""
|
||||
print(f"{self.youtube_id}: delete playlist")
|
||||
self.get_from_es()
|
||||
all_youtube_id = [
|
||||
i["youtube_id"]
|
||||
for i in self.json_data["playlist_entries"]
|
||||
if i["downloaded"]
|
||||
]
|
||||
for youtube_id in all_youtube_id:
|
||||
ta_video.YoutubeVideo(youtube_id).delete_media_file()
|
||||
|
||||
self.delete_metadata()
|
||||
|
||||
def create(self, name):
|
||||
self.json_data = {
|
||||
"playlist_id": self.youtube_id,
|
||||
"playlist_active": False,
|
||||
"playlist_name": name,
|
||||
"playlist_last_refresh": int(datetime.now().timestamp()),
|
||||
"playlist_entries": [],
|
||||
"playlist_type": "custom",
|
||||
"playlist_channel": None,
|
||||
"playlist_channel_id": None,
|
||||
"playlist_description": False,
|
||||
"playlist_thumbnail": False,
|
||||
"playlist_subscribed": False,
|
||||
}
|
||||
self.upload_to_es()
|
||||
self.get_playlist_art()
|
||||
return True
|
||||
|
||||
def add_video_to_playlist(self, video_id):
|
||||
self.get_from_es()
|
||||
video_metadata = self.get_video_metadata(video_id)
|
||||
video_metadata["idx"] = len(self.json_data["playlist_entries"])
|
||||
|
||||
if not self.playlist_entries_contains(video_id):
|
||||
self.json_data["playlist_entries"].append(video_metadata)
|
||||
self.json_data["playlist_last_refresh"] = int(
|
||||
datetime.now().timestamp()
|
||||
)
|
||||
self.set_playlist_thumbnail()
|
||||
self.upload_to_es()
|
||||
video = ta_video.YoutubeVideo(video_id)
|
||||
video.get_from_es()
|
||||
if "playlist" not in video.json_data:
|
||||
video.json_data["playlist"] = []
|
||||
video.json_data["playlist"].append(self.youtube_id)
|
||||
video.upload_to_es()
|
||||
return True
|
||||
|
||||
def remove_playlist_from_video(self, video_id):
|
||||
video = ta_video.YoutubeVideo(video_id)
|
||||
video.get_from_es()
|
||||
if video.json_data is not None and "playlist" in video.json_data:
|
||||
video.json_data["playlist"].remove(self.youtube_id)
|
||||
video.upload_to_es()
|
||||
|
||||
def move_video(self, video_id, action, hide_watched=False):
|
||||
self.get_from_es()
|
||||
video_index = self.get_video_index(video_id)
|
||||
playlist = self.json_data["playlist_entries"]
|
||||
item = playlist[video_index]
|
||||
playlist.pop(video_index)
|
||||
if action == "remove":
|
||||
self.remove_playlist_from_video(item["youtube_id"])
|
||||
else:
|
||||
if action == "up":
|
||||
while True:
|
||||
video_index = max(0, video_index - 1)
|
||||
if (
|
||||
not hide_watched
|
||||
or video_index == 0
|
||||
or (
|
||||
not self.get_video_is_watched(
|
||||
playlist[video_index]["youtube_id"]
|
||||
)
|
||||
)
|
||||
):
|
||||
break
|
||||
elif action == "down":
|
||||
while True:
|
||||
video_index = min(len(playlist), video_index + 1)
|
||||
if (
|
||||
not hide_watched
|
||||
or video_index == len(playlist)
|
||||
or (
|
||||
not self.get_video_is_watched(
|
||||
playlist[video_index - 1]["youtube_id"]
|
||||
)
|
||||
)
|
||||
):
|
||||
break
|
||||
elif action == "top":
|
||||
video_index = 0
|
||||
else:
|
||||
video_index = len(playlist)
|
||||
playlist.insert(video_index, item)
|
||||
self.json_data["playlist_last_refresh"] = int(
|
||||
datetime.now().timestamp()
|
||||
)
|
||||
|
||||
for i, item in enumerate(playlist):
|
||||
item["idx"] = i
|
||||
|
||||
self.set_playlist_thumbnail()
|
||||
self.upload_to_es()
|
||||
|
||||
return True
|
||||
|
||||
def del_video(self, video_id):
|
||||
playlist = self.json_data["playlist_entries"]
|
||||
|
||||
i = 0
|
||||
while i < len(playlist):
|
||||
if video_id == playlist[i]["youtube_id"]:
|
||||
playlist.pop(i)
|
||||
self.set_playlist_thumbnail()
|
||||
i -= 1
|
||||
i += 1
|
||||
|
||||
def get_video_index(self, video_id):
|
||||
for i, child in enumerate(self.json_data["playlist_entries"]):
|
||||
if child["youtube_id"] == video_id:
|
||||
return i
|
||||
return -1
|
||||
|
||||
def playlist_entries_contains(self, video_id):
|
||||
return (
|
||||
len(
|
||||
list(
|
||||
filter(
|
||||
lambda x: x["youtube_id"] == video_id,
|
||||
self.json_data["playlist_entries"],
|
||||
)
|
||||
)
|
||||
)
|
||||
> 0
|
||||
)
|
||||
|
||||
def get_video_is_watched(self, video_id):
|
||||
video = ta_video.YoutubeVideo(video_id)
|
||||
video.get_from_es()
|
||||
return video.json_data["player"]["watched"]
|
||||
|
||||
def set_playlist_thumbnail(self):
|
||||
playlist = self.json_data["playlist_entries"]
|
||||
self.json_data["playlist_thumbnail"] = False
|
||||
|
||||
for video in playlist:
|
||||
url = ThumbManager(video["youtube_id"]).vid_thumb_path()
|
||||
if url is not None:
|
||||
self.json_data["playlist_thumbnail"] = url
|
||||
break
|
||||
self.get_playlist_art()
|
||||
|
||||
def get_video_metadata(self, video_id):
|
||||
video = ta_video.YoutubeVideo(video_id)
|
||||
video.get_from_es()
|
||||
video_json_data = {
|
||||
"youtube_id": video.json_data["youtube_id"],
|
||||
"title": video.json_data["title"],
|
||||
"uploader": video.json_data["channel"]["channel_name"],
|
||||
"idx": 0,
|
||||
"downloaded": "date_downloaded" in video.json_data
|
||||
and video.json_data["date_downloaded"] > 0,
|
||||
}
|
||||
return video_json_data
|
||||
53
backend/playlist/src/query_building.py
Normal file
53
backend/playlist/src/query_building.py
Normal file
@@ -0,0 +1,53 @@
|
||||
"""build query for playlists"""
|
||||
|
||||
from playlist.src.constants import PlaylistTypesEnum
|
||||
|
||||
|
||||
class QueryBuilder:
|
||||
"""contain functionality"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.request_params = kwargs
|
||||
|
||||
def build_data(self) -> dict:
|
||||
"""build data dict"""
|
||||
data = {}
|
||||
data["query"] = self.build_query()
|
||||
if sort := self.parse_sort():
|
||||
data.update(sort)
|
||||
|
||||
return data
|
||||
|
||||
def build_query(self) -> dict:
|
||||
"""build query key"""
|
||||
must_list = []
|
||||
channel = self.request_params.get("channel")
|
||||
if channel:
|
||||
must_list.append({"match": {"playlist_channel_id": channel[0]}})
|
||||
|
||||
subscribed = self.request_params.get("subscribed")
|
||||
if subscribed:
|
||||
subed_bool = subscribed[0] == "true"
|
||||
must_list.append({"match": {"playlist_subscribed": subed_bool}})
|
||||
|
||||
playlist_type = self.request_params.get("type")
|
||||
if playlist_type:
|
||||
type_list = self.parse_type(playlist_type[0])
|
||||
must_list.append(type_list)
|
||||
|
||||
query = {"bool": {"must": must_list}}
|
||||
|
||||
return query
|
||||
|
||||
def parse_type(self, playlist_type: str) -> dict:
|
||||
"""parse playlist type"""
|
||||
if not hasattr(PlaylistTypesEnum, playlist_type.upper()):
|
||||
raise ValueError(f"'{playlist_type}' not in PlaylistTypesEnum")
|
||||
|
||||
type_parsed = getattr(PlaylistTypesEnum, playlist_type.upper()).value
|
||||
|
||||
return {"match": {"playlist_type.keyword": type_parsed}}
|
||||
|
||||
def parse_sort(self) -> dict:
|
||||
"""return sort"""
|
||||
return {"sort": [{"playlist_name.keyword": {"order": "asc"}}]}
|
||||
0
backend/playlist/tests/__init__.py
Normal file
0
backend/playlist/tests/__init__.py
Normal file
0
backend/playlist/tests/test_src/__init__.py
Normal file
0
backend/playlist/tests/test_src/__init__.py
Normal file
30
backend/playlist/tests/test_src/test_query_building.py
Normal file
30
backend/playlist/tests/test_src/test_query_building.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""test playlist query building"""
|
||||
|
||||
import pytest
|
||||
from playlist.src.query_building import QueryBuilder
|
||||
|
||||
|
||||
def test_build_data():
|
||||
"""test for correct key building"""
|
||||
qb = QueryBuilder(
|
||||
channel=["test_channel"],
|
||||
subscribed=["true"],
|
||||
type=["regular"],
|
||||
)
|
||||
result = qb.build_data()
|
||||
must_list = result["query"]["bool"]["must"]
|
||||
assert "query" in result
|
||||
assert "sort" in result
|
||||
assert result["sort"] == [{"playlist_name.keyword": {"order": "asc"}}]
|
||||
assert {"match": {"playlist_channel_id": "test_channel"}} in must_list
|
||||
assert {"match": {"playlist_subscribed": True}} in must_list
|
||||
|
||||
|
||||
def test_parse_type():
|
||||
"""validate type"""
|
||||
qb = QueryBuilder(type=["regular"])
|
||||
with pytest.raises(ValueError):
|
||||
qb.parse_type("invalid")
|
||||
|
||||
result = qb.parse_type("custom")
|
||||
assert result == {"match": {"playlist_type.keyword": "custom"}}
|
||||
17
backend/playlist/urls.py
Normal file
17
backend/playlist/urls.py
Normal file
@@ -0,0 +1,17 @@
|
||||
"""all playlist API urls"""
|
||||
|
||||
from django.urls import path
|
||||
from playlist import views
|
||||
|
||||
urlpatterns = [
|
||||
path(
|
||||
"",
|
||||
views.PlaylistApiListView.as_view(),
|
||||
name="api-playlist-list",
|
||||
),
|
||||
path(
|
||||
"<slug:playlist_id>/",
|
||||
views.PlaylistApiView.as_view(),
|
||||
name="api-playlist",
|
||||
),
|
||||
]
|
||||
116
backend/playlist/views.py
Normal file
116
backend/playlist/views.py
Normal file
@@ -0,0 +1,116 @@
|
||||
"""all playlist API views"""
|
||||
|
||||
from common.views_base import AdminWriteOnly, ApiBaseView
|
||||
from download.src.subscriptions import PlaylistSubscription
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
from playlist.src.query_building import QueryBuilder
|
||||
from rest_framework import status
|
||||
from rest_framework.response import Response
|
||||
from task.tasks import subscribe_to
|
||||
from user.src.user_config import UserConfig
|
||||
|
||||
|
||||
class PlaylistApiListView(ApiBaseView):
|
||||
"""resolves to /api/playlist/
|
||||
GET: returns list of indexed playlists
|
||||
params:
|
||||
- channel:str=<channel-id>
|
||||
- subscribed: bool
|
||||
- type:enum=regular|custom
|
||||
POST: change subscribe state
|
||||
"""
|
||||
|
||||
search_base = "ta_playlist/_search/"
|
||||
permission_classes = [AdminWriteOnly]
|
||||
|
||||
def get(self, request):
|
||||
"""get request"""
|
||||
try:
|
||||
data = QueryBuilder(**request.GET).build_data()
|
||||
except ValueError as err:
|
||||
return Response({"error": str(err)}, status=400)
|
||||
|
||||
self.data = data
|
||||
self.get_document_list(request)
|
||||
|
||||
return Response(self.response)
|
||||
|
||||
def post(self, request):
|
||||
"""subscribe/unsubscribe to list of playlists"""
|
||||
data = request.data
|
||||
try:
|
||||
to_add = data["data"]
|
||||
except KeyError:
|
||||
message = "missing expected data key"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
pending = []
|
||||
for playlist_item in to_add:
|
||||
playlist_id = playlist_item["playlist_id"]
|
||||
if playlist_item["playlist_subscribed"]:
|
||||
pending.append(playlist_id)
|
||||
else:
|
||||
self._unsubscribe(playlist_id)
|
||||
|
||||
if pending:
|
||||
url_str = " ".join(pending)
|
||||
subscribe_to.delay(url_str, expected_type="playlist")
|
||||
|
||||
return Response(data)
|
||||
|
||||
@staticmethod
|
||||
def _unsubscribe(playlist_id: str):
|
||||
"""unsubscribe"""
|
||||
print(f"[{playlist_id}] unsubscribe from playlist")
|
||||
PlaylistSubscription().change_subscribe(
|
||||
playlist_id, subscribe_status=False
|
||||
)
|
||||
|
||||
|
||||
class PlaylistApiView(ApiBaseView):
|
||||
"""resolves to /api/playlist/<playlist_id>/
|
||||
GET: returns metadata dict of playlist
|
||||
"""
|
||||
|
||||
search_base = "ta_playlist/_doc/"
|
||||
permission_classes = [AdminWriteOnly]
|
||||
valid_custom_actions = ["create", "remove", "up", "down", "top", "bottom"]
|
||||
|
||||
def get(self, request, playlist_id):
|
||||
# pylint: disable=unused-argument
|
||||
"""get request"""
|
||||
self.get_document(playlist_id)
|
||||
return Response(self.response, status=self.status_code)
|
||||
|
||||
def post(self, request, playlist_id):
|
||||
"""post to custom playlist to add a video to list"""
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
if not playlist.is_custom_playlist():
|
||||
message = f"playlist with ID {playlist_id} is not custom"
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
action = request.data.get("action")
|
||||
if action not in self.valid_custom_actions:
|
||||
message = f"invalid action: {action}"
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
video_id = request.data.get("video_id")
|
||||
if action == "create":
|
||||
playlist.add_video_to_playlist(video_id)
|
||||
else:
|
||||
hide = UserConfig(request.user.id).get_value("hide_watched")
|
||||
playlist.move_video(video_id, action, hide_watched=hide)
|
||||
|
||||
return Response({"success": True}, status=status.HTTP_201_CREATED)
|
||||
|
||||
def delete(self, request, playlist_id):
|
||||
"""delete playlist"""
|
||||
print(f"{playlist_id}: delete playlist")
|
||||
delete_videos = request.GET.get("delete-videos", False)
|
||||
if delete_videos:
|
||||
YoutubePlaylist(playlist_id).delete_videos_playlist()
|
||||
else:
|
||||
YoutubePlaylist(playlist_id).delete_metadata()
|
||||
|
||||
return Response({"success": True})
|
||||
12
backend/requirements-dev.txt
Normal file
12
backend/requirements-dev.txt
Normal file
@@ -0,0 +1,12 @@
|
||||
-r requirements.txt
|
||||
black
|
||||
codespell
|
||||
flake8
|
||||
ipython
|
||||
isort
|
||||
pylint
|
||||
pylint-django
|
||||
pytest
|
||||
pytest-django
|
||||
python-dotenv
|
||||
types-requests
|
||||
14
backend/requirements.txt
Normal file
14
backend/requirements.txt
Normal file
@@ -0,0 +1,14 @@
|
||||
apprise==1.8.1
|
||||
celery==5.4.0
|
||||
Django==5.0.7
|
||||
django-auth-ldap==4.8.0
|
||||
django-celery-beat==2.6.0
|
||||
django-cors-headers==4.4.0
|
||||
djangorestframework==3.15.2
|
||||
Pillow==10.4.0
|
||||
redis==5.0.8
|
||||
requests==2.32.3
|
||||
ryd-client==0.0.6
|
||||
uWSGI==2.0.26
|
||||
whitenoise==6.7.0
|
||||
yt-dlp==2024.8.1
|
||||
0
backend/stats/__init__.py
Normal file
0
backend/stats/__init__.py
Normal file
0
backend/stats/migrations/__init__.py
Normal file
0
backend/stats/migrations/__init__.py
Normal file
0
backend/stats/src/__init__.py
Normal file
0
backend/stats/src/__init__.py
Normal file
351
backend/stats/src/aggs.py
Normal file
351
backend/stats/src/aggs.py
Normal file
@@ -0,0 +1,351 @@
|
||||
"""aggregations"""
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.helper import get_duration_str
|
||||
|
||||
|
||||
class AggBase:
|
||||
"""base class for aggregation calls"""
|
||||
|
||||
path: str = ""
|
||||
data: dict = {}
|
||||
name: str = ""
|
||||
|
||||
def get(self):
|
||||
"""make get call"""
|
||||
response, _ = ElasticWrap(self.path).get(self.data)
|
||||
print(f"[agg][{self.name}] took {response.get('took')} ms to process")
|
||||
|
||||
return response.get("aggregations")
|
||||
|
||||
def process(self):
|
||||
"""implement in subclassess"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Video(AggBase):
|
||||
"""get video stats"""
|
||||
|
||||
name = "video_stats"
|
||||
path = "ta_video/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"video_type": {
|
||||
"terms": {"field": "vid_type"},
|
||||
"aggs": {
|
||||
"media_size": {"sum": {"field": "media_size"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
},
|
||||
},
|
||||
"video_active": {
|
||||
"terms": {"field": "active"},
|
||||
"aggs": {
|
||||
"media_size": {"sum": {"field": "media_size"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
},
|
||||
},
|
||||
"video_media_size": {"sum": {"field": "media_size"}},
|
||||
"video_count": {"value_count": {"field": "youtube_id"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process aggregation"""
|
||||
aggregations = self.get()
|
||||
|
||||
duration = int(aggregations["duration"]["value"])
|
||||
response = {
|
||||
"doc_count": aggregations["video_count"]["value"],
|
||||
"media_size": int(aggregations["video_media_size"]["value"]),
|
||||
"duration": duration,
|
||||
"duration_str": get_duration_str(duration),
|
||||
}
|
||||
for bucket in aggregations["video_type"]["buckets"]:
|
||||
duration = int(bucket["duration"].get("value"))
|
||||
response.update(
|
||||
{
|
||||
f"type_{bucket['key']}": {
|
||||
"doc_count": bucket.get("doc_count"),
|
||||
"media_size": int(bucket["media_size"].get("value")),
|
||||
"duration": duration,
|
||||
"duration_str": get_duration_str(duration),
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
for bucket in aggregations["video_active"]["buckets"]:
|
||||
duration = int(bucket["duration"].get("value"))
|
||||
response.update(
|
||||
{
|
||||
f"active_{bucket['key_as_string']}": {
|
||||
"doc_count": bucket.get("doc_count"),
|
||||
"media_size": int(bucket["media_size"].get("value")),
|
||||
"duration": duration,
|
||||
"duration_str": get_duration_str(duration),
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class Channel(AggBase):
|
||||
"""get channel stats"""
|
||||
|
||||
name = "channel_stats"
|
||||
path = "ta_channel/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"channel_count": {"value_count": {"field": "channel_id"}},
|
||||
"channel_active": {"terms": {"field": "channel_active"}},
|
||||
"channel_subscribed": {"terms": {"field": "channel_subscribed"}},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process aggregation"""
|
||||
aggregations = self.get()
|
||||
|
||||
response = {
|
||||
"doc_count": aggregations["channel_count"].get("value"),
|
||||
}
|
||||
for bucket in aggregations["channel_active"]["buckets"]:
|
||||
key = f"active_{bucket['key_as_string']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
for bucket in aggregations["channel_subscribed"]["buckets"]:
|
||||
key = f"subscribed_{bucket['key_as_string']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class Playlist(AggBase):
|
||||
"""get playlist stats"""
|
||||
|
||||
name = "playlist_stats"
|
||||
path = "ta_playlist/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"playlist_count": {"value_count": {"field": "playlist_id"}},
|
||||
"playlist_active": {"terms": {"field": "playlist_active"}},
|
||||
"playlist_subscribed": {"terms": {"field": "playlist_subscribed"}},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process aggregation"""
|
||||
aggregations = self.get()
|
||||
response = {"doc_count": aggregations["playlist_count"].get("value")}
|
||||
for bucket in aggregations["playlist_active"]["buckets"]:
|
||||
key = f"active_{bucket['key_as_string']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
for bucket in aggregations["playlist_subscribed"]["buckets"]:
|
||||
key = f"subscribed_{bucket['key_as_string']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class Download(AggBase):
|
||||
"""get downloads queue stats"""
|
||||
|
||||
name = "download_queue_stats"
|
||||
path = "ta_download/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"status": {"terms": {"field": "status"}},
|
||||
"video_type": {
|
||||
"filter": {"term": {"status": "pending"}},
|
||||
"aggs": {"type_pending": {"terms": {"field": "vid_type"}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process aggregation"""
|
||||
aggregations = self.get()
|
||||
response = {}
|
||||
for bucket in aggregations["status"]["buckets"]:
|
||||
response.update({bucket["key"]: bucket.get("doc_count")})
|
||||
|
||||
for bucket in aggregations["video_type"]["type_pending"]["buckets"]:
|
||||
key = f"pending_{bucket['key']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class WatchProgress(AggBase):
|
||||
"""get watch progress"""
|
||||
|
||||
name = "watch_progress"
|
||||
path = "ta_video/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
name: {
|
||||
"terms": {"field": "player.watched"},
|
||||
"aggs": {
|
||||
"watch_docs": {
|
||||
"filter": {"terms": {"player.watched": [True, False]}},
|
||||
"aggs": {
|
||||
"true_count": {"value_count": {"field": "_index"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"total_duration": {"sum": {"field": "player.duration"}},
|
||||
"total_vids": {"value_count": {"field": "_index"}},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""make the call"""
|
||||
aggregations = self.get()
|
||||
buckets = aggregations[self.name]["buckets"]
|
||||
|
||||
response = {}
|
||||
all_duration = int(aggregations["total_duration"].get("value"))
|
||||
response.update(
|
||||
{
|
||||
"total": {
|
||||
"duration": all_duration,
|
||||
"duration_str": get_duration_str(all_duration),
|
||||
"items": aggregations["total_vids"].get("value"),
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
for bucket in buckets:
|
||||
response.update(self._build_bucket(bucket, all_duration))
|
||||
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _build_bucket(bucket, all_duration):
|
||||
"""parse bucket"""
|
||||
|
||||
duration = int(bucket["watch_docs"]["duration"]["value"])
|
||||
duration_str = get_duration_str(duration)
|
||||
items = bucket["watch_docs"]["true_count"]["value"]
|
||||
if bucket["key_as_string"] == "false":
|
||||
key = "unwatched"
|
||||
else:
|
||||
key = "watched"
|
||||
|
||||
bucket_parsed = {
|
||||
key: {
|
||||
"duration": duration,
|
||||
"duration_str": duration_str,
|
||||
"progress": duration / all_duration if all_duration else 0,
|
||||
"items": items,
|
||||
}
|
||||
}
|
||||
|
||||
return bucket_parsed
|
||||
|
||||
|
||||
class DownloadHist(AggBase):
|
||||
"""get downloads histogram last week"""
|
||||
|
||||
name = "videos_last_week"
|
||||
path = "ta_video/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
name: {
|
||||
"date_histogram": {
|
||||
"field": "date_downloaded",
|
||||
"calendar_interval": "day",
|
||||
"format": "yyyy-MM-dd",
|
||||
"order": {"_key": "desc"},
|
||||
"time_zone": EnvironmentSettings.TZ,
|
||||
},
|
||||
"aggs": {
|
||||
"total_videos": {"value_count": {"field": "youtube_id"}},
|
||||
"media_size": {"sum": {"field": "media_size"}},
|
||||
},
|
||||
}
|
||||
},
|
||||
"query": {
|
||||
"range": {
|
||||
"date_downloaded": {
|
||||
"gte": "now-7d/d",
|
||||
"time_zone": EnvironmentSettings.TZ,
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process query"""
|
||||
aggregations = self.get()
|
||||
buckets = aggregations[self.name]["buckets"]
|
||||
|
||||
response = [
|
||||
{
|
||||
"date": i.get("key_as_string"),
|
||||
"count": i.get("doc_count"),
|
||||
"media_size": i["media_size"].get("value"),
|
||||
}
|
||||
for i in buckets
|
||||
]
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class BiggestChannel(AggBase):
|
||||
"""get channel aggregations"""
|
||||
|
||||
def __init__(self, order):
|
||||
self.data["aggs"][self.name]["multi_terms"]["order"] = {order: "desc"}
|
||||
|
||||
name = "channel_stats"
|
||||
path = "ta_video/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
name: {
|
||||
"multi_terms": {
|
||||
"terms": [
|
||||
{"field": "channel.channel_name.keyword"},
|
||||
{"field": "channel.channel_id"},
|
||||
],
|
||||
"order": {"doc_count": "desc"},
|
||||
},
|
||||
"aggs": {
|
||||
"doc_count": {"value_count": {"field": "_index"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
"media_size": {"sum": {"field": "media_size"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
order_choices = ["doc_count", "duration", "media_size"]
|
||||
|
||||
def process(self):
|
||||
"""process aggregation, order_by validated in the view"""
|
||||
|
||||
aggregations = self.get()
|
||||
buckets = aggregations[self.name]["buckets"]
|
||||
|
||||
response = [
|
||||
{
|
||||
"id": i["key"][1],
|
||||
"name": i["key"][0].title(),
|
||||
"doc_count": i["doc_count"]["value"],
|
||||
"duration": i["duration"]["value"],
|
||||
"duration_str": get_duration_str(int(i["duration"]["value"])),
|
||||
"media_size": i["media_size"]["value"],
|
||||
}
|
||||
for i in buckets
|
||||
]
|
||||
|
||||
return response
|
||||
42
backend/stats/urls.py
Normal file
42
backend/stats/urls.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""all stats API urls"""
|
||||
|
||||
from django.urls import path
|
||||
from stats import views
|
||||
|
||||
urlpatterns = [
|
||||
path(
|
||||
"video/",
|
||||
views.StatVideoView.as_view(),
|
||||
name="api-stats-video",
|
||||
),
|
||||
path(
|
||||
"channel/",
|
||||
views.StatChannelView.as_view(),
|
||||
name="api-stats-channel",
|
||||
),
|
||||
path(
|
||||
"playlist/",
|
||||
views.StatPlaylistView.as_view(),
|
||||
name="api-stats-playlist",
|
||||
),
|
||||
path(
|
||||
"download/",
|
||||
views.StatDownloadView.as_view(),
|
||||
name="api-stats-download",
|
||||
),
|
||||
path(
|
||||
"watch/",
|
||||
views.StatWatchProgress.as_view(),
|
||||
name="api-stats-watch",
|
||||
),
|
||||
path(
|
||||
"downloadhist/",
|
||||
views.StatDownloadHist.as_view(),
|
||||
name="api-stats-downloadhist",
|
||||
),
|
||||
path(
|
||||
"biggestchannels/",
|
||||
views.StatBiggestChannel.as_view(),
|
||||
name="api-stats-biggestchannels",
|
||||
),
|
||||
]
|
||||
104
backend/stats/views.py
Normal file
104
backend/stats/views.py
Normal file
@@ -0,0 +1,104 @@
|
||||
"""all stats API views"""
|
||||
|
||||
from common.views_base import ApiBaseView
|
||||
from rest_framework.response import Response
|
||||
from stats.src.aggs import (
|
||||
BiggestChannel,
|
||||
Channel,
|
||||
Download,
|
||||
DownloadHist,
|
||||
Playlist,
|
||||
Video,
|
||||
WatchProgress,
|
||||
)
|
||||
|
||||
|
||||
class StatVideoView(ApiBaseView):
|
||||
"""resolves to /api/stats/video/
|
||||
GET: return video stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""get stats"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(Video().process())
|
||||
|
||||
|
||||
class StatChannelView(ApiBaseView):
|
||||
"""resolves to /api/stats/channel/
|
||||
GET: return channel stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""get stats"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(Channel().process())
|
||||
|
||||
|
||||
class StatPlaylistView(ApiBaseView):
|
||||
"""resolves to /api/stats/playlist/
|
||||
GET: return playlist stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""get stats"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(Playlist().process())
|
||||
|
||||
|
||||
class StatDownloadView(ApiBaseView):
|
||||
"""resolves to /api/stats/download/
|
||||
GET: return download stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""get stats"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(Download().process())
|
||||
|
||||
|
||||
class StatWatchProgress(ApiBaseView):
|
||||
"""resolves to /api/stats/watchprogress/
|
||||
GET: return watch/unwatch progress stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(WatchProgress().process())
|
||||
|
||||
|
||||
class StatDownloadHist(ApiBaseView):
|
||||
"""resolves to /api/stats/downloadhist/
|
||||
GET: return download video count histogram for last days
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(DownloadHist().process())
|
||||
|
||||
|
||||
class StatBiggestChannel(ApiBaseView):
|
||||
"""resolves to /api/stats/biggestchannels/
|
||||
GET: return biggest channels
|
||||
param: order
|
||||
"""
|
||||
|
||||
order_choices = ["doc_count", "duration", "media_size"]
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
|
||||
order = request.GET.get("order", "doc_count")
|
||||
if order and order not in self.order_choices:
|
||||
message = {"message": f"invalid order parameter {order}"}
|
||||
return Response(message, status=400)
|
||||
|
||||
return Response(BiggestChannel(order).process())
|
||||
7
backend/task/__init__.py
Normal file
7
backend/task/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
"""start celery app"""
|
||||
|
||||
from __future__ import absolute_import, unicode_literals
|
||||
|
||||
from task.celery import app as celery_app
|
||||
|
||||
__all__ = ("celery_app",)
|
||||
22
backend/task/celery.py
Normal file
22
backend/task/celery.py
Normal file
@@ -0,0 +1,22 @@
|
||||
"""initiate celery"""
|
||||
|
||||
import os
|
||||
|
||||
from celery import Celery
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
|
||||
REDIS_HOST = EnvironmentSettings.REDIS_HOST
|
||||
REDIS_PORT = EnvironmentSettings.REDIS_PORT
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
||||
app = Celery(
|
||||
"tasks",
|
||||
broker=f"redis://{REDIS_HOST}:{REDIS_PORT}",
|
||||
backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
|
||||
result_extended=True,
|
||||
)
|
||||
app.config_from_object(
|
||||
"django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE
|
||||
)
|
||||
app.autodiscover_tasks()
|
||||
app.conf.timezone = EnvironmentSettings.TZ
|
||||
34
backend/task/migrations/0001_initial.py
Normal file
34
backend/task/migrations/0001_initial.py
Normal file
@@ -0,0 +1,34 @@
|
||||
# Generated by Django 5.0.7 on 2024-07-22 18:39
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
("django_celery_beat", "0018_improve_crontab_helptext"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="CustomPeriodicTask",
|
||||
fields=[
|
||||
(
|
||||
"periodictask_ptr",
|
||||
models.OneToOneField(
|
||||
auto_created=True,
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
parent_link=True,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
to="django_celery_beat.periodictask",
|
||||
),
|
||||
),
|
||||
("task_config", models.JSONField(default=dict)),
|
||||
],
|
||||
bases=("django_celery_beat.periodictask",),
|
||||
),
|
||||
]
|
||||
0
backend/task/migrations/__init__.py
Normal file
0
backend/task/migrations/__init__.py
Normal file
10
backend/task/models.py
Normal file
10
backend/task/models.py
Normal file
@@ -0,0 +1,10 @@
|
||||
"""task model"""
|
||||
|
||||
from django.db import models
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
|
||||
|
||||
class CustomPeriodicTask(PeriodicTask):
|
||||
"""add custom metadata to task"""
|
||||
|
||||
task_config = models.JSONField(default=dict)
|
||||
0
backend/task/src/__init__.py
Normal file
0
backend/task/src/__init__.py
Normal file
141
backend/task/src/config_schedule.py
Normal file
141
backend/task/src/config_schedule.py
Normal file
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
Functionality:
|
||||
- Handle scheduler config update
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from celery.schedules import crontab
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from django.utils import dateformat
|
||||
from django_celery_beat.models import CrontabSchedule
|
||||
from task.models import CustomPeriodicTask
|
||||
from task.src.task_config import TASK_CONFIG
|
||||
|
||||
|
||||
class ScheduleBuilder:
|
||||
"""build schedule dicts for beat"""
|
||||
|
||||
SCHEDULES = {
|
||||
"update_subscribed": "0 8 *",
|
||||
"download_pending": "0 16 *",
|
||||
"check_reindex": "0 12 *",
|
||||
"thumbnail_check": "0 17 *",
|
||||
"run_backup": "0 18 0",
|
||||
"version_check": "0 11 *",
|
||||
}
|
||||
MSG = "message:setting"
|
||||
|
||||
def __init__(self):
|
||||
self.config = AppConfig().config
|
||||
|
||||
def update_schedule(
|
||||
self, task_name: str, cron_schedule: str, schedule_conf: dict | None
|
||||
) -> None:
|
||||
"""update schedule"""
|
||||
if cron_schedule == "auto":
|
||||
cron_schedule = self.SCHEDULES[task_name]
|
||||
|
||||
if cron_schedule:
|
||||
_ = self.get_set_task(task_name, cron_schedule)
|
||||
|
||||
if schedule_conf:
|
||||
for key, value in schedule_conf.items():
|
||||
self.set_config(task_name, key, value)
|
||||
|
||||
def get_set_task(self, task_name, schedule=False):
|
||||
"""get task"""
|
||||
try:
|
||||
task = CustomPeriodicTask.objects.get(name=task_name)
|
||||
except CustomPeriodicTask.DoesNotExist:
|
||||
description = TASK_CONFIG[task_name].get("title")
|
||||
task = CustomPeriodicTask(
|
||||
name=task_name,
|
||||
task=task_name,
|
||||
description=description,
|
||||
)
|
||||
|
||||
if schedule:
|
||||
task_crontab = self.get_set_cron_tab(schedule)
|
||||
task.crontab = task_crontab
|
||||
task.last_run_at = dateformat.make_aware(datetime.now())
|
||||
task.save()
|
||||
|
||||
return task
|
||||
|
||||
@staticmethod
|
||||
def get_set_cron_tab(schedule: str) -> CrontabSchedule:
|
||||
"""needs to be validated before"""
|
||||
kwargs = dict(zip(["minute", "hour", "day_of_week"], schedule.split()))
|
||||
kwargs.update({"timezone": EnvironmentSettings.TZ})
|
||||
task_crontab, _ = CrontabSchedule.objects.get_or_create(**kwargs)
|
||||
|
||||
return task_crontab
|
||||
|
||||
def set_config(self, task_name: str, key: str, value) -> None:
|
||||
"""set task_config, validate before"""
|
||||
try:
|
||||
task = CustomPeriodicTask.objects.get(name=task_name)
|
||||
task.task_config.update({key: value})
|
||||
task.save()
|
||||
except CustomPeriodicTask.DoesNotExist:
|
||||
pass
|
||||
|
||||
|
||||
class CrontabValidator:
|
||||
"""validate crontab"""
|
||||
|
||||
CONFIG = {
|
||||
"check_reindex": ["days"],
|
||||
"run_backup": ["rotate"],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def validate_fields(cron_fields: str) -> None:
|
||||
"""expect 3 cron fields"""
|
||||
if not len(cron_fields) == 3:
|
||||
raise ValueError("expected three cron schedule fields")
|
||||
|
||||
@staticmethod
|
||||
def validate_minute(minute_field: str):
|
||||
"""expect minute int"""
|
||||
if not minute_field.isdigit():
|
||||
raise ValueError("Invalid value for minutes. Must be an integer.")
|
||||
|
||||
minutes = int(minute_field)
|
||||
if not 0 <= minutes <= 59:
|
||||
raise ValueError("Invalid minutes. Must be between 0 and 59.")
|
||||
|
||||
@staticmethod
|
||||
def validate_cron_tab(minute, hour, day_of_week):
|
||||
"""check if crontab can be created"""
|
||||
try:
|
||||
crontab(minute=minute, hour=hour, day_of_week=day_of_week)
|
||||
except ValueError as err:
|
||||
raise ValueError(f"invalid crontab: {err}") from err
|
||||
|
||||
def validate_cron(self, cron_expression):
|
||||
"""create crontab schedule"""
|
||||
if not cron_expression or cron_expression == "auto":
|
||||
return
|
||||
|
||||
cron_fields = cron_expression.split()
|
||||
self.validate_fields(cron_fields)
|
||||
|
||||
minute, hour, day_of_week = cron_fields
|
||||
self.validate_minute(minute)
|
||||
self.validate_cron_tab(minute, hour, day_of_week)
|
||||
|
||||
def validate_config(self, task_name: str, schedule_config: dict):
|
||||
"""validate config for given task"""
|
||||
if not schedule_config:
|
||||
return
|
||||
|
||||
config_keys = self.CONFIG.get(task_name)
|
||||
if not config_keys:
|
||||
raise ValueError(f"task '{task_name}' doesn't take config")
|
||||
|
||||
for key in schedule_config:
|
||||
if key not in config_keys:
|
||||
raise ValueError(f"invalid config key for task '{task_name}'")
|
||||
141
backend/task/src/notify.py
Normal file
141
backend/task/src/notify.py
Normal file
@@ -0,0 +1,141 @@
|
||||
"""send notifications using apprise"""
|
||||
|
||||
import apprise
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from task.src.task_config import TASK_CONFIG
|
||||
from task.src.task_manager import TaskManager
|
||||
|
||||
|
||||
class Notifications:
|
||||
"""store notifications in ES"""
|
||||
|
||||
GET_PATH = "ta_config/_doc/notify"
|
||||
UPDATE_PATH = "ta_config/_update/notify/"
|
||||
|
||||
def __init__(self, task_name: str):
|
||||
self.task_name = task_name
|
||||
|
||||
def send(self, task_id: str, task_title: str) -> None:
|
||||
"""send notifications"""
|
||||
apobj = apprise.Apprise()
|
||||
urls: list[str] = self.get_urls()
|
||||
if not urls:
|
||||
return
|
||||
|
||||
title, body = self._build_message(task_id, task_title)
|
||||
|
||||
if not body:
|
||||
return
|
||||
|
||||
for url in urls:
|
||||
apobj.add(url)
|
||||
|
||||
apobj.notify(body=body, title=title)
|
||||
|
||||
def _build_message(
|
||||
self, task_id: str, task_title: str
|
||||
) -> tuple[str, str | None]:
|
||||
"""build message to send notification"""
|
||||
task = TaskManager().get_task(task_id)
|
||||
status = task.get("status")
|
||||
title: str = f"[TA] {task_title} process ended with {status}"
|
||||
body: str | None = task.get("result")
|
||||
|
||||
return title, body
|
||||
|
||||
def get_urls(self) -> list[str]:
|
||||
"""get stored urls for task"""
|
||||
response, code = ElasticWrap(self.GET_PATH).get(print_error=False)
|
||||
if not code == 200:
|
||||
return []
|
||||
|
||||
urls = response["_source"].get(self.task_name, [])
|
||||
|
||||
return urls
|
||||
|
||||
def add_url(self, url: str) -> None:
|
||||
"""add url to task notification"""
|
||||
source = (
|
||||
"if (!ctx._source.containsKey(params.task_name)) "
|
||||
+ "{ctx._source[params.task_name] = [params.url]} "
|
||||
+ "else if (!ctx._source[params.task_name].contains(params.url)) "
|
||||
+ "{ctx._source[params.task_name].add(params.url)} "
|
||||
+ "else {ctx.op = 'none'}"
|
||||
)
|
||||
|
||||
data = {
|
||||
"script": {
|
||||
"source": source,
|
||||
"lang": "painless",
|
||||
"params": {"url": url, "task_name": self.task_name},
|
||||
},
|
||||
"upsert": {self.task_name: [url]},
|
||||
}
|
||||
|
||||
_, _ = ElasticWrap(self.UPDATE_PATH).post(data)
|
||||
|
||||
def remove_url(self, url: str) -> tuple[dict, int]:
|
||||
"""remove url from task"""
|
||||
source = (
|
||||
"if (ctx._source.containsKey(params.task_name) "
|
||||
+ "&& ctx._source[params.task_name].contains(params.url)) "
|
||||
+ "{ctx._source[params.task_name]."
|
||||
+ "remove(ctx._source[params.task_name].indexOf(params.url))}"
|
||||
)
|
||||
|
||||
data = {
|
||||
"script": {
|
||||
"source": source,
|
||||
"lang": "painless",
|
||||
"params": {"url": url, "task_name": self.task_name},
|
||||
}
|
||||
}
|
||||
|
||||
response, status_code = ElasticWrap(self.UPDATE_PATH).post(data)
|
||||
if not self.get_urls():
|
||||
_, _ = self.remove_task()
|
||||
|
||||
return response, status_code
|
||||
|
||||
def remove_task(self) -> tuple[dict, int]:
|
||||
"""remove all notifications from task"""
|
||||
source = (
|
||||
"if (ctx._source.containsKey(params.task_name)) "
|
||||
+ "{ctx._source.remove(params.task_name)}"
|
||||
)
|
||||
data = {
|
||||
"script": {
|
||||
"source": source,
|
||||
"lang": "painless",
|
||||
"params": {"task_name": self.task_name},
|
||||
}
|
||||
}
|
||||
|
||||
response, status_code = ElasticWrap(self.UPDATE_PATH).post(data)
|
||||
|
||||
return response, status_code
|
||||
|
||||
|
||||
def get_all_notifications() -> dict[str, list[str]]:
|
||||
"""get all notifications stored"""
|
||||
path = "ta_config/_doc/notify"
|
||||
response, status_code = ElasticWrap(path).get(print_error=False)
|
||||
if not status_code == 200:
|
||||
return {}
|
||||
|
||||
notifications: dict = {}
|
||||
source = response.get("_source")
|
||||
if not source:
|
||||
return notifications
|
||||
|
||||
for task_id, urls in source.items():
|
||||
notifications.update(
|
||||
{
|
||||
task_id: {
|
||||
"urls": urls,
|
||||
"title": TASK_CONFIG[task_id]["title"],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return notifications
|
||||
125
backend/task/src/task_config.py
Normal file
125
backend/task/src/task_config.py
Normal file
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
Functionality:
|
||||
- Static Task config values
|
||||
- Type definitions
|
||||
- separate to avoid circular imports
|
||||
"""
|
||||
|
||||
from typing import TypedDict
|
||||
|
||||
|
||||
class TaskItemConfig(TypedDict):
|
||||
"""describes a task item config"""
|
||||
|
||||
title: str
|
||||
group: str
|
||||
api_start: bool
|
||||
api_stop: bool
|
||||
|
||||
|
||||
UPDATE_SUBSCRIBED: TaskItemConfig = {
|
||||
"title": "Rescan your Subscriptions",
|
||||
"group": "download:scan",
|
||||
"api_start": True,
|
||||
"api_stop": True,
|
||||
}
|
||||
|
||||
DOWNLOAD_PENDING: TaskItemConfig = {
|
||||
"title": "Downloading",
|
||||
"group": "download:run",
|
||||
"api_start": True,
|
||||
"api_stop": True,
|
||||
}
|
||||
|
||||
EXTRACT_DOWNLOAD: TaskItemConfig = {
|
||||
"title": "Add to download queue",
|
||||
"group": "download:add",
|
||||
"api_start": False,
|
||||
"api_stop": True,
|
||||
}
|
||||
|
||||
CHECK_REINDEX: TaskItemConfig = {
|
||||
"title": "Reindex Documents",
|
||||
"group": "reindex:run",
|
||||
"api_start": False,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
MANUAL_IMPORT: TaskItemConfig = {
|
||||
"title": "Manual video import",
|
||||
"group": "setting:import",
|
||||
"api_start": True,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
RUN_BACKUP: TaskItemConfig = {
|
||||
"title": "Index Backup",
|
||||
"group": "setting:backup",
|
||||
"api_start": True,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
RESTORE_BACKUP: TaskItemConfig = {
|
||||
"title": "Restore Backup",
|
||||
"group": "setting:restore",
|
||||
"api_start": False,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
RESCAN_FILESYSTEM: TaskItemConfig = {
|
||||
"title": "Rescan your Filesystem",
|
||||
"group": "setting:filesystemscan",
|
||||
"api_start": True,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
THUMBNAIL_CHECK: TaskItemConfig = {
|
||||
"title": "Check your Thumbnails",
|
||||
"group": "setting:thumbnailcheck",
|
||||
"api_start": True,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
RESYNC_THUMBS: TaskItemConfig = {
|
||||
"title": "Sync Thumbnails to Media Files",
|
||||
"group": "setting:thumbnailsync",
|
||||
"api_start": True,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
INDEX_PLAYLISTS: TaskItemConfig = {
|
||||
"title": "Index Channel Playlist",
|
||||
"group": "channel:indexplaylist",
|
||||
"api_start": False,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
SUBSCRIBE_TO: TaskItemConfig = {
|
||||
"title": "Add Subscription",
|
||||
"group": "subscription:add",
|
||||
"api_start": False,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
VERSION_CHECK: TaskItemConfig = {
|
||||
"title": "Look for new Version",
|
||||
"group": "",
|
||||
"api_start": False,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
TASK_CONFIG: dict[str, TaskItemConfig] = {
|
||||
"update_subscribed": UPDATE_SUBSCRIBED,
|
||||
"download_pending": DOWNLOAD_PENDING,
|
||||
"extract_download": EXTRACT_DOWNLOAD,
|
||||
"check_reindex": CHECK_REINDEX,
|
||||
"manual_import": MANUAL_IMPORT,
|
||||
"run_backup": RUN_BACKUP,
|
||||
"restore_backup": RESTORE_BACKUP,
|
||||
"rescan_filesystem": RESCAN_FILESYSTEM,
|
||||
"thumbnail_check": THUMBNAIL_CHECK,
|
||||
"resync_thumbs": RESYNC_THUMBS,
|
||||
"index_playlists": INDEX_PLAYLISTS,
|
||||
"subscribe_to": SUBSCRIBE_TO,
|
||||
"version_check": VERSION_CHECK,
|
||||
}
|
||||
117
backend/task/src/task_manager.py
Normal file
117
backend/task/src/task_manager.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
functionality:
|
||||
- interact with in redis stored task results
|
||||
- handle threads and locks
|
||||
"""
|
||||
|
||||
from common.src.ta_redis import RedisArchivist, TaskRedis
|
||||
from task.celery import app as celery_app
|
||||
from task.src.task_config import TASK_CONFIG
|
||||
|
||||
|
||||
class TaskManager:
|
||||
"""manage tasks"""
|
||||
|
||||
def get_all_results(self):
|
||||
"""return all task results"""
|
||||
handler = TaskRedis()
|
||||
all_keys = handler.get_all()
|
||||
if not all_keys:
|
||||
return False
|
||||
|
||||
return [handler.get_single(i) for i in all_keys]
|
||||
|
||||
def get_tasks_by_name(self, task_name):
|
||||
"""get all tasks by name"""
|
||||
all_results = self.get_all_results()
|
||||
if not all_results:
|
||||
return False
|
||||
|
||||
return [i for i in all_results if i.get("name") == task_name]
|
||||
|
||||
def get_task(self, task_id):
|
||||
"""get single task"""
|
||||
return TaskRedis().get_single(task_id)
|
||||
|
||||
def is_pending(self, task):
|
||||
"""check if task_name is pending, pass task object"""
|
||||
tasks = self.get_tasks_by_name(task.name)
|
||||
if not tasks:
|
||||
return False
|
||||
|
||||
return bool([i for i in tasks if i.get("status") == "PENDING"])
|
||||
|
||||
def is_stopped(self, task_id):
|
||||
"""check if task_id has received STOP command"""
|
||||
task = self.get_task(task_id)
|
||||
|
||||
return task.get("command") == "STOP"
|
||||
|
||||
def get_pending(self, task_name):
|
||||
"""get all pending tasks of task_name"""
|
||||
tasks = self.get_tasks_by_name(task_name)
|
||||
if not tasks:
|
||||
return False
|
||||
|
||||
return [i for i in tasks if i.get("status") == "PENDING"]
|
||||
|
||||
def init(self, task):
|
||||
"""pass task object from bind task to set initial pending message"""
|
||||
message = {
|
||||
"status": "PENDING",
|
||||
"result": None,
|
||||
"traceback": None,
|
||||
"date_done": False,
|
||||
"name": task.name,
|
||||
"task_id": task.request.id,
|
||||
}
|
||||
TaskRedis().set_key(task.request.id, message)
|
||||
|
||||
def fail_pending(self):
|
||||
"""
|
||||
mark all pending as failed,
|
||||
run at startup to recover from hard reset
|
||||
"""
|
||||
all_results = self.get_all_results()
|
||||
if not all_results:
|
||||
return
|
||||
|
||||
for result in all_results:
|
||||
if result.get("status") == "PENDING":
|
||||
result["status"] = "FAILED"
|
||||
TaskRedis().set_key(result["task_id"], result, expire=True)
|
||||
|
||||
|
||||
class TaskCommand:
|
||||
"""run commands on task"""
|
||||
|
||||
def start(self, task_name):
|
||||
"""start task by task_name, only pass task that don't take args"""
|
||||
task = celery_app.tasks.get(task_name).delay()
|
||||
message = {
|
||||
"task_id": task.id,
|
||||
"status": task.status,
|
||||
"task_name": task.name,
|
||||
}
|
||||
|
||||
return message
|
||||
|
||||
def stop(self, task_id, message_key):
|
||||
"""
|
||||
send stop signal to task_id,
|
||||
needs to be implemented in task to take effect
|
||||
"""
|
||||
print(f"[task][{task_id}]: received STOP signal.")
|
||||
handler = TaskRedis()
|
||||
|
||||
task = handler.get_single(task_id)
|
||||
if not task["name"] in TASK_CONFIG:
|
||||
raise ValueError
|
||||
|
||||
handler.set_command(task_id, "STOP")
|
||||
RedisArchivist().set_message(message_key, "STOP", path=".command")
|
||||
|
||||
def kill(self, task_id):
|
||||
"""send kill signal to task_id"""
|
||||
print(f"[task][{task_id}]: received KILL signal.")
|
||||
celery_app.control.revoke(task_id, terminate=True)
|
||||
312
backend/task/tasks.py
Normal file
312
backend/task/tasks.py
Normal file
@@ -0,0 +1,312 @@
|
||||
"""
|
||||
Functionality:
|
||||
- collect tasks
|
||||
- handle task callbacks
|
||||
- handle task notifications
|
||||
- handle task locking
|
||||
"""
|
||||
|
||||
from appsettings.src.backup import ElasticBackup
|
||||
from appsettings.src.config import ReleaseVersion
|
||||
from appsettings.src.filesystem import Scanner
|
||||
from appsettings.src.index_setup import ElasitIndexWrap
|
||||
from appsettings.src.manual import ImportFolderScanner
|
||||
from appsettings.src.reindex import Reindex, ReindexManual, ReindexPopulate
|
||||
from celery import Task, shared_task
|
||||
from celery.exceptions import Retry
|
||||
from channel.src.index import YoutubeChannel
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
from common.src.urlparser import Parser
|
||||
from download.src.queue import PendingList
|
||||
from download.src.subscriptions import SubscriptionHandler, SubscriptionScanner
|
||||
from download.src.thumbnails import ThumbFilesystem, ThumbValidator
|
||||
from download.src.yt_dlp_handler import VideoDownloader
|
||||
from task.src.notify import Notifications
|
||||
from task.src.task_config import TASK_CONFIG
|
||||
from task.src.task_manager import TaskManager
|
||||
|
||||
|
||||
class BaseTask(Task):
|
||||
"""base class to inherit each class from"""
|
||||
|
||||
# pylint: disable=abstract-method
|
||||
|
||||
def on_failure(self, exc, task_id, args, kwargs, einfo):
|
||||
"""callback for task failure"""
|
||||
print(f"{task_id} Failed callback")
|
||||
message, key = self._build_message(level="error")
|
||||
message.update({"messages": [f"Task failed: {exc}"]})
|
||||
RedisArchivist().set_message(key, message, expire=20)
|
||||
|
||||
def on_success(self, retval, task_id, args, kwargs):
|
||||
"""callback task completed successfully"""
|
||||
print(f"{task_id} success callback")
|
||||
message, key = self._build_message()
|
||||
message.update({"messages": ["Task completed successfully"]})
|
||||
RedisArchivist().set_message(key, message, expire=5)
|
||||
|
||||
def before_start(self, task_id, args, kwargs):
|
||||
"""callback before initiating task"""
|
||||
print(f"{self.name} create callback")
|
||||
message, key = self._build_message()
|
||||
message.update({"messages": ["New task received."]})
|
||||
RedisArchivist().set_message(key, message)
|
||||
|
||||
def after_return(self, status, retval, task_id, args, kwargs, einfo):
|
||||
"""callback after task returns"""
|
||||
print(f"{task_id} return callback")
|
||||
task_title = TASK_CONFIG.get(self.name).get("title")
|
||||
Notifications(self.name).send(task_id, task_title)
|
||||
|
||||
def send_progress(self, message_lines, progress=False, title=False):
|
||||
"""send progress message"""
|
||||
message, key = self._build_message()
|
||||
message.update(
|
||||
{
|
||||
"messages": message_lines,
|
||||
"progress": progress,
|
||||
}
|
||||
)
|
||||
if title:
|
||||
message["title"] = title
|
||||
|
||||
RedisArchivist().set_message(key, message)
|
||||
|
||||
def _build_message(self, level="info"):
|
||||
"""build message dict"""
|
||||
task_id = self.request.id
|
||||
message = TASK_CONFIG.get(self.name).copy()
|
||||
message.update({"level": level, "id": task_id})
|
||||
task_result = TaskManager().get_task(task_id)
|
||||
if task_result:
|
||||
command = task_result.get("command", False)
|
||||
message.update({"command": command})
|
||||
|
||||
key = f"message:{message.get('group')}:{task_id.split('-')[0]}"
|
||||
return message, key
|
||||
|
||||
def is_stopped(self):
|
||||
"""check if task is stopped"""
|
||||
return TaskManager().is_stopped(self.request.id)
|
||||
|
||||
|
||||
@shared_task(name="update_subscribed", bind=True, base=BaseTask)
|
||||
def update_subscribed(self):
|
||||
"""look for missing videos and add to pending"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] rescan already running")
|
||||
self.send_progress("Rescan already in progress.")
|
||||
return None
|
||||
|
||||
manager.init(self)
|
||||
handler = SubscriptionScanner(task=self)
|
||||
missing_videos = handler.scan()
|
||||
auto_start = handler.auto_start
|
||||
if missing_videos:
|
||||
print(missing_videos)
|
||||
extrac_dl.delay(missing_videos, auto_start=auto_start)
|
||||
message = f"Found {len(missing_videos)} videos to add to the queue."
|
||||
return message
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@shared_task(
|
||||
name="download_pending",
|
||||
bind=True,
|
||||
base=BaseTask,
|
||||
max_retries=3,
|
||||
default_retry_delay=10,
|
||||
)
|
||||
def download_pending(self, auto_only=False):
|
||||
"""download latest pending videos"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] download queue already running")
|
||||
self.send_progress("Download Queue is already running.")
|
||||
return None
|
||||
|
||||
manager.init(self)
|
||||
try:
|
||||
downloader = VideoDownloader(task=self)
|
||||
downloaded, failed = downloader.run_queue(auto_only=auto_only)
|
||||
|
||||
if failed:
|
||||
print(f"[task][{self.name}] Videos failed, retry.")
|
||||
self.send_progress("Videos failed, retry.")
|
||||
raise self.retry()
|
||||
|
||||
except Retry as exc:
|
||||
raise exc
|
||||
|
||||
if downloaded:
|
||||
return f"downloaded {downloaded} video(s)."
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@shared_task(name="extract_download", bind=True, base=BaseTask)
|
||||
def extrac_dl(self, youtube_ids, auto_start=False, status="pending"):
|
||||
"""parse list passed and add to pending"""
|
||||
TaskManager().init(self)
|
||||
if isinstance(youtube_ids, str):
|
||||
to_add = Parser(youtube_ids).parse()
|
||||
else:
|
||||
to_add = youtube_ids
|
||||
|
||||
pending_handler = PendingList(youtube_ids=to_add, task=self)
|
||||
pending_handler.parse_url_list()
|
||||
videos_added = pending_handler.add_to_pending(
|
||||
status=status, auto_start=auto_start
|
||||
)
|
||||
|
||||
if auto_start:
|
||||
download_pending.delay(auto_only=True)
|
||||
|
||||
if videos_added:
|
||||
return f"added {len(videos_added)} Videos to Queue"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@shared_task(bind=True, name="check_reindex", base=BaseTask)
|
||||
def check_reindex(self, data=False, extract_videos=False):
|
||||
"""run the reindex main command"""
|
||||
if data:
|
||||
# started from frontend through API
|
||||
print(f"[task][{self.name}] reindex {data}")
|
||||
self.send_progress("Add items to the reindex Queue.")
|
||||
ReindexManual(extract_videos=extract_videos).extract_data(data)
|
||||
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] reindex queue is already running")
|
||||
self.send_progress("Reindex Queue is already running.")
|
||||
return
|
||||
|
||||
manager.init(self)
|
||||
if not data:
|
||||
# started from scheduler
|
||||
populate = ReindexPopulate()
|
||||
print(f"[task][{self.name}] reindex outdated documents")
|
||||
self.send_progress("Add recent documents to the reindex Queue.")
|
||||
populate.get_interval()
|
||||
populate.add_recent()
|
||||
self.send_progress("Add outdated documents to the reindex Queue.")
|
||||
populate.add_outdated()
|
||||
|
||||
handler = Reindex(task=self)
|
||||
handler.reindex_all()
|
||||
|
||||
return handler.build_message()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="manual_import", base=BaseTask)
|
||||
def run_manual_import(self):
|
||||
"""called from settings page, to go through import folder"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] manual import is already running")
|
||||
self.send_progress("Manual import is already running.")
|
||||
return
|
||||
|
||||
manager.init(self)
|
||||
ImportFolderScanner(task=self).scan()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="run_backup", base=BaseTask)
|
||||
def run_backup(self, reason="auto"):
|
||||
"""called from settings page, dump backup to zip file"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] backup is already running")
|
||||
self.send_progress("Backup is already running.")
|
||||
return
|
||||
|
||||
manager.init(self)
|
||||
ElasticBackup(reason=reason, task=self).backup_all_indexes()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="restore_backup", base=BaseTask)
|
||||
def run_restore_backup(self, filename):
|
||||
"""called from settings page, dump backup to zip file"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] restore is already running")
|
||||
self.send_progress("Restore is already running.")
|
||||
return None
|
||||
|
||||
manager.init(self)
|
||||
self.send_progress(["Reset your Index"])
|
||||
ElasitIndexWrap().reset()
|
||||
ElasticBackup(task=self).restore(filename)
|
||||
print("index restore finished")
|
||||
|
||||
return f"backup restore completed: {filename}"
|
||||
|
||||
|
||||
@shared_task(bind=True, name="rescan_filesystem", base=BaseTask)
|
||||
def rescan_filesystem(self):
|
||||
"""check the media folder for mismatches"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] filesystem rescan already running")
|
||||
self.send_progress("Filesystem Rescan is already running.")
|
||||
return
|
||||
|
||||
manager.init(self)
|
||||
handler = Scanner(task=self)
|
||||
handler.scan()
|
||||
handler.apply()
|
||||
ThumbValidator(task=self).validate()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="thumbnail_check", base=BaseTask)
|
||||
def thumbnail_check(self):
|
||||
"""validate thumbnails"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] thumbnail check is already running")
|
||||
self.send_progress("Thumbnail check is already running.")
|
||||
return
|
||||
|
||||
manager.init(self)
|
||||
thumnail = ThumbValidator(task=self)
|
||||
thumnail.validate()
|
||||
thumnail.clean_up()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="resync_thumbs", base=BaseTask)
|
||||
def re_sync_thumbs(self):
|
||||
"""sync thumbnails to mediafiles"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] thumb re-embed is already running")
|
||||
self.send_progress("Thumbnail re-embed is already running.")
|
||||
return
|
||||
|
||||
manager.init(self)
|
||||
ThumbFilesystem(task=self).embed()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="subscribe_to", base=BaseTask)
|
||||
def subscribe_to(self, url_str: str, expected_type: str | bool = False):
|
||||
"""
|
||||
take a list of urls to subscribe to
|
||||
optionally validate expected_type channel / playlist
|
||||
"""
|
||||
SubscriptionHandler(url_str, task=self).subscribe(expected_type)
|
||||
|
||||
|
||||
@shared_task(bind=True, name="index_playlists", base=BaseTask)
|
||||
def index_channel_playlists(self, channel_id):
|
||||
"""add all playlists of channel to index"""
|
||||
channel = YoutubeChannel(channel_id, task=self)
|
||||
channel.index_channel_playlists()
|
||||
|
||||
|
||||
@shared_task(name="version_check")
|
||||
def version_check():
|
||||
"""check for new updates"""
|
||||
ReleaseVersion().check()
|
||||
0
backend/task/tests/__init__.py
Normal file
0
backend/task/tests/__init__.py
Normal file
0
backend/task/tests/test_src/__init__.py
Normal file
0
backend/task/tests/test_src/__init__.py
Normal file
68
backend/task/tests/test_src/test_config_schedule.py
Normal file
68
backend/task/tests/test_src/test_config_schedule.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""test schedule parsing"""
|
||||
|
||||
# flake8: noqa: E402
|
||||
|
||||
import os
|
||||
|
||||
import django
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
||||
django.setup()
|
||||
|
||||
import pytest
|
||||
from task.src.config_schedule import CrontabValidator
|
||||
|
||||
INCORRECT_CRONTAB = [
|
||||
"0 0 * * *",
|
||||
"0 0",
|
||||
"0",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_value", INCORRECT_CRONTAB)
|
||||
def test_invalid_len(invalid_value):
|
||||
"""raise error on invalid crontab"""
|
||||
validator = CrontabValidator()
|
||||
with pytest.raises(ValueError, match="three cron schedule fields"):
|
||||
validator.validate_cron(invalid_value)
|
||||
|
||||
|
||||
NONE_INT_MINUTE = [
|
||||
"* * *",
|
||||
"0,30 * *",
|
||||
"0,1,2 * *",
|
||||
"-1 * *",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_value", NONE_INT_MINUTE)
|
||||
def test_none_int_crontabs(invalid_value):
|
||||
"""raise error on invalid crontab"""
|
||||
validator = CrontabValidator()
|
||||
with pytest.raises(ValueError, match="Must be an integer."):
|
||||
validator.validate_cron(invalid_value)
|
||||
|
||||
|
||||
INVALID_MINUTE = ["60 * *", "61 * *"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_value", INVALID_MINUTE)
|
||||
def test_invalid_minute(invalid_value):
|
||||
"""raise error on invalid crontab"""
|
||||
validator = CrontabValidator()
|
||||
with pytest.raises(ValueError, match="Must be between 0 and 59."):
|
||||
validator.validate_cron(invalid_value)
|
||||
|
||||
|
||||
INVALID_CRONTAB = [
|
||||
"0 /1 *",
|
||||
"0 0/1 *",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_value", INVALID_CRONTAB)
|
||||
def test_invalid_crontab(invalid_value):
|
||||
"""raise error on invalid crontab"""
|
||||
validator = CrontabValidator()
|
||||
with pytest.raises(ValueError, match="invalid crontab"):
|
||||
validator.validate_cron(invalid_value)
|
||||
32
backend/task/urls.py
Normal file
32
backend/task/urls.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""all tasks api URLs"""
|
||||
|
||||
from django.urls import path
|
||||
from task import views
|
||||
|
||||
urlpatterns = [
|
||||
path(
|
||||
"by-name/",
|
||||
views.TaskListView.as_view(),
|
||||
name="api-task-list",
|
||||
),
|
||||
path(
|
||||
"by-name/<slug:task_name>/",
|
||||
views.TaskNameListView.as_view(),
|
||||
name="api-task-name-list",
|
||||
),
|
||||
path(
|
||||
"by-id/<slug:task_id>/",
|
||||
views.TaskIDView.as_view(),
|
||||
name="api-task-id",
|
||||
),
|
||||
path(
|
||||
"schedule/<slug:task_name>/",
|
||||
views.ScheduleView.as_view(),
|
||||
name="api-schedule",
|
||||
),
|
||||
path(
|
||||
"notification/",
|
||||
views.ScheduleNotification.as_view(),
|
||||
name="api-schedule-notification",
|
||||
),
|
||||
]
|
||||
207
backend/task/views.py
Normal file
207
backend/task/views.py
Normal file
@@ -0,0 +1,207 @@
|
||||
"""all task API views"""
|
||||
|
||||
from common.views_base import AdminOnly, ApiBaseView
|
||||
from django.shortcuts import get_object_or_404
|
||||
from rest_framework.response import Response
|
||||
from task.models import CustomPeriodicTask
|
||||
from task.src.config_schedule import CrontabValidator, ScheduleBuilder
|
||||
from task.src.notify import Notifications, get_all_notifications
|
||||
from task.src.task_config import TASK_CONFIG
|
||||
from task.src.task_manager import TaskCommand, TaskManager
|
||||
|
||||
|
||||
class TaskListView(ApiBaseView):
|
||||
"""resolves to /api/task/by-name/
|
||||
GET: return a list of all stored task results
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
all_results = TaskManager().get_all_results()
|
||||
|
||||
return Response(all_results)
|
||||
|
||||
|
||||
class TaskNameListView(ApiBaseView):
|
||||
"""resolves to /api/task/by-name/<task-name>/
|
||||
GET: return a list of stored results of task
|
||||
POST: start new background process
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
def get(self, request, task_name):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
if task_name not in TASK_CONFIG:
|
||||
message = {"message": "invalid task name"}
|
||||
return Response(message, status=404)
|
||||
|
||||
all_results = TaskManager().get_tasks_by_name(task_name)
|
||||
|
||||
return Response(all_results)
|
||||
|
||||
def post(self, request, task_name):
|
||||
"""
|
||||
handle post request
|
||||
404 for invalid task_name
|
||||
400 if task can't be started here without argument
|
||||
"""
|
||||
# pylint: disable=unused-argument
|
||||
task_config = TASK_CONFIG.get(task_name)
|
||||
if not task_config:
|
||||
message = {"message": "invalid task name"}
|
||||
return Response(message, status=404)
|
||||
|
||||
if not task_config.get("api_start"):
|
||||
message = {"message": "can not start task through this endpoint"}
|
||||
return Response(message, status=400)
|
||||
|
||||
message = TaskCommand().start(task_name)
|
||||
|
||||
return Response({"message": message})
|
||||
|
||||
|
||||
class TaskIDView(ApiBaseView):
|
||||
"""resolves to /api/task/by-id/<task-id>/
|
||||
GET: return details of task id
|
||||
POST: send command to task by id
|
||||
"""
|
||||
|
||||
valid_commands = ["stop", "kill"]
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
def get(self, request, task_id):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
task_result = TaskManager().get_task(task_id)
|
||||
if not task_result:
|
||||
message = {"message": "task id not found"}
|
||||
return Response(message, status=404)
|
||||
|
||||
return Response(task_result)
|
||||
|
||||
def post(self, request, task_id):
|
||||
"""post command to task"""
|
||||
command = request.data.get("command")
|
||||
if not command or command not in self.valid_commands:
|
||||
message = {"message": "no valid command found"}
|
||||
return Response(message, status=400)
|
||||
|
||||
task_result = TaskManager().get_task(task_id)
|
||||
if not task_result:
|
||||
message = {"message": "task id not found"}
|
||||
return Response(message, status=404)
|
||||
|
||||
task_conf = TASK_CONFIG.get(task_result.get("name"))
|
||||
if command == "stop":
|
||||
if not task_conf.get("api_stop"):
|
||||
message = {"message": "task can not be stopped"}
|
||||
return Response(message, status=400)
|
||||
|
||||
message_key = self._build_message_key(task_conf, task_id)
|
||||
TaskCommand().stop(task_id, message_key)
|
||||
if command == "kill":
|
||||
if not task_conf.get("api_stop"):
|
||||
message = {"message": "task can not be killed"}
|
||||
return Response(message, status=400)
|
||||
|
||||
TaskCommand().kill(task_id)
|
||||
|
||||
return Response({"message": "command sent"})
|
||||
|
||||
def _build_message_key(self, task_conf, task_id):
|
||||
"""build message key to forward command to notification"""
|
||||
return f"message:{task_conf.get('group')}:{task_id.split('-')[0]}"
|
||||
|
||||
|
||||
class ScheduleView(ApiBaseView):
|
||||
"""resolves to /api/task/schedule/<task-name>/
|
||||
POST: create/update schedule for task with config
|
||||
- example: {"schedule": "0 0 *", "config": {"days": 90}}
|
||||
DEL: delete schedule for task
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
def post(self, request, task_name):
|
||||
"""create/update schedule for task"""
|
||||
cron_schedule = request.data.get("schedule")
|
||||
schedule_config = request.data.get("config")
|
||||
if not cron_schedule and not schedule_config:
|
||||
message = {"message": "expected schedule or config key"}
|
||||
return Response(message, status=400)
|
||||
|
||||
try:
|
||||
validator = CrontabValidator()
|
||||
validator.validate_cron(cron_schedule)
|
||||
validator.validate_config(task_name, schedule_config)
|
||||
except ValueError as err:
|
||||
return Response({"message": str(err)}, status=400)
|
||||
|
||||
ScheduleBuilder().update_schedule(
|
||||
task_name, cron_schedule, schedule_config
|
||||
)
|
||||
message = f"update schedule for task {task_name}"
|
||||
if schedule_config:
|
||||
message += f" with config {schedule_config}"
|
||||
|
||||
return Response({"message": message})
|
||||
|
||||
def delete(self, request, task_name):
|
||||
"""delete schedule by task_name query"""
|
||||
task = get_object_or_404(CustomPeriodicTask, name=task_name)
|
||||
_ = task.delete()
|
||||
|
||||
return Response({"success": True})
|
||||
|
||||
|
||||
class ScheduleNotification(ApiBaseView):
|
||||
"""resolves to /api/task/notification/
|
||||
GET: get all schedule notifications
|
||||
POST: add notification url to task
|
||||
DEL: delete notification
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
|
||||
return Response(get_all_notifications())
|
||||
|
||||
def post(self, request):
|
||||
"""handle create notification"""
|
||||
task_name = request.data.get("task_name")
|
||||
url = request.data.get("url")
|
||||
|
||||
if not TASK_CONFIG.get(task_name):
|
||||
message = {"message": "task_name not found"}
|
||||
return Response(message, status=404)
|
||||
|
||||
if not url:
|
||||
message = {"message": "missing url key"}
|
||||
return Response(message, status=400)
|
||||
|
||||
Notifications(task_name).add_url(url)
|
||||
message = {"task_name": task_name, "url": url}
|
||||
|
||||
return Response(message)
|
||||
|
||||
def delete(self, request):
|
||||
"""handle delete"""
|
||||
|
||||
task_name = request.data.get("task_name")
|
||||
url = request.data.get("url")
|
||||
|
||||
if not TASK_CONFIG.get(task_name):
|
||||
message = {"message": "task_name not found"}
|
||||
return Response(message, status=404)
|
||||
|
||||
if url:
|
||||
response, status_code = Notifications(task_name).remove_url(url)
|
||||
else:
|
||||
response, status_code = Notifications(task_name).remove_task()
|
||||
|
||||
return Response({"response": response, "status_code": status_code})
|
||||
0
backend/user/__init__.py
Normal file
0
backend/user/__init__.py
Normal file
45
backend/user/admin.py
Normal file
45
backend/user/admin.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""custom admin classes"""
|
||||
|
||||
from django.contrib import admin
|
||||
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
|
||||
from django_celery_beat import models as BeatModels
|
||||
from user.models import Account
|
||||
|
||||
|
||||
class HomeAdmin(BaseUserAdmin):
|
||||
"""register in admin page"""
|
||||
|
||||
list_display = ("name", "is_staff", "is_superuser")
|
||||
list_filter = ("is_superuser",)
|
||||
|
||||
fieldsets = (
|
||||
(None, {"fields": ("is_staff", "is_superuser", "password")}),
|
||||
("Personal info", {"fields": ("name",)}),
|
||||
("Groups", {"fields": ("groups",)}),
|
||||
("Permissions", {"fields": ("user_permissions",)}),
|
||||
)
|
||||
add_fieldsets = (
|
||||
(
|
||||
None,
|
||||
{"fields": ("is_staff", "is_superuser", "password1", "password2")},
|
||||
),
|
||||
("Personal info", {"fields": ("name",)}),
|
||||
("Groups", {"fields": ("groups",)}),
|
||||
("Permissions", {"fields": ("user_permissions",)}),
|
||||
)
|
||||
|
||||
search_fields = ("name",)
|
||||
ordering = ("name",)
|
||||
filter_horizontal = ()
|
||||
|
||||
|
||||
admin.site.register(Account, HomeAdmin)
|
||||
admin.site.unregister(
|
||||
[
|
||||
BeatModels.ClockedSchedule,
|
||||
BeatModels.CrontabSchedule,
|
||||
BeatModels.IntervalSchedule,
|
||||
BeatModels.PeriodicTask,
|
||||
BeatModels.SolarSchedule,
|
||||
]
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user