public/tubearchivist
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/tubearchivist.git synced 2026-08-04 23:59:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

renamed django app folder to backend

Browse Source
This commit is contained in:
Simon
2024-08-03 21:58:22 +02:00
parent 1d07386a06
commit a5b492fecd
124 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
backend/appsettings/__init__.py Normal file
View File

687
backend/appsettings/index_mapping.json Normal file
View File

@@ -0,0 +1,687 @@
{
"index_config": [{
"index_name": "config",
"expected_map": {
"config": {
"type": "object",
"enabled": false
}
},
"expected_set": {
"number_of_replicas": "0"
}
},
{
"index_name": "channel",
"expected_map": {
"channel_id": {
"type": "keyword"
},
"channel_name": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
},
"search_as_you_type": {
"type": "search_as_you_type",
"doc_values": false,
"max_shingle_size": 3
}
}
},
"channel_banner_url": {
"type": "keyword",
"index": false
},
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": {
"type": "keyword",
"index": false
},
"channel_description": {
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"channel_tags": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"channel_overwrites": {
"properties": {
"download_format": {
"type": "text"
},
"autodelete_days": {
"type": "long"
},
"index_playlists": {
"type": "boolean"
},
"integrate_sponsorblock": {
"type": "boolean"
},
"subscriptions_channel_size": {
"type": "long"
},
"subscriptions_live_channel_size": {
"type": "long"
},
"subscriptions_shorts_channel_size": {
"type": "long"
}
}
}
},
"expected_set": {
"analysis": {
"normalizer": {
"to_lower": {
"type": "custom",
"filter": ["lowercase"]
}
}
},
"number_of_replicas": "0"
}
},
{
"index_name": "video",
"expected_map": {
"vid_thumb_url": {
"type": "text",
"index": false
},
"vid_thumb_base64": {
"type": "text",
"index": false
},
"date_downloaded": {
"type": "date",
"format": "epoch_second"
},
"channel": {
"properties": {
"channel_id": {
"type": "keyword"
},
"channel_name": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
},
"search_as_you_type": {
"type": "search_as_you_type",
"doc_values": false,
"max_shingle_size": 3
}
}
},
"channel_banner_url": {
"type": "keyword",
"index": false
},
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": {
"type": "keyword",
"index": false
},
"channel_description": {
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"channel_tags": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"channel_overwrites": {
"properties": {
"download_format": {
"type": "text"
},
"autodelete_days": {
"type": "long"
},
"index_playlists": {
"type": "boolean"
},
"integrate_sponsorblock": {
"type": "boolean"
},
"subscriptions_channel_size": {
"type": "long"
},
"subscriptions_live_channel_size": {
"type": "long"
},
"subscriptions_shorts_channel_size": {
"type": "long"
}
}
}
}
},
"description": {
"type": "text"
},
"media_url": {
"type": "keyword",
"index": false
},
"media_size": {
"type": "long"
},
"tags": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
},
"search_as_you_type": {
"type": "search_as_you_type",
"doc_values": false,
"max_shingle_size": 3
}
}
},
"vid_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"youtube_id": {
"type": "keyword"
},
"vid_type": {
"type": "keyword"
},
"published": {
"type": "date"
},
"playlist": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"comment_count": {
"type": "long"
},
"stats": {
"properties": {
"average_rating": {
"type": "float"
},
"dislike_count": {
"type": "long"
},
"like_count": {
"type": "long"
},
"view_count": {
"type": "long"
}
}
},
"player": {
"properties": {
"duration": {
"type": "long"
},
"duration_str": {
"type": "keyword",
"index": false
},
"watched": {
"type": "boolean"
},
"watched_date": {
"type": "date",
"format": "epoch_second"
}
}
},
"subtitles": {
"properties": {
"ext": {
"type": "keyword",
"index": false
},
"lang": {
"type": "keyword",
"index": false
},
"media_url": {
"type": "keyword",
"index": false
},
"name": {
"type": "keyword"
},
"source": {
"type": "keyword"
},
"url": {
"type": "keyword",
"index": false
}
}
},
"streams": {
"properties": {
"type": {
"type": "keyword",
"index": false
},
"index": {
"type": "short",
"index": false
},
"codec": {
"type": "text"
},
"width": {
"type": "short"
},
"height": {
"type": "short"
},
"bitrate": {
"type": "integer"
}
}
},
"sponsorblock": {
"properties": {
"last_refresh": {
"type": "date",
"format": "epoch_second"
},
"has_unlocked": {
"type": "boolean"
},
"is_enabled": {
"type": "boolean"
},
"segments": {
"properties": {
"UUID": {
"type": "keyword"
},
"actionType": {
"type": "keyword"
},
"category": {
"type": "keyword"
},
"locked": {
"type": "short"
},
"segment": {
"type": "float"
},
"videoDuration": {
"type": "float"
},
"votes": {
"type": "long"
}
}
}
}
}
},
"expected_set": {
"analysis": {
"normalizer": {
"to_lower": {
"type": "custom",
"filter": ["lowercase"]
}
}
},
"number_of_replicas": "0"
}
},
{
"index_name": "download",
"expected_map": {
"timestamp": {
"type": "date",
"format": "epoch_second"
},
"channel_id": {
"type": "keyword"
},
"channel_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"status": {
"type": "keyword"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"vid_thumb_url": {
"type": "keyword"
},
"youtube_id": {
"type": "keyword"
},
"vid_type": {
"type": "keyword"
},
"auto_start": {
"type": "boolean"
},
"message": {
"type": "text"
}
},
"expected_set": {
"analysis": {
"normalizer": {
"to_lower": {
"type": "custom",
"filter": ["lowercase"]
}
}
},
"number_of_replicas": "0"
}
},
{
"index_name": "playlist",
"expected_map": {
"playlist_id": {
"type": "keyword"
},
"playlist_description": {
"type": "text"
},
"playlist_name": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
},
"search_as_you_type": {
"type": "search_as_you_type",
"doc_values": false,
"max_shingle_size": 3
}
}
},
"playlist_channel": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"playlist_channel_id": {
"type": "keyword"
},
"playlist_thumbnail": {
"type": "keyword"
},
"playlist_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"playlist_entries": {
"properties": {
"downloaded": {
"type": "boolean"
},
"idx": {
"type": "long"
},
"title": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"uploader": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"youtube_id": {
"type": "keyword"
}
}
}
},
"expected_set": {
"analysis": {
"normalizer": {
"to_lower": {
"type": "custom",
"filter": ["lowercase"]
}
}
},
"number_of_replicas": "0"
}
},
{
"index_name": "subtitle",
"expected_map": {
"youtube_id": {
"type": "keyword"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"subtitle_fragment_id": {
"type": "keyword"
},
"subtitle_channel": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"subtitle_channel_id": {
"type": "keyword"
},
"subtitle_start": {
"type": "text"
},
"subtitle_end": {
"type": "text"
},
"subtitle_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"subtitle_index": {
"type": "long"
},
"subtitle_lang": {
"type": "keyword"
},
"subtitle_source": {
"type": "keyword"
},
"subtitle_line": {
"type": "text",
"analyzer": "english"
}
},
"expected_set": {
"analysis": {
"normalizer": {
"to_lower": {
"type": "custom",
"filter": ["lowercase"]
}
}
},
"number_of_replicas": "0"
}
},
{
"index_name": "comment",
"expected_map": {
"youtube_id": {
"type": "keyword"
},
"comment_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"comment_channel_id": {
"type": "keyword"
},
"comment_comments": {
"properties": {
"comment_id": {
"type": "keyword"
},
"comment_text": {
"type": "text"
},
"comment_timestamp": {
"type": "date",
"format": "epoch_second"
},
"comment_time_text": {
"type": "text"
},
"comment_likecount": {
"type": "long"
},
"comment_is_favorited": {
"type": "boolean"
},
"comment_author": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"comment_author_id": {
"type": "keyword"
},
"comment_author_thumbnail": {
"type": "keyword"
},
"comment_author_is_uploader": {
"type": "boolean"
},
"comment_parent": {
"type": "keyword"
}
}
}
},
"expected_set": {
"analysis": {
"normalizer": {
"to_lower": {
"type": "custom",
"filter": ["lowercase"]
}
}
},
"number_of_replicas": "0"
}
}
]
}

0
backend/appsettings/migrations/__init__.py Normal file
View File

0
backend/appsettings/src/__init__.py Normal file
View File

273
backend/appsettings/src/backup.py Normal file
View File

@@ -0,0 +1,273 @@
"""
Functionality:
- Handle json zip file based backup
- create backup
- restore backup
"""
import json
import os
import zipfile
from datetime import datetime
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap, IndexPaginate
from common.src.helper import get_mapping, ignore_filelist
from task.models import CustomPeriodicTask
class ElasticBackup:
"""dump index to nd-json files for later bulk import"""
INDEX_SPLIT = ["comment"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
BACKUP_DIR = os.path.join(CACHE_DIR, "backup")
def __init__(self, reason=False, task=False) -> None:
self.timestamp = datetime.now().strftime("%Y%m%d")
self.index_config = get_mapping()
self.reason = reason
self.task = task
def backup_all_indexes(self):
"""backup all indexes, add reason to init"""
print("backup all indexes")
if not self.reason:
raise ValueError("missing backup reason in ElasticBackup")
if self.task:
self.task.send_progress(["Scanning your index."])
for index in self.index_config:
index_name = index["index_name"]
print(f"backup: export in progress for {index_name}")
if not self.index_exists(index_name):
print(f"skip backup for not yet existing index {index_name}")
continue
self.backup_index(index_name)
if self.task:
self.task.send_progress(["Compress files to zip archive."])
self.zip_it()
if self.reason == "auto":
self.rotate_backup()
def backup_index(self, index_name):
"""export all documents of a single index"""
paginate_kwargs = {
"data": {"query": {"match_all": {}}},
"keep_source": True,
"callback": BackupCallback,
"task": self.task,
"total": self._get_total(index_name),
}
if index_name in self.INDEX_SPLIT:
paginate_kwargs.update({"size": 200})
paginate = IndexPaginate(f"ta_{index_name}", **paginate_kwargs)
_ = paginate.get_results()
@staticmethod
def _get_total(index_name):
"""get total documents in index"""
path = f"ta_{index_name}/_count"
response, _ = ElasticWrap(path).get()
return response.get("count")
def zip_it(self):
"""pack it up into single zip file"""
file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
to_backup = []
for file in os.listdir(self.BACKUP_DIR):
if file.endswith(".json"):
to_backup.append(os.path.join(self.BACKUP_DIR, file))
backup_file = os.path.join(self.BACKUP_DIR, file_name)
comp = zipfile.ZIP_DEFLATED
with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
for backup_file in to_backup:
zip_f.write(backup_file, os.path.basename(backup_file))
# cleanup
for backup_file in to_backup:
os.remove(backup_file)
def post_bulk_restore(self, file_name):
"""send bulk to es"""
file_path = os.path.join(self.CACHE_DIR, file_name)
with open(file_path, "r", encoding="utf-8") as f:
data = f.read()
if not data.strip():
return
_, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
def get_all_backup_files(self):
"""build all available backup files for view"""
all_backup_files = ignore_filelist(os.listdir(self.BACKUP_DIR))
all_available_backups = [
i
for i in all_backup_files
if i.startswith("ta_") and i.endswith(".zip")
]
all_available_backups.sort(reverse=True)
backup_dicts = []
for filename in all_available_backups:
data = self.build_backup_file_data(filename)
backup_dicts.append(data)
return backup_dicts
def build_backup_file_data(self, filename):
"""build metadata of single backup file"""
file_path = os.path.join(self.BACKUP_DIR, filename)
if not os.path.exists(file_path):
return False
file_split = filename.split("-")
if len(file_split) == 2:
timestamp = file_split[1].strip(".zip")
reason = False
elif len(file_split) == 3:
timestamp = file_split[1]
reason = file_split[2].strip(".zip")
else:
raise ValueError
data = {
"filename": filename,
"file_path": file_path,
"file_size": os.path.getsize(file_path),
"timestamp": timestamp,
"reason": reason,
}
return data
def restore(self, filename):
"""
restore from backup zip file
call reset from ElasitIndexWrap first to start blank
"""
zip_content = self._unpack_zip_backup(filename)
self._restore_json_files(zip_content)
def _unpack_zip_backup(self, filename):
"""extract backup zip and return filelist"""
file_path = os.path.join(self.BACKUP_DIR, filename)
with zipfile.ZipFile(file_path, "r") as z:
zip_content = z.namelist()
z.extractall(self.BACKUP_DIR)
return zip_content
def _restore_json_files(self, zip_content):
"""go through the unpacked files and restore"""
for idx, json_f in enumerate(zip_content):
self._notify_restore(idx, json_f, len(zip_content))
file_name = os.path.join(self.BACKUP_DIR, json_f)
if not json_f.startswith("es_") or not json_f.endswith(".json"):
os.remove(file_name)
continue
print("restoring: " + json_f)
self.post_bulk_restore(file_name)
os.remove(file_name)
def _notify_restore(self, idx, json_f, total_files):
"""notify restore progress"""
message = [f"Restore index from json backup file {json_f}."]
progress = (idx + 1) / total_files
self.task.send_progress(message_lines=message, progress=progress)
@staticmethod
def index_exists(index_name):
"""check if index already exists to skip"""
_, status_code = ElasticWrap(f"ta_{index_name}").get()
exists = status_code == 200
return exists
def rotate_backup(self):
"""delete old backups if needed"""
try:
task = CustomPeriodicTask.objects.get(name="run_backup")
except CustomPeriodicTask.DoesNotExist:
return
rotate = task.task_config.get("rotate")
if not rotate:
return
all_backup_files = self.get_all_backup_files()
auto = [i for i in all_backup_files if i["reason"] == "auto"]
if len(auto) <= rotate:
print("no backup files to rotate")
return
all_to_delete = auto[rotate:]
for to_delete in all_to_delete:
self.delete_file(to_delete["filename"])
def delete_file(self, filename):
"""delete backup file"""
file_path = os.path.join(self.BACKUP_DIR, filename)
if not os.path.exists(file_path):
print(f"backup file not found: {filename}")
return False
print(f"remove old backup file: {file_path}")
os.remove(file_path)
return file_path
class BackupCallback:
"""handle backup ndjson writer as callback for IndexPaginate"""
def __init__(self, source, index_name, counter=0):
self.source = source
self.index_name = index_name
self.counter = counter
self.timestamp = datetime.now().strftime("%Y%m%d")
self.cache_dir = EnvironmentSettings.CACHE_DIR
def run(self):
"""run the junk task"""
file_content = self._build_bulk()
self._write_es_json(file_content)
def _build_bulk(self):
"""build bulk query data from all_results"""
bulk_list = []
for document in self.source:
document_id = document["_id"]
es_index = document["_index"]
action = {"index": {"_index": es_index, "_id": document_id}}
source = document["_source"]
bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(source))
# add last newline
bulk_list.append("\n")
file_content = "\n".join(bulk_list)
return file_content
def _write_es_json(self, file_content):
"""write nd-json file for es _bulk API to disk"""
index = self.index_name.lstrip("ta_")
file_name = f"es_{index}-{self.timestamp}-{self.counter}.json"
file_path = os.path.join(self.cache_dir, "backup", file_name)
with open(file_path, "a+", encoding="utf-8") as f:
f.write(file_content)

245
backend/appsettings/src/config.py Normal file
View File

@@ -0,0 +1,245 @@
"""
Functionality:
- read and write config
- load config variables into redis
"""
from random import randint
from time import sleep
from typing import Literal, TypedDict
import requests
from appsettings.src.snapshot import ElasticSnapshot
from common.src.es_connect import ElasticWrap
from common.src.ta_redis import RedisArchivist
from django.conf import settings
from download.src.yt_dlp_base import CookieHandler
class SubscriptionsConfigType(TypedDict):
"""describes subscriptions config"""
channel_size: int
live_channel_size: int
shorts_channel_size: int
auto_start: bool
class DownloadsConfigType(TypedDict):
"""describes downloads config"""
limit_speed: int
sleep_interval: int
autodelete_days: int
format: str | bool
format_sort: str | bool
add_metadata: bool
add_thumbnail: bool
subtitle: str | bool
subtitle_source: Literal["user", "auto"] | bool
subtitle_index: bool
comment_max: str | bool
comment_sort: Literal["top", "new"]
cookie_import: bool
throttledratelimit: int
extractor_lang: str | bool
integrate_ryd: bool
integrate_sponsorblock: bool
class ApplicationConfigType(TypedDict):
"""describes application config"""
enable_snapshot: bool
class AppConfigType(TypedDict):
"""combined app config type"""
subscriptions: SubscriptionsConfigType
downloads: DownloadsConfigType
application: ApplicationConfigType
class AppConfig:
"""handle application variables"""
ES_PATH = "ta_config/_doc/appsettings"
CONFIG_DEFAULTS: AppConfigType = {
"subscriptions": {
"channel_size": 50,
"live_channel_size": 50,
"shorts_channel_size": 50,
"auto_start": False,
},
"downloads": {
"limit_speed": False,
"sleep_interval": 3,
"autodelete_days": False,
"format": False,
"format_sort": False,
"add_metadata": False,
"add_thumbnail": False,
"subtitle": False,
"subtitle_source": False,
"subtitle_index": False,
"comment_max": False,
"comment_sort": "top",
"cookie_import": False,
"throttledratelimit": False,
"extractor_lang": False,
"integrate_ryd": False,
"integrate_sponsorblock": False,
},
"application": {"enable_snapshot": True},
}
def __init__(self):
self.config = self.get_config()
def get_config(self) -> AppConfigType:
"""get config from ES"""
response, status_code = ElasticWrap(self.ES_PATH).get()
if not status_code == 200:
return self.CONFIG_DEFAULTS
return response["_source"]
def update_config(self, data: dict) -> AppConfigType:
"""update single config value"""
for key, value in data.items():
key_map = key.split(".")
self._validate_key(key_map)
self.config[key_map[0]][key_map[1]] = value
response, status_code = ElasticWrap(self.ES_PATH).post(self.config)
if not status_code == 200:
print(response)
return self.config
def _validate_key(self, key_map: list[str]) -> None:
"""raise valueerror on invalid key"""
exists = self.CONFIG_DEFAULTS.get(key_map[0], {}).get(key_map[1]) # type: ignore # noqa: E501
if exists is None:
raise ValueError(f"trying to access invalid config key: {key_map}")
def post_process_updated(self, data: dict) -> None:
"""apply hooks for some config keys"""
for config_value, updated_value in data:
if config_value == "downloads.cookie_import":
self.process_cookie(updated_value)
if config_value == "application.enable_snapshot" and updated_value:
ElasticSnapshot().setup()
def process_cookie(self, updated_value):
"""import and validate cookie"""
handler = CookieHandler(self.config)
if updated_value:
try:
handler.import_cookie()
except FileNotFoundError:
print("cookie: import failed, file not found")
handler.revoke()
self._fail_message("Cookie file not found.")
return
valid = handler.validate()
if not valid:
handler.revoke()
self._fail_message("Failed to validate cookie file.")
else:
handler.revoke()
@staticmethod
def _fail_message(message_line):
"""notify our failure"""
key = "message:setting"
message = {
"status": key,
"group": "setting:application",
"level": "error",
"title": "Cookie import failed",
"messages": [message_line],
"id": "0000",
}
RedisArchivist().set_message(key, message=message, expire=True)
class ReleaseVersion:
"""compare local version with remote version"""
REMOTE_URL = "https://www.tubearchivist.com/api/release/latest/"
NEW_KEY = "versioncheck:new"
def __init__(self) -> None:
self.local_version: str = settings.TA_VERSION
self.is_unstable: bool = settings.TA_VERSION.endswith("-unstable")
self.remote_version: str = ""
self.is_breaking: bool = False
def check(self) -> None:
"""check version"""
print(f"[{self.local_version}]: look for updates")
self.get_remote_version()
new_version = self._has_update()
if new_version:
message = {
"status": True,
"version": new_version,
"is_breaking": self.is_breaking,
}
RedisArchivist().set_message(self.NEW_KEY, message)
print(f"[{self.local_version}]: found new version {new_version}")
def get_local_version(self) -> str:
"""read version from local"""
return self.local_version
def get_remote_version(self) -> None:
"""read version from remote"""
sleep(randint(0, 60))
response = requests.get(self.REMOTE_URL, timeout=20).json()
self.remote_version = response["release_version"]
self.is_breaking = response["breaking_changes"]
def _has_update(self) -> str | bool:
"""check if there is an update"""
remote_parsed = self._parse_version(self.remote_version)
local_parsed = self._parse_version(self.local_version)
if remote_parsed > local_parsed:
return self.remote_version
if self.is_unstable and local_parsed == remote_parsed:
return self.remote_version
return False
@staticmethod
def _parse_version(version) -> tuple[int, ...]:
"""return version parts"""
clean = version.rstrip("-unstable").lstrip("v")
return tuple((int(i) for i in clean.split(".")))
def is_updated(self) -> str | bool:
"""check if update happened in the mean time"""
message = self.get_update()
if not message:
return False
local_parsed = self._parse_version(self.local_version)
message_parsed = self._parse_version(message.get("version"))
if local_parsed >= message_parsed:
RedisArchivist().del_message(self.NEW_KEY)
return settings.TA_VERSION
return False
def get_update(self) -> dict:
"""return new version dict if available"""
message = RedisArchivist().get_message(self.NEW_KEY)
if not message.get("status"):
return {}
return message

126
backend/appsettings/src/filesystem.py Normal file
View File

@@ -0,0 +1,126 @@
"""
Functionality:
- scan the filesystem to delete or index
"""
import os
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap, IndexPaginate
from common.src.helper import ignore_filelist
from video.src.comments import CommentList
from video.src.index import YoutubeVideo, index_new_video
class Scanner:
"""scan index and filesystem"""
VIDEOS: str = EnvironmentSettings.MEDIA_DIR
def __init__(self, task=False) -> None:
self.task = task
self.to_delete: set[str] = set()
self.to_index: set[str] = set()
def scan(self) -> None:
"""scan the filesystem"""
downloaded: set[str] = self._get_downloaded()
indexed: set[str] = self._get_indexed()
self.to_index = downloaded - indexed
self.to_delete = indexed - downloaded
def _get_downloaded(self) -> set[str]:
"""get downloaded ids"""
if self.task:
self.task.send_progress(["Scan your filesystem for videos."])
downloaded: set = set()
channels = ignore_filelist(os.listdir(self.VIDEOS))
for channel in channels:
folder = os.path.join(self.VIDEOS, channel)
files = ignore_filelist(os.listdir(folder))
downloaded.update({i.split(".")[0] for i in files})
return downloaded
def _get_indexed(self) -> set:
"""get all indexed ids"""
if self.task:
self.task.send_progress(["Get all videos indexed."])
data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
response = IndexPaginate("ta_video", data).get_results()
return {i["youtube_id"] for i in response}
def apply(self) -> None:
"""apply all changes"""
self.delete()
self.index()
self.url_fix()
def delete(self) -> None:
"""delete videos from index"""
if not self.to_delete:
print("nothing to delete")
return
if self.task:
self.task.send_progress(
[f"Remove {len(self.to_delete)} videos from index."]
)
for youtube_id in self.to_delete:
YoutubeVideo(youtube_id).delete_media_file()
def index(self) -> None:
"""index new"""
if not self.to_index:
print("nothing to index")
return
total = len(self.to_index)
for idx, youtube_id in enumerate(self.to_index):
if self.task:
self.task.send_progress(
message_lines=[
f"Index missing video {youtube_id}, {idx + 1}/{total}"
],
progress=(idx + 1) / total,
)
index_new_video(youtube_id)
comment_list = CommentList(task=self.task)
comment_list.add(video_ids=list(self.to_index))
comment_list.index()
def url_fix(self) -> None:
"""
update path v0.3.6 to v0.3.7
fix url not matching channel-videoid pattern
"""
bool_must = (
"doc['media_url'].value == "
+ "(doc['channel.channel_id'].value + '/' + "
+ "doc['youtube_id'].value) + '.mp4'"
)
to_update = (
"ctx._source['media_url'] = "
+ "ctx._source.channel['channel_id'] + '/' + "
+ "ctx._source['youtube_id'] + '.mp4'"
)
data = {
"query": {
"bool": {
"must_not": [{"script": {"script": {"source": bool_must}}}]
}
},
"script": {"source": to_update},
}
response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data)
updated = response.get("updates")
if updated:
print(f"updated {updated} bad media_url")
if self.task:
self.task.send_progress(
[f"Updated {updated} wrong media urls."]
)

212
backend/appsettings/src/index_setup.py Normal file
View File

@@ -0,0 +1,212 @@
"""
functionality:
- setup elastic index at first start
- verify and update index mapping and settings if needed
- backup and restore metadata
"""
from appsettings.src.backup import ElasticBackup
from appsettings.src.config import AppConfig
from appsettings.src.snapshot import ElasticSnapshot
from common.src.es_connect import ElasticWrap
from common.src.helper import get_mapping
class ElasticIndex:
"""interact with a single index"""
def __init__(self, index_name, expected_map=False, expected_set=False):
self.index_name = index_name
self.expected_map = expected_map
self.expected_set = expected_set
self.exists, self.details = self.index_exists()
def index_exists(self):
"""check if index already exists and return mapping if it does"""
response, status_code = ElasticWrap(f"ta_{self.index_name}").get()
exists = status_code == 200
details = response.get(f"ta_{self.index_name}", False)
return exists, details
def validate(self):
"""
check if all expected mappings and settings match
returns True when rebuild is needed
"""
if self.expected_map:
rebuild = self.validate_mappings()
if rebuild:
return rebuild
if self.expected_set:
rebuild = self.validate_settings()
if rebuild:
return rebuild
return False
def validate_mappings(self):
"""check if all mappings are as expected"""
now_map = self.details["mappings"]["properties"]
for key, value in self.expected_map.items():
# nested
if list(value.keys()) == ["properties"]:
for key_n, value_n in value["properties"].items():
if key not in now_map:
print(f"detected mapping change: {key_n}, {value_n}")
return True
if key_n not in now_map[key]["properties"].keys():
print(f"detected mapping change: {key_n}, {value_n}")
return True
if not value_n == now_map[key]["properties"][key_n]:
print(f"detected mapping change: {key_n}, {value_n}")
return True
continue
# not nested
if key not in now_map.keys():
print(f"detected mapping change: {key}, {value}")
return True
if not value == now_map[key]:
print(f"detected mapping change: {key}, {value}")
return True
return False
def validate_settings(self):
"""check if all settings are as expected"""
now_set = self.details["settings"]["index"]
for key, value in self.expected_set.items():
if key not in now_set.keys():
print(key, value)
return True
if not value == now_set[key]:
print(key, value)
return True
return False
def rebuild_index(self):
"""rebuild with new mapping"""
print(f"applying new mappings to index ta_{self.index_name}...")
self.create_blank(for_backup=True)
self.reindex("backup")
self.delete_index(backup=False)
self.create_blank()
self.reindex("restore")
self.delete_index()
def reindex(self, method):
"""create on elastic search"""
if method == "backup":
source = f"ta_{self.index_name}"
destination = f"ta_{self.index_name}_backup"
elif method == "restore":
source = f"ta_{self.index_name}_backup"
destination = f"ta_{self.index_name}"
else:
raise ValueError("invalid method, expected 'backup' or 'restore'")
data = {"source": {"index": source}, "dest": {"index": destination}}
_, _ = ElasticWrap("_reindex?refresh=true").post(data=data)
def delete_index(self, backup=True):
"""delete index passed as argument"""
path = f"ta_{self.index_name}"
if backup:
path = path + "_backup"
_, _ = ElasticWrap(path).delete()
def create_blank(self, for_backup=False):
"""apply new mapping and settings for blank new index"""
print(f"create new blank index with name ta_{self.index_name}...")
path = f"ta_{self.index_name}"
if for_backup:
path = f"{path}_backup"
data = {}
if self.expected_set:
data.update({"settings": self.expected_set})
if self.expected_map:
data.update({"mappings": {"properties": self.expected_map}})
_, _ = ElasticWrap(path).put(data)
class ElasitIndexWrap:
"""interact with all index mapping and setup"""
def __init__(self):
self.index_config = get_mapping()
self.backup_run = False
def setup(self):
"""setup elastic index, run at startup"""
for index in self.index_config:
index_name, expected_map, expected_set = self._config_split(index)
handler = ElasticIndex(index_name, expected_map, expected_set)
if not handler.exists:
handler.create_blank()
continue
rebuild = handler.validate()
if rebuild:
self._check_backup()
handler.rebuild_index()
continue
# else all good
print(f"ta_{index_name} index is created and up to date...")
def reset(self):
"""reset all indexes to blank"""
self.delete_all()
self.create_all_blank()
def delete_all(self):
"""delete all indexes"""
print("reset elastic index")
for index in self.index_config:
index_name, _, _ = self._config_split(index)
handler = ElasticIndex(index_name)
handler.delete_index(backup=False)
def create_all_blank(self):
"""create all blank indexes"""
print("create all new indexes in elastic from template")
for index in self.index_config:
index_name, expected_map, expected_set = self._config_split(index)
handler = ElasticIndex(index_name, expected_map, expected_set)
handler.create_blank()
@staticmethod
def _config_split(index):
"""split index config keys"""
index_name = index["index_name"]
expected_map = index["expected_map"]
expected_set = index["expected_set"]
return index_name, expected_map, expected_set
def _check_backup(self):
"""create backup if needed"""
if self.backup_run:
return
config = AppConfig().config
if config["application"]["enable_snapshot"]:
# take snapshot if enabled
ElasticSnapshot().take_snapshot_now(wait=True)
else:
# fallback to json backup
ElasticBackup(reason="update").backup_all_indexes()
self.backup_run = True

483
backend/appsettings/src/manual.py Normal file
View File

@@ -0,0 +1,483 @@
"""
Functionality:
- Handle manual import task
- Scan and identify media files in import folder
- Process import media files
"""
import json
import os
import re
import shutil
import subprocess
from appsettings.src.config import AppConfig
from common.src.env_settings import EnvironmentSettings
from common.src.helper import ignore_filelist
from download.src.thumbnails import ThumbManager
from PIL import Image
from video.src.comments import CommentList
from video.src.index import YoutubeVideo
from yt_dlp.utils import ISO639Utils
class ImportFolderScanner:
"""import and indexing existing video files
- identify all media files belonging to a video
- identify youtube id
- convert if needed
"""
CONFIG = AppConfig().config
CACHE_DIR = EnvironmentSettings.CACHE_DIR
IMPORT_DIR = os.path.join(CACHE_DIR, "import")
"""All extensions should be in lowercase until better handling is in place.
Described in Issue #502.
"""
EXT_MAP = {
"media": [".mp4", ".mkv", ".webm"],
"metadata": [".json"],
"thumb": [".jpg", ".png", ".webp"],
"subtitle": [".vtt"],
}
def __init__(self, task=False):
self.task = task
self.to_import = False
def scan(self):
"""scan and match media files"""
if self.task:
self.task.send_progress(["Scanning your import folder."])
all_files = self.get_all_files()
self.match_files(all_files)
self.process_videos()
return self.to_import
def get_all_files(self):
"""get all files in /import"""
rel_paths = ignore_filelist(os.listdir(self.IMPORT_DIR))
all_files = [os.path.join(self.IMPORT_DIR, i) for i in rel_paths]
all_files.sort()
return all_files
@staticmethod
def _get_template():
"""base dict for video"""
return {
"media": False,
"video_id": False,
"metadata": False,
"thumb": False,
"subtitle": [],
}
def match_files(self, all_files):
"""loop through all files, join what matches"""
self.to_import = []
current_video = self._get_template()
last_base = False
for file_path in all_files:
base_name, ext = self._detect_base_name(file_path)
key, file_path = self._detect_type(file_path, ext)
if not key or not file_path:
continue
if base_name != last_base:
if last_base:
print(f"manual import: {current_video}")
self.to_import.append(current_video)
current_video = self._get_template()
last_base = base_name
if key == "subtitle":
current_video["subtitle"].append(file_path)
else:
current_video[key] = file_path
if current_video.get("media"):
print(f"manual import: {current_video}")
self.to_import.append(current_video)
def _detect_base_name(self, file_path):
"""extract base_name and ext for matching"""
base_name_raw, ext = os.path.splitext(file_path)
base_name, ext2 = os.path.splitext(base_name_raw)
if ext2:
if ISO639Utils.short2long(ext2.strip(".")) or ext2 == ".info":
# valid secondary extension
return base_name, ext
return base_name_raw, ext
def _detect_type(self, file_path, ext):
"""detect metadata type for file"""
for key, value in self.EXT_MAP.items():
if ext.lower() in value:
return key, file_path
return False, False
def process_videos(self):
"""loop through all videos"""
for idx, current_video in enumerate(self.to_import):
if not current_video["media"]:
print(f"{current_video}: no matching media file found.")
raise ValueError
if self.task:
self._notify(idx, current_video)
self._detect_youtube_id(current_video)
self._dump_thumb(current_video)
self._convert_thumb(current_video)
self._get_subtitles(current_video)
self._convert_video(current_video)
print(f"manual import: {current_video}")
ManualImport(current_video, self.CONFIG).run()
video_ids = [i["video_id"] for i in self.to_import]
comment_list = CommentList(task=self.task)
comment_list.add(video_ids=video_ids)
comment_list.index()
def _notify(self, idx, current_video):
"""send notification back to task"""
filename = os.path.split(current_video["media"])[-1]
if len(filename) > 50:
filename = filename[:50] + "..."
message = [
f"Import queue processing video {idx + 1}/{len(self.to_import)}",
filename,
]
progress = (idx + 1) / len(self.to_import)
self.task.send_progress(message, progress=progress)
def _detect_youtube_id(self, current_video):
"""find video id from filename or json"""
youtube_id = self._extract_id_from_filename(current_video["media"])
if youtube_id:
current_video["video_id"] = youtube_id
return
youtube_id = self._extract_id_from_json(current_video["metadata"])
if youtube_id:
current_video["video_id"] = youtube_id
return
raise ValueError("failed to find video id")
@staticmethod
def _extract_id_from_filename(file_name):
"""
look at the file name for the youtube id
expects filename ending in [<youtube_id>].<ext>
"""
base_name, _ = os.path.splitext(file_name)
id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
if id_search:
youtube_id = id_search.group(1)
return youtube_id
print(f"id extraction failed from filename: {file_name}")
return False
def _extract_id_from_json(self, json_file):
"""open json file and extract id"""
json_path = os.path.join(self.CACHE_DIR, "import", json_file)
with open(json_path, "r", encoding="utf-8") as f:
json_content = f.read()
youtube_id = json.loads(json_content)["id"]
return youtube_id
def _dump_thumb(self, current_video):
"""extract embedded thumb before converting"""
if current_video["thumb"]:
return
media_path = current_video["media"]
_, ext = os.path.splitext(media_path)
new_path = False
if ext == ".mkv":
idx, thumb_type = self._get_mkv_thumb_stream(media_path)
if idx is not None:
new_path = self.dump_mpv_thumb(media_path, idx, thumb_type)
elif ext == ".mp4":
thumb_type = self.get_mp4_thumb_type(media_path)
if thumb_type:
new_path = self.dump_mp4_thumb(media_path, thumb_type)
if new_path:
current_video["thumb"] = new_path
def _get_mkv_thumb_stream(self, media_path):
"""get stream idx of thumbnail for mkv files"""
streams = self._get_streams(media_path)
attachments = [
i for i in streams["streams"] if i["codec_type"] == "attachment"
]
for idx, stream in enumerate(attachments):
tags = stream["tags"]
if "mimetype" in tags and tags["filename"].startswith("cover"):
_, ext = os.path.splitext(tags["filename"])
return idx, ext
return None, None
@staticmethod
def dump_mpv_thumb(media_path, idx, thumb_type):
"""write cover to disk for mkv"""
_, media_ext = os.path.splitext(media_path)
new_path = f"{media_path.rstrip(media_ext)}{thumb_type}"
subprocess.run(
[
"ffmpeg",
"-v",
"quiet",
f"-dump_attachment:t:{idx}",
new_path,
"-i",
media_path,
],
check=False,
)
return new_path
def get_mp4_thumb_type(self, media_path):
"""detect filetype of embedded thumbnail"""
streams = self._get_streams(media_path)
for stream in streams["streams"]:
if stream["codec_name"] in ["png", "jpg"]:
return stream["codec_name"]
return False
def _convert_thumb(self, current_video):
"""convert all thumbnails to jpg"""
if not current_video["thumb"]:
return
thumb_path = current_video["thumb"]
base_path, ext = os.path.splitext(thumb_path)
if ext == ".jpg":
return
new_path = f"{base_path}.jpg"
img_raw = Image.open(thumb_path)
img_raw.convert("RGB").save(new_path)
os.remove(thumb_path)
current_video["thumb"] = new_path
def _get_subtitles(self, current_video):
"""find all subtitles in media file"""
if current_video["subtitle"]:
return
media_path = current_video["media"]
streams = self._get_streams(media_path)
base_path, ext = os.path.splitext(media_path)
if ext == ".webm":
print(f"{media_path}: subtitle extract from webm not supported")
return
for idx, stream in enumerate(streams["streams"]):
if stream["codec_type"] == "subtitle":
lang = ISO639Utils.long2short(stream["tags"]["language"])
sub_path = f"{base_path}.{lang}.vtt"
self._dump_subtitle(idx, media_path, sub_path)
current_video["subtitle"].append(sub_path)
@staticmethod
def _dump_subtitle(idx, media_path, sub_path):
"""extract subtitle from media file"""
subprocess.run(
["ffmpeg", "-i", media_path, "-map", f"0:{idx}", sub_path],
check=True,
)
@staticmethod
def _get_streams(media_path):
"""return all streams from media_path"""
streams_raw = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-show_streams",
"-print_format",
"json",
media_path,
],
capture_output=True,
check=True,
)
streams = json.loads(streams_raw.stdout.decode())
return streams
@staticmethod
def dump_mp4_thumb(media_path, thumb_type):
"""save cover to disk"""
_, ext = os.path.splitext(media_path)
new_path = f"{media_path.rstrip(ext)}.{thumb_type}"
subprocess.run(
[
"ffmpeg",
"-i",
media_path,
"-map",
"0:v",
"-map",
"-0:V",
"-c",
"copy",
new_path,
],
check=True,
)
return new_path
def _convert_video(self, current_video):
"""convert if needed"""
current_path = current_video["media"]
base_path, ext = os.path.splitext(current_path)
if ext == ".mp4":
return
new_path = base_path + ".mp4"
subprocess.run(
[
"ffmpeg",
"-i",
current_path,
new_path,
"-loglevel",
"warning",
"-stats",
],
check=True,
)
current_video["media"] = new_path
os.remove(current_path)
class ManualImport:
"""import single identified video"""
def __init__(self, current_video, config):
self.current_video = current_video
self.config = config
def run(self):
"""run all"""
json_data = self.index_metadata()
self._move_to_archive(json_data)
self._cleanup(json_data)
def index_metadata(self):
"""get metadata from yt or json"""
video_id = self.current_video["video_id"]
video = YoutubeVideo(video_id)
video.build_json(
youtube_meta_overwrite=self._get_info_json(),
media_path=self.current_video["media"],
)
if not video.json_data:
print(f"{video_id}: manual import failed, and no metadata found.")
raise ValueError
video.check_subtitles(subtitle_files=self.current_video["subtitle"])
video.upload_to_es()
if video.offline_import and self.current_video["thumb"]:
old_path = self.current_video["thumb"]
thumbs = ThumbManager(video_id)
new_path = thumbs.vid_thumb_path(absolute=True, create_folder=True)
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
else:
url = video.json_data["vid_thumb_url"]
ThumbManager(video_id).download_video_thumb(url)
return video.json_data
def _get_info_json(self):
"""read info_json from file"""
if not self.current_video["metadata"]:
return False
with open(self.current_video["metadata"], "r", encoding="utf-8") as f:
info_json = json.loads(f.read())
return info_json
def _move_to_archive(self, json_data):
"""move identified media file to archive"""
videos = EnvironmentSettings.MEDIA_DIR
host_uid = EnvironmentSettings.HOST_UID
host_gid = EnvironmentSettings.HOST_GID
channel, file = os.path.split(json_data["media_url"])
channel_folder = os.path.join(videos, channel)
if not os.path.exists(channel_folder):
os.makedirs(channel_folder)
if host_uid and host_gid:
os.chown(channel_folder, host_uid, host_gid)
old_path = self.current_video["media"]
new_path = os.path.join(channel_folder, file)
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
if host_uid and host_gid:
os.chown(new_path, host_uid, host_gid)
base_name, _ = os.path.splitext(new_path)
for old_path in self.current_video["subtitle"]:
lang = old_path.split(".")[-2]
new_path = f"{base_name}.{lang}.vtt"
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
def _cleanup(self, json_data):
"""cleanup leftover files"""
meta_data = self.current_video["metadata"]
if meta_data and os.path.exists(meta_data):
os.remove(meta_data)
thumb = self.current_video["thumb"]
if thumb and os.path.exists(thumb):
os.remove(thumb)
for subtitle_file in self.current_video["subtitle"]:
if os.path.exists(subtitle_file):
os.remove(subtitle_file)
channel_info = os.path.join(
EnvironmentSettings.CACHE_DIR,
"import",
f"{json_data['channel']['channel_id']}.info.json",
)
if os.path.exists(channel_info):
os.remove(channel_info)

563
backend/appsettings/src/reindex.py Normal file
View File

@@ -0,0 +1,563 @@
"""
functionality:
- periodically refresh documents
- index and update in es
"""
import json
import os
from datetime import datetime
from time import sleep
from typing import Callable, TypedDict
from appsettings.src.config import AppConfig
from channel.src.index import YoutubeChannel
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap, IndexPaginate
from common.src.ta_redis import RedisQueue
from download.src.subscriptions import ChannelSubscription
from download.src.thumbnails import ThumbManager
from download.src.yt_dlp_base import CookieHandler
from playlist.src.index import YoutubePlaylist
from task.models import CustomPeriodicTask
from video.src.comments import Comments
from video.src.index import YoutubeVideo
class ReindexConfigType(TypedDict):
"""represents config type"""
index_name: str
queue_name: str
active_key: str
refresh_key: str
class ReindexBase:
"""base config class for reindex task"""
REINDEX_CONFIG: dict[str, ReindexConfigType] = {
"video": {
"index_name": "ta_video",
"queue_name": "reindex:ta_video",
"active_key": "active",
"refresh_key": "vid_last_refresh",
},
"channel": {
"index_name": "ta_channel",
"queue_name": "reindex:ta_channel",
"active_key": "channel_active",
"refresh_key": "channel_last_refresh",
},
"playlist": {
"index_name": "ta_playlist",
"queue_name": "reindex:ta_playlist",
"active_key": "playlist_active",
"refresh_key": "playlist_last_refresh",
},
}
MULTIPLY = 1.2
DAYS3 = 60 * 60 * 24 * 3
def __init__(self):
self.config = AppConfig().config
self.now = int(datetime.now().timestamp())
def populate(self, all_ids, reindex_config: ReindexConfigType):
"""add all to reindex ids to redis queue"""
if not all_ids:
return
RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
class ReindexPopulate(ReindexBase):
"""add outdated and recent documents to reindex queue"""
INTERVAL_DEFAIULT: int = 90
def __init__(self):
super().__init__()
self.interval = self.INTERVAL_DEFAIULT
def get_interval(self) -> None:
"""get reindex days interval from task"""
try:
task = CustomPeriodicTask.objects.get(name="check_reindex")
except CustomPeriodicTask.DoesNotExist:
return
task_config = task.task_config
if task_config.get("days"):
self.interval = task_config.get("days")
def add_recent(self) -> None:
"""add recent videos to refresh"""
gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
must_list = [
{"term": {"active": {"value": True}}},
{"range": {"published": {"gte": gte}}},
]
data = {
"size": 10000,
"query": {"bool": {"must": must_list}},
"sort": [{"published": {"order": "desc"}}],
}
response, _ = ElasticWrap("ta_video/_search").get(data=data)
hits = response["hits"]["hits"]
if not hits:
return
all_ids = [i["_source"]["youtube_id"] for i in hits]
reindex_config: ReindexConfigType = self.REINDEX_CONFIG["video"]
self.populate(all_ids, reindex_config)
def add_outdated(self) -> None:
"""add outdated documents"""
for reindex_config in self.REINDEX_CONFIG.values():
total_hits = self._get_total_hits(reindex_config)
daily_should = self._get_daily_should(total_hits)
all_ids = self._get_outdated_ids(reindex_config, daily_should)
self.populate(all_ids, reindex_config)
@staticmethod
def _get_total_hits(reindex_config: ReindexConfigType) -> int:
"""get total hits from index"""
index_name = reindex_config["index_name"]
active_key = reindex_config["active_key"]
data = {
"query": {"term": {active_key: {"value": True}}},
"_source": False,
}
total = IndexPaginate(index_name, data, keep_source=True).get_results()
return len(total)
def _get_daily_should(self, total_hits: int) -> int:
"""calc how many should reindex daily"""
daily_should = int((total_hits // self.interval + 1) * self.MULTIPLY)
if daily_should >= 10000:
daily_should = 9999
return daily_should
def _get_outdated_ids(
self, reindex_config: ReindexConfigType, daily_should: int
) -> list[str]:
"""get outdated from index_name"""
index_name = reindex_config["index_name"]
refresh_key = reindex_config["refresh_key"]
now_lte = str(self.now - self.interval * 24 * 60 * 60)
must_list = [
{"match": {reindex_config["active_key"]: True}},
{"range": {refresh_key: {"lte": now_lte}}},
]
data = {
"size": daily_should,
"query": {"bool": {"must": must_list}},
"sort": [{refresh_key: {"order": "asc"}}],
"_source": False,
}
response, _ = ElasticWrap(f"{index_name}/_search").get(data=data)
all_ids = [i["_id"] for i in response["hits"]["hits"]]
return all_ids
class ReindexManual(ReindexBase):
"""
manually add ids to reindex queue from API
data_example = {
"video": ["video1", "video2", "video3"],
"channel": ["channel1", "channel2", "channel3"],
"playlist": ["playlist1", "playlist2"],
}
extract_videos to also reindex all videos of channel/playlist
"""
def __init__(self, extract_videos=False):
super().__init__()
self.extract_videos = extract_videos
self.data = False
def extract_data(self, data) -> None:
"""process data"""
self.data = data
for key, values in self.data.items():
reindex_config = self.REINDEX_CONFIG.get(key)
if not reindex_config:
print(f"reindex type {key} not valid")
raise ValueError
self.process_index(reindex_config, values)
def process_index(
self, index_config: ReindexConfigType, values: list[str]
) -> None:
"""process values per index"""
index_name = index_config["index_name"]
if index_name == "ta_video":
self._add_videos(values)
elif index_name == "ta_channel":
self._add_channels(values)
elif index_name == "ta_playlist":
self._add_playlists(values)
def _add_videos(self, values: list[str]) -> None:
"""add list of videos to reindex queue"""
if not values:
return
queue_name = self.REINDEX_CONFIG["video"]["queue_name"]
RedisQueue(queue_name).add_list(values)
def _add_channels(self, values: list[str]) -> None:
"""add list of channels to reindex queue"""
queue_name = self.REINDEX_CONFIG["channel"]["queue_name"]
RedisQueue(queue_name).add_list(values)
if self.extract_videos:
for channel_id in values:
all_videos = self._get_channel_videos(channel_id)
self._add_videos(all_videos)
def _add_playlists(self, values: list[str]) -> None:
"""add list of playlists to reindex queue"""
queue_name = self.REINDEX_CONFIG["playlist"]["queue_name"]
RedisQueue(queue_name).add_list(values)
if self.extract_videos:
for playlist_id in values:
all_videos = self._get_playlist_videos(playlist_id)
self._add_videos(all_videos)
def _get_channel_videos(self, channel_id: str) -> list[str]:
"""get all videos from channel"""
data = {
"query": {"term": {"channel.channel_id": {"value": channel_id}}},
"_source": ["youtube_id"],
}
all_results = IndexPaginate("ta_video", data).get_results()
return [i["youtube_id"] for i in all_results]
def _get_playlist_videos(self, playlist_id: str) -> list[str]:
"""get all videos from playlist"""
data = {
"query": {"term": {"playlist.keyword": {"value": playlist_id}}},
"_source": ["youtube_id"],
}
all_results = IndexPaginate("ta_video", data).get_results()
return [i["youtube_id"] for i in all_results]
class Reindex(ReindexBase):
"""reindex all documents from redis queue"""
def __init__(self, task=False):
super().__init__()
self.task = task
self.processed = {
"videos": 0,
"channels": 0,
"playlists": 0,
}
def reindex_all(self) -> None:
"""reindex all in queue"""
if not self.cookie_is_valid():
print("[reindex] cookie invalid, exiting...")
return
for name, index_config in self.REINDEX_CONFIG.items():
if not RedisQueue(index_config["queue_name"]).length():
continue
self.reindex_type(name, index_config)
def reindex_type(self, name: str, index_config: ReindexConfigType) -> None:
"""reindex all of a single index"""
reindex = self._get_reindex_map(index_config["index_name"])
queue = RedisQueue(index_config["queue_name"])
while True:
total = queue.max_score()
youtube_id, idx = queue.get_next()
if not youtube_id or not idx or not total:
break
if self.task:
self._notify(name, total, idx)
reindex(youtube_id)
sleep_interval = self.config["downloads"].get("sleep_interval", 0)
sleep(sleep_interval)
def _get_reindex_map(self, index_name: str) -> Callable:
"""return def to run for index"""
def_map = {
"ta_video": self._reindex_single_video,
"ta_channel": self._reindex_single_channel,
"ta_playlist": self._reindex_single_playlist,
}
return def_map[index_name]
def _notify(self, name: str, total: int, idx: int) -> None:
"""send notification back to task"""
message = [f"Reindexing {name.title()}s {idx}/{total}"]
progress = idx / total
self.task.send_progress(message, progress=progress)
def _reindex_single_video(self, youtube_id: str) -> None:
"""refresh data for single video"""
video = YoutubeVideo(youtube_id)
# read current state
video.get_from_es()
if not video.json_data:
return
es_meta = video.json_data.copy()
# get new
media_url = os.path.join(
EnvironmentSettings.MEDIA_DIR, es_meta["media_url"]
)
video.build_json(media_path=media_url)
if not video.youtube_meta:
video.deactivate()
return
video.delete_subtitles(subtitles=es_meta.get("subtitles"))
video.check_subtitles()
# add back
video.json_data["player"] = es_meta.get("player")
video.json_data["date_downloaded"] = es_meta.get("date_downloaded")
video.json_data["channel"] = es_meta.get("channel")
if es_meta.get("playlist"):
video.json_data["playlist"] = es_meta.get("playlist")
video.upload_to_es()
thumb_handler = ThumbManager(youtube_id)
thumb_handler.delete_video_thumb()
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
Comments(youtube_id, config=self.config).reindex_comments()
self.processed["videos"] += 1
def _reindex_single_channel(self, channel_id: str) -> None:
"""refresh channel data and sync to videos"""
# read current state
channel = YoutubeChannel(channel_id)
channel.get_from_es()
if not channel.json_data:
return
es_meta = channel.json_data.copy()
# get new
channel.get_from_youtube()
if not channel.youtube_meta:
channel.deactivate()
channel.get_from_es()
channel.sync_to_videos()
return
channel.process_youtube_meta()
channel.get_channel_art()
# add back
channel.json_data["channel_subscribed"] = es_meta["channel_subscribed"]
overwrites = es_meta.get("channel_overwrites")
if overwrites:
channel.json_data["channel_overwrites"] = overwrites
channel.upload_to_es()
channel.sync_to_videos()
ChannelFullScan(channel_id).scan()
self.processed["channels"] += 1
def _reindex_single_playlist(self, playlist_id: str) -> None:
"""refresh playlist data"""
playlist = YoutubePlaylist(playlist_id)
playlist.get_from_es()
if (
not playlist.json_data
or playlist.json_data["playlist_type"] == "custom"
):
return
is_active = playlist.update_playlist()
if not is_active:
playlist.deactivate()
return
self.processed["playlists"] += 1
def cookie_is_valid(self) -> bool:
"""return true if cookie is enabled and valid"""
if not self.config["downloads"]["cookie_import"]:
# is not activated, continue reindex
return True
valid = CookieHandler(self.config).validate()
return valid
def build_message(self) -> str:
"""build progress message"""
message = ""
for key, value in self.processed.items():
if value:
message = message + f"{value} {key}, "
if message:
message = f"reindexed {message.rstrip(', ')}"
return message
class ReindexProgress(ReindexBase):
"""
get progress of reindex task
request_type: key of self.REINDEX_CONFIG
request_id: id of request_type
return = {
"state": "running" | "queued" | False
"total_queued": int
"in_queue_name": "queue_name"
}
"""
def __init__(self, request_type=False, request_id=False):
super().__init__()
self.request_type = request_type
self.request_id = request_id
def get_progress(self) -> dict:
"""get progress from task"""
queue_name, request_type = self._get_queue_name()
total = self._get_total_in_queue(queue_name)
progress = {
"total_queued": total,
"type": request_type,
}
state = self._get_state(total, queue_name)
progress.update(state)
return progress
def _get_queue_name(self):
"""return queue_name, queue_type, raise exception on error"""
if not self.request_type:
return "all", "all"
reindex_config = self.REINDEX_CONFIG.get(self.request_type)
if not reindex_config:
print(f"reindex_config not found: {self.request_type}")
raise ValueError
return reindex_config["queue_name"], self.request_type
def _get_total_in_queue(self, queue_name):
"""get all items in queue"""
total = 0
if queue_name == "all":
queues = [i["queue_name"] for i in self.REINDEX_CONFIG.values()]
for queue in queues:
total += len(RedisQueue(queue).get_all())
else:
total += len(RedisQueue(queue_name).get_all())
return total
def _get_state(self, total, queue_name):
"""get state based on request_id"""
state_dict = {}
if self.request_id:
state = RedisQueue(queue_name).in_queue(self.request_id)
state_dict.update({"id": self.request_id, "state": state})
return state_dict
if total:
state = "running"
else:
state = "empty"
state_dict.update({"state": state})
return state_dict
class ChannelFullScan:
"""
update from v0.3.0 to v0.3.1
full scan of channel to fix vid_type mismatch
"""
def __init__(self, channel_id):
self.channel_id = channel_id
self.to_update = False
def scan(self):
"""match local with remote"""
print(f"{self.channel_id}: start full scan")
all_local_videos = self._get_all_local()
all_remote_videos = self._get_all_remote()
self.to_update = []
for video in all_local_videos:
video_id = video["youtube_id"]
remote_match = [i for i in all_remote_videos if i[0] == video_id]
if not remote_match:
print(f"{video_id}: no remote match found")
continue
expected_type = remote_match[0][-1]
if video["vid_type"] != expected_type:
self.to_update.append(
{
"video_id": video_id,
"vid_type": expected_type,
}
)
self.update()
def _get_all_remote(self):
"""get all channel videos"""
sub = ChannelSubscription()
all_remote_videos = sub.get_last_youtube_videos(
self.channel_id, limit=False
)
return all_remote_videos
def _get_all_local(self):
"""get all local indexed channel_videos"""
channel = YoutubeChannel(self.channel_id)
all_local_videos = channel.get_channel_videos()
return all_local_videos
def update(self):
"""build bulk query for updates"""
if not self.to_update:
print(f"{self.channel_id}: nothing to update")
return
print(f"{self.channel_id}: fixing {len(self.to_update)} videos")
bulk_list = []
for video in self.to_update:
action = {
"update": {"_id": video.get("video_id"), "_index": "ta_video"}
}
source = {"doc": {"vid_type": video.get("vid_type")}}
bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(source))
# add last newline
bulk_list.append("\n")
data = "\n".join(bulk_list)
_, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)

285
backend/appsettings/src/snapshot.py Normal file
View File

@@ -0,0 +1,285 @@
"""
functionality:
- handle snapshots in ES
"""
from datetime import datetime
from time import sleep
from zoneinfo import ZoneInfo
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap
from common.src.helper import get_mapping
class ElasticSnapshot:
"""interact with snapshots on ES"""
REPO = "ta_snapshot"
REPO_SETTINGS = {
"compress": "true",
"chunk_size": "1g",
"location": EnvironmentSettings.ES_SNAPSHOT_DIR,
}
POLICY = "ta_daily"
def __init__(self):
self.all_indices = self._get_all_indices()
def _get_all_indices(self):
"""return all indices names managed by TA"""
mapping = get_mapping()
all_indices = [f"ta_{i['index_name']}" for i in mapping]
return all_indices
def setup(self):
"""setup the snapshot in ES, create or update if needed"""
print("snapshot: run setup")
repo_exists = self._check_repo_exists()
if not repo_exists:
self.create_repo()
policy_exists = self._check_policy_exists()
if not policy_exists:
self.create_policy()
is_outdated = self._needs_startup_snapshot()
if is_outdated:
_ = self.take_snapshot_now()
def _check_repo_exists(self):
"""check if expected repo already exists"""
path = f"_snapshot/{self.REPO}"
response, statuscode = ElasticWrap(path).get()
if statuscode == 200:
print(f"snapshot: repo {self.REPO} already created")
matching = response[self.REPO]["settings"] == self.REPO_SETTINGS
if not matching:
print(f"snapshot: update repo settings {self.REPO_SETTINGS}")
return matching
print(f"snapshot: setup repo {self.REPO} config {self.REPO_SETTINGS}")
return False
def create_repo(self):
"""create filesystem repo"""
path = f"_snapshot/{self.REPO}"
data = {
"type": "fs",
"settings": self.REPO_SETTINGS,
}
response, statuscode = ElasticWrap(path).post(data=data)
if statuscode == 200:
print(f"snapshot: repo setup correctly: {response}")
def _check_policy_exists(self):
"""check if snapshot policy is set correctly"""
policy = self._get_policy()
expected_policy = self._build_policy_data()
if not policy:
print(f"snapshot: create policy {self.POLICY} {expected_policy}")
return False
if policy["policy"] != expected_policy:
print(f"snapshot: update policy settings {expected_policy}")
return False
print("snapshot: policy is set.")
return True
def _get_policy(self):
"""get policy from es"""
path = f"_slm/policy/{self.POLICY}"
response, statuscode = ElasticWrap(path).get()
if statuscode != 200:
return False
return response[self.POLICY]
def create_policy(self):
"""create snapshot lifetime policy"""
path = f"_slm/policy/{self.POLICY}"
data = self._build_policy_data()
response, statuscode = ElasticWrap(path).put(data)
if statuscode == 200:
print(f"snapshot: policy setup correctly: {response}")
def _build_policy_data(self):
"""build policy dict from config"""
at_12 = datetime.now().replace(hour=12, minute=0, second=0)
hour = at_12.astimezone(ZoneInfo("UTC")).hour
return {
"schedule": f"0 0 {hour} * * ?",
"name": f"<{self.POLICY}_>",
"repository": self.REPO,
"config": {
"indices": self.all_indices,
"ignore_unavailable": True,
"include_global_state": True,
},
"retention": {
"expire_after": "30d",
"min_count": 5,
"max_count": 50,
},
}
def _needs_startup_snapshot(self):
"""check if last snapshot is expired"""
snap_dicts = self._get_all_snapshots()
if not snap_dicts:
print("snapshot: create initial snapshot")
return True
last_stamp = snap_dicts[0]["end_stamp"]
now = int(datetime.now().timestamp())
outdated = (now - last_stamp) / 60 / 60 > 24
if outdated:
print("snapshot: is outdated, create new now")
print("snapshot: last snapshot is up-to-date")
return outdated
def take_snapshot_now(self, wait=False):
"""execute daily snapshot now"""
path = f"_slm/policy/{self.POLICY}/_execute"
response, statuscode = ElasticWrap(path).post()
if statuscode == 200:
print(f"snapshot: executing now: {response}")
if wait:
self._wait_for_snapshot(response["snapshot_name"])
return response
def _wait_for_snapshot(self, snapshot_name):
"""return after snapshot_name completes"""
path = f"_snapshot/{self.REPO}/{snapshot_name}"
while True:
# wait for task to be created
sleep(1)
_, statuscode = ElasticWrap(path).get()
if statuscode == 200:
break
while True:
# wait for snapshot success
response, statuscode = ElasticWrap(path).get()
snapshot_state = response["snapshots"][0]["state"]
if snapshot_state == "SUCCESS":
break
print(f"snapshot: {snapshot_name} in state {snapshot_state}")
print("snapshot: wait to complete")
sleep(5)
print(f"snapshot: completed - {response}")
def get_snapshot_stats(self):
"""get snapshot info for frontend"""
snapshot_info = self._build_policy_details()
if snapshot_info:
snapshot_info.update({"snapshots": self._get_all_snapshots()})
return snapshot_info
def get_single_snapshot(self, snapshot_id):
"""get single snapshot metadata"""
path = f"_snapshot/{self.REPO}/{snapshot_id}"
response, statuscode = ElasticWrap(path).get()
if statuscode == 404:
print(f"snapshots: not found: {snapshot_id}")
return False
snapshot = response["snapshots"][0]
return self._parse_single_snapshot(snapshot)
def _get_all_snapshots(self):
"""get a list of all registered snapshots"""
path = f"_snapshot/{self.REPO}/*?sort=start_time&order=desc"
response, statuscode = ElasticWrap(path).get()
if statuscode == 404:
print("snapshots: not configured")
return False
all_snapshots = response["snapshots"]
if not all_snapshots:
print("snapshots: no snapshots found")
return False
snap_dicts = []
for snapshot in all_snapshots:
snap_dict = self._parse_single_snapshot(snapshot)
snap_dicts.append(snap_dict)
return snap_dicts
def _parse_single_snapshot(self, snapshot):
"""extract relevant metadata from single snapshot"""
snap_dict = {
"id": snapshot["snapshot"],
"state": snapshot["state"],
"es_version": snapshot["version"],
"start_date": self._date_converter(snapshot["start_time"]),
"end_date": self._date_converter(snapshot["end_time"]),
"end_stamp": snapshot["end_time_in_millis"] // 1000,
"duration_s": snapshot["duration_in_millis"] // 1000,
}
return snap_dict
def _build_policy_details(self):
"""get additional policy details"""
policy = self._get_policy()
if not policy:
return False
next_exec = policy["next_execution_millis"] // 1000
next_exec_date = datetime.fromtimestamp(next_exec)
next_exec_str = next_exec_date.strftime("%Y-%m-%d %H:%M")
expire_after = policy["policy"]["retention"]["expire_after"]
policy_metadata = {
"next_exec": next_exec,
"next_exec_str": next_exec_str,
"expire_after": expire_after,
}
return policy_metadata
@staticmethod
def _date_converter(date_utc):
"""convert datetime string"""
date = datetime.strptime(date_utc, "%Y-%m-%dT%H:%M:%S.%fZ")
utc_date = date.replace(tzinfo=ZoneInfo("UTC"))
converted = utc_date.astimezone(ZoneInfo(EnvironmentSettings.TZ))
converted_str = converted.strftime("%Y-%m-%d %H:%M")
return converted_str
def restore_all(self, snapshot_name):
"""restore snapshot by name"""
for index in self.all_indices:
_, _ = ElasticWrap(index).delete()
path = f"_snapshot/{self.REPO}/{snapshot_name}/_restore"
data = {"indices": "*"}
response, statuscode = ElasticWrap(path).post(data=data)
if statuscode == 200:
print(f"snapshot: executing now: {response}")
return response
print(f"snapshot: failed to restore, {statuscode} {response}")
return False
def delete_single_snapshot(self, snapshot_id):
"""delete single snapshot from index"""
path = f"_snapshot/{self.REPO}/{snapshot_id}"
response, statuscode = ElasticWrap(path).delete()
if statuscode == 200:
print(f"snapshot: deleting {snapshot_id} {response}")
return response
print(f"snapshot: failed to delete, {statuscode} {response}")
return False

42
backend/appsettings/urls.py Normal file
View File

@@ -0,0 +1,42 @@
"""all app settings API urls"""
from appsettings import views
from django.urls import path
urlpatterns = [
path(
"config/",
views.AppConfigApiView.as_view(),
name="api-config",
),
path(
"snapshot/",
views.SnapshotApiListView.as_view(),
name="api-snapshot-list",
),
path(
"snapshot/<slug:snapshot_id>/",
views.SnapshotApiView.as_view(),
name="api-snapshot",
),
path(
"backup/",
views.BackupApiListView.as_view(),
name="api-backup-list",
),
path(
"backup/<str:filename>/",
views.BackupApiView.as_view(),
name="api-backup",
),
path(
"cookie/",
views.CookieView.as_view(),
name="api-cookie",
),
path(
"token/",
views.TokenView.as_view(),
name="api-token",
),
]

252
backend/appsettings/views.py Normal file
View File

@@ -0,0 +1,252 @@
"""all app settings API views"""
from appsettings.src.backup import ElasticBackup
from appsettings.src.config import AppConfig
from appsettings.src.snapshot import ElasticSnapshot
from common.src.ta_redis import RedisArchivist
from common.views_base import AdminOnly, ApiBaseView
from download.src.yt_dlp_base import CookieHandler
from rest_framework.response import Response
from task.src.task_manager import TaskCommand
from task.tasks import run_restore_backup
class AppConfigApiView(ApiBaseView):
"""resolves to /api/appsettings/config/
GET: return app settings
POST: update app settings
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request):
"""get config"""
response = AppConfig().config
return Response(response)
@staticmethod
def post(request):
"""
update config values
data object where key is flatted CONFIG_DEFAULTS separated by '.', e.g.
{"subscriptions.channel_size": 5, "subscriptions.live_channel_size": 5}
"""
data = request.data
try:
config = AppConfig().update_config(data)
except ValueError as err:
return Response({"error": str(err)}, status=400)
return Response(config)
class SnapshotApiListView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/
GET: returns snapshot config plus list of existing snapshots
POST: take snapshot now
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request):
"""handle get request"""
# pylint: disable=unused-argument
snapshots = ElasticSnapshot().get_snapshot_stats()
return Response(snapshots)
@staticmethod
def post(request):
"""take snapshot now with post request"""
# pylint: disable=unused-argument
response = ElasticSnapshot().take_snapshot_now()
return Response(response)
class SnapshotApiView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/<snapshot-id>/
GET: return a single snapshot
POST: restore snapshot
DELETE: delete a snapshot
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request, snapshot_id):
"""handle get request"""
# pylint: disable=unused-argument
snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
if not snapshot:
return Response({"message": "snapshot not found"}, status=404)
return Response(snapshot)
@staticmethod
def post(request, snapshot_id):
"""restore snapshot with post request"""
# pylint: disable=unused-argument
response = ElasticSnapshot().restore_all(snapshot_id)
if not response:
message = {"message": "failed to restore snapshot"}
return Response(message, status=400)
return Response(response)
@staticmethod
def delete(request, snapshot_id):
"""delete snapshot from index"""
# pylint: disable=unused-argument
response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
if not response:
message = {"message": "failed to delete snapshot"}
return Response(message, status=400)
return Response(response)
class BackupApiListView(ApiBaseView):
"""resolves to /api/appsettings/backup/
GET: returns list of available zip backups
POST: take zip backup now
"""
permission_classes = [AdminOnly]
task_name = "run_backup"
@staticmethod
def get(request):
"""handle get request"""
# pylint: disable=unused-argument
backup_files = ElasticBackup().get_all_backup_files()
return Response(backup_files)
def post(self, request):
"""handle post request"""
# pylint: disable=unused-argument
response = TaskCommand().start(self.task_name)
message = {
"message": "backup task started",
"task_id": response["task_id"],
}
return Response(message)
class BackupApiView(ApiBaseView):
"""resolves to /api/appsettings/backup/<filename>/
GET: return a single backup
POST: restore backup
DELETE: delete backup
"""
permission_classes = [AdminOnly]
task_name = "restore_backup"
@staticmethod
def get(request, filename):
"""get single backup"""
# pylint: disable=unused-argument
backup_file = ElasticBackup().build_backup_file_data(filename)
if not backup_file:
message = {"message": "file not found"}
return Response(message, status=404)
return Response(backup_file)
def post(self, request, filename):
"""restore backup file"""
# pylint: disable=unused-argument
task = run_restore_backup.delay(filename)
message = {
"message": "backup restore task started",
"filename": filename,
"task_id": task.id,
}
return Response(message)
@staticmethod
def delete(request, filename):
"""delete backup file"""
# pylint: disable=unused-argument
backup_file = ElasticBackup().delete_file(filename)
if not backup_file:
message = {"message": "file not found"}
return Response(message, status=404)
message = {"message": f"file {filename} deleted"}
return Response(message)
class CookieView(ApiBaseView):
"""resolves to /api/appsettings/cookie/
GET: check if cookie is enabled
POST: verify validity of cookie
PUT: import cookie
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request):
"""handle get request"""
# pylint: disable=unused-argument
config = AppConfig().config
valid = RedisArchivist().get_message("cookie:valid")
response = {"cookie_enabled": config["downloads"]["cookie_import"]}
response.update(valid)
return Response(response)
@staticmethod
def post(request):
"""handle post request"""
# pylint: disable=unused-argument
config = AppConfig().config
validated = CookieHandler(config).validate()
return Response({"cookie_validated": validated})
@staticmethod
def put(request):
"""handle put request"""
# pylint: disable=unused-argument
config = AppConfig().config
cookie = request.data.get("cookie")
if not cookie:
message = "missing cookie key in request data"
print(message)
return Response({"message": message}, status=400)
print(f"cookie preview:\n\n{cookie[:300]}")
handler = CookieHandler(config)
handler.set_cookie(cookie)
validated = handler.validate()
if not validated:
handler.revoke()
message = {"cookie_import": "fail", "cookie_validated": validated}
print(f"cookie: {message}")
return Response({"message": message}, status=400)
message = {"cookie_import": "done", "cookie_validated": validated}
return Response(message)
class TokenView(ApiBaseView):
"""resolves to /api/appsettings/token/
DELETE: revoke the token
"""
permission_classes = [AdminOnly]
@staticmethod
def delete(request):
"""delete the token, new will get created automatically"""
print("revoke API token")
request.user.auth_token.delete()
return Response({"success": True})