mirror of
https://git.vectorsigma.ru/public/tubearchivist.git
synced 2026-08-07 23:00:00 +00:00
renamed django app folder to backend
This commit is contained in:
Simon
0
backend/common/src/__init__.py
Normal file
0
backend/common/src/__init__.py
Normal file
117
backend/common/src/env_settings.py
Normal file
117
backend/common/src/env_settings.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
Functionality:
|
||||
- read and write application config backed by ES
|
||||
- encapsulate persistence of application properties
|
||||
"""
|
||||
|
||||
from os import environ
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
print("loading local dotenv")
|
||||
load_dotenv(".env")
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
class EnvironmentSettings:
|
||||
"""
|
||||
Handle settings for the application that are driven from the environment.
|
||||
These will not change when the user is using the application.
|
||||
These settings are only provided only on startup.
|
||||
"""
|
||||
|
||||
HOST_UID: int = int(environ.get("HOST_UID", False))
|
||||
HOST_GID: int = int(environ.get("HOST_GID", False))
|
||||
ENABLE_CAST: bool = bool(environ.get("ENABLE_CAST"))
|
||||
TZ: str = str(environ.get("TZ", "UTC"))
|
||||
TA_PORT: int = int(environ.get("TA_PORT", False))
|
||||
TA_UWSGI_PORT: int = int(environ.get("TA_UWSGI_PORT", False))
|
||||
TA_USERNAME: str = str(environ.get("TA_USERNAME"))
|
||||
TA_PASSWORD: str = str(environ.get("TA_PASSWORD"))
|
||||
|
||||
# Application Paths
|
||||
MEDIA_DIR: str = str(environ.get("TA_MEDIA_DIR", "/youtube"))
|
||||
APP_DIR: str = str(environ.get("TA_APP_DIR", "/app"))
|
||||
CACHE_DIR: str = str(environ.get("TA_CACHE_DIR", "/cache"))
|
||||
|
||||
# Redis
|
||||
REDIS_HOST: str = str(environ.get("REDIS_HOST"))
|
||||
REDIS_PORT: int = int(environ.get("REDIS_PORT", 6379))
|
||||
REDIS_NAME_SPACE: str = str(environ.get("REDIS_NAME_SPACE", "ta:"))
|
||||
|
||||
# ElasticSearch
|
||||
ES_URL: str = str(environ.get("ES_URL"))
|
||||
ES_PASS: str = str(environ.get("ELASTIC_PASSWORD"))
|
||||
ES_USER: str = str(environ.get("ELASTIC_USER", "elastic"))
|
||||
ES_SNAPSHOT_DIR: str = str(
|
||||
environ.get(
|
||||
"ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
|
||||
)
|
||||
)
|
||||
ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL"))
|
||||
|
||||
def get_cache_root(self):
|
||||
"""get root for web server"""
|
||||
if self.CACHE_DIR.startswith("/"):
|
||||
return self.CACHE_DIR
|
||||
|
||||
return f"/{self.CACHE_DIR}"
|
||||
|
||||
def get_media_root(self):
|
||||
"""get root for media folder"""
|
||||
if self.MEDIA_DIR.startswith("/"):
|
||||
return self.MEDIA_DIR
|
||||
|
||||
return f"/{self.MEDIA_DIR}"
|
||||
|
||||
def print_generic(self):
|
||||
"""print generic env vars"""
|
||||
print(
|
||||
f"""
|
||||
HOST_UID: {self.HOST_UID}
|
||||
HOST_GID: {self.HOST_GID}
|
||||
TZ: {self.TZ}
|
||||
ENABLE_CAST: {self.ENABLE_CAST}
|
||||
TA_PORT: {self.TA_PORT}
|
||||
TA_UWSGI_PORT: {self.TA_UWSGI_PORT}
|
||||
TA_USERNAME: {self.TA_USERNAME}
|
||||
TA_PASSWORD: *****"""
|
||||
)
|
||||
|
||||
def print_paths(self):
|
||||
"""debug paths set"""
|
||||
print(
|
||||
f"""
|
||||
MEDIA_DIR: {self.MEDIA_DIR}
|
||||
APP_DIR: {self.APP_DIR}
|
||||
CACHE_DIR: {self.CACHE_DIR}"""
|
||||
)
|
||||
|
||||
def print_redis_conf(self):
|
||||
"""debug redis conf paths"""
|
||||
print(
|
||||
f"""
|
||||
REDIS_HOST: {self.REDIS_HOST}
|
||||
REDIS_PORT: {self.REDIS_PORT}
|
||||
REDIS_NAME_SPACE: {self.REDIS_NAME_SPACE}"""
|
||||
)
|
||||
|
||||
def print_es_paths(self):
|
||||
"""debug es conf"""
|
||||
print(
|
||||
f"""
|
||||
ES_URL: {self.ES_URL}
|
||||
ES_PASS: *****
|
||||
ES_USER: {self.ES_USER}
|
||||
ES_SNAPSHOT_DIR: {self.ES_SNAPSHOT_DIR}
|
||||
ES_DISABLE_VERIFY_SSL: {self.ES_DISABLE_VERIFY_SSL}"""
|
||||
)
|
||||
|
||||
def print_all(self):
|
||||
"""print all"""
|
||||
self.print_generic()
|
||||
self.print_paths()
|
||||
self.print_redis_conf()
|
||||
self.print_es_paths()
|
||||
231
backend/common/src/es_connect.py
Normal file
231
backend/common/src/es_connect.py
Normal file
@@ -0,0 +1,231 @@
|
||||
"""
|
||||
functionality:
|
||||
- wrapper around requests to call elastic search
|
||||
- reusable search_after to extract total index
|
||||
"""
|
||||
|
||||
# pylint: disable=missing-timeout
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
import urllib3
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
|
||||
|
||||
class ElasticWrap:
|
||||
"""makes all calls to elastic search
|
||||
returns response json and status code tuple
|
||||
"""
|
||||
|
||||
def __init__(self, path: str):
|
||||
self.url: str = f"{EnvironmentSettings.ES_URL}/{path}"
|
||||
self.auth: tuple[str, str] = (
|
||||
EnvironmentSettings.ES_USER,
|
||||
EnvironmentSettings.ES_PASS,
|
||||
)
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
def get(
|
||||
self,
|
||||
data: bool | dict = False,
|
||||
timeout: int = 10,
|
||||
print_error: bool = True,
|
||||
) -> tuple[dict, int]:
|
||||
"""get data from es"""
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"auth": self.auth,
|
||||
"timeout": timeout,
|
||||
}
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
kwargs["verify"] = False
|
||||
|
||||
if data:
|
||||
kwargs["json"] = data
|
||||
|
||||
response = requests.get(self.url, **kwargs)
|
||||
|
||||
if print_error and not response.ok:
|
||||
print(response.text)
|
||||
|
||||
return response.json(), response.status_code
|
||||
|
||||
def post(
|
||||
self, data: bool | dict = False, ndjson: bool = False
|
||||
) -> tuple[dict, int]:
|
||||
"""post data to es"""
|
||||
|
||||
kwargs: dict[str, Any] = {"auth": self.auth}
|
||||
|
||||
if ndjson and data:
|
||||
kwargs.update(
|
||||
{
|
||||
"headers": {"Content-type": "application/x-ndjson"},
|
||||
"data": data,
|
||||
}
|
||||
)
|
||||
elif data:
|
||||
kwargs.update(
|
||||
{
|
||||
"headers": {"Content-type": "application/json"},
|
||||
"data": json.dumps(data),
|
||||
}
|
||||
)
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
kwargs["verify"] = False
|
||||
|
||||
response = requests.post(self.url, **kwargs)
|
||||
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
|
||||
return response.json(), response.status_code
|
||||
|
||||
def put(
|
||||
self,
|
||||
data: bool | dict = False,
|
||||
refresh: bool = False,
|
||||
) -> tuple[dict, Any]:
|
||||
"""put data to es"""
|
||||
|
||||
if refresh:
|
||||
self.url = f"{self.url}/?refresh=true"
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"json": data,
|
||||
"auth": self.auth,
|
||||
}
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
kwargs["verify"] = False
|
||||
|
||||
response = requests.put(self.url, **kwargs)
|
||||
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
print(data)
|
||||
raise ValueError("failed to add item to index")
|
||||
|
||||
return response.json(), response.status_code
|
||||
|
||||
def delete(
|
||||
self,
|
||||
data: bool | dict = False,
|
||||
refresh: bool = False,
|
||||
) -> tuple[dict, Any]:
|
||||
"""delete document from es"""
|
||||
|
||||
if refresh:
|
||||
self.url = f"{self.url}/?refresh=true"
|
||||
|
||||
kwargs: dict[str, Any] = {"auth": self.auth}
|
||||
|
||||
if data:
|
||||
kwargs["json"] = data
|
||||
|
||||
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||
kwargs["verify"] = False
|
||||
|
||||
response = requests.delete(self.url, **kwargs)
|
||||
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
|
||||
return response.json(), response.status_code
|
||||
|
||||
|
||||
class IndexPaginate:
|
||||
"""use search_after to go through whole index
|
||||
kwargs:
|
||||
- size: int, overwrite DEFAULT_SIZE
|
||||
- keep_source: bool, keep _source key from es results
|
||||
- callback: obj, Class implementing run method callback for every loop
|
||||
- task: task object to send notification
|
||||
- total: int, total items in index for progress message
|
||||
"""
|
||||
|
||||
DEFAULT_SIZE = 500
|
||||
|
||||
def __init__(self, index_name, data, **kwargs):
|
||||
self.index_name = index_name
|
||||
self.data = data
|
||||
self.pit_id = False
|
||||
self.kwargs = kwargs
|
||||
|
||||
def get_results(self):
|
||||
"""get all results, add task and total for notifications"""
|
||||
self.get_pit()
|
||||
self.validate_data()
|
||||
all_results = self.run_loop()
|
||||
self.clean_pit()
|
||||
return all_results
|
||||
|
||||
def get_pit(self):
|
||||
"""get pit for index"""
|
||||
path = f"{self.index_name}/_pit?keep_alive=10m"
|
||||
response, _ = ElasticWrap(path).post()
|
||||
self.pit_id = response["id"]
|
||||
|
||||
def validate_data(self):
|
||||
"""add pit and size to data"""
|
||||
if not self.data:
|
||||
self.data = {}
|
||||
|
||||
if "query" not in self.data.keys():
|
||||
self.data.update({"query": {"match_all": {}}})
|
||||
|
||||
if "sort" not in self.data.keys():
|
||||
self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
|
||||
|
||||
self.data["size"] = self.kwargs.get("size") or self.DEFAULT_SIZE
|
||||
self.data["pit"] = {"id": self.pit_id, "keep_alive": "10m"}
|
||||
|
||||
def run_loop(self):
|
||||
"""loop through results until last hit"""
|
||||
all_results = []
|
||||
counter = 0
|
||||
while True:
|
||||
response, _ = ElasticWrap("_search").get(data=self.data)
|
||||
all_hits = response["hits"]["hits"]
|
||||
if not all_hits:
|
||||
break
|
||||
|
||||
for hit in all_hits:
|
||||
if self.kwargs.get("keep_source"):
|
||||
all_results.append(hit)
|
||||
else:
|
||||
all_results.append(hit["_source"])
|
||||
|
||||
if self.kwargs.get("callback"):
|
||||
self.kwargs.get("callback")(
|
||||
all_hits, self.index_name, counter=counter
|
||||
).run()
|
||||
|
||||
if self.kwargs.get("task"):
|
||||
print(f"{self.index_name}: processing page {counter}")
|
||||
self._notify(len(all_results))
|
||||
|
||||
counter += 1
|
||||
|
||||
# update search_after with last hit data
|
||||
self.data["search_after"] = all_hits[-1]["sort"]
|
||||
|
||||
return all_results
|
||||
|
||||
def _notify(self, processed):
|
||||
"""send notification on task"""
|
||||
total = self.kwargs.get("total")
|
||||
progress = processed / total
|
||||
index_clean = self.index_name.lstrip("ta_").title()
|
||||
message = [f"Processing {index_clean}s {processed}/{total}"]
|
||||
self.kwargs.get("task").send_progress(message, progress=progress)
|
||||
|
||||
def clean_pit(self):
|
||||
"""delete pit from elastic search"""
|
||||
ElasticWrap("_pit").delete(data={"id": self.pit_id})
|
||||
11
backend/common/src/health.py
Normal file
11
backend/common/src/health.py
Normal file
@@ -0,0 +1,11 @@
|
||||
from django.http import HttpResponse
|
||||
|
||||
|
||||
class HealthCheckMiddleware:
|
||||
def __init__(self, get_response):
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request):
|
||||
if request.path == "/health":
|
||||
return HttpResponse("ok")
|
||||
return self.get_response(request)
|
||||
269
backend/common/src/helper.py
Normal file
269
backend/common/src/helper.py
Normal file
@@ -0,0 +1,269 @@
|
||||
"""
|
||||
Loose collection of helper functions
|
||||
- don't import AppConfig class here to avoid circular imports
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import string
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import IndexPaginate
|
||||
|
||||
|
||||
def ignore_filelist(filelist: list[str]) -> list[str]:
|
||||
"""ignore temp files for os.listdir sanitizer"""
|
||||
to_ignore = [
|
||||
"@eaDir",
|
||||
"Icon\r\r",
|
||||
"Network Trash Folder",
|
||||
"Temporary Items",
|
||||
]
|
||||
cleaned: list[str] = []
|
||||
for file_name in filelist:
|
||||
if file_name.startswith(".") or file_name in to_ignore:
|
||||
continue
|
||||
|
||||
cleaned.append(file_name)
|
||||
|
||||
return cleaned
|
||||
|
||||
|
||||
def randomizor(length: int) -> str:
|
||||
"""generate random alpha numeric string"""
|
||||
pool: str = string.digits + string.ascii_letters
|
||||
return "".join(random.choice(pool) for i in range(length))
|
||||
|
||||
|
||||
def requests_headers() -> dict[str, str]:
|
||||
"""build header with random user agent for requests outside of yt-dlp"""
|
||||
|
||||
chrome_versions = (
|
||||
"90.0.4430.212",
|
||||
"90.0.4430.24",
|
||||
"90.0.4430.70",
|
||||
"90.0.4430.72",
|
||||
"90.0.4430.85",
|
||||
"90.0.4430.93",
|
||||
"91.0.4472.101",
|
||||
"91.0.4472.106",
|
||||
"91.0.4472.114",
|
||||
"91.0.4472.124",
|
||||
"91.0.4472.164",
|
||||
"91.0.4472.19",
|
||||
"91.0.4472.77",
|
||||
"92.0.4515.107",
|
||||
"92.0.4515.115",
|
||||
"92.0.4515.131",
|
||||
"92.0.4515.159",
|
||||
"92.0.4515.43",
|
||||
"93.0.4556.0",
|
||||
"93.0.4577.15",
|
||||
"93.0.4577.63",
|
||||
"93.0.4577.82",
|
||||
"94.0.4606.41",
|
||||
"94.0.4606.54",
|
||||
"94.0.4606.61",
|
||||
"94.0.4606.71",
|
||||
"94.0.4606.81",
|
||||
"94.0.4606.85",
|
||||
"95.0.4638.17",
|
||||
"95.0.4638.50",
|
||||
"95.0.4638.54",
|
||||
"95.0.4638.69",
|
||||
"95.0.4638.74",
|
||||
"96.0.4664.18",
|
||||
"96.0.4664.45",
|
||||
"96.0.4664.55",
|
||||
"96.0.4664.93",
|
||||
"97.0.4692.20",
|
||||
)
|
||||
template = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
+ f"Chrome/{random.choice(chrome_versions)} Safari/537.36"
|
||||
)
|
||||
|
||||
return {"User-Agent": template}
|
||||
|
||||
|
||||
def date_parser(timestamp: int | str) -> str:
|
||||
"""return formatted date string"""
|
||||
if isinstance(timestamp, int):
|
||||
date_obj = datetime.fromtimestamp(timestamp)
|
||||
elif isinstance(timestamp, str):
|
||||
date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
|
||||
else:
|
||||
raise TypeError(f"invalid timestamp: {timestamp}")
|
||||
|
||||
return date_obj.date().isoformat()
|
||||
|
||||
|
||||
def time_parser(timestamp: str) -> float:
|
||||
"""return seconds from timestamp, false on empty"""
|
||||
if not timestamp:
|
||||
return False
|
||||
|
||||
if timestamp.isnumeric():
|
||||
return int(timestamp)
|
||||
|
||||
hours, minutes, seconds = timestamp.split(":", maxsplit=3)
|
||||
return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
|
||||
|
||||
|
||||
def clear_dl_cache(cache_dir: str) -> int:
|
||||
"""clear leftover files from dl cache"""
|
||||
print("clear download cache")
|
||||
download_cache_dir = os.path.join(cache_dir, "download")
|
||||
leftover_files = ignore_filelist(os.listdir(download_cache_dir))
|
||||
for cached in leftover_files:
|
||||
to_delete = os.path.join(download_cache_dir, cached)
|
||||
os.remove(to_delete)
|
||||
|
||||
return len(leftover_files)
|
||||
|
||||
|
||||
def get_mapping() -> dict:
|
||||
"""read index_mapping.json and get expected mapping and settings"""
|
||||
with open("appsettings/index_mapping.json", "r", encoding="utf-8") as f:
|
||||
index_config: dict = json.load(f).get("index_config")
|
||||
|
||||
return index_config
|
||||
|
||||
|
||||
def is_shorts(youtube_id: str) -> bool:
|
||||
"""check if youtube_id is a shorts video, bot not it it's not a shorts"""
|
||||
shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
|
||||
cookies = {"SOCS": "CAI"}
|
||||
response = requests.head(
|
||||
shorts_url, cookies=cookies, headers=requests_headers(), timeout=10
|
||||
)
|
||||
|
||||
return response.status_code == 200
|
||||
|
||||
|
||||
def get_duration_sec(file_path: str) -> int:
|
||||
"""get duration of media file from file path"""
|
||||
|
||||
duration = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
"default=noprint_wrappers=1:nokey=1",
|
||||
file_path,
|
||||
],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
duration_raw = duration.stdout.decode().strip()
|
||||
if duration_raw == "N/A":
|
||||
return 0
|
||||
|
||||
duration_sec = int(float(duration_raw))
|
||||
return duration_sec
|
||||
|
||||
|
||||
def get_duration_str(seconds: int) -> str:
|
||||
"""Return a human-readable duration string from seconds."""
|
||||
if not seconds:
|
||||
return "NA"
|
||||
|
||||
units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
|
||||
duration_parts = []
|
||||
|
||||
for unit_label, unit_seconds in units:
|
||||
if seconds >= unit_seconds:
|
||||
unit_count, seconds = divmod(seconds, unit_seconds)
|
||||
duration_parts.append(f"{unit_count:02}{unit_label}")
|
||||
|
||||
duration_parts[0] = duration_parts[0].lstrip("0")
|
||||
|
||||
return " ".join(duration_parts)
|
||||
|
||||
|
||||
def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
|
||||
"""parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
|
||||
allowed_hosts: list[str] = [
|
||||
"localhost",
|
||||
"tubearchivist",
|
||||
]
|
||||
csrf_trusted_origins: list[str] = [
|
||||
"http://localhost",
|
||||
"http://tubearchivist",
|
||||
]
|
||||
for host in ta_host.split():
|
||||
host_clean = host.strip()
|
||||
if not host_clean.startswith("http"):
|
||||
host_clean = f"http://{host_clean}"
|
||||
|
||||
parsed = urlparse(host_clean)
|
||||
allowed_hosts.append(f"{parsed.hostname}")
|
||||
csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}")
|
||||
|
||||
return allowed_hosts, csrf_trusted_origins
|
||||
|
||||
|
||||
def get_stylesheets() -> list:
|
||||
"""Get all valid stylesheets from /static/css"""
|
||||
app_root = EnvironmentSettings.APP_DIR
|
||||
try:
|
||||
stylesheets = os.listdir(os.path.join(app_root, "static/css"))
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
|
||||
stylesheets.remove("style.css")
|
||||
stylesheets.sort()
|
||||
stylesheets = list(filter(lambda x: x.endswith(".css"), stylesheets))
|
||||
return stylesheets
|
||||
|
||||
|
||||
def check_stylesheet(stylesheet: str):
|
||||
"""Check if a stylesheet exists. Return dark.css as a fallback"""
|
||||
if stylesheet in get_stylesheets():
|
||||
return stylesheet
|
||||
|
||||
return "dark.css"
|
||||
|
||||
|
||||
def is_missing(
|
||||
to_check: str | list[str],
|
||||
index_name: str = "ta_video,ta_download",
|
||||
on_key: str = "youtube_id",
|
||||
) -> list[str]:
|
||||
"""id or list of ids that are missing from index_name"""
|
||||
if isinstance(to_check, str):
|
||||
to_check = [to_check]
|
||||
|
||||
data = {
|
||||
"query": {"terms": {on_key: to_check}},
|
||||
"_source": [on_key],
|
||||
}
|
||||
result = IndexPaginate(index_name, data=data).get_results()
|
||||
existing_ids = [i[on_key] for i in result]
|
||||
dl = [i for i in to_check if i not in existing_ids]
|
||||
|
||||
return dl
|
||||
|
||||
|
||||
def get_channel_overwrites() -> dict[str, dict[str, Any]]:
|
||||
"""get overwrites indexed my channel_id"""
|
||||
data = {
|
||||
"query": {
|
||||
"bool": {"must": [{"exists": {"field": "channel_overwrites"}}]}
|
||||
},
|
||||
"_source": ["channel_id", "channel_overwrites"],
|
||||
}
|
||||
result = IndexPaginate("ta_channel", data).get_results()
|
||||
overwrites = {i["channel_id"]: i["channel_overwrites"] for i in result}
|
||||
|
||||
return overwrites
|
||||
145
backend/common/src/index_generic.py
Normal file
145
backend/common/src/index_generic.py
Normal file
@@ -0,0 +1,145 @@
|
||||
"""
|
||||
functionality:
|
||||
- generic base class to inherit from for video, channel and playlist
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from user.src.user_config import UserConfig
|
||||
|
||||
|
||||
class YouTubeItem:
|
||||
"""base class for youtube"""
|
||||
|
||||
es_path = False
|
||||
index_name = ""
|
||||
yt_base = ""
|
||||
yt_obs: dict[str, bool | str] = {
|
||||
"skip_download": True,
|
||||
"noplaylist": True,
|
||||
}
|
||||
|
||||
def __init__(self, youtube_id):
|
||||
self.youtube_id = youtube_id
|
||||
self.es_path = f"{self.index_name}/_doc/{youtube_id}"
|
||||
self.config = AppConfig().config
|
||||
self.youtube_meta = False
|
||||
self.json_data = False
|
||||
|
||||
def build_yt_url(self):
|
||||
"""build youtube url"""
|
||||
return self.yt_base + self.youtube_id
|
||||
|
||||
def get_from_youtube(self):
|
||||
"""use yt-dlp to get meta data from youtube"""
|
||||
print(f"{self.youtube_id}: get metadata from youtube")
|
||||
obs_request = self.yt_obs.copy()
|
||||
if self.config["downloads"]["extractor_lang"]:
|
||||
langs = self.config["downloads"]["extractor_lang"]
|
||||
langs_list = [i.strip() for i in langs.split(",")]
|
||||
obs_request["extractor_args"] = {"youtube": {"lang": langs_list}}
|
||||
|
||||
url = self.build_yt_url()
|
||||
self.youtube_meta = YtWrap(obs_request, self.config).extract(url)
|
||||
|
||||
def get_from_es(self):
|
||||
"""get indexed data from elastic search"""
|
||||
print(f"{self.youtube_id}: get metadata from es")
|
||||
response, _ = ElasticWrap(f"{self.es_path}").get()
|
||||
source = response.get("_source")
|
||||
self.json_data = source
|
||||
|
||||
def upload_to_es(self):
|
||||
"""add json_data to elastic"""
|
||||
_, _ = ElasticWrap(self.es_path).put(self.json_data, refresh=True)
|
||||
|
||||
def deactivate(self):
|
||||
"""deactivate document in es"""
|
||||
print(f"{self.youtube_id}: deactivate document")
|
||||
key_match = {
|
||||
"ta_video": "active",
|
||||
"ta_channel": "channel_active",
|
||||
"ta_playlist": "playlist_active",
|
||||
}
|
||||
path = f"{self.index_name}/_update/{self.youtube_id}?refresh=true"
|
||||
data = {
|
||||
"script": f"ctx._source.{key_match.get(self.index_name)} = false"
|
||||
}
|
||||
_, _ = ElasticWrap(path).post(data)
|
||||
|
||||
def del_in_es(self):
|
||||
"""delete item from elastic search"""
|
||||
print(f"{self.youtube_id}: delete from es")
|
||||
_, _ = ElasticWrap(self.es_path).delete(refresh=True)
|
||||
|
||||
|
||||
class Pagination:
|
||||
"""
|
||||
figure out the pagination based on page size and total_hits
|
||||
"""
|
||||
|
||||
def __init__(self, request):
|
||||
self.request = request
|
||||
self.page_get = False
|
||||
self.params = False
|
||||
self.get_params()
|
||||
self.page_size = self.get_page_size()
|
||||
self.pagination = self.first_guess()
|
||||
|
||||
def get_params(self):
|
||||
"""process url query parameters"""
|
||||
query_dict = self.request.GET.copy()
|
||||
self.page_get = int(query_dict.get("page", 0))
|
||||
|
||||
_ = query_dict.pop("page", False)
|
||||
self.params = query_dict.urlencode()
|
||||
|
||||
def get_page_size(self):
|
||||
"""get default or user modified page_size"""
|
||||
return UserConfig(self.request.user.id).get_value("page_size")
|
||||
|
||||
def first_guess(self):
|
||||
"""build first guess before api call"""
|
||||
page_get = self.page_get
|
||||
page_from = 0
|
||||
if page_get in [0, 1]:
|
||||
prev_pages = False
|
||||
elif page_get > 1:
|
||||
page_from = (page_get - 1) * self.page_size
|
||||
prev_pages = [
|
||||
i for i in range(page_get - 1, page_get - 6, -1) if i > 1
|
||||
]
|
||||
prev_pages.reverse()
|
||||
pagination = {
|
||||
"page_size": self.page_size,
|
||||
"page_from": page_from,
|
||||
"prev_pages": prev_pages,
|
||||
"current_page": page_get,
|
||||
"max_hits": False,
|
||||
"params": self.params,
|
||||
}
|
||||
|
||||
return pagination
|
||||
|
||||
def validate(self, total_hits):
|
||||
"""validate pagination with total_hits after making api call"""
|
||||
page_get = self.page_get
|
||||
max_pages = math.ceil(total_hits / self.page_size)
|
||||
if total_hits >= 10000:
|
||||
# es returns maximal 10000 results
|
||||
self.pagination["max_hits"] = True
|
||||
max_pages = max_pages - 1
|
||||
|
||||
if page_get < max_pages and max_pages > 1:
|
||||
self.pagination["last_page"] = max_pages
|
||||
else:
|
||||
self.pagination["last_page"] = False
|
||||
next_pages = [
|
||||
i for i in range(page_get + 1, page_get + 6) if 1 < i < max_pages
|
||||
]
|
||||
|
||||
self.pagination["next_pages"] = next_pages
|
||||
self.pagination["total_hits"] = total_hits
|
||||
182
backend/common/src/search_processor.py
Normal file
182
backend/common/src/search_processor.py
Normal file
@@ -0,0 +1,182 @@
|
||||
"""
|
||||
Functionality:
|
||||
- processing search results for frontend
|
||||
- this is duplicated code from home.src.frontend.searching.SearchHandler
|
||||
"""
|
||||
|
||||
import urllib.parse
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.helper import date_parser, get_duration_str
|
||||
from download.src.thumbnails import ThumbManager
|
||||
|
||||
|
||||
class SearchProcess:
|
||||
"""process search results"""
|
||||
|
||||
def __init__(self, response):
|
||||
self.response = response
|
||||
self.processed = False
|
||||
|
||||
def process(self):
|
||||
"""detect type and process"""
|
||||
print(self.response)
|
||||
if "_source" in self.response.keys():
|
||||
# single
|
||||
self.processed = self._process_result(self.response)
|
||||
|
||||
elif "hits" in self.response.keys():
|
||||
# multiple
|
||||
self.processed = []
|
||||
all_sources = self.response["hits"]["hits"]
|
||||
for result in all_sources:
|
||||
self.processed.append(self._process_result(result))
|
||||
|
||||
return self.processed
|
||||
|
||||
def _process_result(self, result):
|
||||
"""detect which type of data to process"""
|
||||
index = result["_index"]
|
||||
processed = False
|
||||
if index == "ta_video":
|
||||
processed = self._process_video(result["_source"])
|
||||
if index == "ta_channel":
|
||||
processed = self._process_channel(result["_source"])
|
||||
if index == "ta_playlist":
|
||||
processed = self._process_playlist(result["_source"])
|
||||
if index == "ta_download":
|
||||
processed = self._process_download(result["_source"])
|
||||
if index == "ta_comment":
|
||||
processed = self._process_comment(result["_source"])
|
||||
if index == "ta_subtitle":
|
||||
processed = self._process_subtitle(result)
|
||||
|
||||
if isinstance(processed, dict):
|
||||
processed.update(
|
||||
{
|
||||
"_index": index,
|
||||
"_score": round(result.get("_score") or 0, 2),
|
||||
}
|
||||
)
|
||||
|
||||
return processed
|
||||
|
||||
@staticmethod
|
||||
def _process_channel(channel_dict):
|
||||
"""run on single channel"""
|
||||
channel_id = channel_dict["channel_id"]
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
art_base = f"{cache_root}/channels/{channel_id}"
|
||||
date_str = date_parser(channel_dict["channel_last_refresh"])
|
||||
channel_dict.update(
|
||||
{
|
||||
"channel_last_refresh": date_str,
|
||||
"channel_banner_url": f"{art_base}_banner.jpg",
|
||||
"channel_thumb_url": f"{art_base}_thumb.jpg",
|
||||
"channel_tvart_url": f"{art_base}_tvart.jpg",
|
||||
}
|
||||
)
|
||||
|
||||
return dict(sorted(channel_dict.items()))
|
||||
|
||||
def _process_video(self, video_dict):
|
||||
"""run on single video dict"""
|
||||
video_id = video_dict["youtube_id"]
|
||||
media_url = urllib.parse.quote(video_dict["media_url"])
|
||||
vid_last_refresh = date_parser(video_dict["vid_last_refresh"])
|
||||
published = date_parser(video_dict["published"])
|
||||
vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
|
||||
channel = self._process_channel(video_dict["channel"])
|
||||
|
||||
if "subtitles" in video_dict:
|
||||
for idx, _ in enumerate(video_dict["subtitles"]):
|
||||
url = video_dict["subtitles"][idx]["media_url"]
|
||||
video_dict["subtitles"][idx]["media_url"] = f"/media/{url}"
|
||||
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
media_root = EnvironmentSettings().get_media_root()
|
||||
|
||||
video_dict.update(
|
||||
{
|
||||
"channel": channel,
|
||||
"media_url": f"{media_root}/{media_url}",
|
||||
"vid_last_refresh": vid_last_refresh,
|
||||
"published": published,
|
||||
"vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
|
||||
}
|
||||
)
|
||||
|
||||
return dict(sorted(video_dict.items()))
|
||||
|
||||
@staticmethod
|
||||
def _process_playlist(playlist_dict):
|
||||
"""run on single playlist dict"""
|
||||
playlist_id = playlist_dict["playlist_id"]
|
||||
playlist_last_refresh = date_parser(
|
||||
playlist_dict["playlist_last_refresh"]
|
||||
)
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
playlist_thumbnail = f"{cache_root}/playlists/{playlist_id}.jpg"
|
||||
playlist_dict.update(
|
||||
{
|
||||
"playlist_thumbnail": playlist_thumbnail,
|
||||
"playlist_last_refresh": playlist_last_refresh,
|
||||
}
|
||||
)
|
||||
|
||||
return dict(sorted(playlist_dict.items()))
|
||||
|
||||
def _process_download(self, download_dict):
|
||||
"""run on single download item"""
|
||||
video_id = download_dict["youtube_id"]
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
|
||||
published = date_parser(download_dict["published"])
|
||||
|
||||
download_dict.update(
|
||||
{
|
||||
"vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
|
||||
"published": published,
|
||||
}
|
||||
)
|
||||
return dict(sorted(download_dict.items()))
|
||||
|
||||
def _process_comment(self, comment_dict):
|
||||
"""run on all comments, create reply thread"""
|
||||
all_comments = comment_dict["comment_comments"]
|
||||
processed_comments = []
|
||||
|
||||
for comment in all_comments:
|
||||
if comment["comment_parent"] == "root":
|
||||
comment.update({"comment_replies": []})
|
||||
processed_comments.append(comment)
|
||||
else:
|
||||
processed_comments[-1]["comment_replies"].append(comment)
|
||||
|
||||
return processed_comments
|
||||
|
||||
def _process_subtitle(self, result):
|
||||
"""take complete result dict to extract highlight"""
|
||||
subtitle_dict = result["_source"]
|
||||
highlight = result.get("highlight")
|
||||
if highlight:
|
||||
# replace lines with the highlighted markdown
|
||||
subtitle_line = highlight.get("subtitle_line")[0]
|
||||
subtitle_dict.update({"subtitle_line": subtitle_line})
|
||||
|
||||
thumb_path = ThumbManager(subtitle_dict["youtube_id"]).vid_thumb_path()
|
||||
subtitle_dict.update({"vid_thumb_url": f"/cache/{thumb_path}"})
|
||||
|
||||
return subtitle_dict
|
||||
|
||||
|
||||
def process_aggs(response):
|
||||
"""convert aggs duration to str"""
|
||||
|
||||
if response.get("aggregations"):
|
||||
aggs = response["aggregations"]
|
||||
if "total_duration" in aggs:
|
||||
duration_sec = int(aggs["total_duration"]["value"])
|
||||
aggs["total_duration"].update(
|
||||
{"value_str": get_duration_str(duration_sec)}
|
||||
)
|
||||
382
backend/common/src/searching.py
Normal file
382
backend/common/src/searching.py
Normal file
@@ -0,0 +1,382 @@
|
||||
"""
|
||||
Functionality:
|
||||
- handle search to populate results to view
|
||||
- cache youtube video thumbnails and channel artwork
|
||||
- parse values in hit_cleanup for frontend
|
||||
- calculate pagination values
|
||||
"""
|
||||
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.search_processor import SearchProcess
|
||||
|
||||
|
||||
class SearchForm:
|
||||
"""build query from search form data"""
|
||||
|
||||
def multi_search(self, search_query):
|
||||
"""searching through index"""
|
||||
path, query, query_type = SearchParser(search_query).run()
|
||||
response, _ = ElasticWrap(path).get(data=query)
|
||||
search_results = SearchProcess(response).process()
|
||||
all_results = self.build_results(search_results)
|
||||
|
||||
return {"results": all_results, "queryType": query_type}
|
||||
|
||||
@staticmethod
|
||||
def build_results(search_results):
|
||||
"""build the all_results dict"""
|
||||
video_results = []
|
||||
channel_results = []
|
||||
playlist_results = []
|
||||
fulltext_results = []
|
||||
if search_results:
|
||||
for result in search_results:
|
||||
if result["_index"] == "ta_video":
|
||||
video_results.append(result)
|
||||
elif result["_index"] == "ta_channel":
|
||||
channel_results.append(result)
|
||||
elif result["_index"] == "ta_playlist":
|
||||
playlist_results.append(result)
|
||||
elif result["_index"] == "ta_subtitle":
|
||||
fulltext_results.append(result)
|
||||
|
||||
all_results = {
|
||||
"video_results": video_results,
|
||||
"channel_results": channel_results,
|
||||
"playlist_results": playlist_results,
|
||||
"fulltext_results": fulltext_results,
|
||||
}
|
||||
|
||||
return all_results
|
||||
|
||||
|
||||
class SearchParser:
|
||||
"""handle structured searches"""
|
||||
|
||||
def __init__(self, search_query):
|
||||
self.query_words = search_query.lower().split()
|
||||
self.query_map = {"term": [], "fuzzy": []}
|
||||
self.append_to = "term"
|
||||
|
||||
def run(self):
|
||||
"""collection, return path and query dict for es"""
|
||||
print(f"query words: {self.query_words}")
|
||||
query_type = self._find_map()
|
||||
self._run_words()
|
||||
self._delete_unset()
|
||||
self._match_data_types()
|
||||
|
||||
path, query = QueryBuilder(self.query_map, query_type).run()
|
||||
|
||||
return path, query, query_type
|
||||
|
||||
def _find_map(self):
|
||||
"""find query in keyword map"""
|
||||
first_word = self.query_words[0]
|
||||
key_word_map = self._get_map()
|
||||
|
||||
if ":" in first_word:
|
||||
index_match, query_string = first_word.split(":")
|
||||
if index_match in key_word_map:
|
||||
self.query_map.update(key_word_map.get(index_match))
|
||||
self.query_words[0] = query_string
|
||||
return index_match
|
||||
|
||||
self.query_map.update(key_word_map.get("simple"))
|
||||
print(f"query_map: {self.query_map}")
|
||||
|
||||
return "simple"
|
||||
|
||||
@staticmethod
|
||||
def _get_map():
|
||||
"""return map to build on"""
|
||||
return {
|
||||
"simple": {
|
||||
"index": "ta_video,ta_channel,ta_playlist",
|
||||
},
|
||||
"video": {
|
||||
"index": "ta_video",
|
||||
"channel": [],
|
||||
"active": [],
|
||||
},
|
||||
"channel": {
|
||||
"index": "ta_channel",
|
||||
"active": [],
|
||||
"subscribed": [],
|
||||
},
|
||||
"playlist": {
|
||||
"index": "ta_playlist",
|
||||
"active": [],
|
||||
"subscribed": [],
|
||||
},
|
||||
"full": {
|
||||
"index": "ta_subtitle",
|
||||
"lang": [],
|
||||
"source": [],
|
||||
},
|
||||
}
|
||||
|
||||
def _run_words(self):
|
||||
"""append word by word"""
|
||||
for word in self.query_words:
|
||||
if ":" in word:
|
||||
keyword, search_string = word.split(":")
|
||||
if keyword in self.query_map:
|
||||
self.append_to = keyword
|
||||
word = search_string
|
||||
|
||||
if word:
|
||||
self.query_map[self.append_to].append(word)
|
||||
|
||||
def _delete_unset(self):
|
||||
"""delete unset keys"""
|
||||
new_query_map = {}
|
||||
for key, value in self.query_map.items():
|
||||
if value:
|
||||
new_query_map.update({key: value})
|
||||
self.query_map = new_query_map
|
||||
|
||||
def _match_data_types(self):
|
||||
"""match values with data types"""
|
||||
for key, value in self.query_map.items():
|
||||
if key in ["term", "channel"]:
|
||||
self.query_map[key] = " ".join(self.query_map[key])
|
||||
if key in ["active", "subscribed"]:
|
||||
self.query_map[key] = "yes" in value
|
||||
|
||||
|
||||
class QueryBuilder:
|
||||
"""build query for ES from form data"""
|
||||
|
||||
def __init__(self, query_map, query_type):
|
||||
self.query_map = query_map
|
||||
self.query_type = query_type
|
||||
|
||||
def run(self):
|
||||
"""build query"""
|
||||
path = self._build_path()
|
||||
query = self.build_query()
|
||||
print(f"es path: {path}")
|
||||
print(f"query: {query}")
|
||||
|
||||
return path, query
|
||||
|
||||
def _build_path(self):
|
||||
"""build es index search path"""
|
||||
return f"{self.query_map.get('index')}/_search"
|
||||
|
||||
def build_query(self):
|
||||
"""build query based on query_type"""
|
||||
|
||||
exec_map = {
|
||||
"simple": self._build_simple,
|
||||
"video": self._build_video,
|
||||
"channel": self._build_channel,
|
||||
"playlist": self._build_playlist,
|
||||
"full": self._build_fulltext,
|
||||
}
|
||||
|
||||
build_must_list = exec_map[self.query_type]
|
||||
|
||||
if self.query_type == "full":
|
||||
query = build_must_list()
|
||||
else:
|
||||
query = {
|
||||
"size": 30,
|
||||
"query": {"bool": {"must": build_must_list()}},
|
||||
}
|
||||
|
||||
return query
|
||||
|
||||
def _get_fuzzy(self):
|
||||
"""return fuziness valuee"""
|
||||
fuzzy_value = self.query_map.get("fuzzy", ["auto"])[0]
|
||||
if fuzzy_value == "no":
|
||||
return 0
|
||||
|
||||
if not fuzzy_value.isdigit():
|
||||
return "auto"
|
||||
|
||||
if int(fuzzy_value) > 2:
|
||||
return "2"
|
||||
|
||||
return fuzzy_value
|
||||
|
||||
def _build_simple(self):
|
||||
"""build simple cross index query"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": term,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"channel_name._2gram",
|
||||
"channel_name._3gram",
|
||||
"channel_name.search_as_you_type",
|
||||
"playlist_name._2gram",
|
||||
"playlist_name._3gram",
|
||||
"playlist_name.search_as_you_type",
|
||||
"title._2gram",
|
||||
"title._3gram",
|
||||
"title.search_as_you_type",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return must_list
|
||||
|
||||
def _build_video(self):
|
||||
"""build video query"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": term,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"title._2gram^2",
|
||||
"title._3gram^2",
|
||||
"title.search_as_you_type^2",
|
||||
"tags",
|
||||
"category",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (active := self.query_map.get("active")) is not None:
|
||||
must_list.append({"term": {"active": {"value": active}}})
|
||||
|
||||
if (channel := self.query_map.get("channel")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": channel,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"channel.channel_name._2gram",
|
||||
"channel.channel_name._3gram",
|
||||
"channel.channel_name.search_as_you_type",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return must_list
|
||||
|
||||
def _build_channel(self):
|
||||
"""build query for channel"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": term,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"channel_description",
|
||||
"channel_name._2gram^2",
|
||||
"channel_name._3gram^2",
|
||||
"channel_name.search_as_you_type^2",
|
||||
"channel_tags",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (active := self.query_map.get("active")) is not None:
|
||||
must_list.append({"term": {"channel_active": {"value": active}}})
|
||||
|
||||
if (subscribed := self.query_map.get("subscribed")) is not None:
|
||||
must_list.append(
|
||||
{"term": {"channel_subscribed": {"value": subscribed}}}
|
||||
)
|
||||
|
||||
return must_list
|
||||
|
||||
def _build_playlist(self):
|
||||
"""build query for playlist"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": term,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"playlist_description",
|
||||
"playlist_name._2gram^2",
|
||||
"playlist_name._3gram^2",
|
||||
"playlist_name.search_as_you_type^2",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (active := self.query_map.get("active")) is not None:
|
||||
must_list.append({"term": {"playlist_active": {"value": active}}})
|
||||
|
||||
if (subscribed := self.query_map.get("subscribed")) is not None:
|
||||
must_list.append(
|
||||
{"term": {"playlist_subscribed": {"value": subscribed}}}
|
||||
)
|
||||
|
||||
return must_list
|
||||
|
||||
def _build_fulltext(self):
|
||||
"""build query for fulltext search"""
|
||||
must_list = []
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"match": {
|
||||
"subtitle_line": {
|
||||
"query": term,
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (lang := self.query_map.get("lang")) is not None:
|
||||
must_list.append({"term": {"subtitle_lang": {"value": lang[0]}}})
|
||||
|
||||
if (source := self.query_map.get("source")) is not None:
|
||||
must_list.append(
|
||||
{"term": {"subtitle_source": {"value": source[0]}}}
|
||||
)
|
||||
|
||||
query = {
|
||||
"size": 30,
|
||||
"query": {"bool": {"must": must_list}},
|
||||
"highlight": {
|
||||
"fields": {
|
||||
"subtitle_line": {
|
||||
"number_of_fragments": 0,
|
||||
"pre_tags": ['<span class="settings-current">'],
|
||||
"post_tags": ["</span>"],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
return query
|
||||
241
backend/common/src/ta_redis.py
Normal file
241
backend/common/src/ta_redis.py
Normal file
@@ -0,0 +1,241 @@
|
||||
"""
|
||||
functionality:
|
||||
- interact with redis
|
||||
- hold temporary download queue in redis
|
||||
- interact with celery tasks results
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import redis
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
|
||||
|
||||
class RedisBase:
|
||||
"""connection base for redis"""
|
||||
|
||||
NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE
|
||||
|
||||
def __init__(self):
|
||||
self.conn = redis.Redis(
|
||||
host=EnvironmentSettings.REDIS_HOST,
|
||||
port=EnvironmentSettings.REDIS_PORT,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
|
||||
class RedisArchivist(RedisBase):
|
||||
"""collection of methods to interact with redis"""
|
||||
|
||||
CHANNELS: list[str] = [
|
||||
"download",
|
||||
"add",
|
||||
"rescan",
|
||||
"subchannel",
|
||||
"subplaylist",
|
||||
"playlistscan",
|
||||
"setting",
|
||||
]
|
||||
|
||||
def set_message(
|
||||
self,
|
||||
key: str,
|
||||
message: dict,
|
||||
path: str = ".",
|
||||
expire: bool | int = False,
|
||||
save: bool = False,
|
||||
) -> None:
|
||||
"""write new message to redis"""
|
||||
self.conn.execute_command(
|
||||
"JSON.SET", self.NAME_SPACE + key, path, json.dumps(message)
|
||||
)
|
||||
|
||||
if expire:
|
||||
if isinstance(expire, bool):
|
||||
secs: int = 20
|
||||
else:
|
||||
secs = expire
|
||||
self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
|
||||
|
||||
if save:
|
||||
self.bg_save()
|
||||
|
||||
def bg_save(self) -> None:
|
||||
"""save to aof"""
|
||||
try:
|
||||
self.conn.bgsave()
|
||||
except redis.exceptions.ResponseError:
|
||||
pass
|
||||
|
||||
def get_message(self, key: str) -> dict:
|
||||
"""get message dict from redis"""
|
||||
reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
|
||||
if reply:
|
||||
return json.loads(reply)
|
||||
|
||||
return {"status": False}
|
||||
|
||||
def list_keys(self, query: str) -> list:
|
||||
"""return all key matches"""
|
||||
reply = self.conn.execute_command(
|
||||
"KEYS", self.NAME_SPACE + query + "*"
|
||||
)
|
||||
if not reply:
|
||||
return []
|
||||
|
||||
return [i.lstrip(self.NAME_SPACE) for i in reply]
|
||||
|
||||
def list_items(self, query: str) -> list:
|
||||
"""list all matches"""
|
||||
all_matches = self.list_keys(query)
|
||||
if not all_matches:
|
||||
return []
|
||||
|
||||
return [self.get_message(i) for i in all_matches]
|
||||
|
||||
def del_message(self, key: str) -> bool:
|
||||
"""delete key from redis"""
|
||||
response = self.conn.execute_command("DEL", self.NAME_SPACE + key)
|
||||
return response
|
||||
|
||||
|
||||
class RedisQueue(RedisBase):
|
||||
"""
|
||||
dynamically interact with queues in redis using sorted set
|
||||
- low score number is first in queue
|
||||
- add new items with high score number
|
||||
|
||||
queue names in use:
|
||||
download:channel channels during download
|
||||
download:playlist:full playlists during dl for full refresh
|
||||
download:playlist:quick playlists during dl for quick refresh
|
||||
download:video videos during downloads
|
||||
index:comment videos needing comment indexing
|
||||
reindex:ta_video reindex videos
|
||||
reindex:ta_channel reindex channels
|
||||
reindex:ta_playlist reindex playlists
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, queue_name: str):
|
||||
super().__init__()
|
||||
self.key = f"{self.NAME_SPACE}{queue_name}"
|
||||
|
||||
def get_all(self) -> list[str]:
|
||||
"""return all elements in list"""
|
||||
result = self.conn.zrange(self.key, 0, -1)
|
||||
return result
|
||||
|
||||
def length(self) -> int:
|
||||
"""return total elements in list"""
|
||||
return self.conn.zcard(self.key)
|
||||
|
||||
def in_queue(self, element) -> str | bool:
|
||||
"""check if element is in list"""
|
||||
result = self.conn.zrank(self.key, element)
|
||||
if result is not None:
|
||||
return "in_queue"
|
||||
|
||||
return False
|
||||
|
||||
def add(self, to_add: str) -> None:
|
||||
"""add single item to queue"""
|
||||
if not to_add:
|
||||
return
|
||||
|
||||
next_score = self._get_next_score()
|
||||
self.conn.zadd(self.key, {to_add: next_score})
|
||||
|
||||
def add_list(self, to_add: list) -> None:
|
||||
"""add list to queue"""
|
||||
if not to_add:
|
||||
return
|
||||
|
||||
next_score = self._get_next_score()
|
||||
mapping = {i[1]: next_score + i[0] for i in enumerate(to_add)}
|
||||
self.conn.zadd(self.key, mapping)
|
||||
|
||||
def max_score(self) -> int | None:
|
||||
"""get max score"""
|
||||
last = self.conn.zrange(self.key, -1, -1, withscores=True)
|
||||
if not last:
|
||||
return None
|
||||
|
||||
return int(last[0][1])
|
||||
|
||||
def _get_next_score(self) -> float:
|
||||
"""get next score in queue to append"""
|
||||
last = self.conn.zrange(self.key, -1, -1, withscores=True)
|
||||
if not last:
|
||||
return 1.0
|
||||
|
||||
return last[0][1] + 1
|
||||
|
||||
def get_next(self) -> tuple[str | None, int | None]:
|
||||
"""return next element in the queue, if available"""
|
||||
result = self.conn.zpopmin(self.key)
|
||||
if not result:
|
||||
return None, None
|
||||
|
||||
item, idx = result[0][0], int(result[0][1])
|
||||
|
||||
return item, idx
|
||||
|
||||
def clear(self) -> None:
|
||||
"""delete list from redis"""
|
||||
self.conn.delete(self.key)
|
||||
|
||||
|
||||
class TaskRedis(RedisBase):
|
||||
"""interact with redis tasks"""
|
||||
|
||||
BASE: str = "celery-task-meta-"
|
||||
EXPIRE: int = 60 * 60 * 24
|
||||
COMMANDS: list[str] = ["STOP", "KILL"]
|
||||
|
||||
def get_all(self) -> list:
|
||||
"""return all tasks"""
|
||||
all_keys = self.conn.execute_command("KEYS", f"{self.BASE}*")
|
||||
return [i.replace(self.BASE, "") for i in all_keys]
|
||||
|
||||
def get_single(self, task_id: str) -> dict:
|
||||
"""return content of single task"""
|
||||
result = self.conn.execute_command("GET", self.BASE + task_id)
|
||||
if not result:
|
||||
return {}
|
||||
|
||||
return json.loads(result)
|
||||
|
||||
def set_key(
|
||||
self, task_id: str, message: dict, expire: bool | int = False
|
||||
) -> None:
|
||||
"""set value for lock, initial or update"""
|
||||
key: str = f"{self.BASE}{task_id}"
|
||||
self.conn.execute_command("SET", key, json.dumps(message))
|
||||
|
||||
if expire:
|
||||
self.conn.execute_command("EXPIRE", key, self.EXPIRE)
|
||||
|
||||
def set_command(self, task_id: str, command: str) -> None:
|
||||
"""set task command"""
|
||||
if command not in self.COMMANDS:
|
||||
print(f"{command} not in valid commands {self.COMMANDS}")
|
||||
raise ValueError
|
||||
|
||||
message = self.get_single(task_id)
|
||||
if not message:
|
||||
print(f"{task_id} not found")
|
||||
raise KeyError
|
||||
|
||||
message.update({"command": command})
|
||||
self.set_key(task_id, message)
|
||||
|
||||
def del_task(self, task_id: str) -> None:
|
||||
"""delete task result by id"""
|
||||
self.conn.execute_command("DEL", f"{self.BASE}{task_id}")
|
||||
|
||||
def del_all(self) -> None:
|
||||
"""delete all task results"""
|
||||
all_tasks = self.get_all()
|
||||
for task_id in all_tasks:
|
||||
self.del_task(task_id)
|
||||
138
backend/common/src/urlparser.py
Normal file
138
backend/common/src/urlparser.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""
|
||||
Functionality:
|
||||
- detect valid youtube ids and links from multi line string
|
||||
- identify vid_type if possible
|
||||
"""
|
||||
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from download.src.yt_dlp_base import YtWrap
|
||||
from video.src.constants import VideoTypeEnum
|
||||
|
||||
|
||||
class Parser:
|
||||
"""take a multi line string and detect valid youtube ids"""
|
||||
|
||||
def __init__(self, url_str):
|
||||
self.url_list = [i.strip() for i in url_str.split()]
|
||||
|
||||
def parse(self):
|
||||
"""parse the list"""
|
||||
ids = []
|
||||
for url in self.url_list:
|
||||
parsed = urlparse(url)
|
||||
if parsed.netloc:
|
||||
# is url
|
||||
identified = self.process_url(parsed)
|
||||
else:
|
||||
# is not url
|
||||
identified = self._find_valid_id(url)
|
||||
|
||||
if "vid_type" not in identified:
|
||||
identified.update(self._detect_vid_type(parsed.path))
|
||||
|
||||
ids.append(identified)
|
||||
|
||||
return ids
|
||||
|
||||
def process_url(self, parsed):
|
||||
"""process as url"""
|
||||
if parsed.netloc == "youtu.be":
|
||||
# shortened
|
||||
youtube_id = parsed.path.strip("/")
|
||||
return self._validate_expected(youtube_id, "video")
|
||||
|
||||
if "youtube.com" not in parsed.netloc:
|
||||
message = f"invalid domain: {parsed.netloc}"
|
||||
raise ValueError(message)
|
||||
|
||||
query_parsed = parse_qs(parsed.query)
|
||||
if "v" in query_parsed:
|
||||
# video from v query str
|
||||
youtube_id = query_parsed["v"][0]
|
||||
return self._validate_expected(youtube_id, "video")
|
||||
|
||||
if "list" in query_parsed:
|
||||
# playlist from list query str
|
||||
youtube_id = query_parsed["list"][0]
|
||||
return self._validate_expected(youtube_id, "playlist")
|
||||
|
||||
all_paths = parsed.path.strip("/").split("/")
|
||||
if all_paths[0] == "shorts":
|
||||
# is shorts video
|
||||
item = self._validate_expected(all_paths[1], "video")
|
||||
item.update({"vid_type": VideoTypeEnum.SHORTS.value})
|
||||
return item
|
||||
|
||||
if all_paths[0] == "channel":
|
||||
return self._validate_expected(all_paths[1], "channel")
|
||||
|
||||
# detect channel
|
||||
channel_id = self._extract_channel_name(parsed.geturl())
|
||||
return {"type": "channel", "url": channel_id}
|
||||
|
||||
def _validate_expected(self, youtube_id, expected_type):
|
||||
"""raise value error if not matching"""
|
||||
matched = self._find_valid_id(youtube_id)
|
||||
if matched["type"] != expected_type:
|
||||
raise ValueError(
|
||||
f"{youtube_id} not of expected type {expected_type}"
|
||||
)
|
||||
|
||||
return {"type": expected_type, "url": youtube_id}
|
||||
|
||||
def _find_valid_id(self, id_str):
|
||||
"""detect valid id from length of string"""
|
||||
if id_str in ("LL", "WL"):
|
||||
return {"type": "playlist", "url": id_str}
|
||||
|
||||
if id_str.startswith("@"):
|
||||
url = f"https://www.youtube.com/{id_str}"
|
||||
channel_id = self._extract_channel_name(url)
|
||||
return {"type": "channel", "url": channel_id}
|
||||
|
||||
len_id_str = len(id_str)
|
||||
if len_id_str == 11:
|
||||
item_type = "video"
|
||||
elif len_id_str == 24:
|
||||
item_type = "channel"
|
||||
elif len_id_str in (34, 26, 18) or id_str.startswith("TA_playlist_"):
|
||||
item_type = "playlist"
|
||||
else:
|
||||
raise ValueError(f"not a valid id_str: {id_str}")
|
||||
|
||||
return {"type": item_type, "url": id_str}
|
||||
|
||||
@staticmethod
|
||||
def _extract_channel_name(url):
|
||||
"""find channel id from channel name with yt-dlp help"""
|
||||
obs_request = {
|
||||
"check_formats": None,
|
||||
"skip_download": True,
|
||||
"extract_flat": True,
|
||||
"playlistend": 0,
|
||||
}
|
||||
url_info = YtWrap(obs_request).extract(url)
|
||||
channel_id = url_info.get("channel_id", False)
|
||||
if channel_id:
|
||||
return channel_id
|
||||
|
||||
url = url_info.get("url", False)
|
||||
if url:
|
||||
# handle old channel name redirect with url path split
|
||||
channel_id = urlparse(url).path.strip("/").split("/")[1]
|
||||
|
||||
return channel_id
|
||||
|
||||
print(f"failed to extract channel id from {url}")
|
||||
raise ValueError
|
||||
|
||||
def _detect_vid_type(self, path):
|
||||
"""try to match enum from path, needs to be serializable"""
|
||||
last = path.strip("/").split("/")[-1]
|
||||
try:
|
||||
vid_type = VideoTypeEnum(last).value
|
||||
except ValueError:
|
||||
vid_type = VideoTypeEnum.UNKNOWN.value
|
||||
|
||||
return {"vid_type": vid_type}
|
||||
105
backend/common/src/watched.py
Normal file
105
backend/common/src/watched.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
functionality:
|
||||
- handle watched state for videos, channels and playlists
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.urlparser import Parser
|
||||
|
||||
|
||||
class WatchState:
|
||||
"""handle watched checkbox for videos and channels"""
|
||||
|
||||
def __init__(self, youtube_id, is_watched):
|
||||
self.youtube_id = youtube_id
|
||||
self.is_watched = is_watched
|
||||
self.stamp = int(datetime.now().timestamp())
|
||||
self.pipeline = f"_ingest/pipeline/watch_{youtube_id}"
|
||||
|
||||
def change(self):
|
||||
"""change watched state of item(s)"""
|
||||
print(f"{self.youtube_id}: change watched state to {self.is_watched}")
|
||||
url_type = self._dedect_type()
|
||||
if url_type == "video":
|
||||
self.change_vid_state()
|
||||
return
|
||||
|
||||
self._add_pipeline()
|
||||
path = f"ta_video/_update_by_query?pipeline=watch_{self.youtube_id}"
|
||||
data = self._build_update_data(url_type)
|
||||
_, _ = ElasticWrap(path).post(data)
|
||||
self._delete_pipeline()
|
||||
|
||||
def _dedect_type(self):
|
||||
"""find youtube id type"""
|
||||
url_process = Parser(self.youtube_id).parse()
|
||||
url_type = url_process[0]["type"]
|
||||
return url_type
|
||||
|
||||
def change_vid_state(self):
|
||||
"""change watched state of video"""
|
||||
path = f"ta_video/_update/{self.youtube_id}"
|
||||
data = {
|
||||
"doc": {
|
||||
"player": {
|
||||
"watched": self.is_watched,
|
||||
"watched_date": self.stamp,
|
||||
}
|
||||
}
|
||||
}
|
||||
response, status_code = ElasticWrap(path).post(data=data)
|
||||
if status_code != 200:
|
||||
print(response)
|
||||
raise ValueError("failed to mark video as watched")
|
||||
|
||||
def _build_update_data(self, url_type):
|
||||
"""build update by query data based on url_type"""
|
||||
term_key_map = {
|
||||
"channel": "channel.channel_id",
|
||||
"playlist": "playlist.keyword",
|
||||
}
|
||||
term_key = term_key_map.get(url_type)
|
||||
|
||||
return {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"term": {term_key: {"value": self.youtube_id}}},
|
||||
{
|
||||
"term": {
|
||||
"player.watched": {
|
||||
"value": not self.is_watched
|
||||
}
|
||||
}
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def _add_pipeline(self):
|
||||
"""add ingest pipeline"""
|
||||
data = {
|
||||
"description": f"{self.youtube_id}: watched {self.is_watched}",
|
||||
"processors": [
|
||||
{
|
||||
"set": {
|
||||
"field": "player.watched",
|
||||
"value": self.is_watched,
|
||||
}
|
||||
},
|
||||
{
|
||||
"set": {
|
||||
"field": "player.watched_date",
|
||||
"value": self.stamp,
|
||||
}
|
||||
},
|
||||
],
|
||||
}
|
||||
_, _ = ElasticWrap(self.pipeline).put(data)
|
||||
|
||||
def _delete_pipeline(self):
|
||||
"""delete pipeline"""
|
||||
ElasticWrap(self.pipeline).delete()
|
||||
Reference in New Issue
Block a user