renamed django app folder to backend

backend/task/src/config_schedule.py

@@ -0,0 +1,141 @@
+"""
+Functionality:
+- Handle scheduler config update
+"""
+
+from datetime import datetime
+
+from appsettings.src.config import AppConfig
+from celery.schedules import crontab
+from common.src.env_settings import EnvironmentSettings
+from django.utils import dateformat
+from django_celery_beat.models import CrontabSchedule
+from task.models import CustomPeriodicTask
+from task.src.task_config import TASK_CONFIG
+
+
+class ScheduleBuilder:
+    """build schedule dicts for beat"""
+
+    SCHEDULES = {
+        "update_subscribed": "0 8 *",
+        "download_pending": "0 16 *",
+        "check_reindex": "0 12 *",
+        "thumbnail_check": "0 17 *",
+        "run_backup": "0 18 0",
+        "version_check": "0 11 *",
+    }
+    MSG = "message:setting"
+
+    def __init__(self):
+        self.config = AppConfig().config
+
+    def update_schedule(
+        self, task_name: str, cron_schedule: str, schedule_conf: dict | None
+    ) -> None:
+        """update schedule"""
+        if cron_schedule == "auto":
+            cron_schedule = self.SCHEDULES[task_name]
+
+        if cron_schedule:
+            _ = self.get_set_task(task_name, cron_schedule)
+
+        if schedule_conf:
+            for key, value in schedule_conf.items():
+                self.set_config(task_name, key, value)
+
+    def get_set_task(self, task_name, schedule=False):
+        """get task"""
+        try:
+            task = CustomPeriodicTask.objects.get(name=task_name)
+        except CustomPeriodicTask.DoesNotExist:
+            description = TASK_CONFIG[task_name].get("title")
+            task = CustomPeriodicTask(
+                name=task_name,
+                task=task_name,
+                description=description,
+            )
+
+        if schedule:
+            task_crontab = self.get_set_cron_tab(schedule)
+            task.crontab = task_crontab
+            task.last_run_at = dateformat.make_aware(datetime.now())
+            task.save()
+
+        return task
+
+    @staticmethod
+    def get_set_cron_tab(schedule: str) -> CrontabSchedule:
+        """needs to be validated before"""
+        kwargs = dict(zip(["minute", "hour", "day_of_week"], schedule.split()))
+        kwargs.update({"timezone": EnvironmentSettings.TZ})
+        task_crontab, _ = CrontabSchedule.objects.get_or_create(**kwargs)
+
+        return task_crontab
+
+    def set_config(self, task_name: str, key: str, value) -> None:
+        """set task_config, validate before"""
+        try:
+            task = CustomPeriodicTask.objects.get(name=task_name)
+            task.task_config.update({key: value})
+            task.save()
+        except CustomPeriodicTask.DoesNotExist:
+            pass
+
+
+class CrontabValidator:
+    """validate crontab"""
+
+    CONFIG = {
+        "check_reindex": ["days"],
+        "run_backup": ["rotate"],
+    }
+
+    @staticmethod
+    def validate_fields(cron_fields: str) -> None:
+        """expect 3 cron fields"""
+        if not len(cron_fields) == 3:
+            raise ValueError("expected three cron schedule fields")
+
+    @staticmethod
+    def validate_minute(minute_field: str):
+        """expect minute int"""
+        if not minute_field.isdigit():
+            raise ValueError("Invalid value for minutes. Must be an integer.")
+
+        minutes = int(minute_field)
+        if not 0 <= minutes <= 59:
+            raise ValueError("Invalid minutes. Must be between 0 and 59.")
+
+    @staticmethod
+    def validate_cron_tab(minute, hour, day_of_week):
+        """check if crontab can be created"""
+        try:
+            crontab(minute=minute, hour=hour, day_of_week=day_of_week)
+        except ValueError as err:
+            raise ValueError(f"invalid crontab: {err}") from err
+
+    def validate_cron(self, cron_expression):
+        """create crontab schedule"""
+        if not cron_expression or cron_expression == "auto":
+            return
+
+        cron_fields = cron_expression.split()
+        self.validate_fields(cron_fields)
+
+        minute, hour, day_of_week = cron_fields
+        self.validate_minute(minute)
+        self.validate_cron_tab(minute, hour, day_of_week)
+
+    def validate_config(self, task_name: str, schedule_config: dict):
+        """validate config for given task"""
+        if not schedule_config:
+            return
+
+        config_keys = self.CONFIG.get(task_name)
+        if not config_keys:
+            raise ValueError(f"task '{task_name}' doesn't take config")
+
+        for key in schedule_config:
+            if key not in config_keys:
+                raise ValueError(f"invalid config key for task '{task_name}'")

backend/task/src/notify.py

@@ -0,0 +1,141 @@
+"""send notifications using apprise"""
+
+import apprise
+from common.src.es_connect import ElasticWrap
+from task.src.task_config import TASK_CONFIG
+from task.src.task_manager import TaskManager
+
+
+class Notifications:
+    """store notifications in ES"""
+
+    GET_PATH = "ta_config/_doc/notify"
+    UPDATE_PATH = "ta_config/_update/notify/"
+
+    def __init__(self, task_name: str):
+        self.task_name = task_name
+
+    def send(self, task_id: str, task_title: str) -> None:
+        """send notifications"""
+        apobj = apprise.Apprise()
+        urls: list[str] = self.get_urls()
+        if not urls:
+            return
+
+        title, body = self._build_message(task_id, task_title)
+
+        if not body:
+            return
+
+        for url in urls:
+            apobj.add(url)
+
+        apobj.notify(body=body, title=title)
+
+    def _build_message(
+        self, task_id: str, task_title: str
+    ) -> tuple[str, str | None]:
+        """build message to send notification"""
+        task = TaskManager().get_task(task_id)
+        status = task.get("status")
+        title: str = f"[TA] {task_title} process ended with {status}"
+        body: str | None = task.get("result")
+
+        return title, body
+
+    def get_urls(self) -> list[str]:
+        """get stored urls for task"""
+        response, code = ElasticWrap(self.GET_PATH).get(print_error=False)
+        if not code == 200:
+            return []
+
+        urls = response["_source"].get(self.task_name, [])
+
+        return urls
+
+    def add_url(self, url: str) -> None:
+        """add url to task notification"""
+        source = (
+            "if (!ctx._source.containsKey(params.task_name)) "
+            + "{ctx._source[params.task_name] = [params.url]} "
+            + "else if (!ctx._source[params.task_name].contains(params.url)) "
+            + "{ctx._source[params.task_name].add(params.url)} "
+            + "else {ctx.op = 'none'}"
+        )
+
+        data = {
+            "script": {
+                "source": source,
+                "lang": "painless",
+                "params": {"url": url, "task_name": self.task_name},
+            },
+            "upsert": {self.task_name: [url]},
+        }
+
+        _, _ = ElasticWrap(self.UPDATE_PATH).post(data)
+
+    def remove_url(self, url: str) -> tuple[dict, int]:
+        """remove url from task"""
+        source = (
+            "if (ctx._source.containsKey(params.task_name) "
+            + "&& ctx._source[params.task_name].contains(params.url)) "
+            + "{ctx._source[params.task_name]."
+            + "remove(ctx._source[params.task_name].indexOf(params.url))}"
+        )
+
+        data = {
+            "script": {
+                "source": source,
+                "lang": "painless",
+                "params": {"url": url, "task_name": self.task_name},
+            }
+        }
+
+        response, status_code = ElasticWrap(self.UPDATE_PATH).post(data)
+        if not self.get_urls():
+            _, _ = self.remove_task()
+
+        return response, status_code
+
+    def remove_task(self) -> tuple[dict, int]:
+        """remove all notifications from task"""
+        source = (
+            "if (ctx._source.containsKey(params.task_name)) "
+            + "{ctx._source.remove(params.task_name)}"
+        )
+        data = {
+            "script": {
+                "source": source,
+                "lang": "painless",
+                "params": {"task_name": self.task_name},
+            }
+        }
+
+        response, status_code = ElasticWrap(self.UPDATE_PATH).post(data)
+
+        return response, status_code
+
+
+def get_all_notifications() -> dict[str, list[str]]:
+    """get all notifications stored"""
+    path = "ta_config/_doc/notify"
+    response, status_code = ElasticWrap(path).get(print_error=False)
+    if not status_code == 200:
+        return {}
+
+    notifications: dict = {}
+    source = response.get("_source")
+    if not source:
+        return notifications
+
+    for task_id, urls in source.items():
+        notifications.update(
+            {
+                task_id: {
+                    "urls": urls,
+                    "title": TASK_CONFIG[task_id]["title"],
+                }
+            }
+        )
+
+    return notifications

backend/task/src/task_config.py

@@ -0,0 +1,125 @@
+"""
+Functionality:
+- Static Task config values
+- Type definitions
+- separate to avoid circular imports
+"""
+
+from typing import TypedDict
+
+
+class TaskItemConfig(TypedDict):
+    """describes a task item config"""
+
+    title: str
+    group: str
+    api_start: bool
+    api_stop: bool
+
+
+UPDATE_SUBSCRIBED: TaskItemConfig = {
+    "title": "Rescan your Subscriptions",
+    "group": "download:scan",
+    "api_start": True,
+    "api_stop": True,
+}
+
+DOWNLOAD_PENDING: TaskItemConfig = {
+    "title": "Downloading",
+    "group": "download:run",
+    "api_start": True,
+    "api_stop": True,
+}
+
+EXTRACT_DOWNLOAD: TaskItemConfig = {
+    "title": "Add to download queue",
+    "group": "download:add",
+    "api_start": False,
+    "api_stop": True,
+}
+
+CHECK_REINDEX: TaskItemConfig = {
+    "title": "Reindex Documents",
+    "group": "reindex:run",
+    "api_start": False,
+    "api_stop": False,
+}
+
+MANUAL_IMPORT: TaskItemConfig = {
+    "title": "Manual video import",
+    "group": "setting:import",
+    "api_start": True,
+    "api_stop": False,
+}
+
+RUN_BACKUP: TaskItemConfig = {
+    "title": "Index Backup",
+    "group": "setting:backup",
+    "api_start": True,
+    "api_stop": False,
+}
+
+RESTORE_BACKUP: TaskItemConfig = {
+    "title": "Restore Backup",
+    "group": "setting:restore",
+    "api_start": False,
+    "api_stop": False,
+}
+
+RESCAN_FILESYSTEM: TaskItemConfig = {
+    "title": "Rescan your Filesystem",
+    "group": "setting:filesystemscan",
+    "api_start": True,
+    "api_stop": False,
+}
+
+THUMBNAIL_CHECK: TaskItemConfig = {
+    "title": "Check your Thumbnails",
+    "group": "setting:thumbnailcheck",
+    "api_start": True,
+    "api_stop": False,
+}
+
+RESYNC_THUMBS: TaskItemConfig = {
+    "title": "Sync Thumbnails to Media Files",
+    "group": "setting:thumbnailsync",
+    "api_start": True,
+    "api_stop": False,
+}
+
+INDEX_PLAYLISTS: TaskItemConfig = {
+    "title": "Index Channel Playlist",
+    "group": "channel:indexplaylist",
+    "api_start": False,
+    "api_stop": False,
+}
+
+SUBSCRIBE_TO: TaskItemConfig = {
+    "title": "Add Subscription",
+    "group": "subscription:add",
+    "api_start": False,
+    "api_stop": False,
+}
+
+VERSION_CHECK: TaskItemConfig = {
+    "title": "Look for new Version",
+    "group": "",
+    "api_start": False,
+    "api_stop": False,
+}
+
+TASK_CONFIG: dict[str, TaskItemConfig] = {
+    "update_subscribed": UPDATE_SUBSCRIBED,
+    "download_pending": DOWNLOAD_PENDING,
+    "extract_download": EXTRACT_DOWNLOAD,
+    "check_reindex": CHECK_REINDEX,
+    "manual_import": MANUAL_IMPORT,
+    "run_backup": RUN_BACKUP,
+    "restore_backup": RESTORE_BACKUP,
+    "rescan_filesystem": RESCAN_FILESYSTEM,
+    "thumbnail_check": THUMBNAIL_CHECK,
+    "resync_thumbs": RESYNC_THUMBS,
+    "index_playlists": INDEX_PLAYLISTS,
+    "subscribe_to": SUBSCRIBE_TO,
+    "version_check": VERSION_CHECK,
+}

backend/task/src/task_manager.py

@@ -0,0 +1,117 @@
+"""
+functionality:
+- interact with in redis stored task results
+- handle threads and locks
+"""
+
+from common.src.ta_redis import RedisArchivist, TaskRedis
+from task.celery import app as celery_app
+from task.src.task_config import TASK_CONFIG
+
+
+class TaskManager:
+    """manage tasks"""
+
+    def get_all_results(self):
+        """return all task results"""
+        handler = TaskRedis()
+        all_keys = handler.get_all()
+        if not all_keys:
+            return False
+
+        return [handler.get_single(i) for i in all_keys]
+
+    def get_tasks_by_name(self, task_name):
+        """get all tasks by name"""
+        all_results = self.get_all_results()
+        if not all_results:
+            return False
+
+        return [i for i in all_results if i.get("name") == task_name]
+
+    def get_task(self, task_id):
+        """get single task"""
+        return TaskRedis().get_single(task_id)
+
+    def is_pending(self, task):
+        """check if task_name is pending, pass task object"""
+        tasks = self.get_tasks_by_name(task.name)
+        if not tasks:
+            return False
+
+        return bool([i for i in tasks if i.get("status") == "PENDING"])
+
+    def is_stopped(self, task_id):
+        """check if task_id has received STOP command"""
+        task = self.get_task(task_id)
+
+        return task.get("command") == "STOP"
+
+    def get_pending(self, task_name):
+        """get all pending tasks of task_name"""
+        tasks = self.get_tasks_by_name(task_name)
+        if not tasks:
+            return False
+
+        return [i for i in tasks if i.get("status") == "PENDING"]
+
+    def init(self, task):
+        """pass task object from bind task to set initial pending message"""
+        message = {
+            "status": "PENDING",
+            "result": None,
+            "traceback": None,
+            "date_done": False,
+            "name": task.name,
+            "task_id": task.request.id,
+        }
+        TaskRedis().set_key(task.request.id, message)
+
+    def fail_pending(self):
+        """
+        mark all pending as failed,
+        run at startup to recover from hard reset
+        """
+        all_results = self.get_all_results()
+        if not all_results:
+            return
+
+        for result in all_results:
+            if result.get("status") == "PENDING":
+                result["status"] = "FAILED"
+                TaskRedis().set_key(result["task_id"], result, expire=True)
+
+
+class TaskCommand:
+    """run commands on task"""
+
+    def start(self, task_name):
+        """start task by task_name, only pass task that don't take args"""
+        task = celery_app.tasks.get(task_name).delay()
+        message = {
+            "task_id": task.id,
+            "status": task.status,
+            "task_name": task.name,
+        }
+
+        return message
+
+    def stop(self, task_id, message_key):
+        """
+        send stop signal to task_id,
+        needs to be implemented in task to take effect
+        """
+        print(f"[task][{task_id}]: received STOP signal.")
+        handler = TaskRedis()
+
+        task = handler.get_single(task_id)
+        if not task["name"] in TASK_CONFIG:
+            raise ValueError
+
+        handler.set_command(task_id, "STOP")
+        RedisArchivist().set_message(message_key, "STOP", path=".command")
+
+    def kill(self, task_id):
+        """send kill signal to task_id"""
+        print(f"[task][{task_id}]: received KILL signal.")
+        celery_app.control.revoke(task_id, terminate=True)