renamed django app folder to backend

backend/task/__init__.py

@@ -0,0 +1,7 @@
+"""start celery app"""
+
+from __future__ import absolute_import, unicode_literals
+
+from task.celery import app as celery_app
+
+__all__ = ("celery_app",)

backend/task/celery.py

@@ -0,0 +1,22 @@
+"""initiate celery"""
+
+import os
+
+from celery import Celery
+from common.src.env_settings import EnvironmentSettings
+
+REDIS_HOST = EnvironmentSettings.REDIS_HOST
+REDIS_PORT = EnvironmentSettings.REDIS_PORT
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+app = Celery(
+    "tasks",
+    broker=f"redis://{REDIS_HOST}:{REDIS_PORT}",
+    backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
+    result_extended=True,
+)
+app.config_from_object(
+    "django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE
+)
+app.autodiscover_tasks()
+app.conf.timezone = EnvironmentSettings.TZ

backend/task/migrations/0001_initial.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.0.7 on 2024-07-22 18:39
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("django_celery_beat", "0018_improve_crontab_helptext"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CustomPeriodicTask",
+            fields=[
+                (
+                    "periodictask_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="django_celery_beat.periodictask",
+                    ),
+                ),
+                ("task_config", models.JSONField(default=dict)),
+            ],
+            bases=("django_celery_beat.periodictask",),
+        ),
+    ]

backend/task/models.py

@@ -0,0 +1,10 @@
+"""task model"""
+
+from django.db import models
+from django_celery_beat.models import PeriodicTask
+
+
+class CustomPeriodicTask(PeriodicTask):
+    """add custom metadata to task"""
+
+    task_config = models.JSONField(default=dict)

backend/task/src/config_schedule.py

@@ -0,0 +1,141 @@
+"""
+Functionality:
+- Handle scheduler config update
+"""
+
+from datetime import datetime
+
+from appsettings.src.config import AppConfig
+from celery.schedules import crontab
+from common.src.env_settings import EnvironmentSettings
+from django.utils import dateformat
+from django_celery_beat.models import CrontabSchedule
+from task.models import CustomPeriodicTask
+from task.src.task_config import TASK_CONFIG
+
+
+class ScheduleBuilder:
+    """build schedule dicts for beat"""
+
+    SCHEDULES = {
+        "update_subscribed": "0 8 *",
+        "download_pending": "0 16 *",
+        "check_reindex": "0 12 *",
+        "thumbnail_check": "0 17 *",
+        "run_backup": "0 18 0",
+        "version_check": "0 11 *",
+    }
+    MSG = "message:setting"
+
+    def __init__(self):
+        self.config = AppConfig().config
+
+    def update_schedule(
+        self, task_name: str, cron_schedule: str, schedule_conf: dict | None
+    ) -> None:
+        """update schedule"""
+        if cron_schedule == "auto":
+            cron_schedule = self.SCHEDULES[task_name]
+
+        if cron_schedule:
+            _ = self.get_set_task(task_name, cron_schedule)
+
+        if schedule_conf:
+            for key, value in schedule_conf.items():
+                self.set_config(task_name, key, value)
+
+    def get_set_task(self, task_name, schedule=False):
+        """get task"""
+        try:
+            task = CustomPeriodicTask.objects.get(name=task_name)
+        except CustomPeriodicTask.DoesNotExist:
+            description = TASK_CONFIG[task_name].get("title")
+            task = CustomPeriodicTask(
+                name=task_name,
+                task=task_name,
+                description=description,
+            )
+
+        if schedule:
+            task_crontab = self.get_set_cron_tab(schedule)
+            task.crontab = task_crontab
+            task.last_run_at = dateformat.make_aware(datetime.now())
+            task.save()
+
+        return task
+
+    @staticmethod
+    def get_set_cron_tab(schedule: str) -> CrontabSchedule:
+        """needs to be validated before"""
+        kwargs = dict(zip(["minute", "hour", "day_of_week"], schedule.split()))
+        kwargs.update({"timezone": EnvironmentSettings.TZ})
+        task_crontab, _ = CrontabSchedule.objects.get_or_create(**kwargs)
+
+        return task_crontab
+
+    def set_config(self, task_name: str, key: str, value) -> None:
+        """set task_config, validate before"""
+        try:
+            task = CustomPeriodicTask.objects.get(name=task_name)
+            task.task_config.update({key: value})
+            task.save()
+        except CustomPeriodicTask.DoesNotExist:
+            pass
+
+
+class CrontabValidator:
+    """validate crontab"""
+
+    CONFIG = {
+        "check_reindex": ["days"],
+        "run_backup": ["rotate"],
+    }
+
+    @staticmethod
+    def validate_fields(cron_fields: str) -> None:
+        """expect 3 cron fields"""
+        if not len(cron_fields) == 3:
+            raise ValueError("expected three cron schedule fields")
+
+    @staticmethod
+    def validate_minute(minute_field: str):
+        """expect minute int"""
+        if not minute_field.isdigit():
+            raise ValueError("Invalid value for minutes. Must be an integer.")
+
+        minutes = int(minute_field)
+        if not 0 <= minutes <= 59:
+            raise ValueError("Invalid minutes. Must be between 0 and 59.")
+
+    @staticmethod
+    def validate_cron_tab(minute, hour, day_of_week):
+        """check if crontab can be created"""
+        try:
+            crontab(minute=minute, hour=hour, day_of_week=day_of_week)
+        except ValueError as err:
+            raise ValueError(f"invalid crontab: {err}") from err
+
+    def validate_cron(self, cron_expression):
+        """create crontab schedule"""
+        if not cron_expression or cron_expression == "auto":
+            return
+
+        cron_fields = cron_expression.split()
+        self.validate_fields(cron_fields)
+
+        minute, hour, day_of_week = cron_fields
+        self.validate_minute(minute)
+        self.validate_cron_tab(minute, hour, day_of_week)
+
+    def validate_config(self, task_name: str, schedule_config: dict):
+        """validate config for given task"""
+        if not schedule_config:
+            return
+
+        config_keys = self.CONFIG.get(task_name)
+        if not config_keys:
+            raise ValueError(f"task '{task_name}' doesn't take config")
+
+        for key in schedule_config:
+            if key not in config_keys:
+                raise ValueError(f"invalid config key for task '{task_name}'")

backend/task/src/notify.py

@@ -0,0 +1,141 @@
+"""send notifications using apprise"""
+
+import apprise
+from common.src.es_connect import ElasticWrap
+from task.src.task_config import TASK_CONFIG
+from task.src.task_manager import TaskManager
+
+
+class Notifications:
+    """store notifications in ES"""
+
+    GET_PATH = "ta_config/_doc/notify"
+    UPDATE_PATH = "ta_config/_update/notify/"
+
+    def __init__(self, task_name: str):
+        self.task_name = task_name
+
+    def send(self, task_id: str, task_title: str) -> None:
+        """send notifications"""
+        apobj = apprise.Apprise()
+        urls: list[str] = self.get_urls()
+        if not urls:
+            return
+
+        title, body = self._build_message(task_id, task_title)
+
+        if not body:
+            return
+
+        for url in urls:
+            apobj.add(url)
+
+        apobj.notify(body=body, title=title)
+
+    def _build_message(
+        self, task_id: str, task_title: str
+    ) -> tuple[str, str | None]:
+        """build message to send notification"""
+        task = TaskManager().get_task(task_id)
+        status = task.get("status")
+        title: str = f"[TA] {task_title} process ended with {status}"
+        body: str | None = task.get("result")
+
+        return title, body
+
+    def get_urls(self) -> list[str]:
+        """get stored urls for task"""
+        response, code = ElasticWrap(self.GET_PATH).get(print_error=False)
+        if not code == 200:
+            return []
+
+        urls = response["_source"].get(self.task_name, [])
+
+        return urls
+
+    def add_url(self, url: str) -> None:
+        """add url to task notification"""
+        source = (
+            "if (!ctx._source.containsKey(params.task_name)) "
+            + "{ctx._source[params.task_name] = [params.url]} "
+            + "else if (!ctx._source[params.task_name].contains(params.url)) "
+            + "{ctx._source[params.task_name].add(params.url)} "
+            + "else {ctx.op = 'none'}"
+        )
+
+        data = {
+            "script": {
+                "source": source,
+                "lang": "painless",
+                "params": {"url": url, "task_name": self.task_name},
+            },
+            "upsert": {self.task_name: [url]},
+        }
+
+        _, _ = ElasticWrap(self.UPDATE_PATH).post(data)
+
+    def remove_url(self, url: str) -> tuple[dict, int]:
+        """remove url from task"""
+        source = (
+            "if (ctx._source.containsKey(params.task_name) "
+            + "&& ctx._source[params.task_name].contains(params.url)) "
+            + "{ctx._source[params.task_name]."
+            + "remove(ctx._source[params.task_name].indexOf(params.url))}"
+        )
+
+        data = {
+            "script": {
+                "source": source,
+                "lang": "painless",
+                "params": {"url": url, "task_name": self.task_name},
+            }
+        }
+
+        response, status_code = ElasticWrap(self.UPDATE_PATH).post(data)
+        if not self.get_urls():
+            _, _ = self.remove_task()
+
+        return response, status_code
+
+    def remove_task(self) -> tuple[dict, int]:
+        """remove all notifications from task"""
+        source = (
+            "if (ctx._source.containsKey(params.task_name)) "
+            + "{ctx._source.remove(params.task_name)}"
+        )
+        data = {
+            "script": {
+                "source": source,
+                "lang": "painless",
+                "params": {"task_name": self.task_name},
+            }
+        }
+
+        response, status_code = ElasticWrap(self.UPDATE_PATH).post(data)
+
+        return response, status_code
+
+
+def get_all_notifications() -> dict[str, list[str]]:
+    """get all notifications stored"""
+    path = "ta_config/_doc/notify"
+    response, status_code = ElasticWrap(path).get(print_error=False)
+    if not status_code == 200:
+        return {}
+
+    notifications: dict = {}
+    source = response.get("_source")
+    if not source:
+        return notifications
+
+    for task_id, urls in source.items():
+        notifications.update(
+            {
+                task_id: {
+                    "urls": urls,
+                    "title": TASK_CONFIG[task_id]["title"],
+                }
+            }
+        )
+
+    return notifications

backend/task/src/task_config.py

@@ -0,0 +1,125 @@
+"""
+Functionality:
+- Static Task config values
+- Type definitions
+- separate to avoid circular imports
+"""
+
+from typing import TypedDict
+
+
+class TaskItemConfig(TypedDict):
+    """describes a task item config"""
+
+    title: str
+    group: str
+    api_start: bool
+    api_stop: bool
+
+
+UPDATE_SUBSCRIBED: TaskItemConfig = {
+    "title": "Rescan your Subscriptions",
+    "group": "download:scan",
+    "api_start": True,
+    "api_stop": True,
+}
+
+DOWNLOAD_PENDING: TaskItemConfig = {
+    "title": "Downloading",
+    "group": "download:run",
+    "api_start": True,
+    "api_stop": True,
+}
+
+EXTRACT_DOWNLOAD: TaskItemConfig = {
+    "title": "Add to download queue",
+    "group": "download:add",
+    "api_start": False,
+    "api_stop": True,
+}
+
+CHECK_REINDEX: TaskItemConfig = {
+    "title": "Reindex Documents",
+    "group": "reindex:run",
+    "api_start": False,
+    "api_stop": False,
+}
+
+MANUAL_IMPORT: TaskItemConfig = {
+    "title": "Manual video import",
+    "group": "setting:import",
+    "api_start": True,
+    "api_stop": False,
+}
+
+RUN_BACKUP: TaskItemConfig = {
+    "title": "Index Backup",
+    "group": "setting:backup",
+    "api_start": True,
+    "api_stop": False,
+}
+
+RESTORE_BACKUP: TaskItemConfig = {
+    "title": "Restore Backup",
+    "group": "setting:restore",
+    "api_start": False,
+    "api_stop": False,
+}
+
+RESCAN_FILESYSTEM: TaskItemConfig = {
+    "title": "Rescan your Filesystem",
+    "group": "setting:filesystemscan",
+    "api_start": True,
+    "api_stop": False,
+}
+
+THUMBNAIL_CHECK: TaskItemConfig = {
+    "title": "Check your Thumbnails",
+    "group": "setting:thumbnailcheck",
+    "api_start": True,
+    "api_stop": False,
+}
+
+RESYNC_THUMBS: TaskItemConfig = {
+    "title": "Sync Thumbnails to Media Files",
+    "group": "setting:thumbnailsync",
+    "api_start": True,
+    "api_stop": False,
+}
+
+INDEX_PLAYLISTS: TaskItemConfig = {
+    "title": "Index Channel Playlist",
+    "group": "channel:indexplaylist",
+    "api_start": False,
+    "api_stop": False,
+}
+
+SUBSCRIBE_TO: TaskItemConfig = {
+    "title": "Add Subscription",
+    "group": "subscription:add",
+    "api_start": False,
+    "api_stop": False,
+}
+
+VERSION_CHECK: TaskItemConfig = {
+    "title": "Look for new Version",
+    "group": "",
+    "api_start": False,
+    "api_stop": False,
+}
+
+TASK_CONFIG: dict[str, TaskItemConfig] = {
+    "update_subscribed": UPDATE_SUBSCRIBED,
+    "download_pending": DOWNLOAD_PENDING,
+    "extract_download": EXTRACT_DOWNLOAD,
+    "check_reindex": CHECK_REINDEX,
+    "manual_import": MANUAL_IMPORT,
+    "run_backup": RUN_BACKUP,
+    "restore_backup": RESTORE_BACKUP,
+    "rescan_filesystem": RESCAN_FILESYSTEM,
+    "thumbnail_check": THUMBNAIL_CHECK,
+    "resync_thumbs": RESYNC_THUMBS,
+    "index_playlists": INDEX_PLAYLISTS,
+    "subscribe_to": SUBSCRIBE_TO,
+    "version_check": VERSION_CHECK,
+}

backend/task/src/task_manager.py

@@ -0,0 +1,117 @@
+"""
+functionality:
+- interact with in redis stored task results
+- handle threads and locks
+"""
+
+from common.src.ta_redis import RedisArchivist, TaskRedis
+from task.celery import app as celery_app
+from task.src.task_config import TASK_CONFIG
+
+
+class TaskManager:
+    """manage tasks"""
+
+    def get_all_results(self):
+        """return all task results"""
+        handler = TaskRedis()
+        all_keys = handler.get_all()
+        if not all_keys:
+            return False
+
+        return [handler.get_single(i) for i in all_keys]
+
+    def get_tasks_by_name(self, task_name):
+        """get all tasks by name"""
+        all_results = self.get_all_results()
+        if not all_results:
+            return False
+
+        return [i for i in all_results if i.get("name") == task_name]
+
+    def get_task(self, task_id):
+        """get single task"""
+        return TaskRedis().get_single(task_id)
+
+    def is_pending(self, task):
+        """check if task_name is pending, pass task object"""
+        tasks = self.get_tasks_by_name(task.name)
+        if not tasks:
+            return False
+
+        return bool([i for i in tasks if i.get("status") == "PENDING"])
+
+    def is_stopped(self, task_id):
+        """check if task_id has received STOP command"""
+        task = self.get_task(task_id)
+
+        return task.get("command") == "STOP"
+
+    def get_pending(self, task_name):
+        """get all pending tasks of task_name"""
+        tasks = self.get_tasks_by_name(task_name)
+        if not tasks:
+            return False
+
+        return [i for i in tasks if i.get("status") == "PENDING"]
+
+    def init(self, task):
+        """pass task object from bind task to set initial pending message"""
+        message = {
+            "status": "PENDING",
+            "result": None,
+            "traceback": None,
+            "date_done": False,
+            "name": task.name,
+            "task_id": task.request.id,
+        }
+        TaskRedis().set_key(task.request.id, message)
+
+    def fail_pending(self):
+        """
+        mark all pending as failed,
+        run at startup to recover from hard reset
+        """
+        all_results = self.get_all_results()
+        if not all_results:
+            return
+
+        for result in all_results:
+            if result.get("status") == "PENDING":
+                result["status"] = "FAILED"
+                TaskRedis().set_key(result["task_id"], result, expire=True)
+
+
+class TaskCommand:
+    """run commands on task"""
+
+    def start(self, task_name):
+        """start task by task_name, only pass task that don't take args"""
+        task = celery_app.tasks.get(task_name).delay()
+        message = {
+            "task_id": task.id,
+            "status": task.status,
+            "task_name": task.name,
+        }
+
+        return message
+
+    def stop(self, task_id, message_key):
+        """
+        send stop signal to task_id,
+        needs to be implemented in task to take effect
+        """
+        print(f"[task][{task_id}]: received STOP signal.")
+        handler = TaskRedis()
+
+        task = handler.get_single(task_id)
+        if not task["name"] in TASK_CONFIG:
+            raise ValueError
+
+        handler.set_command(task_id, "STOP")
+        RedisArchivist().set_message(message_key, "STOP", path=".command")
+
+    def kill(self, task_id):
+        """send kill signal to task_id"""
+        print(f"[task][{task_id}]: received KILL signal.")
+        celery_app.control.revoke(task_id, terminate=True)

backend/task/tasks.py

@@ -0,0 +1,312 @@
+"""
+Functionality:
+- collect tasks
+- handle task callbacks
+- handle task notifications
+- handle task locking
+"""
+
+from appsettings.src.backup import ElasticBackup
+from appsettings.src.config import ReleaseVersion
+from appsettings.src.filesystem import Scanner
+from appsettings.src.index_setup import ElasitIndexWrap
+from appsettings.src.manual import ImportFolderScanner
+from appsettings.src.reindex import Reindex, ReindexManual, ReindexPopulate
+from celery import Task, shared_task
+from celery.exceptions import Retry
+from channel.src.index import YoutubeChannel
+from common.src.ta_redis import RedisArchivist
+from common.src.urlparser import Parser
+from download.src.queue import PendingList
+from download.src.subscriptions import SubscriptionHandler, SubscriptionScanner
+from download.src.thumbnails import ThumbFilesystem, ThumbValidator
+from download.src.yt_dlp_handler import VideoDownloader
+from task.src.notify import Notifications
+from task.src.task_config import TASK_CONFIG
+from task.src.task_manager import TaskManager
+
+
+class BaseTask(Task):
+    """base class to inherit each class from"""
+
+    # pylint: disable=abstract-method
+
+    def on_failure(self, exc, task_id, args, kwargs, einfo):
+        """callback for task failure"""
+        print(f"{task_id} Failed callback")
+        message, key = self._build_message(level="error")
+        message.update({"messages": [f"Task failed: {exc}"]})
+        RedisArchivist().set_message(key, message, expire=20)
+
+    def on_success(self, retval, task_id, args, kwargs):
+        """callback task completed successfully"""
+        print(f"{task_id} success callback")
+        message, key = self._build_message()
+        message.update({"messages": ["Task completed successfully"]})
+        RedisArchivist().set_message(key, message, expire=5)
+
+    def before_start(self, task_id, args, kwargs):
+        """callback before initiating task"""
+        print(f"{self.name} create callback")
+        message, key = self._build_message()
+        message.update({"messages": ["New task received."]})
+        RedisArchivist().set_message(key, message)
+
+    def after_return(self, status, retval, task_id, args, kwargs, einfo):
+        """callback after task returns"""
+        print(f"{task_id} return callback")
+        task_title = TASK_CONFIG.get(self.name).get("title")
+        Notifications(self.name).send(task_id, task_title)
+
+    def send_progress(self, message_lines, progress=False, title=False):
+        """send progress message"""
+        message, key = self._build_message()
+        message.update(
+            {
+                "messages": message_lines,
+                "progress": progress,
+            }
+        )
+        if title:
+            message["title"] = title
+
+        RedisArchivist().set_message(key, message)
+
+    def _build_message(self, level="info"):
+        """build message dict"""
+        task_id = self.request.id
+        message = TASK_CONFIG.get(self.name).copy()
+        message.update({"level": level, "id": task_id})
+        task_result = TaskManager().get_task(task_id)
+        if task_result:
+            command = task_result.get("command", False)
+            message.update({"command": command})
+
+        key = f"message:{message.get('group')}:{task_id.split('-')[0]}"
+        return message, key
+
+    def is_stopped(self):
+        """check if task is stopped"""
+        return TaskManager().is_stopped(self.request.id)
+
+
+@shared_task(name="update_subscribed", bind=True, base=BaseTask)
+def update_subscribed(self):
+    """look for missing videos and add to pending"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] rescan already running")
+        self.send_progress("Rescan already in progress.")
+        return None
+
+    manager.init(self)
+    handler = SubscriptionScanner(task=self)
+    missing_videos = handler.scan()
+    auto_start = handler.auto_start
+    if missing_videos:
+        print(missing_videos)
+        extrac_dl.delay(missing_videos, auto_start=auto_start)
+        message = f"Found {len(missing_videos)} videos to add to the queue."
+        return message
+
+    return None
+
+
+@shared_task(
+    name="download_pending",
+    bind=True,
+    base=BaseTask,
+    max_retries=3,
+    default_retry_delay=10,
+)
+def download_pending(self, auto_only=False):
+    """download latest pending videos"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] download queue already running")
+        self.send_progress("Download Queue is already running.")
+        return None
+
+    manager.init(self)
+    try:
+        downloader = VideoDownloader(task=self)
+        downloaded, failed = downloader.run_queue(auto_only=auto_only)
+
+        if failed:
+            print(f"[task][{self.name}] Videos failed, retry.")
+            self.send_progress("Videos failed, retry.")
+            raise self.retry()
+
+    except Retry as exc:
+        raise exc
+
+    if downloaded:
+        return f"downloaded {downloaded} video(s)."
+
+    return None
+
+
+@shared_task(name="extract_download", bind=True, base=BaseTask)
+def extrac_dl(self, youtube_ids, auto_start=False, status="pending"):
+    """parse list passed and add to pending"""
+    TaskManager().init(self)
+    if isinstance(youtube_ids, str):
+        to_add = Parser(youtube_ids).parse()
+    else:
+        to_add = youtube_ids
+
+    pending_handler = PendingList(youtube_ids=to_add, task=self)
+    pending_handler.parse_url_list()
+    videos_added = pending_handler.add_to_pending(
+        status=status, auto_start=auto_start
+    )
+
+    if auto_start:
+        download_pending.delay(auto_only=True)
+
+    if videos_added:
+        return f"added {len(videos_added)} Videos to Queue"
+
+    return None
+
+
+@shared_task(bind=True, name="check_reindex", base=BaseTask)
+def check_reindex(self, data=False, extract_videos=False):
+    """run the reindex main command"""
+    if data:
+        # started from frontend through API
+        print(f"[task][{self.name}] reindex {data}")
+        self.send_progress("Add items to the reindex Queue.")
+        ReindexManual(extract_videos=extract_videos).extract_data(data)
+
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] reindex queue is already running")
+        self.send_progress("Reindex Queue is already running.")
+        return
+
+    manager.init(self)
+    if not data:
+        # started from scheduler
+        populate = ReindexPopulate()
+        print(f"[task][{self.name}] reindex outdated documents")
+        self.send_progress("Add recent documents to the reindex Queue.")
+        populate.get_interval()
+        populate.add_recent()
+        self.send_progress("Add outdated documents to the reindex Queue.")
+        populate.add_outdated()
+
+    handler = Reindex(task=self)
+    handler.reindex_all()
+
+    return handler.build_message()
+
+
+@shared_task(bind=True, name="manual_import", base=BaseTask)
+def run_manual_import(self):
+    """called from settings page, to go through import folder"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] manual import is already running")
+        self.send_progress("Manual import is already running.")
+        return
+
+    manager.init(self)
+    ImportFolderScanner(task=self).scan()
+
+
+@shared_task(bind=True, name="run_backup", base=BaseTask)
+def run_backup(self, reason="auto"):
+    """called from settings page, dump backup to zip file"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] backup is already running")
+        self.send_progress("Backup is already running.")
+        return
+
+    manager.init(self)
+    ElasticBackup(reason=reason, task=self).backup_all_indexes()
+
+
+@shared_task(bind=True, name="restore_backup", base=BaseTask)
+def run_restore_backup(self, filename):
+    """called from settings page, dump backup to zip file"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] restore is already running")
+        self.send_progress("Restore is already running.")
+        return None
+
+    manager.init(self)
+    self.send_progress(["Reset your Index"])
+    ElasitIndexWrap().reset()
+    ElasticBackup(task=self).restore(filename)
+    print("index restore finished")
+
+    return f"backup restore completed: {filename}"
+
+
+@shared_task(bind=True, name="rescan_filesystem", base=BaseTask)
+def rescan_filesystem(self):
+    """check the media folder for mismatches"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] filesystem rescan already running")
+        self.send_progress("Filesystem Rescan is already running.")
+        return
+
+    manager.init(self)
+    handler = Scanner(task=self)
+    handler.scan()
+    handler.apply()
+    ThumbValidator(task=self).validate()
+
+
+@shared_task(bind=True, name="thumbnail_check", base=BaseTask)
+def thumbnail_check(self):
+    """validate thumbnails"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] thumbnail check is already running")
+        self.send_progress("Thumbnail check is already running.")
+        return
+
+    manager.init(self)
+    thumnail = ThumbValidator(task=self)
+    thumnail.validate()
+    thumnail.clean_up()
+
+
+@shared_task(bind=True, name="resync_thumbs", base=BaseTask)
+def re_sync_thumbs(self):
+    """sync thumbnails to mediafiles"""
+    manager = TaskManager()
+    if manager.is_pending(self):
+        print(f"[task][{self.name}] thumb re-embed is already running")
+        self.send_progress("Thumbnail re-embed is already running.")
+        return
+
+    manager.init(self)
+    ThumbFilesystem(task=self).embed()
+
+
+@shared_task(bind=True, name="subscribe_to", base=BaseTask)
+def subscribe_to(self, url_str: str, expected_type: str | bool = False):
+    """
+    take a list of urls to subscribe to
+    optionally validate expected_type channel / playlist
+    """
+    SubscriptionHandler(url_str, task=self).subscribe(expected_type)
+
+
+@shared_task(bind=True, name="index_playlists", base=BaseTask)
+def index_channel_playlists(self, channel_id):
+    """add all playlists of channel to index"""
+    channel = YoutubeChannel(channel_id, task=self)
+    channel.index_channel_playlists()
+
+
+@shared_task(name="version_check")
+def version_check():
+    """check for new updates"""
+    ReleaseVersion().check()

backend/task/tests/test_src/test_config_schedule.py

@@ -0,0 +1,68 @@
+"""test schedule parsing"""
+
+# flake8: noqa: E402
+
+import os
+
+import django
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+django.setup()
+
+import pytest
+from task.src.config_schedule import CrontabValidator
+
+INCORRECT_CRONTAB = [
+    "0 0 * * *",
+    "0 0",
+    "0",
+]
+
+
+@pytest.mark.parametrize("invalid_value", INCORRECT_CRONTAB)
+def test_invalid_len(invalid_value):
+    """raise error on invalid crontab"""
+    validator = CrontabValidator()
+    with pytest.raises(ValueError, match="three cron schedule fields"):
+        validator.validate_cron(invalid_value)
+
+
+NONE_INT_MINUTE = [
+    "* * *",
+    "0,30 * *",
+    "0,1,2 * *",
+    "-1 * *",
+]
+
+
+@pytest.mark.parametrize("invalid_value", NONE_INT_MINUTE)
+def test_none_int_crontabs(invalid_value):
+    """raise error on invalid crontab"""
+    validator = CrontabValidator()
+    with pytest.raises(ValueError, match="Must be an integer."):
+        validator.validate_cron(invalid_value)
+
+
+INVALID_MINUTE = ["60 * *", "61 * *"]
+
+
+@pytest.mark.parametrize("invalid_value", INVALID_MINUTE)
+def test_invalid_minute(invalid_value):
+    """raise error on invalid crontab"""
+    validator = CrontabValidator()
+    with pytest.raises(ValueError, match="Must be between 0 and 59."):
+        validator.validate_cron(invalid_value)
+
+
+INVALID_CRONTAB = [
+    "0 /1 *",
+    "0 0/1 *",
+]
+
+
+@pytest.mark.parametrize("invalid_value", INVALID_CRONTAB)
+def test_invalid_crontab(invalid_value):
+    """raise error on invalid crontab"""
+    validator = CrontabValidator()
+    with pytest.raises(ValueError, match="invalid crontab"):
+        validator.validate_cron(invalid_value)

backend/task/urls.py

@@ -0,0 +1,32 @@
+"""all tasks api URLs"""
+
+from django.urls import path
+from task import views
+
+urlpatterns = [
+    path(
+        "by-name/",
+        views.TaskListView.as_view(),
+        name="api-task-list",
+    ),
+    path(
+        "by-name/<slug:task_name>/",
+        views.TaskNameListView.as_view(),
+        name="api-task-name-list",
+    ),
+    path(
+        "by-id/<slug:task_id>/",
+        views.TaskIDView.as_view(),
+        name="api-task-id",
+    ),
+    path(
+        "schedule/<slug:task_name>/",
+        views.ScheduleView.as_view(),
+        name="api-schedule",
+    ),
+    path(
+        "notification/",
+        views.ScheduleNotification.as_view(),
+        name="api-schedule-notification",
+    ),
+]

backend/task/views.py

@@ -0,0 +1,207 @@
+"""all task API views"""
+
+from common.views_base import AdminOnly, ApiBaseView
+from django.shortcuts import get_object_or_404
+from rest_framework.response import Response
+from task.models import CustomPeriodicTask
+from task.src.config_schedule import CrontabValidator, ScheduleBuilder
+from task.src.notify import Notifications, get_all_notifications
+from task.src.task_config import TASK_CONFIG
+from task.src.task_manager import TaskCommand, TaskManager
+
+
+class TaskListView(ApiBaseView):
+    """resolves to /api/task/by-name/
+    GET: return a list of all stored task results
+    """
+
+    permission_classes = [AdminOnly]
+
+    def get(self, request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        all_results = TaskManager().get_all_results()
+
+        return Response(all_results)
+
+
+class TaskNameListView(ApiBaseView):
+    """resolves to /api/task/by-name/<task-name>/
+    GET: return a list of stored results of task
+    POST: start new background process
+    """
+
+    permission_classes = [AdminOnly]
+
+    def get(self, request, task_name):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        if task_name not in TASK_CONFIG:
+            message = {"message": "invalid task name"}
+            return Response(message, status=404)
+
+        all_results = TaskManager().get_tasks_by_name(task_name)
+
+        return Response(all_results)
+
+    def post(self, request, task_name):
+        """
+        handle post request
+        404 for invalid task_name
+        400 if task can't be started here without argument
+        """
+        # pylint: disable=unused-argument
+        task_config = TASK_CONFIG.get(task_name)
+        if not task_config:
+            message = {"message": "invalid task name"}
+            return Response(message, status=404)
+
+        if not task_config.get("api_start"):
+            message = {"message": "can not start task through this endpoint"}
+            return Response(message, status=400)
+
+        message = TaskCommand().start(task_name)
+
+        return Response({"message": message})
+
+
+class TaskIDView(ApiBaseView):
+    """resolves to /api/task/by-id/<task-id>/
+    GET: return details of task id
+    POST: send command to task by id
+    """
+
+    valid_commands = ["stop", "kill"]
+    permission_classes = [AdminOnly]
+
+    def get(self, request, task_id):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        task_result = TaskManager().get_task(task_id)
+        if not task_result:
+            message = {"message": "task id not found"}
+            return Response(message, status=404)
+
+        return Response(task_result)
+
+    def post(self, request, task_id):
+        """post command to task"""
+        command = request.data.get("command")
+        if not command or command not in self.valid_commands:
+            message = {"message": "no valid command found"}
+            return Response(message, status=400)
+
+        task_result = TaskManager().get_task(task_id)
+        if not task_result:
+            message = {"message": "task id not found"}
+            return Response(message, status=404)
+
+        task_conf = TASK_CONFIG.get(task_result.get("name"))
+        if command == "stop":
+            if not task_conf.get("api_stop"):
+                message = {"message": "task can not be stopped"}
+                return Response(message, status=400)
+
+            message_key = self._build_message_key(task_conf, task_id)
+            TaskCommand().stop(task_id, message_key)
+        if command == "kill":
+            if not task_conf.get("api_stop"):
+                message = {"message": "task can not be killed"}
+                return Response(message, status=400)
+
+            TaskCommand().kill(task_id)
+
+        return Response({"message": "command sent"})
+
+    def _build_message_key(self, task_conf, task_id):
+        """build message key to forward command to notification"""
+        return f"message:{task_conf.get('group')}:{task_id.split('-')[0]}"
+
+
+class ScheduleView(ApiBaseView):
+    """resolves to /api/task/schedule/<task-name>/
+    POST: create/update schedule for task with config
+    - example: {"schedule": "0 0 *", "config": {"days": 90}}
+    DEL: delete schedule for task
+    """
+
+    permission_classes = [AdminOnly]
+
+    def post(self, request, task_name):
+        """create/update schedule for task"""
+        cron_schedule = request.data.get("schedule")
+        schedule_config = request.data.get("config")
+        if not cron_schedule and not schedule_config:
+            message = {"message": "expected schedule or config key"}
+            return Response(message, status=400)
+
+        try:
+            validator = CrontabValidator()
+            validator.validate_cron(cron_schedule)
+            validator.validate_config(task_name, schedule_config)
+        except ValueError as err:
+            return Response({"message": str(err)}, status=400)
+
+        ScheduleBuilder().update_schedule(
+            task_name, cron_schedule, schedule_config
+        )
+        message = f"update schedule for task {task_name}"
+        if schedule_config:
+            message += f" with config {schedule_config}"
+
+        return Response({"message": message})
+
+    def delete(self, request, task_name):
+        """delete schedule by task_name query"""
+        task = get_object_or_404(CustomPeriodicTask, name=task_name)
+        _ = task.delete()
+
+        return Response({"success": True})
+
+
+class ScheduleNotification(ApiBaseView):
+    """resolves to /api/task/notification/
+    GET: get all schedule notifications
+    POST: add notification url to task
+    DEL: delete notification
+    """
+
+    def get(self, request):
+        """handle get request"""
+
+        return Response(get_all_notifications())
+
+    def post(self, request):
+        """handle create notification"""
+        task_name = request.data.get("task_name")
+        url = request.data.get("url")
+
+        if not TASK_CONFIG.get(task_name):
+            message = {"message": "task_name not found"}
+            return Response(message, status=404)
+
+        if not url:
+            message = {"message": "missing url key"}
+            return Response(message, status=400)
+
+        Notifications(task_name).add_url(url)
+        message = {"task_name": task_name, "url": url}
+
+        return Response(message)
+
+    def delete(self, request):
+        """handle delete"""
+
+        task_name = request.data.get("task_name")
+        url = request.data.get("url")
+
+        if not TASK_CONFIG.get(task_name):
+            message = {"message": "task_name not found"}
+            return Response(message, status=404)
+
+        if url:
+            response, status_code = Notifications(task_name).remove_url(url)
+        else:
+            response, status_code = Notifications(task_name).remove_task()
+
+        return Response({"response": response, "status_code": status_code})