From 34203b70ae591d1ab58c61e97d1b033632507b74 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 25 Jan 2025 00:54:47 +0700 Subject: [PATCH] add appsettings sync at startup --- backend/appsettings/src/config.py | 74 +++++++++++++------ .../config/management/commands/ta_startup.py | 36 +++++++-- 2 files changed, 82 insertions(+), 28 deletions(-) diff --git a/backend/appsettings/src/config.py b/backend/appsettings/src/config.py index 8778f038..251065e2 100644 --- a/backend/appsettings/src/config.py +++ b/backend/appsettings/src/config.py @@ -27,22 +27,22 @@ class SubscriptionsConfigType(TypedDict): class DownloadsConfigType(TypedDict): """describes downloads config""" - limit_speed: int - sleep_interval: int - autodelete_days: int - format: str | bool - format_sort: str | bool + limit_speed: int | None + sleep_interval: int | None + autodelete_days: int | None + format: str | None + format_sort: str | None add_metadata: bool add_thumbnail: bool - subtitle: str | bool - subtitle_source: Literal["user", "auto"] | bool + subtitle: str | None + subtitle_source: Literal["user", "auto"] | None subtitle_index: bool - comment_max: str | bool - comment_sort: Literal["top", "new"] + comment_max: str | None + comment_sort: Literal["top", "new"] | None cookie_import: bool potoken: bool - throttledratelimit: int - extractor_lang: str | bool + throttledratelimit: int | None + extractor_lang: str | None integrate_ryd: bool integrate_sponsorblock: bool @@ -65,30 +65,32 @@ class AppConfig: """handle application variables""" ES_PATH = "ta_config/_doc/appsettings" + ES_UPDATE_PATH = "ta_config/_update/appsettings" CONFIG_DEFAULTS: AppConfigType = { "subscriptions": { "channel_size": 50, "live_channel_size": 50, "shorts_channel_size": 50, "auto_start": False, + "new_config": 100, }, "downloads": { - "limit_speed": False, + "limit_speed": None, "sleep_interval": 10, - "autodelete_days": False, - "format": False, - "format_sort": False, + "autodelete_days": None, + "format": None, + "format_sort": None, "add_metadata": False, "add_thumbnail": False, - "subtitle": False, - "subtitle_source": False, + "subtitle": None, + "subtitle_source": None, "subtitle_index": False, - "comment_max": False, + "comment_max": None, "comment_sort": "top", "cookie_import": False, "potoken": False, - "throttledratelimit": False, - "extractor_lang": False, + "throttledratelimit": None, + "extractor_lang": None, "integrate_ryd": False, "integrate_sponsorblock": False, }, @@ -102,7 +104,7 @@ class AppConfig: """get config from ES""" response, status_code = ElasticWrap(self.ES_PATH).get() if not status_code == 200: - return self.CONFIG_DEFAULTS + raise ValueError(f"no config found at {self.ES_PATH}") return response["_source"] @@ -119,6 +121,13 @@ class AppConfig: return self.config + def _update_config_dict(self, to_update) -> None: + """none validated partial update for defaults sync""" + data = {"doc": to_update} + response, status_code = ElasticWrap(self.ES_UPDATE_PATH).post(data) + if not status_code == 200: + print(f"update failed: {response}, {status_code}") + def _validate_key(self, key_map: list[str]) -> None: """raise valueerror on invalid key""" exists = self.CONFIG_DEFAULTS.get(key_map[0], {}).get(key_map[1]) # type: ignore # noqa: E501 @@ -145,6 +154,29 @@ class AppConfig: } RedisArchivist().set_message(key, message=message, expire=True) + def sync_defaults(self): + """sync defaults at startup, needs to be called with __new__""" + return ElasticWrap(self.ES_PATH).post(self.CONFIG_DEFAULTS) + + def add_new_defaults(self) -> list[str]: + """add new default config values to ES, called at startup""" + updated = [] + for key, value in self.CONFIG_DEFAULTS.items(): + if key not in self.config: + # complete new key + self._update_config_dict({key: value}) + updated.append(str({key: value})) + continue + + for sub_key, sub_value in value.items(): # type: ignore + if sub_key not in self.config[key]: + # new partial key + to_update = {key: {sub_key: sub_value}} + self._update_config_dict(to_update) + updated.append(str(to_update)) + + return updated + class ReleaseVersion: """compare local version with remote version""" diff --git a/backend/config/management/commands/ta_startup.py b/backend/config/management/commands/ta_startup.py index c995f5c0..7e852141 100644 --- a/backend/config/management/commands/ta_startup.py +++ b/backend/config/management/commands/ta_startup.py @@ -9,7 +9,7 @@ from datetime import datetime from random import randint from time import sleep -from appsettings.src.config import ReleaseVersion +from appsettings.src.config import AppConfig, ReleaseVersion from appsettings.src.index_setup import ElasitIndexWrap from appsettings.src.snapshot import ElasticSnapshot from common.src.env_settings import EnvironmentSettings @@ -51,9 +51,10 @@ class Command(BaseCommand): self._snapshot_check() self._create_default_schedules() self._update_schedule_tz() + self._init_app_config() def _mig_app_settings(self) -> None: - """update from v0.4.10 to v0.5.0, migrate application settings""" + """update from v0.4.13 to v0.5.0, migrate application settings""" self.stdout.write("[MIGRATION] move appconfig to ES") try: config = RedisArchivist().get_message("config") @@ -187,10 +188,7 @@ class Command(BaseCommand): ElasticSnapshot().setup() def _create_default_schedules(self) -> None: - """ - create default schedules for new installations - needs to be called after _mig_schedule_store - """ + """create default schedules for new installations""" self.stdout.write("[9] create initial schedules") init_has_run = CustomPeriodicTask.objects.filter( name="version_check" @@ -242,7 +240,7 @@ class Command(BaseCommand): def _update_schedule_tz(self) -> None: """update timezone for Schedule instances""" - self.stdout.write("[9] validate schedules TZ") + self.stdout.write("[10] validate schedules TZ") tz = EnvironmentSettings.TZ to_update = CrontabSchedule.objects.exclude(timezone=tz) @@ -257,3 +255,27 @@ class Command(BaseCommand): self.style.SUCCESS(f" ✓ updated {updated} schedules to {tz}.") ) PeriodicTasks.update_changed() + + def _init_app_config(self) -> None: + """init default app config to ES""" + self.stdout.write("[11] Check AppConfig") + try: + _ = AppConfig().config + self.stdout.write( + self.style.SUCCESS(" skip completed appsettings init") + ) + updated_defaults = AppConfig().add_new_defaults() + for new_default in updated_defaults: + self.stdout.write( + self.style.SUCCESS(f" added new default: {new_default}") + ) + + except ValueError: + handler = AppConfig.__new__(AppConfig) + _, status_code = handler.sync_defaults() + self.stdout.write( + self.style.SUCCESS(" ✓ Created default appsettings.") + ) + self.stdout.write( + self.style.SUCCESS(f" Status code: {status_code}") + )