From 81bf614afaf2eaeda6bb206b04678d2ae855e2e2 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 14:30:39 +0200 Subject: [PATCH 1/4] fix version check not triggering --- tubearchivist/config/management/commands/ta_startup.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index 80f8edf1..b3e72106 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -12,6 +12,7 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError from django_celery_beat.models import CrontabSchedule from home.models import CustomPeriodicTask +from home.tasks import version_check from home.src.es.connect import ElasticWrap from home.src.es.index_setup import ElasitIndexWrap from home.src.es.snapshot import ElasticSnapshot @@ -147,6 +148,14 @@ class Command(BaseCommand): else: self.stdout.write(self.style.SUCCESS(" no new update found")) + version_task = CustomPeriodicTask.objects.filter(name="version_check") + if not version_task.exists(): + return + + if not version_task.first().last_run_at: + self.style.SUCCESS(" ✓ send initial version check task") + version_check.delay() + def _mig_index_setup(self): """migration: validate index mappings""" self.stdout.write("[MIGRATION] validate index mappings") From 437cd72f453f237e2f4baa348cdf224eac0b92a5 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 16:57:28 +0200 Subject: [PATCH 2/4] update TZ in schedule when changed --- .../config/management/commands/ta_startup.py | 48 +++++++++++++++++-- tubearchivist/home/src/ta/config_schedule.py | 4 ++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index b3e72106..ce74f547 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -5,14 +5,15 @@ Functionality: """ import os +from datetime import datetime from random import randint from time import sleep from django.conf import settings from django.core.management.base import BaseCommand, CommandError -from django_celery_beat.models import CrontabSchedule +from django.utils import dateformat +from django_celery_beat.models import CrontabSchedule, PeriodicTasks from home.models import CustomPeriodicTask -from home.tasks import version_check from home.src.es.connect import ElasticWrap from home.src.es.index_setup import ElasitIndexWrap from home.src.es.snapshot import ElasticSnapshot @@ -24,6 +25,7 @@ from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_config import TASK_CONFIG from home.src.ta.task_manager import TaskManager +from home.tasks import version_check TOPIC = """ @@ -52,7 +54,9 @@ class Command(BaseCommand): self._mig_snapshot_check() self._mig_schedule_store() self._mig_custom_playlist() + self._mig_add_missing_timestamp() self._create_default_schedules() + self._update_schedule_tz() def _sync_redis_state(self): """make sure redis gets new config.json values""" @@ -325,6 +329,22 @@ class Command(BaseCommand): sleep(60) raise CommandError(message) + def _mig_add_missing_timestamp(self) -> None: + """ + add missing timestamp for versioncheck + migrate from v0.4.8 to v0.4.9 + """ + version_tasks = CustomPeriodicTask.objects.filter(name="version_check") + if not version_tasks.exists(): + return + + version_task = version_tasks.first() + if not version_task.last_run_at: + self.style.SUCCESS(" ✓ send initial version check task") + version_check.delay() + version_task.last_run_at = dateformat.make_aware(datetime.now()) + version_task.save() + def _create_default_schedules(self) -> None: """ create default schedules for new installations @@ -348,6 +368,7 @@ class Command(BaseCommand): "check_reindex", schedule=builder.SCHEDULES["check_reindex"] ) check_reindex.task_config.update({"days": 90}) + check_reindex.last_run_at = dateformat.make_aware(datetime.now()) check_reindex.save() self.stdout.write( self.style.SUCCESS( @@ -358,20 +379,39 @@ class Command(BaseCommand): thumbnail_check = builder.get_set_task( "thumbnail_check", schedule=builder.SCHEDULES["thumbnail_check"] ) + thumbnail_check.last_run_at = dateformat.make_aware(datetime.now()) + thumbnail_check.save() self.stdout.write( self.style.SUCCESS( f" ✓ created new default schedule: {thumbnail_check}" ) ) daily_random = f"{randint(0, 59)} {randint(0, 23)} *" - version_check = builder.get_set_task( + version_check_task = builder.get_set_task( "version_check", schedule=daily_random ) self.stdout.write( self.style.SUCCESS( - f" ✓ created new default schedule: {version_check}" + f" ✓ created new default schedule: {version_check_task}" ) ) self.stdout.write( self.style.SUCCESS(" ✓ all default schedules created") ) + + def _update_schedule_tz(self) -> None: + """update timezone for Schedule instances""" + tz = EnvironmentSettings.TZ + to_update = CrontabSchedule.objects.exclude(timezone=tz) + + if not to_update.exists(): + self.stdout.write( + self.style.SUCCESS(" all schedules have correct TZ") + ) + return + + updated = to_update.update(timezone=tz) + self.stdout.write( + self.style.SUCCESS(f" ✓ updated {updated} schedules to {tz}.") + ) + PeriodicTasks.update_changed() diff --git a/tubearchivist/home/src/ta/config_schedule.py b/tubearchivist/home/src/ta/config_schedule.py index 24b81ca9..cc64466b 100644 --- a/tubearchivist/home/src/ta/config_schedule.py +++ b/tubearchivist/home/src/ta/config_schedule.py @@ -3,6 +3,9 @@ Functionality: - Handle scheduler config update """ +from datetime import datetime + +from django.utils import dateformat from django_celery_beat.models import CrontabSchedule from home.models import CustomPeriodicTask from home.src.ta.config import AppConfig @@ -64,6 +67,7 @@ class ScheduleBuilder: if schedule: task_crontab = self.get_set_cron_tab(schedule) task.crontab = task_crontab + task.last_run_at = dateformat.make_aware(datetime.now()) task.save() return task From 93b921582ddcaa3d3ae7c4b78cf86794bd4b9505 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 16:58:04 +0200 Subject: [PATCH 3/4] bump requirements --- tubearchivist/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 6859991b..cc4d6be5 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,6 +1,6 @@ apprise==1.8.0 celery==5.4.0 -Django==5.0.6 +Django==5.0.7 django-auth-ldap==4.8.0 django-celery-beat==2.6.0 django-cors-headers==4.4.0 @@ -11,4 +11,4 @@ requests==2.32.3 ryd-client==0.0.6 uWSGI==2.0.26 whitenoise==6.7.0 -yt-dlp==2024.7.2 +yt-dlp==2024.7.9 From b0b72a6e6eb190e767345a6d30968ecb65a1f951 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Jul 2024 16:58:50 +0200 Subject: [PATCH 4/4] up ES memory default --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index be1497a4..6e82f010 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,7 +45,7 @@ services: restart: unless-stopped environment: - "ELASTIC_PASSWORD=verysecret" # matching Elasticsearch password - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - "xpack.security.enabled=true" - "discovery.type=single-node" - "path.repo=/usr/share/elasticsearch/data/snapshot"