public/tubearchivist
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/tubearchivist.git synced 2026-08-04 21:39:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

reset reindex counter on new added to queue, #478

Browse Source
This commit is contained in:
Simon
2023-06-22 23:29:05 +07:00
parent 879497d25a
commit 88e199ef9c
1 changed files with 13 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
tubearchivist/home/src/index/reindex.py
View File

@@ -52,6 +52,7 @@ class ReindexBase:
def __init__(self): def __init__(self):
self.config = AppConfig().config self.config = AppConfig().config
self.now = int(datetime.now().timestamp()) self.now = int(datetime.now().timestamp())
self.total = None
def populate(self, all_ids, reindex_config): def populate(self, all_ids, reindex_config):
"""add all to reindex ids to redis queue""" """add all to reindex ids to redis queue"""
@@ -59,6 +60,7 @@ class ReindexBase:
return return
RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids) RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
self.total = None
class ReindexPopulate(ReindexBase): class ReindexPopulate(ReindexBase):
@@ -236,19 +238,19 @@ class Reindex(ReindexBase):
if not RedisQueue(index_config["queue_name"]).has_item(): if not RedisQueue(index_config["queue_name"]).has_item():
continue continue
total = RedisQueue(index_config["queue_name"]).length() self.total = RedisQueue(index_config["queue_name"]).length()
while True: while True:
has_next = self.reindex_index(name, index_config, total) has_next = self.reindex_index(name, index_config)
if not has_next: if not has_next:
break break
def reindex_index(self, name, index_config, total): def reindex_index(self, name, index_config):
"""reindex all of a single index""" """reindex all of a single index"""
reindex = self.get_reindex_map(index_config["index_name"]) reindex = self.get_reindex_map(index_config["index_name"])
youtube_id = RedisQueue(index_config["queue_name"]).get_next() youtube_id = RedisQueue(index_config["queue_name"]).get_next()
if youtube_id: if youtube_id:
if self.task: if self.task:
self._notify(name, index_config, total) self._notify(name, index_config)
reindex(youtube_id) reindex(youtube_id)
sleep_interval = self.config["downloads"].get("sleep_interval", 0) sleep_interval = self.config["downloads"].get("sleep_interval", 0)
sleep(sleep_interval) sleep(sleep_interval)
@@ -265,12 +267,15 @@ class Reindex(ReindexBase):
return def_map.get(index_name) return def_map.get(index_name)
def _notify(self, name, index_config, total): def _notify(self, name, index_config):
"""send notification back to task""" """send notification back to task"""
if self.total is None:
self.total = RedisQueue(index_config["queue_name"]).length()
remaining = RedisQueue(index_config["queue_name"]).length() remaining = RedisQueue(index_config["queue_name"]).length()
idx = total - remaining idx = self.total - remaining
message = [f"Reindexing {name.title()}s {idx}/{total}"] message = [f"Reindexing {name.title()}s {idx}/{self.total}"]
progress = idx / total progress = idx / self.total
self.task.send_progress(message, progress=progress) self.task.send_progress(message, progress=progress)
def _reindex_single_video(self, youtube_id): def _reindex_single_video(self, youtube_id):