public/tubearchivist
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/tubearchivist.git synced 2026-08-04 21:39:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

paginate channel migration index update

Browse Source
This commit is contained in:
Simon
2023-07-29 17:05:01 +07:00
parent aea403a874
commit 75a63c4828
1 changed files with 9 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tubearchivist/config/management/commands/ta_migpath.py
View File

@@ -98,7 +98,8 @@ class FolderMigration:
def migrate_videos(self, to_migrate): def migrate_videos(self, to_migrate):
"""migrate all videos of channel""" """migrate all videos of channel"""
for video in to_migrate: total = len(to_migrate)
for idx, video in enumerate(to_migrate):
new_media_url = self._move_video_file(video) new_media_url = self._move_video_file(video)
if not new_media_url: if not new_media_url:
continue continue
@@ -113,6 +114,9 @@ class FolderMigration:
self.bulk_list.append(json.dumps(action)) self.bulk_list.append(json.dumps(action))
self.bulk_list.append(json.dumps(source)) self.bulk_list.append(json.dumps(source))
if idx % 1000 == 0:
print(f"processing migration [{idx}/{total}]")
self.send_bulk()
def _move_video_file(self, video): def _move_video_file(self, video):
"""move video file to new location""" """move video file to new location"""
@@ -158,11 +162,14 @@ class FolderMigration:
return return
self.bulk_list.append("\n") self.bulk_list.append("\n")
path = "_bulk?refresh=true"
data = "\n".join(self.bulk_list) data = "\n".join(self.bulk_list)
response, status = ElasticWrap("_bulk").post(data=data, ndjson=True) response, status = ElasticWrap(path).post(data=data, ndjson=True)
if not status == 200: if not status == 200:
print(response) print(response)
self.bulk_list = []
def delete_old(self): def delete_old(self):
"""delete old empty folders""" """delete old empty folders"""
all_folders = ignore_filelist(os.listdir(self.videos)) all_folders = ignore_filelist(os.listdir(self.videos))