channel migration take 2

tubearchivist/config/management/commands/ta_migpath.py

@@ -25,99 +25,83 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         """run commands"""
         self.stdout.write(TOPIC)
-        need_migration = self.channels_need_migration()
+
-        if not need_migration:
+        handler = FolderMigration()
+        to_migrate = handler.get_to_migrate()
+        if not to_migrate:
             self.stdout.write(
-                self.style.SUCCESS("    no channel migration needed")
+                self.style.SUCCESS("    no channel migration needed\n")
             )
             return
 
+        self.stdout.write(self.style.SUCCESS("    migrating channels"))
+        total_channels = handler.create_folders(to_migrate)
         self.stdout.write(
-            self.style.SUCCESS(f"    migrating {len(need_migration)} channels")
+            self.style.SUCCESS(f"    created {total_channels} channels")
-        )
-        for channel in need_migration:
-            channel_name = channel["channel_name"]
-            channel_id = channel["channel_id"]
-            self.stdout.write(
-                self.style.SUCCESS(
-                    f"    migrating {channel_name} [{channel_id}]"
-                )
-            )
-            ChannelMigration(channel).migrate()
-
-        self.stdout.write(
-            self.style.SUCCESS("    ✓ channel migration completed")
         )
 
-    def channels_need_migration(self):
+        self.stdout.write(
-        """get channels that need migration"""
+            self.style.SUCCESS(f"    migrating {len(to_migrate)} videos")
-        all_indexed = self._get_channel_indexed()
+        )
-        all_folders = self._get_channel_folders()
+        handler.migrate_videos(to_migrate)
-        need_migration = []
+        self.stdout.write(self.style.SUCCESS("    update videos in index"))
-        for channel in all_indexed:
+        handler.send_bulk()
-            if channel["channel_id"] not in all_folders:
-                need_migration.append(channel)
 
-        return need_migration
+        self.stdout.write(self.style.SUCCESS("    cleanup old folders"))
+        handler.delete_old()
 
-    def _get_channel_indexed(self):
+        self.stdout.write(self.style.SUCCESS("    ✓ migration completed\n"))
-        """get all channels indexed"""
-        all_results = IndexPaginate("ta_channel", False).get_results()
-
-        return all_results
-
-    def _get_channel_folders(self):
-        """get all channel folders"""
-        base_folder = AppConfig().config["application"]["videos"]
-        existing_folders = ignore_filelist(os.listdir(base_folder))
-
-        return existing_folders
 
 
-class ChannelMigration:
+class FolderMigration:
-    """migrate single channel"""
+    """migrate video archive folder"""
 
-    def __init__(self, channel):
+    def __init__(self):
-        self.channel = channel
         self.config = AppConfig().config
         self.videos = self.config["application"]["videos"]
         self.bulk_list = []
 
-    def migrate(self):
+    def get_to_migrate(self):
-        """run migration"""
+        """get videos to migrate"""
-        self._create_new_folder()
+        script = (
-        all_videos = self.get_channel_videos()
+            "doc['media_url'].value == "
-        self.migrate_videos(all_videos)
+            + "doc['channel.channel_id'].value + '/'"
-        self.send_bulk()
+            + " + doc['youtube_id'].value + '.mp4'"
-        self.delete_old(all_videos)
+        )
+        data = {
+            "query": {"bool": {"must_not": [{"script": {"script": script}}]}},
+            "_source": [
+                "youtube_id",
+                "media_url",
+                "channel.channel_id",
+                "subtitles",
+            ],
+        }
+        response = IndexPaginate("ta_video", data).get_results()
 
-    def _create_new_folder(self):
+        return response
-        """create new channel id folder"""
+
+    def create_folders(self, to_migrate):
+        """create required channel folders"""
         host_uid = self.config["application"]["HOST_UID"]
         host_gid = self.config["application"]["HOST_GID"]
-        new_path = os.path.join(self.videos, self.channel["channel_id"])
+        all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate}
-        if not os.path.exists(new_path):
+
-            os.mkdir(new_path)
+        for channel_id in all_channel_ids:
+            new_folder = os.path.join(self.videos, channel_id)
+            os.makedirs(new_folder, exist_ok=True)
             if host_uid and host_gid:
-                os.chown(new_path, host_uid, host_gid)
+                os.chown(new_folder, host_uid, host_gid)
 
-    def get_channel_videos(self):
+        return len(all_channel_ids)
-        """get all videos of channel"""
-        data = {
-            "query": {
-                "term": {
-                    "channel.channel_id": {"value": self.channel["channel_id"]}
-                }
-            }
-        }
-        all_videos = IndexPaginate("ta_video", data).get_results()
 
-        return all_videos
+    def migrate_videos(self, to_migrate):
-
-    def migrate_videos(self, all_videos):
         """migrate all videos of channel"""
-        for video in all_videos:
+        for video in to_migrate:
             new_media_url = self._move_video_file(video)
+            if not new_media_url:
+                continue
+
             all_subtitles = self._move_subtitles(video)
             action = {
                 "update": {"_id": video["youtube_id"], "_index": "ta_video"}
@@ -137,9 +121,10 @@ class ChannelMigration:
             return False
 
         new_media_url = os.path.join(
-            self.channel["channel_id"], video["youtube_id"] + ".mp4"
+            video["channel"]["channel_id"], video["youtube_id"] + ".mp4"
         )
-        os.rename(old_path, os.path.join(self.videos, new_media_url))
+        new_path = os.path.join(self.videos, new_media_url)
+        os.rename(old_path, new_path)
 
         return new_media_url
 
@@ -155,11 +140,12 @@ class ChannelMigration:
                 print(f"did not find expected subtitle at {old_path}")
                 continue
 
-            ext = ".".join(old_path.split(".")[-2:])
             new_media_url = os.path.join(
-                self.channel["channel_id"], video["youtube_id"] + f".{ext}"
+                video["channel"]["channel_id"],
+                f"{video.get('youtube_id')}.{subtitle.get('lang')}.vtt",
             )
-            os.rename(old_path, os.path.join(self.videos, new_media_url))
+            new_path = os.path.join(self.videos, new_media_url)
+            os.rename(old_path, new_path)
             subtitle["media_url"] = new_media_url
 
         return all_subtitles
@@ -176,15 +162,10 @@ class ChannelMigration:
         if not status == 200:
             print(response)
 
-    def delete_old(self, all_videos):
+    def delete_old(self):
-        """delete old folder path if empty"""
+        """delete old empty folders"""
-        if not all_videos:
+        all_folders = ignore_filelist(os.listdir(self.videos))
-            return
+        for folder in all_folders:
-
+            folder_path = os.path.join(self.videos, folder)
-        channel_name = os.path.split(all_videos[0]["media_url"])[0]
+            if not ignore_filelist(os.listdir(folder_path)):
-        old_path = os.path.join(self.videos, channel_name)
+                os.rmdir(folder_path)
-        if os.path.exists(old_path) and not os.listdir(old_path):
-            os.rmdir(old_path)
-            return
-
-        print(f"failed to clean up old folder {old_path}")