Update s3_migration.py

s3_migration.py

@@ -40,3 +40,50 @@ def main():
 
 if __name__ == "__main__":
     main()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+import boto3
+import multiprocessing
+import os
+
+def upload_file_to_s3(file_path, s3_bucket, s3_key):
+    s3 = boto3.client('s3')
+    s3.upload_file(file_path, s3_bucket, s3_key)
+
+def migrate_files_to_s3(folder_path, s3_bucket, s3_folder, num_processes):
+    files = [os.path.join(folder_path, file) for file in os.listdir(folder_path)]
+
+    def worker(files):
+        for file in files:
+            s3_key = os.path.join(s3_folder, os.path.basename(file))
+            upload_file_to_s3(file, s3_bucket, s3_key)
+
+    with multiprocessing.Pool(processes=num_processes) as pool:
+        chunk_size = len(files) // num_processes
+        for i in range(0, len(files), chunk_size):
+            chunk = files[i:i+chunk_size]
+            pool.apply_async(worker, args=(chunk,))
+        pool.close()
+        pool.join()
+
+if __name__ == '__main__':
+    folder_path = 'PRT'
+    s3_bucket = 'MPT-2'
+    s3_folder = 'Ortus'
+    num_processes = 4  # Adjust the number of processes as needed
+
+    migrate_files_to_s3(folder_path, s3_bucket, s3_folder, num_processes)