Create s3_migration.py

s3_migration.py

@@ -0,0 +1,42 @@
+import os
+import boto3
+from multiprocessing import Pool, cpu_count
+
+# Configuration
+LOCAL_FOLDER = 'PRT'  # The local folder with files
+BUCKET_NAME = 'your-s3-bucket-name'  # Your S3 bucket name
+REGION = 'your-region'  # Your AWS region, e.g., 'us-east-1'
+
+# Initialize S3 client
+s3_client = boto3.client('s3', region_name=REGION)
+
+def upload_file(file_path):
+    """
+    Upload a file to S3.
+    :param file_path: Path of the file to upload
+    """
+    try:
+        # Define the key for the file in S3 (optional: you can keep the same name or change it)
+        s3_key = os.path.relpath(file_path, LOCAL_FOLDER)
+        print(f"Uploading {file_path} to s3://{BUCKET_NAME}/{s3_key}")
+        s3_client.upload_file(file_path, BUCKET_NAME, s3_key)
+        print(f"Upload of {file_path} completed successfully.")
+    except Exception as e:
+        print(f"Error uploading {file_path}: {e}")
+
+def main():
+    # Get all files in the LOCAL_FOLDER
+    files_to_upload = [
+        os.path.join(LOCAL_FOLDER, f) for f in os.listdir(LOCAL_FOLDER)
+        if os.path.isfile(os.path.join(LOCAL_FOLDER, f))
+    ]
+    
+    # Determine the number of processes to use
+    num_processes = min(cpu_count(), len(files_to_upload))
+
+    # Use multiprocessing Pool to upload files in parallel
+    with Pool(processes=num_processes) as pool:
+        pool.map(upload_file, files_to_upload)
+
+if __name__ == "__main__":
+    main()