concurnecy

app.py

@@ -17,7 +17,7 @@ from concurrent.futures import ThreadPoolExecutor
 import random
 import string
 
-from experiments.upload_to_s3 import upload_multiple_files
+from upload_to_s3 import upload_multiple_files
 load_dotenv()
 
 # Access the variables

upload_to_s3.py

@@ -0,0 +1,78 @@
+from dotenv import load_dotenv
+import boto3
+import os
+from botocore.exceptions import ClientError
+import time
+from boto3.s3.transfer import TransferConfig# Configuration
+REGION = "us-east-1"
+BUCKET = "shortsai-us"  # Replace with your bucket name
+S3_KEY = "test/IMG_8280.MOV"  # S3 path for the file
+LOCAL_FILE_PATH = "/Users/georgia.bucea/products/ShortsAI/21_aug/mcp_video-3028_singular_display.MOV"  # Replace with your local file path
+
+load_dotenv()
+
+# Access the variables
+
+
+from concurrent.futures import ThreadPoolExecutor
+
+def upload_multiple_files(file_list, bucket):
+
+    uploaded_files = []
+    
+    def upload_to_s3(file_path, bucket, s3_key):
+        """Upload a local file to an S3 bucket."""
+
+        start_time = time.time()
+        try:
+
+            config = TransferConfig(
+                multipart_threshold=16*1024*1024,  # 8MB threshold
+                max_concurrency=30,  # Parallel upload threads
+                multipart_chunksize=8*1024*1024,  # Chunk size
+                use_threads=True,
+            )
+            # Initialize S3 client
+            s3 = boto3.client("s3",endpoint_url='https://s3-accelerate.amazonaws.com', region_name=REGION )
+
+            # Verify file exists
+            if not os.path.exists(file_path):
+                raise FileNotFoundError(f"File not found: {file_path}")
+
+            # # Upload file
+            s3.upload_file(file_path, bucket, s3_key,ExtraArgs={'ACL': 'bucket-owner-full-control'}, Config=config)
+            print(f"Successfully uploaded {file_path} to s3://{bucket}/{s3_key}")
+
+            uploaded_files.append(s3_key)
+            end_time = time.time()
+            # Calculate elapsed time
+            elapsed_time = end_time - start_time
+
+            print(f"In {elapsed_time} time ")
+            return True
+
+        except FileNotFoundError as e:
+            print(f"Error: {e}")
+            return False
+        except ClientError as e:
+            print(f"AWS Client Error: {e}")
+            return False
+        except Exception as e:
+            print(f"Unexpected error: {e}")
+            return False
+
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        executor.map(lambda f: upload_to_s3(f[0], bucket, f[1]), file_list)
+    return uploaded_files
+
+
+if __name__ == "__main__":
+    # upload_to_s3(LOCAL_FILE_PATH,BUCKET,S3_KEY)
+    bucket = os.getenv("BUCKET")
+
+
+    file_list = [
+    ("/Users/georgia.bucea/products/ShortsAI/21_aug/mcp_video-3028_singular_display.MOV", "test/21_aug/mcp_video-3028_singular_display.MOV"),
+    ]
+
+    uploaded = upload_multiple_files(file_list, bucket)