from dotenv import load_dotenv import boto3 import os from botocore.exceptions import ClientError import time from boto3.s3.transfer import TransferConfig# Configuration REGION = "us-east-1" BUCKET = "shortsai-us" # Replace with your bucket name S3_KEY = "test/IMG_8280.MOV" # S3 path for the file LOCAL_FILE_PATH = "/Users/georgia.bucea/products/ShortsAI/21_aug/mcp_video-3028_singular_display.MOV" # Replace with your local file path import os def list_files_in_folder(folder_path: str) -> list: """ Reads all files in the specified folder (non-recursive, excludes subdirectories). :param folder_path: Path to the folder. :return: List of file names. """ try: # Check if folder exists if not os.path.isdir(folder_path): raise ValueError(f"Directory does not exist: {folder_path}") # List all entries in the folder, filter for files only files = [f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))] return files except Exception as e: print(f"Error reading folder: {e}") return [] load_dotenv() # Access the variables from concurrent.futures import ThreadPoolExecutor def upload_multiple_files(file_list, bucket): uploaded_files = [] def upload_to_s3(file_path, bucket, s3_key): """Upload a local file to an S3 bucket.""" start_time = time.time() try: config = TransferConfig( multipart_threshold=16*1024*1024, # 8MB threshold max_concurrency=30, # Parallel upload threads multipart_chunksize=8*1024*1024, # Chunk size use_threads=True, ) # Initialize S3 client s3 = boto3.client("s3",endpoint_url='https://s3-accelerate.amazonaws.com', region_name=REGION ) # Verify file exists if not os.path.exists(file_path): raise FileNotFoundError(f"File not found: {file_path}") # # Upload file s3.upload_file(file_path, bucket, s3_key,ExtraArgs={'ACL': 'bucket-owner-full-control'}, Config=config) print(f"Successfully uploaded {file_path} to s3://{bucket}/{s3_key}") uploaded_files.append(s3_key) end_time = time.time() # Calculate elapsed time elapsed_time = end_time - start_time print(f"In {elapsed_time} time ") return True except FileNotFoundError as e: print(f"Error: {e}") return False except ClientError as e: print(f"AWS Client Error: {e}") return False except Exception as e: print(f"Unexpected error: {e}") return False with ThreadPoolExecutor(max_workers=30) as executor: executor.map(lambda f: upload_to_s3(f[0], bucket, f[1]), file_list) return uploaded_files if __name__ == "__main__": # upload_to_s3(LOCAL_FILE_PATH,BUCKET,S3_KEY) bucket = os.getenv("BUCKET") folder="/Users/georgia.bucea/products/ShortsAI/16_aug" files=list_files_in_folder("/Users/georgia.bucea/products/ShortsAI/16_aug") files=[(folder+ "/"+os.path.basename(path), "testall16/"+os.path.basename(path)) for path in files] print(files) # file_list = [ # ("/Users/georgia.bucea/products/ShortsAI/21_aug/mcp_video-3028_singular_display.MOV", "test/21_aug/mcp_video-3028_singular_display.MOV"), # ] uploaded = upload_multiple_files(files, bucket)