upload_to_s3.py

from dotenv import load_dotenv
import boto3
import os
from botocore.exceptions import ClientError
import time
from boto3.s3.transfer import TransferConfig# Configuration
REGION = "us-east-1"
BUCKET = "shortsai-us"  # Replace with your bucket name
S3_KEY = "test/IMG_8280.MOV"  # S3 path for the file
LOCAL_FILE_PATH = "/Users/georgia.bucea/products/ShortsAI/21_aug/mcp_video-3028_singular_display.MOV"  # Replace with your local file path

import os

def list_files_in_folder(folder_path: str) -> list:
    """
    Reads all files in the specified folder (non-recursive, excludes subdirectories).
    
    :param folder_path: Path to the folder.
    :return: List of file names.
    """
    try:
        # Check if folder exists
        if not os.path.isdir(folder_path):
            raise ValueError(f"Directory does not exist: {folder_path}")
        
        # List all entries in the folder, filter for files only
        files = [f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))]
        return files
    except Exception as e:
        print(f"Error reading folder: {e}")
        return []



load_dotenv()

# Access the variables


from concurrent.futures import ThreadPoolExecutor

def upload_multiple_files(file_list, bucket):

    uploaded_files = []
    
    def upload_to_s3(file_path, bucket, s3_key):
        """Upload a local file to an S3 bucket."""

        start_time = time.time()
        try:

            config = TransferConfig(
                multipart_threshold=16*1024*1024,  # 8MB threshold
                max_concurrency=30,  # Parallel upload threads
                multipart_chunksize=8*1024*1024,  # Chunk size
                use_threads=True,
            )
            # Initialize S3 client
            s3 = boto3.client("s3",endpoint_url='https://s3-accelerate.amazonaws.com', region_name=REGION )

            # Verify file exists
            if not os.path.exists(file_path):
                raise FileNotFoundError(f"File not found: {file_path}")

            # # Upload file
            s3.upload_file(file_path, bucket, s3_key,ExtraArgs={'ACL': 'bucket-owner-full-control'}, Config=config)
            print(f"Successfully uploaded {file_path} to s3://{bucket}/{s3_key}")

            uploaded_files.append(s3_key)
            end_time = time.time()
            # Calculate elapsed time
            elapsed_time = end_time - start_time

            print(f"In {elapsed_time} time ")
            return True

        except FileNotFoundError as e:
            print(f"Error: {e}")
            return False
        except ClientError as e:
            print(f"AWS Client Error: {e}")
            return False
        except Exception as e:
            print(f"Unexpected error: {e}")
            return False

    with ThreadPoolExecutor(max_workers=30) as executor:
        executor.map(lambda f: upload_to_s3(f[0], bucket, f[1]), file_list)
    return uploaded_files


if __name__ == "__main__":
    # upload_to_s3(LOCAL_FILE_PATH,BUCKET,S3_KEY)
    bucket = os.getenv("BUCKET")

    folder="/Users/georgia.bucea/products/ShortsAI/16_aug"

    files=list_files_in_folder("/Users/georgia.bucea/products/ShortsAI/16_aug")
    files=[(folder+ "/"+os.path.basename(path), "testall16/"+os.path.basename(path)) for path in files]
    print(files)

    # file_list = [
    # ("/Users/georgia.bucea/products/ShortsAI/21_aug/mcp_video-3028_singular_display.MOV", "test/21_aug/mcp_video-3028_singular_display.MOV"),
    # ]

    uploaded = upload_multiple_files(files, bucket)