File size: 1,219 Bytes
cea4a4b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
import boto3
import os
from dotenv import load_dotenv
from src.utils import print_execution_time
load_dotenv()
def s3_connect():
s3 = boto3.resource(
service_name="s3",
region_name=os.environ["AWS_DEFAULT_REGION"],
aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
)
return boto3.client("s3")
def s3_resource():
s3 = boto3.resource(
service_name="s3",
region_name=os.environ["AWS_DEFAULT_REGION"],
aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
)
return s3
def list_objects_from_prefix(
prefix: str, client=s3_connect(), add_global_folder: bool = True
):
if add_global_folder:
prefix = add_global_s3_folder(prefix)
client = s3_connect()
response = client.list_objects_v2(Bucket="creative-features", Prefix=prefix)
if "Contents" in response.keys():
list_of_paths = [content["Key"] for content in response["Contents"]]
else:
list_of_paths = []
return list_of_paths
def add_global_s3_folder(path):
return f"{os.getenv('S3_PATH')}/{path}"
|