src/utils/s3/read_file_from_s3.py

import sys

sys.path.append("./.")

import pandas as pd
import json

import os
from dotenv import load_dotenv

load_dotenv()

from src.utils.s3.s3_connect import s3_connect
from src.utils.csv import read_csv_and_eval

from src.utils import *


def read_csv_from_s3(path: str, client=s3_connect(), bucket="lebesgue-common-bucket", evaluate=True, *args, **kwargs):

    object = client.get_object(Bucket=bucket, Key=path)

    if evaluate:
        df = read_csv_and_eval(object["Body"], *args, **kwargs)
    else:
        df = pd.read_csv(object["Body"], *args, **kwargs)

    return df


def read_csv(path: str, s3: bool = False, pd_args: dict = {}, s3_args: dict = {}):
    if s3:
        return read_csv_from_s3(path=path, **s3_args)
    else:
        return read_csv_and_eval(path=path, **pd_args)