import sys sys.path.append("./.") import pandas as pd import json import os from dotenv import load_dotenv load_dotenv() from src.utils.s3.s3_connect import s3_connect from src.utils.csv import read_csv_and_eval from src.utils import * def read_csv_from_s3(path: str, client=s3_connect(), bucket="lebesgue-common-bucket", evaluate=True, *args, **kwargs): object = client.get_object(Bucket=bucket, Key=path) if evaluate: df = read_csv_and_eval(object["Body"], *args, **kwargs) else: df = pd.read_csv(object["Body"], *args, **kwargs) return df def read_csv(path: str, s3: bool = False, pd_args: dict = {}, s3_args: dict = {}): if s3: return read_csv_from_s3(path=path, **s3_args) else: return read_csv_and_eval(path=path, **pd_args)