scripts/csvToJson.py

import pandas as pd
import json
from src.utils.logger import setup_logger

logger = setup_logger("CSVtoJSON")

logger.info("Starting conversion from EvalQuestions.csv to test_set.json")

df = pd.read_csv("data/csvFiles/small_test_set.csv")

df.drop(columns=["createdAt","updatedAt"],inplace=True, errors="ignore")

data = {"questions": df.to_dict(orient="records")}

with open("data/small_test_set.json", "w", encoding="utf-8") as f:
    json.dump(data, f, indent=4)