from datasets import load_dataset, Dataset
import os

from app import SPLIT

# grab our dataset as it is rightnow
src_dataset: Dataset = load_dataset(os.environ["GPTAT_DATASET_NAME"], use_auth_token=True)["train"]


# If the dataset is gated/private, make sure you have run huggingface-cli login
dataset: Dataset = load_dataset(os.environ['RESPONSE_DATASET_NAME'])[SPLIT]
dataset.to_json("data/responses.json")