dataset.py

from datasets import Dataset
import pandas as pd
import os

# Load your documents into a pandas DataFrame
files = []
for filename in os.listdir("pieces"):
    if filename.endswith(".txt"):
        with open(os.path.join("pieces", filename), "r") as f:
            content = f.read()
            files.append({"filename": filename, "content": content})

df = pd.DataFrame(files)
dataset = Dataset.from_pandas(df)

# Push to Hugging Face
dataset.push_to_hub("shaamil101/met-documents")