src/data/load_data.py

from datasets import load_dataset
import pandas as pd
import os

dataset = load_dataset("SetFit/bbc-news")

train_df = pd.DataFrame(dataset["train"])
test_df = pd.DataFrame(dataset["test"])

df = pd.concat([train_df, test_df], ignore_index=True)

os.makedirs("data/raw", exist_ok=True)

df.to_csv("data/raw/bbc-text.csv", index=False)

print(df.shape)
print(df.head())