src/models/dataset_prep.py

import os
from datasets import load_dataset
from collections import Counter
from src.config import HF_ENDPOINT, HF_TOKEN


if HF_ENDPOINT:
    os.environ["HF_ENDPOINT"] = HF_ENDPOINT

def fetch_and_inspect_data():
    try:
        dataset = load_dataset("mediabiasgroup/BABE", token=HF_TOKEN)
        print(Counter(dataset["train"]["label"]))
        print(Counter(dataset["train"]["label_opinion"]))
    except Exception as e:
        print(f"Failed to load dataset: {e}")

if __name__ == "__main__":
    fetch_and_inspect_data()