data/dataset_manager.py

from huggingface_hub import HfApi
import gradio as gr
import pandas as pd
import os
import hashlib
from datasets import Dataset, concatenate_datasets, load_dataset
from huggingface_hub import login
import json
import uuid
from datetime import datetime, UTC
import torch
from huggingface_hub import snapshot_download
import numpy as np

from config.settings import *
from data.contexts import CONTEXTS

HF_TOKEN = os.environ.get("MyJulySecretToken")  # store your token as a secret in Spaces
login(HF_TOKEN)



api = HfApi(token = HF_TOKEN)

def load_hf_dataset():
    """Load existing HF Dataset or create empty one if not exists."""
    try:
        ds = load_dataset(HF_DATASET_NAME, split="train")
    except:
        # Dataset does not exist yet
        df = pd.DataFrame(columns=["user_id", "gender", "audio_file", "score"])
        ds = Dataset.from_pandas(df)
        ds.push_to_hub(HF_DATASET_NAME, private=True)
    return ds