dataset_loader.py

from huggingface_hub import Repository
from huggingface_hub import login
import os

def load_dataset():
    login(token = os.environ['HUB_TOKEN'])

    repo = Repository(
        local_dir="agent_function",
        repo_type="dataset",
        clone_from="gheng/bnm_scam_awareness",
        token=True,
        git_email='zhiheng_dev@dahreply.ai'
    )
    repo.git_pull()

load_dataset()