dataset_loader.py

import os

from dotenv import load_dotenv
from huggingface_hub import Repository, login

load_dotenv()


def load_dataset():
    token = os.getenv("HUB_TOKEN")
    login(token=token)

    repo = Repository(
        local_dir="service_data",
        repo_type="dataset",
        clone_from="cjzhi98/llm_graph_service",
        token=True,
    )
    repo.git_pull()