dataset_loader.py

from huggingface_hub import Repository
from huggingface_hub import login
import os
from dotenv import load_dotenv

load_dotenv()


def load_dataset():
    login(token=os.getenv("HUB_TOKEN"))

    repo = Repository(
        local_dir="klse_cahtbot_data",
        repo_type="dataset",
        clone_from="dahreply/klse-bot",
        git_user=os.getenv("GIT_USER"),
        git_email=os.getenv("GIT_EMAIL"),
        token=True,
    )
    repo.git_pull()