load_docs.py

import os
import json
from huggingface_hub import hf_hub_download

def load_documents():
    path = hf_hub_download(
        repo_id=os.getenv("HF_DATASET_REPO"),
        filename="docs.json",
        repo_type="dataset",
        token=os.getenv("HF_TOKEN"),
    )
    with open(path) as f:
        return json.load(f)

documents = load_documents()