pull.py

import os
import sys
from huggingface_hub import snapshot_download

REPO_ID = os.environ.get("DATASET_REPO_ID")
TOKEN = os.environ.get("HF_TOKEN")

if not REPO_ID or not TOKEN:
    print("Secrets not set. Skipping download.")
    sys.exit(0)

try:
    snapshot_download(
        repo_id=REPO_ID,
        repo_type="dataset",
        local_dir="/app/data",
        allow_patterns=["paperclip_app/*"],
        ignore_patterns=["paperclip_app/instances/default/db/*"],
        token=TOKEN
    )
    print("Download complete.")
except Exception as e:
    print(f"Starting fresh: {e}")