File size: 821 Bytes
3045bc4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | import os
import sys
from huggingface_hub import HfApi, snapshot_download
from pathlib import Path
REPO_ID = os.environ.get("DATASET_REPO_ID")
TOKEN = os.environ.get("HF_TOKEN")
# We only want to pull/push from this subfolder
TARGET_DIR = Path("/app/data/paperclip_app")
if not REPO_ID or not TOKEN:
print("Secrets not set. Skipping download.")
sys.exit(0)
print(f"Checking for existing Paperclip data in {REPO_ID}...")
try:
# This downloads only the 'paperclip_app' folder from your dataset if it exists
snapshot_download(
repo_id=REPO_ID,
repo_type="dataset",
local_dir="/app/data",
allow_patterns="paperclip_app/*",
token=TOKEN
)
print("Download complete.")
except Exception as e:
print(f"No existing paperclip data found (Starting fresh): {e}")
|