| import os |
| import sys |
| from huggingface_hub import snapshot_download |
|
|
| REPO_ID = os.environ.get("DATASET_REPO_ID") |
| TOKEN = os.environ.get("HF_TOKEN") |
|
|
| if not REPO_ID or not TOKEN: |
| print("Secrets not set. Skipping download.") |
| sys.exit(0) |
|
|
| try: |
| snapshot_download( |
| repo_id=REPO_ID, |
| repo_type="dataset", |
| local_dir="/app/data", |
| allow_patterns=["paperclip_app/*"], |
| ignore_patterns=["paperclip_app/instances/default/db/*"], |
| token=TOKEN |
| ) |
| print("Download complete.") |
| except Exception as e: |
| print(f"Starting fresh: {e}") |
