upload_to_hf.py

from huggingface_hub import HfApi, CommitOperationDelete

api = HfApi()
REPO = "richiam/ProtoPure"
BASE_IGNORE = ["*.pyc", "__pycache__", ".git", "*.html", "data/clustering/**/*.png"]

# Step 0: Delete stale HTML and PNG files from HF to free storage
print("Deleting stale HTML/PNG files from HF...")
all_files = list(api.list_repo_files(REPO, repo_type="space"))
to_delete = [f for f in all_files if f.endswith(".html") or
             (f.endswith(".png") and "clustering" in f)]
if to_delete:
    ops = [CommitOperationDelete(path_in_repo=f) for f in to_delete]
    api.create_commit(
        repo_id=REPO,
        repo_type="space",
        operations=ops,
        commit_message=f"Remove {len(to_delete)} stale HTML/PNG files to free storage",
    )
    print(f"  Deleted {len(to_delete)} files")
else:
    print("  Nothing to delete")

# Step 1: Upload app + non-clustering data
api.upload_folder(
    folder_path="/data/ralmadamonter/llm_dashboard",
    repo_id=REPO,
    repo_type="space",
    ignore_patterns=BASE_IGNORE + ["data/clustering/"],
)
print("Phase 1 done (app + non-clustering data)")

# Step 2: Upload clustering data in per-model batches
import os
CLUSTERING_DIR = "/data/ralmadamonter/llm_dashboard/data/clustering"
for entry in sorted(os.listdir(CLUSTERING_DIR)):
    entry_path = os.path.join(CLUSTERING_DIR, entry)
    if not os.path.isdir(entry_path):
        continue
    print(f"Uploading {entry}...")
    api.upload_folder(
        folder_path=entry_path,
        repo_id=REPO,
        repo_type="space",
        path_in_repo=f"data/clustering/{entry}",
        ignore_patterns=["*.html", "*.png"],
    )
    print(f"  Done: {entry}")

print("All done")