add scripts to delete old versions

delete-past-version.sh

@@ -62,7 +62,71 @@ done
 
 if [[ ${#paths_to_remove[@]} -eq 0 ]]; then
     echo ""
-    echo "Nothing to remove. All versions are >= $remove_version_up_to."
+    echo "Nothing to remove from the working tree."
+    echo "Checking for orphaned LFS files on Hugging Face server..."
+    echo ""
+
+    REMOTE_URL=$(git remote get-url origin 2>/dev/null || true)
+    if [[ -z "$REMOTE_URL" ]]; then
+        echo "No remote found. Nothing to do."
+        exit 0
+    fi
+
+    REPO_ID=$(echo "$REMOTE_URL" | sed -E 's|.*[:/](.+/[^/.]+)(\.git)?$|\1|')
+
+    python3 << PYEOF
+import re
+import sys
+import os
+from huggingface_hub import HfApi
+
+api = HfApi()
+
+# Check authentication: try HF_TOKEN env var, then saved token
+token = os.environ.get("HF_TOKEN") or api.token
+if not token:
+    print("ERROR: Not authenticated with Hugging Face.")
+    print("Please do ONE of the following:")
+    print("  1. Run: hf auth login")
+    print("  2. Set env var: export HF_TOKEN=your_token_here")
+    print("     (get a token at https://huggingface.co/settings/tokens)")
+    sys.exit(1)
+
+repo_id = "$REPO_ID"
+remove_up_to = "$remove_version_up_to"
+
+def parse_version(v):
+    parts = v.split(".")
+    return int(parts[0]) * 100000 + int(parts[1]) * 10000 + int(parts[2]) * 10
+
+cutoff = parse_version(remove_up_to)
+
+print(f"Listing LFS files in {repo_id}...")
+all_lfs = list(api.list_lfs_files(repo_id, token=token))
+print(f"Found {len(all_lfs)} LFS files total")
+
+old_files = []
+for f in all_lfs:
+    m = re.search(r"(\d+\.\d+\.\d+)", f.filename)
+    if m and parse_version(m.group(1)) < cutoff:
+        old_files.append(f)
+
+if not old_files:
+    print("No orphaned LFS files found. Storage is already clean.")
+    sys.exit(0)
+
+total_size = sum(f.size for f in old_files)
+print(f"Found {len(old_files)} orphaned LFS files ({total_size / 1024**3:.1f} GB)")
+print("Deleting them from the Hub (this may take a while)...")
+api.permanently_delete_lfs_files(repo_id, old_files, token=token)
+print(f"Done! Freed {total_size / 1024**3:.1f} GB of storage.")
+PYEOF
+
+    echo ""
+    echo "================================================================"
+    echo "Done! Orphaned LFS files have been permanently deleted from"
+    echo "the Hugging Face server."
+    echo "================================================================"
     exit 0
 fi
 
@@ -140,6 +204,14 @@ else
     exit 1
 fi
 
+# ============================================================
+# Prune orphaned LFS objects locally
+# (objects that are no longer referenced by any commit)
+# ============================================================
+echo ""
+echo "Pruning orphaned LFS objects locally..."
+git lfs prune
+
 # ============================================================
 # Stage, commit, and force-push
 # ============================================================
@@ -151,12 +223,74 @@ echo "Committing..."
 git commit -m "Remove old versions (< $remove_version_up_to) to free Hugging Face storage"
 
 echo ""
-echo "Force-pushing to origin..."
+echo "Force-pushing git history to origin..."
 git push origin --force --all
 git push origin --force --tags
 
+echo ""
+echo "Pushing LFS state to origin..."
+git lfs push --all origin
+
+# ============================================================
+# Permanently delete orphaned LFS files from Hugging Face
+# (This is the key step that actually frees server-side storage)
+# ============================================================
+echo ""
+echo "Deleting orphaned LFS files from Hugging Face server..."
+
+# Extract repo ID from remote URL (e.g., git@hf.co2:user/repo -> user/repo)
+REPO_ID=$(echo "$REMOTE_URL" | sed -E 's|.*[:/](.+/[^/.]+)(\.git)?$|\1|')
+
+python3 << PYEOF
+import re
+import sys
+import os
+from huggingface_hub import HfApi
+
+api = HfApi()
+
+token = os.environ.get("HF_TOKEN") or api.token
+if not token:
+    print("ERROR: Not authenticated with Hugging Face.")
+    print("Please do ONE of the following:")
+    print("  1. Run: hf auth login")
+    print("  2. Set env var: export HF_TOKEN=your_token_here")
+    print("     (get a token at https://huggingface.co/settings/tokens)")
+    sys.exit(1)
+
+repo_id = "$REPO_ID"
+remove_up_to = "$remove_version_up_to"
+
+def parse_version(v):
+    parts = v.split(".")
+    return int(parts[0]) * 100000 + int(parts[1]) * 10000 + int(parts[2]) * 10
+
+cutoff = parse_version(remove_up_to)
+
+print(f"Listing LFS files in {repo_id}...")
+all_lfs = list(api.list_lfs_files(repo_id, token=token))
+print(f"Found {len(all_lfs)} LFS files total")
+
+old_files = []
+for f in all_lfs:
+    m = re.search(r"(\d+\.\d+\.\d+)", f.filename)
+    if m and parse_version(m.group(1)) < cutoff:
+        old_files.append(f)
+
+if not old_files:
+    print("No orphaned LFS files found on the server. Storage is already clean.")
+    sys.exit(0)
+
+total_size = sum(f.size for f in old_files)
+print(f"Found {len(old_files)} orphaned LFS files ({total_size / 1024**3:.1f} GB)")
+print("Deleting them from the Hub (this may take a while)...")
+api.permanently_delete_lfs_files(repo_id, old_files, token=token)
+print(f"Done! Freed {total_size / 1024**3:.1f} GB of storage.")
+PYEOF
+
 echo ""
 echo "================================================================"
-echo "Done! Old versions (< $remove_version_up_to) have been removed"
-echo "and the repo has been force-pushed."
+echo "All done! Old versions (< $remove_version_up_to) have been removed,"
+echo "and orphaned LFS files have been permanently deleted from"
+echo "the Hugging Face server."
 echo "================================================================"

generate-vad-asr.py

@@ -467,6 +467,17 @@ def to_file(
         print("<br/>", file=f)
         print("<br/>", file=f)
 
+        print(
+            "<strong>Note about storage</strong> "
+            "Hugging Face provides limited storage for open-source projects. "
+            "To stay within the limit, we <strong>delete old versions frequently</strong> "
+            "and only keep the most recent ones. "
+            "If you need a specific older version, please build it from source.",
+            file=f,
+        )
+        print("<br/>", file=f)
+        print("<br/>", file=f)
+
         # Table: one row per unique model
         seen = set()
         model_rows = []