feat(deploy): KI-152 — upload_to_hf.py syncs deletions, not just adds

After upload_folder, walk local tree + list_repo_files, then issue a second
commit that deletes orphan remote files. Prevents ghost brochures lingering
on the Space after KI-126/143/144-style renames. Adds --dry-run + --no-upload
flags. HF_PROTECTED guards .gitattributes / .gitignore / README.md.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

tools/upload_to_hf.py

@@ -6,18 +6,27 @@ huggingface.co/spaces/rohitsar567/InsuranceBot. Post-D-019 the large data
 dataset rohitsar567/insurance-bot-data — the Space's Dockerfile pulls it via
 huggingface_hub.snapshot_download at image build time.
 
+KI-152 (2026-05-15): operates in SYNC mode. After `upload_folder` (add/replace),
+walks the local tree, lists the remote repo, and issues a second commit that
+DELETES any remote file that no longer exists locally. Prevents ghost files
+(renamed/deleted brochures) from lingering on the Space and being served by
+the Docker image after a rebuild. See KI-126 / KI-143 / KI-144 / KI-151.
+
 Run:
-  python tools/upload_to_hf.py
+  python tools/upload_to_hf.py            # full upload + sync delete
+  python tools/upload_to_hf.py --dry-run  # report orphans, no delete commit
 """
 
 from __future__ import annotations
 
+import argparse
+import fnmatch
 import os
 import sys
 from pathlib import Path
 
 from dotenv import load_dotenv
-from huggingface_hub import HfApi
+from huggingface_hub import CommitOperationDelete, HfApi
 
 ROOT = Path(__file__).resolve().parent.parent
 load_dotenv(ROOT / ".env")
@@ -62,8 +71,97 @@ IGNORE = [
     "kb/calculations/chunk_*",  # Sweep markdown
 ]
 
+# HF-managed files that must never be deleted by the sync step. `.gitattributes`
+# is required by HF for LFS pointer files and is autogenerated/maintained by
+# the platform; deleting it breaks future LFS uploads.
+HF_PROTECTED = {".gitattributes", ".gitignore", "README.md"}
+
+
+def _matches_ignore(rel_path: str) -> bool:
+    """Return True if `rel_path` matches any IGNORE pattern."""
+    # fnmatch doesn't understand `**` the way gitignore does, but the HF hub
+    # `ignore_patterns` arg uses the same fnmatch semantics with `**` handled
+    # specially. Mirror that: replace `**/` => match any depth.
+    for pat in IGNORE:
+        if fnmatch.fnmatch(rel_path, pat):
+            return True
+        # Handle `**/` prefix → match the suffix at any depth.
+        if pat.startswith("**/"):
+            suffix = pat[3:]
+            if fnmatch.fnmatch(rel_path, suffix):
+                return True
+            if fnmatch.fnmatch(rel_path, f"*/{suffix}"):
+                return True
+        # Handle `dir/**` → match anything under dir/.
+        if pat.endswith("/**"):
+            prefix = pat[:-3]
+            if rel_path == prefix or rel_path.startswith(prefix + "/"):
+                return True
+    return False
+
+
+def _compute_local_files() -> set[str]:
+    """Walk ROOT (no symlink follow), apply IGNORE, return relative paths."""
+    local: set[str] = set()
+    for dirpath, dirnames, filenames in os.walk(ROOT, followlinks=False):
+        # Prune ignored directories in-place to avoid descending into them.
+        rel_dir = os.path.relpath(dirpath, ROOT)
+        if rel_dir == ".":
+            rel_dir = ""
+        kept_dirs = []
+        for d in dirnames:
+            sub_rel = f"{rel_dir}/{d}" if rel_dir else d
+            if _matches_ignore(sub_rel) or _matches_ignore(f"{sub_rel}/"):
+                continue
+            kept_dirs.append(d)
+        dirnames[:] = kept_dirs
+
+        for fn in filenames:
+            sub_rel = f"{rel_dir}/{fn}" if rel_dir else fn
+            if _matches_ignore(sub_rel):
+                continue
+            local.add(sub_rel)
+    return local
+
+
+def _compute_orphans(api: HfApi) -> list[str]:
+    """Return remote files that should be deleted (no local counterpart)."""
+    remote_files = set(api.list_repo_files(repo_id=REPO_ID, repo_type=REPO_TYPE))
+    local_files = _compute_local_files()
+
+    orphans = []
+    for rf in sorted(remote_files - local_files):
+        if rf in HF_PROTECTED:
+            continue
+        # An ignored-pattern remote file is still an orphan (uploaded by an
+        # older script version) — deleting it is correct hygiene. But never
+        # touch HF_PROTECTED.
+        orphans.append(rf)
+    return orphans
+
+
+def _print_orphan_summary(orphans: list[str], action: str) -> None:
+    print(f"✓ Orphan {action}: {len(orphans)} paths")
+    for o in orphans[:10]:
+        print(f"    - {o}")
+    if len(orphans) > 10:
+        print(f"    ... and {len(orphans) - 10} more")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Report orphans without issuing the delete commit (still uploads unless --no-upload).",
+    )
+    parser.add_argument(
+        "--no-upload",
+        action="store_true",
+        help="Skip the upload_folder step; only compute/report/delete orphans.",
+    )
+    args = parser.parse_args()
 
-def main():
     token = os.environ.get("HF_TOKEN", "")
     if not token:
         print("ERROR: HF_TOKEN not in environment (or .env)")
@@ -73,16 +171,46 @@ def main():
     print(f"Uploading {ROOT} (code only) -> https://huggingface.co/spaces/{REPO_ID}")
     print("(data is fetched from the companion dataset at Docker build time)")
 
-    api.upload_folder(
-        folder_path=str(ROOT),
-        repo_id=REPO_ID,
-        repo_type=REPO_TYPE,
-        ignore_patterns=IGNORE,
-        commit_message="Deploy: Stack A (NIM brain + Maverick judge + Sarvam voice). D-019.",
-    )
+    if args.dry_run:
+        print("[--dry-run] No mutating calls will be made. Orphan list only.")
+
+    # ---- Step 1: upload_folder (add + replace) -------------------------------
+    if args.no_upload or args.dry_run:
+        if args.dry_run:
+            print("[--dry-run] Skipping upload_folder.")
+        else:
+            print("[--no-upload] Skipping upload_folder.")
+    else:
+        api.upload_folder(
+            folder_path=str(ROOT),
+            repo_id=REPO_ID,
+            repo_type=REPO_TYPE,
+            ignore_patterns=IGNORE,
+            commit_message="Deploy: Stack A (NIM brain + Maverick judge + Sarvam voice). D-019.",
+        )
+        print("✓ Upload complete (folder upload).")
+
+    # ---- Step 2: compute + delete orphans -----------------------------------
+    orphans = _compute_orphans(api)
+
+    if not orphans:
+        print("✓ Orphan sync: 0 orphans (remote in sync with local).")
+    elif args.dry_run:
+        _print_orphan_summary(orphans, action="would delete (dry-run)")
+    else:
+        operations = [CommitOperationDelete(path_in_repo=p) for p in orphans]
+        api.create_commit(
+            repo_id=REPO_ID,
+            repo_type=REPO_TYPE,
+            operations=operations,
+            commit_message=(
+                f"chore(deploy): KI-152 sync — delete {len(orphans)} orphan file(s) "
+                "(brochure renames/removals)"
+            ),
+        )
+        _print_orphan_summary(orphans, action="delete")
 
     print()
-    print("✓ Upload complete.")
     print(f"  Space:    https://huggingface.co/spaces/{REPO_ID}")
     print(f"  Embed:    https://rohitsar567-insurancebot.hf.space")
     print()