scripts/deploy/upload_inspector.py

#!/usr/bin/env python3
"""Upload the Inspector to its target Hugging Face Space.

Usage::
https://huggingface.co/hetchyy

    python scripts/deploy/upload_inspector.py dev   # → hetchyy/quranic-inspector-dev
    python scripts/deploy/upload_inspector.py prod  # → hetchyy/quranic-universal-audio

Build steps:

1. Stage the Space build context: every git-tracked file, HARDLINKED into a
   temp tree (metadata-only — no byte copy; falls back to a copy only across
   volumes), plus two Space-specific overlays (root ``Dockerfile`` + frontmatter
   ``README.md``). The image's contents are defined solely by
   ``inspector/Dockerfile`` + ``.dockerignore`` (both tracked, both consumed
   identically here and by ``docker-publish.yml``'s repo-root build), so there
   is no hand-maintained file allowlist to drift from the Dockerfile's ``COPY``
   set — ``.dockerignore`` prunes at build time. The frontend ``dist/`` is
   built inside the image (Dockerfile stage 1); nothing is pre-built here.
2. ``upload_folder`` to the Space repo (Hugging Face git+xet under the hood),
   with ``delete_patterns="*"`` so the Space tree mirrors the staged tree.

Because the staged tree is the whole tracked repo, the Space build context
equals the ``context: .`` context that ``docker-publish.yml`` builds on every
push — a green CI image build is a faithful guarantee the Space will compile.

Auth: reads ``HF_TOKEN`` (or ``INSPECTOR_HF_TOKEN``) from ``$REPO/.env`` via
``qua_shared._env.load_repo_env``.

Idempotent: re-running uploads only the changed files (the Hub diffs by
content hash). Safe to run from a fresh checkout.
"""

from __future__ import annotations

import argparse
import os
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path

# Files NOT pushed to the Space repo: the uncompressed qpc_hafs.json is >10 MB
# and would be auto-LFS-promoted, leaving a 133-byte pointer in the Space build
# context that defeats the Dockerfile COPY. The committed ``.gz`` (~1.5 MB) is
# what the image and the HF Jobs consume; the uncompressed copy stays in the
# repo for local-dev tools that read it directly.
_SKIP_FROM_STAGE = ("data/qpc_hafs.json",)

# Git-LFS pointer files start with this magic line. If an asset got auto-LFS'd
# (by extension or size) it ships as a ~133-byte pointer instead of real bytes,
# silently breaking the image's Dockerfile COPY at runtime. The deploy guard
# fails loudly when any staged file is a pointer rather than its content.
_LFS_POINTER_MAGIC = b"version https://git-lfs.github.com/spec/"

from huggingface_hub import HfApi
from huggingface_hub.errors import HfHubHTTPError

# HF rate limit is a fixed window (q=2500; w=300s) shared per-token across CI,
# local dev, and bucket reads, so any concurrent burst can 429 the first call
# of a deploy. These bound a polite Retry-After honoring backoff.
_RATE_LIMIT_RETRIES = 5
_RATE_LIMIT_DEFAULT_WAIT = 30  # seconds, when no Retry-After header is sent

# Bring in the .env loader so HF_TOKEN is available.
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
from qua_shared._env import load_repo_env, repo_root  # noqa: E402

SPACE_REPOS = {
    "dev": "hetchyy/quranic-inspector-dev",
    "prod": "hetchyy/quranic-universal-audio",
}

# README frontmatter shipped to the Space. Mirrors the runbook §1 setup.
SPACE_README = """---
title: Quranic Universal Audio{suffix}
emoji: 🎙️
colorFrom: yellow
colorTo: indigo
short_description: Visualize, edit & verify Qur'anic recitation timestamps
sdk: docker
app_port: 7860
pinned: false
hf_oauth: true
hf_oauth_expiration_minutes: 480
---
"""


def _git_tracked_files(repo: Path) -> list[str]:
    """Repo-relative paths of every git-tracked file (the Space build context)."""
    out = subprocess.run(
        ["git", "ls-files", "-z"],
        cwd=repo,
        check=True,
        capture_output=True,
        text=True,
    )
    return [p for p in out.stdout.split("\0") if p]


def _assert_no_lfs_pointers(stage_root: Path) -> None:
    """Abort the deploy if any staged file is a Git-LFS pointer, not its bytes.

    An asset that got auto-LFS-promoted (by extension or size) is staged as a
    ~133-byte pointer text file. Shipping that to the Space leaves the Dockerfile
    ``COPY`` pointing at a stub instead of real content — a silent runtime break
    that CI's image build doesn't catch. Fail loudly here so the operator sees it
    before the upload, not after the Space starts serving garbage.
    """
    pointers: list[str] = []
    n = len(_LFS_POINTER_MAGIC)
    for path in stage_root.rglob("*"):
        if not path.is_file():
            continue
        with path.open("rb") as fh:
            head = fh.read(n)
        if head == _LFS_POINTER_MAGIC:
            pointers.append(str(path.relative_to(stage_root)))
    if pointers:
        listing = "\n  ".join(sorted(pointers))
        raise RuntimeError(
            "Refusing to deploy: the following staged files are Git-LFS "
            "pointers, not their real bytes (they were auto-LFS-promoted and "
            "would ship to the Space as ~133-byte stubs, breaking the "
            "Dockerfile COPY at runtime):\n  "
            f"{listing}\n"
            "Fix by un-LFS'ing the asset, shipping a non-LFS variant (e.g. a "
            "compressed .gz), or inlining it — see _SKIP_FROM_STAGE above."
        )


def _link_or_copy(src: Path, dst: Path) -> None:
    """Hardlink ``src`` → ``dst`` (metadata-only, near-instant); fall back to a
    byte copy only on EXDEV / a filesystem that can't hardlink.

    The staged tree is READ-ONLY input to ``upload_folder`` + the LFS-pointer
    guard, so sharing the worktree's inode is safe and skips the O(all-files)
    byte copy that dominated deploy time on Windows. The temp dir is on the same
    volume as the repo (both on the system drive), so the hardlink path is taken;
    removing the temp tree just drops the extra link, never the worktree file.
    """
    try:
        os.link(src, dst)
    except OSError:
        shutil.copy2(src, dst)


def _stage(repo: Path, stage_root: Path, env: str, branch: str) -> None:
    """Stage the Space build context: every git-tracked file, hardlinked.

    The image's contents are defined solely by ``inspector/Dockerfile`` +
    ``.dockerignore`` (both tracked, both consumed identically here and by
    ``docker-publish.yml``'s ``context: .`` build), so there is no hand-curated
    allowlist to drift from the Dockerfile's ``COPY`` set — ``.dockerignore``
    prunes at build time. Two Space-specific overlays are written on top of the
    tracked tree:

    * ``Dockerfile`` at the root — the Hub docker SDK reads it there, but ours
      lives under ``inspector/``.
    * ``README.md`` — the Space frontmatter, written last so it wins over the
      repo's tracked root ``README.md``.

    The frontend ``dist/`` is built inside the image (Dockerfile stage 1), so
    nothing is pre-built here.
    """
    skip = set(_SKIP_FROM_STAGE)
    for rel in _git_tracked_files(repo):
        if rel in skip:
            continue
        src = repo / rel
        if not src.is_file():
            continue  # tracked but absent in the worktree (e.g. deleted)
        dst = stage_root / rel
        dst.parent.mkdir(parents=True, exist_ok=True)
        _link_or_copy(src, dst)

    _link_or_copy(repo / "inspector" / "Dockerfile", stage_root / "Dockerfile")

    suffix = " (dev)" if env == "dev" else ""
    (stage_root / "README.md").write_text(
        SPACE_README.format(suffix=suffix, branch=branch),
        encoding="utf-8",
    )

    _assert_no_lfs_pointers(stage_root)


def _retry_on_429(label: str, fn, *args, **kwargs):
    """Call ``fn``, retrying on HTTP 429 with a Retry-After honoring backoff.

    The Hub rate limit is a per-token fixed window shared across every process
    using the token (CI, local dev, bucket reads), so a transient 429 here is
    expected and retryable — not a fatal deploy error.
    """
    for attempt in range(1, _RATE_LIMIT_RETRIES + 1):
        try:
            return fn(*args, **kwargs)
        except HfHubHTTPError as exc:
            resp = exc.response
            is_429 = resp is not None and resp.status_code == 429
            if not is_429 or attempt == _RATE_LIMIT_RETRIES:
                raise
            wait = int(resp.headers.get("Retry-After", _RATE_LIMIT_DEFAULT_WAIT))
            print(
                f"==> 429 rate-limited on {label}; sleeping {wait}s "
                f"(attempt {attempt}/{_RATE_LIMIT_RETRIES})",
                file=sys.stderr,
            )
            time.sleep(wait)


def _upload(stage_root: Path, repo_id: str, token: str, commit_msg: str) -> str:
    api = HfApi(token=token)
    # The prod/dev Spaces are permanent — skip the create_repo call (it 409s on
    # every deploy and still spends a request) unless the Space is actually
    # missing, e.g. a first-time contributor bootstrap.
    if not _retry_on_429("repo_exists", api.repo_exists, repo_id=repo_id, repo_type="space"):
        _retry_on_429(
            "create_repo",
            api.create_repo,
            repo_id=repo_id,
            repo_type="space",
            space_sdk="docker",
            private=True,
            exist_ok=True,
        )
    _retry_on_429(
        "upload_folder",
        api.upload_folder,
        folder_path=str(stage_root),
        repo_id=repo_id,
        repo_type="space",
        commit_message=commit_msg,
        delete_patterns="*",
    )
    # Code-only pushes don't change the Dockerfile hash, so HF doesn't
    # rebuild the container automatically — the new commit shows up in
    # the Space repo but the running container still serves the previous
    # bundle. Force a factory reboot so the new code actually runs.
    _retry_on_429("restart_space", api.restart_space, repo_id=repo_id, factory_reboot=True)
    return f"https://huggingface.co/spaces/{repo_id}"


def _commit_sha(repo: Path) -> str:
    try:
        out = subprocess.run(
            ["git", "rev-parse", "--short", "HEAD"],
            cwd=repo,
            check=True,
            capture_output=True,
            text=True,
        )
        return out.stdout.strip()
    except Exception:
        return "unknown"


def main(argv: list[str] | None = None) -> int:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("env", choices=("dev", "prod"))
    p.add_argument(
        "--dry-run",
        action="store_true",
        help="Stage but don't upload. Prints stage path and exits.",
    )
    p.add_argument(
        "--verify-boot",
        action="store_true",
        help=(
            "Before uploading, build the staged context and boot it offline "
            "(filesystem fixtures), asserting /healthz is 200. Aborts the "
            "deploy if the image won't build or boot. Needs Docker."
        ),
    )
    args = p.parse_args(argv)

    load_repo_env()
    token = os.environ.get("INSPECTOR_HF_TOKEN") or os.environ.get("HF_TOKEN")
    if not token:
        print(
            "ERROR: HF_TOKEN / INSPECTOR_HF_TOKEN missing in env. "
            "Add it to .env at the worktree root.",
            file=sys.stderr,
        )
        return 2

    repo = repo_root()
    repo_id = SPACE_REPOS[args.env]
    branch = "dev" if args.env == "dev" else "main"

    print(f"==> Deploying Inspector to {args.env} ({repo_id})")
    sha = _commit_sha(repo)
    commit_msg = f"deploy: inspector {args.env} @ {sha}"

    with tempfile.TemporaryDirectory(prefix=f"inspector-stage-{args.env}-") as tmp:
        stage_root = Path(tmp)
        print(f"==> Staging Space tree in {stage_root}")
        _stage(repo, stage_root, args.env, branch)

        if args.verify_boot:
            print("==> Verifying the staged image builds and boots...")
            smoke = repo / "scripts" / "deploy" / "smoke_boot.py"
            subprocess.run(
                [sys.executable, str(smoke), "--context", str(stage_root)],
                check=True,
            )

        if args.dry_run:
            print(f"==> Dry run; stage at {stage_root} (not deleted on dry-run)")
            # Move out of the temp context so it survives.
            keep = repo / ".local" / "inspector-stage" / args.env
            keep.parent.mkdir(parents=True, exist_ok=True)
            if keep.exists():
                shutil.rmtree(keep)
            shutil.copytree(stage_root, keep)
            print(f"    Copied to {keep}")
            return 0

        print(f"==> Uploading to {repo_id}")
        url = _upload(stage_root, repo_id, token, commit_msg)
        print(f"==> Done. Space: {url}")

    return 0


if __name__ == "__main__":
    sys.exit(main())