scripts/whats_deployed.py

#!/usr/bin/env python
"""
What is actually deployed, right now, on every Space.

    .venv/bin/python scratch/whats_deployed.py

Answers the question that was unanswerable during the 2026-08-25 MCP outage:
which commit is each Space really running, and does any local branch have it?

For each Space it prints the live commit, the runtime stage, and — the part that
matters — a DRIFT verdict:

  in sync      the deployed sha is the tip of its source branch
  behind       source branch has newer commits not deployed yet
  DRIFT        the deployed sha is on NO local branch. Something was pushed
               straight to the Space. The next normal deploy will revert it.

DRIFT is the condition that caused the outage's second-order problem: hotfix
4b87c36 lived only on the prod Space for ~10 hours while every local branch
still carried the bug.

Exit code is 1 if any Space is in DRIFT or not RUNNING, so this can gate a
deploy script later.
"""

import subprocess
import time
import sys

# Space -> the branch it is supposed to be deployed from.
SPACES = [
    ("thoughtspot-demoprep/mcp",      "deploy/mcp",            "MCP  PROD"),
    ("thoughtspot-demoprep/test-mcp", "deploy/mcp",            "MCP  test"),
    ("thoughtspot-dp/demoprep",       "origin/main",           "app  PROD"),
    ("thoughtspot-dp/test-demoprep",  "spike/portal-template", "app  test"),
]


def git(*args, _tries: int = 5) -> str:
    """Run git, retrying if the host is temporarily out of process slots.

    macOS fork() raises BlockingIOError when the per-user process table is
    full (e.g. thousands of zombie browser helpers). That is a host condition,
    not a git failure — back off and retry rather than crashing the report.
    """
    for attempt in range(_tries):
        try:
            return subprocess.run(("git",) + args, capture_output=True,
                                  text=True).stdout.strip()
        except BlockingIOError:
            if attempt == _tries - 1:
                raise
            time.sleep(0.5 * (attempt + 1))
    return ""


def hf_token() -> str:
    for line in open(".env"):
        line = line.strip()
        if line.startswith("HF_TOKEN="):
            return line.split("=", 1)[1].strip().strip('"').strip("'")
    sys.exit("ERROR: HF_TOKEN not found in .env")


def main() -> None:
    from huggingface_hub import HfApi

    api = HfApi(token=hf_token())
    git("fetch", "--all", "--quiet")

    print(f"{'SPACE':<34} {'TIER':<10} {'DEPLOYED':<9} {'STAGE':<14} SOURCE / DRIFT")
    print("-" * 104)

    problems = 0
    for repo_id, source, tier in SPACES:
        try:
            info = api.space_info(repo_id)
        except Exception as exc:
            print(f"{repo_id:<34} {tier:<10} {'?':<9} {'UNREACHABLE':<14} {type(exc).__name__}")
            problems += 1
            continue

        sha = (info.sha or "")[:7]
        stage = getattr(info.runtime, "stage", "?") or "?"

        # Does any local branch contain the deployed commit?
        if sha and git("cat-file", "-t", sha) == "commit":
            holders = [b.strip().lstrip("* +").strip()
                       for b in git("branch", "--contains", sha).splitlines() if b.strip()]
            if not holders:
                verdict = "DRIFT — on no local branch! pushed direct to Space"
                problems += 1
            else:
                tip = git("rev-parse", "--short=7", source)
                if tip == sha:
                    verdict = f"in sync with {source}"
                else:
                    ahead = git("rev-list", "--count", f"{sha}..{source}")
                    behind = git("rev-list", "--count", f"{source}..{sha}")
                    if behind != "0":
                        verdict = (f"DRIFT — {behind} commit(s) deployed that {source} "
                                   f"lacks (+{ahead} undeployed)")
                        problems += 1
                    else:
                        verdict = f"behind {source} by {ahead} commit(s)"
        else:
            verdict = f"deployed sha {sha} UNKNOWN locally — run git fetch --all"
            problems += 1

        if stage != "RUNNING":
            problems += 1

        print(f"{repo_id:<34} {tier:<10} {sha:<9} {stage:<14} {verdict}")

    print()
    if problems:
        print(f"{problems} issue(s). A DRIFT verdict means: merge the Space's commit back "
              f"into its source branch BEFORE the next deploy, or it gets reverted.")
    else:
        print("All Spaces running and in sync with their source branches.")

    print("\nNote: 'RUNNING' only means the container booted. It does NOT mean builds "
          "work — GET / returned ok throughout the 2026-08-25 total build outage.")
    sys.exit(1 if problems else 0)


if __name__ == "__main__":
    main()