Spaces:
Running
Running
File size: 4,816 Bytes
ae9ba39 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | #!/usr/bin/env python
"""
What is actually deployed, right now, on every Space.
.venv/bin/python scratch/whats_deployed.py
Answers the question that was unanswerable during the 2026-08-25 MCP outage:
which commit is each Space really running, and does any local branch have it?
For each Space it prints the live commit, the runtime stage, and — the part that
matters — a DRIFT verdict:
in sync the deployed sha is the tip of its source branch
behind source branch has newer commits not deployed yet
DRIFT the deployed sha is on NO local branch. Something was pushed
straight to the Space. The next normal deploy will revert it.
DRIFT is the condition that caused the outage's second-order problem: hotfix
4b87c36 lived only on the prod Space for ~10 hours while every local branch
still carried the bug.
Exit code is 1 if any Space is in DRIFT or not RUNNING, so this can gate a
deploy script later.
"""
import subprocess
import time
import sys
# Space -> the branch it is supposed to be deployed from.
SPACES = [
("thoughtspot-demoprep/mcp", "deploy/mcp", "MCP PROD"),
("thoughtspot-demoprep/test-mcp", "deploy/mcp", "MCP test"),
("thoughtspot-dp/demoprep", "origin/main", "app PROD"),
("thoughtspot-dp/test-demoprep", "spike/portal-template", "app test"),
]
def git(*args, _tries: int = 5) -> str:
"""Run git, retrying if the host is temporarily out of process slots.
macOS fork() raises BlockingIOError when the per-user process table is
full (e.g. thousands of zombie browser helpers). That is a host condition,
not a git failure — back off and retry rather than crashing the report.
"""
for attempt in range(_tries):
try:
return subprocess.run(("git",) + args, capture_output=True,
text=True).stdout.strip()
except BlockingIOError:
if attempt == _tries - 1:
raise
time.sleep(0.5 * (attempt + 1))
return ""
def hf_token() -> str:
for line in open(".env"):
line = line.strip()
if line.startswith("HF_TOKEN="):
return line.split("=", 1)[1].strip().strip('"').strip("'")
sys.exit("ERROR: HF_TOKEN not found in .env")
def main() -> None:
from huggingface_hub import HfApi
api = HfApi(token=hf_token())
git("fetch", "--all", "--quiet")
print(f"{'SPACE':<34} {'TIER':<10} {'DEPLOYED':<9} {'STAGE':<14} SOURCE / DRIFT")
print("-" * 104)
problems = 0
for repo_id, source, tier in SPACES:
try:
info = api.space_info(repo_id)
except Exception as exc:
print(f"{repo_id:<34} {tier:<10} {'?':<9} {'UNREACHABLE':<14} {type(exc).__name__}")
problems += 1
continue
sha = (info.sha or "")[:7]
stage = getattr(info.runtime, "stage", "?") or "?"
# Does any local branch contain the deployed commit?
if sha and git("cat-file", "-t", sha) == "commit":
holders = [b.strip().lstrip("* +").strip()
for b in git("branch", "--contains", sha).splitlines() if b.strip()]
if not holders:
verdict = "DRIFT — on no local branch! pushed direct to Space"
problems += 1
else:
tip = git("rev-parse", "--short=7", source)
if tip == sha:
verdict = f"in sync with {source}"
else:
ahead = git("rev-list", "--count", f"{sha}..{source}")
behind = git("rev-list", "--count", f"{source}..{sha}")
if behind != "0":
verdict = (f"DRIFT — {behind} commit(s) deployed that {source} "
f"lacks (+{ahead} undeployed)")
problems += 1
else:
verdict = f"behind {source} by {ahead} commit(s)"
else:
verdict = f"deployed sha {sha} UNKNOWN locally — run git fetch --all"
problems += 1
if stage != "RUNNING":
problems += 1
print(f"{repo_id:<34} {tier:<10} {sha:<9} {stage:<14} {verdict}")
print()
if problems:
print(f"{problems} issue(s). A DRIFT verdict means: merge the Space's commit back "
f"into its source branch BEFORE the next deploy, or it gets reverted.")
else:
print("All Spaces running and in sync with their source branches.")
print("\nNote: 'RUNNING' only means the container booted. It does NOT mean builds "
"work — GET / returned ok throughout the 2026-08-25 total build outage.")
sys.exit(1 if problems else 0)
if __name__ == "__main__":
main()
|