File size: 5,284 Bytes
1413cec | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 | #!/usr/bin/env python3
"""Auto-fire chain: runs once the GPU A/B job COMPLETES.
Steps (idempotent, re-runnable):
1. Download both GPU frozen-eval JSONs + confirm checkpoints published.
2. Regenerate the per-genre prospective figure from the GPU readouts.
3. Refresh artifacts/paper_data.yaml A/B macros (triplet n, GPU val totals) from
the GPU frozen evals.
4. Launch the stats-collector job (reuse mode) to emit paper_stats_bundle.json.
5. Commit the refreshed artifacts + figures locally.
Prints a compact completion report. Never touches the gate or frozen manifests.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
AB_REPO = "mattbitzesty/pino-pimt-representation-ab"
PY = str(ROOT / ".venv/bin/python")
def log(m): print(f"[autofire] {m}", flush=True)
def run(cmd, **kw):
log("$ " + " ".join(str(c) for c in cmd))
subprocess.run([str(c) for c in cmd], cwd=ROOT, check=True, **kw)
def hf_dl(name, dest):
from huggingface_hub import hf_hub_download
p = hf_hub_download(AB_REPO, name, repo_type="model", token=os.environ["HF_TOKEN"])
Path(dest).write_bytes(Path(p).read_bytes())
log(f"downloaded {name} -> {dest}")
def main() -> int:
token = os.environ.get("HF_TOKEN")
if not token:
log("ERROR: HF_TOKEN not set"); return 1
# 1. download GPU frozen evals
evals = {}
for arm, fname in [("morgan", "frozen_eval_morgan_gpu.json"),
("openpom_256", "frozen_eval_openpom_256_gpu.json")]:
dest = ROOT / "artifacts" / fname
try:
hf_dl(fname, dest)
evals[arm] = json.loads(dest.read_text())
except Exception as e:
log(f"WARN: {fname} not available yet ({type(e).__name__})")
if len(evals) < 2:
log("ERROR: both GPU frozen evals required; aborting"); return 1
# 2. per-genre prospective figure (GPU readouts)
run([PY, "scripts/generate_prospective_genre_figure.py",
"--evals", "artifacts/frozen_eval_morgan_gpu.json", "artifacts/frozen_eval_openpom_256_gpu.json",
"--labels", "morgan", "openpom_256",
"--output", "figures/fig_prospective_by_genre.pdf"])
# 3. refresh paper_data A/B macros
m, p = evals["morgan"], evals["openpom_256"]
mt, pt = m["substitution_triplets"], p["substitution_triplets"]
macros_patch = {
"abTripletN": mt.get("n_triplets"),
"abMorganTripletAcc": mt.get("accuracy"),
"abPomTripletAcc": pt.get("accuracy"),
"abMorganProspectiveCos": m["prospective_formulas"].get("mean_family_profile_cosine"),
"abPomProspectiveCos": p["prospective_formulas"].get("mean_family_profile_cosine"),
"abMorganValTotal": m.get("final_val_total"),
"abPomValTotal": p.get("final_val_total"),
}
pd_path = ROOT / "artifacts/paper_data.yaml"
txt = pd_path.read_text()
import re
for k, v in macros_patch.items():
if v is None: continue
txt = re.sub(rf"^ {k}: .*$", f" {k}: {v}", txt, flags=re.M)
pd_path.write_text(txt)
log("refreshed paper_data.yaml A/B macros: " + json.dumps(macros_patch))
# 4. commit refreshed artifacts + figures
run(["git", "add", "artifacts/paper_data.yaml", "figures/fig_prospective_by_genre.pdf",
"artifacts/frozen_eval_morgan_gpu.json", "artifacts/frozen_eval_openpom_256_gpu.json"])
run(["git", "commit", "-q", "-m",
"feat: GPU A/B readout — per-genre prospective figure + refreshed paper_data macros\n\n"
"Auto-fired on GPU job completion. Downloaded both GPU frozen_eval JSONs (20-triplet "
"readout), regenerated the per-genre prospective figure, and refreshed paper_data.yaml "
"A/B macros (triplet n + accuracies + prospective cosines + val totals) to the GPU run."])
# 5. launch stats-collector (reuse mode)
log("launching stats-collector job (reuse mode)...")
subprocess.run([
"hf", "jobs", "run", "--detach", "--flavor", "t4-medium", "--timeout", "1h",
"--secrets", "HF_TOKEN",
"-e", "PINO_EPOCHS=20", "-e", "PINO_BATCH=32",
"pytorch/pytorch:2.4.0-cuda12.1-cudnn9-runtime", "--", "bash", "-lc",
"set -e; pip install -q huggingface_hub; "
"python - <<'PY2'\n"
"import os\nfrom huggingface_hub import snapshot_download\n"
"snapshot_download('mattbitzesty/pino-source-code', repo_type='model', local_dir='/workspace/src', token=os.environ.get('HF_TOKEN'))\n"
"PY2\n"
"cd /workspace/src && pip install -q -e . && python scripts/hf_stats_job.py"
], check=False)
log("stats-collector launch attempted (check hf jobs ps)")
# report
log("=== COMPLETION REPORT ===")
log(f"triplets: morgan {mt.get('n_correct')}/{mt.get('n_triplets')} ({mt.get('accuracy')}) | "
f"pom {pt.get('n_correct')}/{pt.get('n_triplets')} ({pt.get('accuracy')}) [chance 0.5]")
log(f"prospective cos: morgan {macros_patch['abMorganProspectiveCos']} | pom {macros_patch['abPomProspectiveCos']}")
log(f"val_total: morgan {macros_patch['abMorganValTotal']} | pom {macros_patch['abPomValTotal']}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|