push artifacts/scripts/hf_push.py

artifacts/scripts/hf_push.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Push Ingot (SN97) build-pod artifacts to HF repo eyad-silx/progress.
+
+Modes: --small (docs/logs/scripts), --big (GGUFs + QAT states), default = both.
+Only uploads new/changed files (size+mtime_ns fingerprint), records sha256 in
+state for the MANIFEST. State: /root/.hf_push_state.json. Log: /root/hf_push.log.
+"""
+import argparse, glob, hashlib, json, os, sys
+from datetime import datetime, timezone
+from pathlib import Path
+from huggingface_hub import HfApi
+
+REPO = "eyad-silx/progress"
+TOKEN = Path("/root/.hf_token").read_text().strip()
+STATE_P = Path("/root/.hf_push_state.json")
+LOG_P = Path("/root/hf_push.log")
+
+README = """# Ingot (SN97) challenger build — progress proof
+
+Automated backup of build/train/eval artifacts from the H100 pod (RunPod, 1x H100 NVL).
+Pushed by `hf_push.py` — checks hourly and uploads any new/changed artifact.
+
+- `king-ix/` — incumbent king GGUF (Ingot-IX-27B-Q2) + provenance files (baseline)
+- `chal-identity/`, `chal-v1/`, `chal-n2/`, `chal-norms/` — challenger GGUF iterations
+  (scale-refit / QAT variants; same 27B Q2 architecture as the king)
+- `states/` — QAT state checkpoints (`t0..tN.safetensors`)
+- `artifacts/` — `HANDOFF.md`, build/train/eval scripts, run logs, sample draws,
+  measurement results (`measure*.jsonl`, `pick*.jsonl`, `drawn.jsonl`)
+- `MANIFEST.md` — size + sha256 + upload time of every artifact (regenerated each push)
+- `PROGRESS.md` — push history
+
+The GGUFs are ~6.8 GB each (PQ2_0 quantized 27B linear-attention models).
+"""
+
+def now():
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+def log(evt, **kw):
+    line = json.dumps({"t": now(), "event": evt, **kw})
+    with open(LOG_P, "a") as f:
+        f.write(line + "\n")
+    print(line, flush=True)
+
+def sha256(p):
+    h = hashlib.sha256()
+    with open(p, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 24), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+def load_state():
+    return json.loads(STATE_P.read_text()) if STATE_P.exists() else {}
+
+def save_state(s):
+    STATE_P.write_text(json.dumps(s, indent=1, sort_keys=True))
+
+def small_tracks():
+    t = [("/root/HANDOFF.md", "artifacts/HANDOFF.md")]
+    for pat, dest in [("/root/*.py", "artifacts/scripts/"),
+                      ("/root/*.sh", "artifacts/scripts/"),
+                      ("/root/*.jsonl", "artifacts/"),
+                      ("/root/*.json", "artifacts/"),
+                      ("/root/*.log", "artifacts/logs/")]:
+        for p in sorted(glob.glob(pat)):
+            name = os.path.basename(p)
+            if name in ("hf_push.log", "hf_push_nohup.log"):
+                continue
+            t.append((p, dest + name))
+    for extra in ("README.md", "LICENSE", "NOTICE.txt"):
+        p = f"/root/models/king-ix/{extra}"
+        if os.path.exists(p):
+            t.append((p, f"king-ix/{extra}"))
+    return t
+
+def big_tracks():
+    # priority order: newest challenger first (pod lifetime is uncertain)
+    t = [("/root/models/chal-identity/Ingot-IX-27B-Q2.gguf", "chal-identity/Ingot-IX-27B-Q2.gguf"),
+         ("/root/models/chal-v1/chal-v1.gguf", "chal-v1/chal-v1.gguf"),
+         ("/root/models/chal-n2/chal-norms.gguf", "chal-n2/chal-norms.gguf"),
+         ("/root/models/chal-norms/chal-norms.gguf", "chal-norms/chal-norms.gguf"),
+         ("/root/models/king-ix/Ingot-IX-27B-Q2.gguf", "king-ix/Ingot-IX-27B-Q2.gguf")]
+    for p in sorted(glob.glob("/root/states/*.safetensors")):
+        t.append((p, "states/" + os.path.basename(p)))
+    return t
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--small", action="store_true")
+    ap.add_argument("--big", action="store_true")
+    args = ap.parse_args()
+    if args.small or args.big:
+        tracks = (small_tracks() if args.small else []) + (big_tracks() if args.big else [])
+    else:
+        tracks = small_tracks() + big_tracks()
+
+    state = load_state()
+    api = HfApi(token=TOKEN)
+    results = []
+    for local, repo in tracks:
+        if not os.path.exists(local):
+            continue
+        st = os.stat(local)
+        fp = f"{st.st_size}:{st.st_mtime_ns}"
+        prev = state.get(repo)
+        if prev and prev["fp"] == fp:
+            results.append((repo, "unchanged"))
+            continue
+        try:
+            d = sha256(local)
+            api.upload_file(path_or_fileobj=local, path_in_repo=repo, repo_id=REPO,
+                            repo_type="model", commit_message=f"push {repo}")
+            state[repo] = {"fp": fp, "sha256": d, "size": st.st_size, "uploaded_at": now()}
+            save_state(state)
+            log("uploaded", local=local, repo=repo, sha256=d, size=st.st_size)
+            results.append((repo, "uploaded"))
+        except Exception as e:
+            log("error", local=local, repo=repo, err=repr(e)[:500])
+            results.append((repo, "FAILED"))
+
+    # regenerate docs (manifest / progress / readme) from state
+    try:
+        manifest = ["# Manifest — eyad-silx/progress", "",
+                    "| repo path | size (bytes) | sha256 | uploaded (UTC) |",
+                    "|---|---|---|---|"]
+        for repo in sorted(state):
+            v = state[repo]
+            manifest.append(f"| `{repo}` | {v['size']:,} | `{v['sha256']}` | {v['uploaded_at']} |")
+        Path("/tmp/MANIFEST.md").write_text("\n".join(manifest) + "\n")
+        api.upload_file(path_or_fileobj="/tmp/MANIFEST.md", path_in_repo="MANIFEST.md",
+                        repo_id=REPO, repo_type="model", commit_message="update MANIFEST.md")
+
+        hist = [json.loads(l) for l in LOG_P.read_text().splitlines()[-80:] if l.strip()]
+        prog = ["# Progress — push history", "", f"Last run: {now()}", ""]
+        prog += [f"- `{h['t']}` **{h['event']}** {h.get('repo', h.get('local', ''))}" for h in hist]
+        Path("/tmp/PROGRESS.md").write_text("\n".join(prog) + "\n")
+        api.upload_file(path_or_fileobj="/tmp/PROGRESS.md", path_in_repo="PROGRESS.md",
+                        repo_id=REPO, repo_type="model", commit_message="update PROGRESS.md")
+
+        Path("/tmp/README_REPO.md").write_text(README)
+        api.upload_file(path_or_fileobj="/tmp/README_REPO.md", path_in_repo="README.md",
+                        repo_id=REPO, repo_type="model", commit_message="README")
+    except Exception as e:
+        log("error", step="docs", err=repr(e)[:500])
+
+    n_up = sum(1 for r in results if r[1] == "uploaded")
+    n_un = sum(1 for r in results if r[1] == "unchanged")
+    n_fail = sum(1 for r in results if r[1] == "FAILED")
+    log("run_done", uploaded=n_up, unchanged=n_un, failed=n_fail)
+    sys.exit(1 if n_fail else 0)
+
+if __name__ == "__main__":
+    main()