File size: 8,642 Bytes
9477b5c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 | """Upload BLT-Reasoner pilot artifacts to a public HF repo.
Token is read from stdin so it never appears in command-line arguments,
process listings, or shell history on the box. Run as:
cat token.txt | python3 -m experiments.blt_reasoner.scripts.hf_upload_pilot \
--repo LauraGG/blt-reasoner-pilot1 \
--pilot_dir /home/ubuntu/work/blt_pilot1 \
--code_dir /home/ubuntu/experiments/blt_reasoner
Uploads (each in its own folder inside the repo):
ckpts/ckpt-step{2000,4000,6000,8000,...} — all saved local ckpts
code/ — full blt_reasoner source tree
logs/run.log, logs/metrics.jsonl, logs/auto_eval.log, logs/interim_*.log
ablations/*.json — interim ablation results
README.md — auto-generated state summary
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import sys
from pathlib import Path
def build_readme(pilot_dir: Path, code_dir: Path, repo: str) -> str:
lines = []
lines.append(f"# BLT-Reasoner Pilot 1 — checkpoints + code\n")
lines.append(
"Compute-constrained latent reasoning pilot on Qwen2.5-1.5B-Instruct + GSM8K. "
"Continuous M-step latent loop + strict y→only-z bottleneck + InfoNCE z↔y "
"identifiability loss. See `code/README.md` for architecture details and "
"`HANDOFF_DACOT_PROPOSAL_2026-05-16.md` (in the main repo) for full motivation.\n"
)
# Inventory
ckpts = sorted([p for p in (pilot_dir).glob("ckpt-step*") if p.is_dir()],
key=lambda p: int(p.name.replace("ckpt-step", "")))
lines.append("## Checkpoints (LoRA adapter + projector + InfoNCE head)\n")
lines.append("Each ckpt is ~25 MB — only the trained adapter/projector/head; "
"the base Qwen2.5-1.5B-Instruct is loaded fresh from HF on resume.\n")
lines.append("| step | K_train | files |")
lines.append("|---|---|---|")
for c in ckpts:
s = int(c.name.replace("ckpt-step", ""))
if s < 4000: k = 4
elif s < 8000: k = 8
else: k = 16
lines.append(f"| {s} | {k} | `ckpts/{c.name}/model/`, `projector.pt`, `head.pt` |")
lines.append("")
# Ablations
abls = []
for c in ckpts:
for f in c.glob("ablation_*.json"):
abls.append((c.name, f))
if abls:
lines.append("## Pre-registered z-ablation results\n")
lines.append(
"Pre-registered success criterion: `Δ_random ≥ 15 pp AND Δ_zero ≥ 25 pp` "
"on GSM8K-test. Below are the interim results captured during training.\n"
)
lines.append("| ckpt | K_eval | n | acc(normal) | acc(random) | acc(zero) | Δ_random | Δ_zero |")
lines.append("|---|---|---|---|---|---|---|---|")
for cname, fpath in sorted(abls):
try:
d = json.loads(Path(fpath).read_text())
r = d.get("results", {})
row = [
cname,
str(d.get("K", "?")),
str(d.get("n", "?")),
f"{r.get('normal', {}).get('acc', float('nan')):.3f}",
f"{r.get('random', {}).get('acc', float('nan')):.3f}",
f"{r.get('zero', {}).get('acc', float('nan')):.3f}",
f"{d.get('delta_normal_minus_random', float('nan')):+.3f}",
f"{d.get('delta_normal_minus_zero', float('nan')):+.3f}",
]
lines.append("| " + " | ".join(row) + " |")
except Exception as e:
lines.append(f"| {cname} | (parse error: {e}) |")
lines.append("")
# Resume instructions
lines.append("## Resume training on a fresh instance\n")
lines.append("```bash\n"
"git clone <main-repo-with-experiments/blt_reasoner> # or pull the code/ subdir here\n"
"pip install transformers peft bitsandbytes datasets safetensors huggingface_hub\n"
"python3 -m experiments.blt_reasoner.train \\\n"
f" --config experiments/blt_reasoner/configs/pilot_qwen15b_gsm8k.json \\\n"
f" --resume_from {repo}:ckpts/ckpt-step6000\n"
"```\n"
"Notes:\n"
"- The `--resume_from` flag (in `train.py`) accepts either a local ckpt path or "
f"a `{repo}:ckpts/ckpt-stepN` HF-Hub reference.\n"
"- **Optimizer state is not preserved** across resume. Expect a short loss spike "
"(~100–300 steps) while Adam moments re-stabilize. The latent geometry (LoRA "
"weights, projector, head) survives intact.\n"
"- The base model `Qwen/Qwen2.5-1.5B-Instruct` is fetched automatically.\n"
)
lines.append("## Logs and intermediate artifacts\n"
"- `logs/run.log` — full training log\n"
"- `logs/metrics.jsonl` — per-step loss/metric breakdown\n"
"- `logs/auto_eval.log` — poller daemon log (auto-eval on train exit)\n"
"- `logs/interim_*.log` — interim ablation logs\n"
"- `code/` — full `experiments/blt_reasoner/` source tree at upload time\n")
return "\n".join(lines)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--repo", required=True, help="e.g., LauraGG/blt-reasoner-pilot1")
parser.add_argument("--pilot_dir", required=True, help="e.g., /home/ubuntu/work/blt_pilot1")
parser.add_argument("--code_dir", required=True, help="e.g., /home/ubuntu/experiments/blt_reasoner")
parser.add_argument("--private", action="store_true")
args = parser.parse_args()
token = sys.stdin.read().strip()
if not token.startswith("hf_"):
print("[upload] stdin did not contain an hf_ token; aborting", file=sys.stderr)
sys.exit(2)
from huggingface_hub import HfApi
api = HfApi(token=token)
print(f"[upload] creating repo {args.repo} (private={args.private})", flush=True)
api.create_repo(repo_id=args.repo, repo_type="model", private=args.private, exist_ok=True)
pilot = Path(args.pilot_dir)
code = Path(args.code_dir)
# Stage layout in a tmp dir, then upload as a single folder commit.
stage = Path("/tmp/blt_upload_stage")
if stage.exists():
shutil.rmtree(stage)
stage.mkdir(parents=True)
# Ckpts
(stage / "ckpts").mkdir()
for c in sorted(pilot.glob("ckpt-step*")):
if c.is_dir():
shutil.copytree(c, stage / "ckpts" / c.name)
print(f"[upload] staged {c.name}", flush=True)
# Code
if code.exists():
shutil.copytree(code, stage / "code",
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
print(f"[upload] staged code dir", flush=True)
# Logs
(stage / "logs").mkdir()
for name in ("run.log", "metrics.jsonl", "auto_eval.log",
"interim_ablation.log",
"interim_ablation_K4.log",
"interim_ablation_K8.log",
"interim_ablation_K16_step8000.log",
"run_attempt1_oom.log",
"run_attempt2.log"):
p = pilot / name
if p.exists():
shutil.copy(p, stage / "logs" / name)
print(f"[upload] staged log {name}", flush=True)
# Also stash all ablation_*.json under ablations/ at the top of the staged tree,
# alongside their per-ckpt copies (which are in ckpts/ckpt-stepN/ already).
(stage / "ablations").mkdir()
for c in sorted(pilot.glob("ckpt-step*")):
for f in c.glob("ablation_*.json"):
shutil.copy(f, stage / "ablations" / f"{c.name}__{f.name}")
# README
readme = build_readme(pilot, code, args.repo)
(stage / "README.md").write_text(readme)
print(f"[upload] staged README.md ({len(readme)} chars)", flush=True)
# Final size
total_bytes = sum(p.stat().st_size for p in stage.rglob("*") if p.is_file())
print(f"[upload] total staged size = {total_bytes/1e6:.1f} MB", flush=True)
print(f"[upload] pushing to {args.repo} ...", flush=True)
api.upload_folder(
folder_path=str(stage),
repo_id=args.repo,
repo_type="model",
commit_message="BLT-Reasoner pilot 1: ckpts + code + logs + ablations",
)
print(f"[upload] DONE — https://huggingface.co/{args.repo}", flush=True)
shutil.rmtree(stage)
if __name__ == "__main__":
main()
|