File size: 5,692 Bytes
1d1c6db | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | """Write models.json / config.json (sha256) / provenance.json / metrics/evaluation.json of the staging repo from the gate,
test and bench results in metrics/. python -m tools.write_meta /home/ubuntu/hf/Julia-2"""
import hashlib, json, sys, time
from pathlib import Path
repo = Path(sys.argv[1]); M = repo / "metrics"
def J(p):
p = M / p; return json.loads(p.read_text()) if p.exists() else None
def sha(p): return hashlib.sha256(Path(p).read_bytes()).hexdigest()
def size_gb(p): return round(Path(p).stat().st_size / 1e9, 2)
gates = {p: J(f"gate_{p}.json") for p in ("bf16", "int8", "int4")}
bench = {p: J(f"bench_cuda_{p}.json") for p in ("bf16", "int8", "int4")}
tests = {p: J(f"tests_cuda_{p}.json") for p in ("bf16", "int8", "int4")}
cfg = json.loads((repo / "config.json").read_text()); cfg["weights_sha256"] = sha(repo / "model.safetensors")
variants = {}
for p, folder in (("bf16", "."), ("int8", "int8"), ("int4", "int4")):
g = gates[p] or {}; b = bench[p] or {}
agree = {k.replace("_vs_bf16", ""): v["argmax_agree"] for k, v in g.items() if k.endswith("_vs_bf16")} if p != "bf16" else {"text_vs_training_reference": (g.get("text_vs_j2ref") or {}).get("argmax_agree")}
maxdp = {k.replace("_vs_bf16", ""): v["maxdp_p95"] for k, v in g.items() if k.endswith("_vs_bf16")} if p != "bf16" else {}
passed = p == "bf16" or (agree.get("text", 0) >= 0.99)
variants[f"julia-2-{p}"] = {"path": folder, "precision": p, "weights_file": "model.safetensors", "weights_sha256": sha(repo / folder / "model.safetensors"), "size_gb": size_gb(repo / folder / "model.safetensors"),
"scheme": {"bf16": "bf16 weights (readout, audio projector fp32)", "int8": "weight-only per-channel symmetric int8 (fp16 scales); norms/embeddings/readout/projectors bf16", "int4": "weight-only asymmetric 4-bit, groups of 128 (fp16 scale+zero), trunk layers 0-11 int8; norms/embeddings/readout/projectors bf16"}[p],
"argmax_agreement_vs_bf16": agree, "maxdp_p95_vs_bf16": maxdp, "accuracy": {"audio_tts_states_354q": g.get("audio_acc"), "live_calls_v2_final": g.get("live_final_acc"), "live_alert_recall_scam": g.get("live_alert_recall_scam"), "live_false_alarm_legit": g.get("live_false_alarm_legit")},
"gate": "PASS" if passed else "FAIL (not for publication)", "tests": {k: v.get("ok") for k, v in (tests[p] or {}).items() if isinstance(v, dict)},
"latency_ms_rtx_pro_6000": {k: v for k, v in b.items() if k.endswith("_ms") or k.endswith("_ms_total")}, "peak_mem_gb_gpu": b.get("peak_mem_gb"),
"recommended_hardware": {"bf16": "any CUDA GPU with >= 6 GB (bf16); CPU with AMX/AVX512-BF16 (bf16 compute) or fp32 fallback", "int8": "GPUs with 3-4 GB; CPU (RAM 3 GB + dequant cache)", "int4": "GPUs with 2-3 GB, laptops; CPU"}[p], "max_length": 32768, "status": "draft"}
models = {"format_version": 2, "default": "julia-2-bf16", "checkpoint": cfg.get("checkpoint"), "models": variants}
(repo / "models.json").write_text(json.dumps(models, indent=2))
(repo / "config.json").write_text(json.dumps(cfg, indent=2))
prov = json.loads((repo / "provenance.json").read_text())
prov.update({"checkpoint": "julia2_audio_v1 (base_v2 trunk byte-identical + pruned EmbeddingGemma-2 audio tower + projector, stem p9_real2)", "frozen": time.strftime("%Y-%m-%d"), "weights_sha256": cfg["weights_sha256"],
"variants_sha256": {k: v["weights_sha256"] for k, v in variants.items()},
"audio_training": ["text self-distillation (trunk frozen): chunk-pooled hidden-state MSE at layers 6/12/18/24 + KL(answers on audio || answers on transcript)",
"speech: LibriSpeech, MLS-pt, FLEURS en/pt-BR, People's Speech clean (all CC-BY-4.0); TTS renders (Kokoro-82M, Apache-2.0) of our decision states, spoken-fact notes and LLM-written call scripts",
"pruning of the audio tower (drop layers 2,7; FFN 3072; 6 heads) recovered by distillation from the unpruned stem"],
"third_party": "see NOTICE and licenses/ (trunk Apache-2.0; EmbeddingGemma-2 audio tower Apache-2.0)"})
(repo / "provenance.json").write_text(json.dumps(prov, indent=2))
ev = J("evaluation.json") or {}
ev["checkpoint"] = cfg.get("checkpoint")
ev["audio"] = {"stem": "p9_real2 (pruned EG2 tower 199.4M + projector 8.4M; 12.5 tokens/s)", "audio_vs_text_tts_states_354q": {"audio": 0.701, "text": 0.749, "whisper_large_v3_turbo_cascade": 0.715, "phone_band_10dB": 0.72, "agreement_with_text": 0.797},
"real_speech_minds14_cc_by": {"intent14_audio": 0.753, "intent14_text": 0.935, "intent14_cascade": 0.947, "yes_no_audio": 0.784, "yes_no_text": 0.903, "by_locale_intent14": {"en-US": 0.886, "en-GB": 0.812, "en-AU": 0.794, "pt-PT": 0.525}},
"spoken_facts_268q": {"audio": 0.937, "text": 0.94}, "live_calls_v2_115": {"audio": {"scam": 0.826, "alert": 0.878, "pressure": 0.904, "impersonation": 0.661, "credentials": 0.574, "intent": 0.759, "alert_recall_scam": 0.941, "false_alarm_legit": 0.362, "time_to_correct_turns": 2.22, "flip_rate_after_decisive": 0.0167, "streaming_equals_full_argmax": 0.9955},
"text_transcript": {"scam": 0.809, "alert": 0.887, "pressure": 0.896, "impersonation": 0.643, "credentials": 0.591, "intent": 0.714}},
"evidence": "runs/evals/audio_gap_p9_real2.json, audio_minds_p9.json, audio_facts_p9.json, live_calls_p9_v2_fast.json (j2 harness)"}
ev["runtime"] = {"gates": gates, "tests": tests, "bench_cuda": bench, "bench_cpu": J("bench_cpu_bf16.json"), "text_regression_vs_base_v2": "bit-identical logits on 800 val rows (evals/audio_text_regress.py)"}
(M / "evaluation.json").write_text(json.dumps(ev, indent=2))
print(json.dumps({k: (v["gate"], v["argmax_agreement_vs_bf16"]) for k, v in variants.items()}, indent=1))
|