| """Compare existing pinned CPU/MPS probe outputs without additional inference.""" |
|
|
| import json |
| from pathlib import Path |
|
|
| cpu = [json.loads(line) for line in Path("reports/local-probes.jsonl").read_text().splitlines()] |
| mps = [json.loads(line) for line in Path("reports/mps-cli-probes.jsonl").read_text().splitlines()] |
| agreements = heads = 0 |
| max_delta = 0.0 |
| for a, b in zip(cpu, mps): |
| assert (a["fixture_id"], a["schema_hash"], a["input_hash"], a["revision"]) == ( |
| b["fixture_id"], |
| b["schema_hash"], |
| b["input_hash"], |
| b["revision"], |
| ) |
| assert not a["error"] and not b["error"] |
| for name, head in a["heads"].items(): |
| heads += 1 |
| agreements += head["selected_label"] == b["heads"][name]["selected_label"] |
| max_delta = max( |
| max_delta, |
| max( |
| abs(p - b["heads"][name]["probabilities"][label]) |
| for label, p in head["probabilities"].items() |
| ), |
| ) |
| result = { |
| "fixture_count": len(cpu), |
| "head_comparisons": heads, |
| "label_agreements": agreements, |
| "max_absolute_probability_difference": max_delta, |
| "timing_conclusion": "not estimated here; fresh-shape compile costs and background CPU run make these provisional timings unsuitable for a warm-speed claim", |
| } |
| Path("reports/mps-probe-parity.json").write_text(json.dumps(result, indent=2) + "\n") |
| print(json.dumps(result, indent=2)) |
|
|
