tools/verify_readme_metrics.py

"""Verify every metric quoted in the release README against its source artifact.

Read-only. Each claim is compared at the precision at which the README states it.
Writes nothing; prints a report suitable for pasting into FINAL_RELEASE_VERIFICATION.md.

Key names were discovered by walking the artifacts, NOT assumed: several live under nested
paths (e.g. change metrics are `metrics.pooled.iou`, grounding is
`results.head_threshold.mean_best_iou`, VLM is `why_usable_verified.adapted_test.*`).

WHERE THE ARTIFACTS COME FROM
-----------------------------
This tool reads the project's `artifacts/` directory, which is NOT part of the public release
(the release ships the six trained weights on the Hugging Face Hub, not the full artifacts tree).
Point it at a checkout that has the artifacts:

    SATQUERY_ARTIFACTS_ROOT=/path/to/satquery-ai python verify_readme_metrics.py

Without that variable it defaults to `../..` relative to this file, which is the layout the tool
was authored in.
"""
import json
import os
import sys

ROOT = os.environ.get(
    "SATQUERY_ARTIFACTS_ROOT",
    os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")),
)

# (label, relative path, dotted key, value as printed in the README)
CLAIMS = [
    ("change pooled IoU", "artifacts/change/eval_test/eval_result.json",
     "metrics.pooled.iou", "0.8122"),
    ("change macro IoU", "artifacts/change/eval_test/eval_result.json",
     "metrics.macro.miou", "0.8457"),
    ("change pooled F1", "artifacts/change/eval_test/eval_result.json",
     "metrics.pooled.f1", "0.8964"),
    ("grounding canonical head_threshold mean_best_IoU",
     "artifacts/grounding/remoteclip_grounding_v001/eval_result_canonical.json",
     "results.head_threshold.mean_best_iou", "0.2838"),
    ("grounding canonical head_threshold recall@0.5",
     "artifacts/grounding/remoteclip_grounding_v001/eval_result_canonical.json",
     "results.head_threshold.recall.0.50", "0.2198"),
    ("grounding matched6 head_threshold mean_best_IoU",
     "artifacts/grounding/remoteclip_grounding_v001/eval_result_matched6.json",
     "results.head_threshold.mean_best_iou", "0.2566"),
    ("grounding matched6 head_threshold recall@0.5",
     "artifacts/grounding/remoteclip_grounding_v001/eval_result_matched6.json",
     "results.head_threshold.recall.0.50", "0.1938"),
    ("grounding head_argmax mean_best_IoU (canonical)",
     "artifacts/grounding/remoteclip_grounding_v001/eval_result_canonical.json",
     "results.head_argmax.mean_best_iou", "0.1215"),
    ("grounding zero-shot baseline IoU (canonical)",
     "artifacts/grounding/remoteclip_grounding_v001/eval_result_canonical.json",
     "results.zero_shot_matched.mean_best_iou", "0.0972"),
    ("optical-SAR fusion accuracy",
     "artifacts/optical_sar/fusion_head_production_v001/pre_registered_115_metric.json",
     "accuracy", "0.931"),
    ("optical-SAR fusion macro_F1",
     "artifacts/optical_sar/fusion_head_production_v001/pre_registered_115_metric.json",
     "macro_f1", "0.434161"),
    ("change_vqa test accuracy", "artifacts/change_vqa/run/PROMOTION.json",
     "verification.test_accuracy", "0.697626"),
    ("change_vqa test macro_F1", "artifacts/change_vqa/run/PROMOTION.json",
     "verification.test_macro_f1", "0.378373"),
    ("change_vqa test2 accuracy", "artifacts/change_vqa/run/PROMOTION.json",
     "verification.test2_accuracy", "0.651469"),
    ("change_vqa test2 macro_F1", "artifacts/change_vqa/run/PROMOTION.json",
     "verification.test2_macro_f1", "0.372309"),
    ("router overall ungated accuracy", "artifacts/router/threshold_sweep_val.json",
     "overall_ungated_accuracy", "0.965116"),
    ("calibration ECE before scaling", "artifacts/calibration_v001.json",
     "metrics.ece_before", "0.013755"),
    ("calibration ECE after scaling", "artifacts/calibration_v001.json",
     "metrics.ece_after", "0.014929"),
    ("VLM adapter exact_match", "artifacts/vlm/phase6_closure.json",
     "why_usable_verified.adapted_test.exact_match", "0.963"),
    ("VLM adapter F1", "artifacts/vlm/phase6_closure.json",
     "why_usable_verified.adapted_test.f1", "0.96432"),
]


def get(obj, dotted):
    """Resolve a dotted path, preferring the LONGEST matching key at each step.

    Needed because some artifact keys themselves contain dots (e.g. the recall
    dict is keyed "0.10"/"0.25"/"0.50"), so a naive split(".") walk would break
    `results.head_threshold.recall.0.50` into ...recall -> 0 -> 50 and fail.
    """
    return _get(obj, dotted.split("."))


def _get(cur, parts):
    if not parts:
        return cur, True
    if isinstance(cur, dict):
        for i in range(len(parts), 0, -1):          # longest key first
            key = ".".join(parts[:i])
            if key in cur:
                return _get(cur[key], parts[i:])
        return None, False
    if isinstance(cur, list):
        return _get(cur[int(parts[0])], parts[1:])
    return None, False


def decimals(s):
    return len(s.split(".")[1]) if "." in s else 0


fails = 0
print(f"{'STATUS':8} {'claim':48} {'artifact':>14} {'readme':>12}  source")
print("-" * 118)
for label, rel, key, claimed in CLAIMS:
    path = os.path.join(ROOT, rel)
    if not os.path.exists(path):
        print(f"{'NOFILE':8} {label:48} {'':>14} {claimed:>12}  {rel}")
        fails += 1
        continue
    data = json.load(open(path, encoding="utf-8"))
    val, found = get(data, key)
    if not found:
        print(f"{'NOKEY':8} {label:48} {'':>14} {claimed:>12}  {rel}#{key}")
        fails += 1
        continue
    ok = round(float(val), decimals(claimed)) == float(claimed)
    if not ok:
        fails += 1
    print(f"{'MATCH' if ok else 'DIFFER':8} {label:48} {val:>14} {claimed:>12}  {rel}#{key}")

# --- artifact-declared statuses that the README also asserts ---
print()
print("=== status assertions ===")
d = json.load(open(os.path.join(ROOT, "artifacts/vlm/phase6_closure.json"), encoding="utf-8"))
hl = d.get("headline", "")
print(f"  VLM headline contains ACCEPTANCE-REJECTED : {'ACCEPTANCE-REJECTED' in hl}")
print(f"  VLM status                               : {d.get('status')}")
r = json.load(open(os.path.join(ROOT, "artifacts/router/threshold_sweep_val.json"), encoding="utf-8"))
print(f"  router corpus_limited                    : {r.get('corpus_limited')}")
print(f"  router n_val                             : {r.get('n_val')}")
c = json.load(open(os.path.join(ROOT, "artifacts/calibration_v001.json"), encoding="utf-8"))
print(f"  calibration temperature (temperature_scaling.temperature) : "
      f"{c.get('temperature_scaling', {}).get('temperature')}")
print(f"  calibration ece_improvement              : "
      f"{c.get('metrics', {}).get('ece_improvement')}  (negative => calibration did NOT help)")

print()
print(f"RESULT: {'ALL CLAIMS VERIFIED' if fails == 0 else str(fails) + ' CLAIM(S) FAILED'}")
sys.exit(0 if fails == 0 else 1)