build_gold_queue.py

#!/usr/bin/env python3
"""Build the gold-controls queue from local human-adjudicated spans.

Joins outputs/gliner_datause_v3_probe_human473.jsonl (key, set, surface)
to analysis/v24_sample_review/gliner2_review.jsonl
(spans[].key -> input passage, start/end, origin). Scores are the
singlepass infer-head H100 predictions
(outputs/gliner-datause-catchall-infer-probe/human473_predictions.jsonl);
truth labels are deliberately excluded (blind controls).
Splits: annotator190 (set=annotator) + jdc283 (set=jdc).

    uv run python human_labeling/build_gold_queue.py [--out human_labeling/queue_human473.json]
"""

import argparse
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

from probe_labels import decide  # noqa: E402

REPO = Path(__file__).resolve().parent.parent
HUMAN = REPO / "outputs" / "gliner_datause_v3_probe_human473.jsonl"
REVIEW = REPO / "analysis" / "v24_sample_review" / "gliner2_review.jsonl"


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default=str(REPO / "human_labeling" / "queue_human473.json"))
    a = ap.parse_args()

    # singlepass (infer-head) rescore, published H100 predictions; 23 of 473
    # out-of-grid spans absent here -> head_score None, band "unscored"
    SP = REPO / "outputs" / "gliner-datause-catchall-infer-probe" / "human473_predictions.jsonl"
    sp = {}
    if SP.exists():
        for line in SP.read_text().splitlines():
            if line.strip():
                o = json.loads(line)
                sp[o["key"]] = float(o.get("head_score"))
    # review spans by key -> (passage, start, end, origin)
    ctx: dict[str, tuple] = {}
    for line in REVIEW.read_text().splitlines():
        if not line.strip():
            continue
        row = json.loads(line)
        for s in row.get("spans", []):
            if s.get("key"):
                ctx[s["key"]] = (row.get("input", ""), s.get("start"),
                                 s.get("end"), row.get("origin"))
    items: list[dict] = []
    missing: list[str] = []
    from collections import Counter
    for line in HUMAN.read_text().splitlines():
        if not line.strip():
            continue
        r = json.loads(line)
        key = r["key"]
        subset = "annotator190" if r.get("set") == "annotator" else "jdc283"
        score = sp.get(key)  # singlepass infer-head (H100); None = out-of-grid
        if key in ctx:
            passage, start, end, origin = ctx[key]
            items.append({
                "key": key, "surface": r.get("surface"), "ctx": passage,
                "ctx_missing": False, "head_score": score,
                "start": start, "end": end,
                "band": decide(score, origin), "origin": origin,
                "specificity": "", "split": "holdout",
                "queue": "human473", "subset": subset,
                "scored_by": "rafmacalaba/gliner-datause-catchall-infer-probe (H100)",
            })
        else:
            missing.append(key)
    Path(a.out).write_text("\n".join(json.dumps(i) for i in items) + "\n")
    print(f"queued={len(items)} missing_ctx={len(missing)} "
          f"scored={sum(1 for i in items if i['head_score'] is not None)} "
          f"{Counter(i['subset'] for i in items)} -> {a.out}")
    if missing:
        print("missing:", missing[:5])


if __name__ == "__main__":
    main()