#!/usr/bin/env python3 """Build the gold-controls queue from local human-adjudicated spans. Joins outputs/gliner_datause_v3_probe_human473.jsonl (key, set, surface) to analysis/v24_sample_review/gliner2_review.jsonl (spans[].key -> input passage, start/end, origin). Scores are the singlepass infer-head H100 predictions (outputs/gliner-datause-catchall-infer-probe/human473_predictions.jsonl); truth labels are deliberately excluded (blind controls). Splits: annotator190 (set=annotator) + jdc283 (set=jdc). uv run python human_labeling/build_gold_queue.py [--out human_labeling/queue_human473.json] """ import argparse import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from probe_labels import decide # noqa: E402 REPO = Path(__file__).resolve().parent.parent HUMAN = REPO / "outputs" / "gliner_datause_v3_probe_human473.jsonl" REVIEW = REPO / "analysis" / "v24_sample_review" / "gliner2_review.jsonl" def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--out", default=str(REPO / "human_labeling" / "queue_human473.json")) a = ap.parse_args() # singlepass (infer-head) rescore, published H100 predictions; 23 of 473 # out-of-grid spans absent here -> head_score None, band "unscored" SP = REPO / "outputs" / "gliner-datause-catchall-infer-probe" / "human473_predictions.jsonl" sp = {} if SP.exists(): for line in SP.read_text().splitlines(): if line.strip(): o = json.loads(line) sp[o["key"]] = float(o.get("head_score")) # review spans by key -> (passage, start, end, origin) ctx: dict[str, tuple] = {} for line in REVIEW.read_text().splitlines(): if not line.strip(): continue row = json.loads(line) for s in row.get("spans", []): if s.get("key"): ctx[s["key"]] = (row.get("input", ""), s.get("start"), s.get("end"), row.get("origin")) items: list[dict] = [] missing: list[str] = [] from collections import Counter for line in HUMAN.read_text().splitlines(): if not line.strip(): continue r = json.loads(line) key = r["key"] subset = "annotator190" if r.get("set") == "annotator" else "jdc283" score = sp.get(key) # singlepass infer-head (H100); None = out-of-grid if key in ctx: passage, start, end, origin = ctx[key] items.append({ "key": key, "surface": r.get("surface"), "ctx": passage, "ctx_missing": False, "head_score": score, "start": start, "end": end, "band": decide(score, origin), "origin": origin, "specificity": "", "split": "holdout", "queue": "human473", "subset": subset, "scored_by": "rafmacalaba/gliner-datause-catchall-infer-probe (H100)", }) else: missing.append(key) Path(a.out).write_text("\n".join(json.dumps(i) for i in items) + "\n") print(f"queued={len(items)} missing_ctx={len(missing)} " f"scored={sum(1 for i in items if i['head_score'] is not None)} " f"{Counter(i['subset'] for i in items)} -> {a.out}") if missing: print("missing:", missing[:5]) if __name__ == "__main__": main()