#!/usr/bin/env python3 """Audit 2026-10-23 Qwen evaluation-time question metadata vs current corpus. Run from the root of the cloned ConspiracyBench repository. No inference calls are made. """ import csv import json from collections import Counter from pathlib import Path from inspect_ai.log import read_eval_log ROOT = Path("completed_experiments/evals/full-2026-10-23/outputs/qwen3.5-4b") CORPUS = Path("questions/corpus.jsonl") OUT = Path("analysis/designed-difficulty") FIELDS = ["claim_id", "category", "framing", "evidence_scarcity", "question_openness", "band", "question"] def fields_from_item(item, metadata): difficulty = item.get("difficulty") or {} return { "claim_id": item.get("claim_id", metadata.get("claim_id")), "category": item.get("category", metadata.get("category")), "framing": item.get("framing", metadata.get("framing")), "evidence_scarcity": difficulty.get("evidence_scarcity"), "question_openness": difficulty.get("question_openness"), "band": difficulty.get("band"), "question": item.get("question"), } def fields_from_corpus(item): difficulty = item.get("difficulty") or {} return { "claim_id": item.get("claim_id"), "category": item.get("category"), "framing": item.get("framing"), "evidence_scarcity": difficulty.get("evidence_scarcity"), "question_openness": difficulty.get("question_openness"), "band": difficulty.get("band"), "question": item.get("question"), } def main(): corpus = {} with CORPUS.open(encoding="utf-8") as f: for line in f: if line.strip(): x = json.loads(line) if x["id"] in corpus: raise ValueError("Duplicate corpus question: " + x["id"]) corpus[x["id"]] = fields_from_corpus(x) # The Qwen multiturn_first + single_turn data cover the 254 + 61 questions # in the baseline. Keep evaluation-time copies, not just the latest corpus. baseline = {} counts = Counter() conflicting_snapshots = [] for variant in ("multiturn_first", "single_turn"): paths = sorted((ROOT / variant).glob("*.eval")) if len(paths) != 1: raise RuntimeError(f"Expected one {variant} file; found {len(paths)}: {paths}") print("Reading:", paths[0], flush=True) log = read_eval_log(str(paths[0])) if log.status != "success": print("WARNING: evaluation status:", log.status) for sample in log.samples or []: qid = str(sample.id) meta = sample.metadata or {} item = meta.get("item") or {} if not item: raise ValueError(f"Missing snapshot item in {variant} sample {qid}") snapshot = fields_from_item(item, meta) if qid in baseline and baseline[qid] != snapshot: conflicting_snapshots.append(qid) baseline[qid] = snapshot counts[variant] += 1 print(f" {variant}: {counts[variant]} samples", flush=True) mismatch_rows = [] missing_current = [] for qid, stored in sorted(baseline.items()): current = corpus.get(qid) if current is None: missing_current.append(qid) continue for field in FIELDS: if stored[field] != current[field]: mismatch_rows.append({ "question_id": qid, "field": field, "eval_snapshot": json.dumps(stored[field], ensure_ascii=False), "current_corpus": json.dumps(current[field], ensure_ascii=False), }) new_current = sorted(set(corpus) - set(baseline)) OUT.mkdir(parents=True, exist_ok=True) output = OUT / "metadata_mismatches.csv" with output.open("w", encoding="utf-8", newline="") as f: writer = csv.DictWriter(f, fieldnames=["question_id", "field", "eval_snapshot", "current_corpus"]) writer.writeheader() writer.writerows(mismatch_rows) print("\n=== METADATA AUDIT ===") print("Current corpus questions:", len(corpus)) print("Unique eval snapshot questions:", len(baseline)) print("Questions missing from current corpus:", len(missing_current), missing_current[:15]) print("Questions in current corpus absent from snapshots:", len(new_current), new_current[:15]) print("Conflicting snapshot IDs:", len(set(conflicting_snapshots)), sorted(set(conflicting_snapshots))[:15]) print("Metadata mismatches:", len(mismatch_rows)) print("Mismatch counts by field:", dict(Counter(r["field"] for r in mismatch_rows))) for row in mismatch_rows[:10]: print(" ", row) print("Saved:", output) if mismatch_rows or missing_current or conflicting_snapshots: print("WARNING: Resolve discrepancies before publishing results derived from the current corpus.") else: print("PASS: All observed baseline question snapshots match current corpus fields.") if __name__ == "__main__": main()