stationhouse's picture jiwon9703's picture
Add exploratory analysis of designed difficulty and model divergence (#449)
a91390f
Raw History Blame Contribute Delete
5.09 kB
#!/usr/bin/env python3
"""Audit 2026-10-23 Qwen evaluation-time question metadata vs current corpus.
Run from the root of the cloned ConspiracyBench repository.
No inference calls are made.
"""
import csv
import json
from collections import Counter
from pathlib import Path
from inspect_ai.log import read_eval_log
ROOT = Path("completed_experiments/evals/full-2026-10-23/outputs/qwen3.5-4b")
CORPUS = Path("questions/corpus.jsonl")
OUT = Path("analysis/designed-difficulty")
FIELDS = ["claim_id", "category", "framing", "evidence_scarcity", "question_openness", "band", "question"]
def fields_from_item(item, metadata):
difficulty = item.get("difficulty") or {}
return {
"claim_id": item.get("claim_id", metadata.get("claim_id")),
"category": item.get("category", metadata.get("category")),
"framing": item.get("framing", metadata.get("framing")),
"evidence_scarcity": difficulty.get("evidence_scarcity"),
"question_openness": difficulty.get("question_openness"),
"band": difficulty.get("band"),
"question": item.get("question"),
}
def fields_from_corpus(item):
difficulty = item.get("difficulty") or {}
return {
"claim_id": item.get("claim_id"),
"category": item.get("category"),
"framing": item.get("framing"),
"evidence_scarcity": difficulty.get("evidence_scarcity"),
"question_openness": difficulty.get("question_openness"),
"band": difficulty.get("band"),
"question": item.get("question"),
}
def main():
corpus = {}
with CORPUS.open(encoding="utf-8") as f:
for line in f:
if line.strip():
x = json.loads(line)
if x["id"] in corpus:
raise ValueError("Duplicate corpus question: " + x["id"])
corpus[x["id"]] = fields_from_corpus(x)
# The Qwen multiturn_first + single_turn data cover the 254 + 61 questions
# in the baseline. Keep evaluation-time copies, not just the latest corpus.
baseline = {}
counts = Counter()
conflicting_snapshots = []
for variant in ("multiturn_first", "single_turn"):
paths = sorted((ROOT / variant).glob("*.eval"))
if len(paths) != 1:
raise RuntimeError(f"Expected one {variant} file; found {len(paths)}: {paths}")
print("Reading:", paths[0], flush=True)
log = read_eval_log(str(paths[0]))
if log.status != "success":
print("WARNING: evaluation status:", log.status)
for sample in log.samples or []:
qid = str(sample.id)
meta = sample.metadata or {}
item = meta.get("item") or {}
if not item:
raise ValueError(f"Missing snapshot item in {variant} sample {qid}")
snapshot = fields_from_item(item, meta)
if qid in baseline and baseline[qid] != snapshot:
conflicting_snapshots.append(qid)
baseline[qid] = snapshot
counts[variant] += 1
print(f" {variant}: {counts[variant]} samples", flush=True)
mismatch_rows = []
missing_current = []
for qid, stored in sorted(baseline.items()):
current = corpus.get(qid)
if current is None:
missing_current.append(qid)
continue
for field in FIELDS:
if stored[field] != current[field]:
mismatch_rows.append({
"question_id": qid,
"field": field,
"eval_snapshot": json.dumps(stored[field], ensure_ascii=False),
"current_corpus": json.dumps(current[field], ensure_ascii=False),
})
new_current = sorted(set(corpus) - set(baseline))
OUT.mkdir(parents=True, exist_ok=True)
output = OUT / "metadata_mismatches.csv"
with output.open("w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=["question_id", "field", "eval_snapshot", "current_corpus"])
writer.writeheader()
writer.writerows(mismatch_rows)
print("\n=== METADATA AUDIT ===")
print("Current corpus questions:", len(corpus))
print("Unique eval snapshot questions:", len(baseline))
print("Questions missing from current corpus:", len(missing_current), missing_current[:15])
print("Questions in current corpus absent from snapshots:", len(new_current), new_current[:15])
print("Conflicting snapshot IDs:", len(set(conflicting_snapshots)), sorted(set(conflicting_snapshots))[:15])
print("Metadata mismatches:", len(mismatch_rows))
print("Mismatch counts by field:", dict(Counter(r["field"] for r in mismatch_rows)))
for row in mismatch_rows[:10]:
print(" ", row)
print("Saved:", output)
if mismatch_rows or missing_current or conflicting_snapshots:
print("WARNING: Resolve discrepancies before publishing results derived from the current corpus.")
else:
print("PASS: All observed baseline question snapshots match current corpus fields.")
if __name__ == "__main__":
main()