Spaces:
Running
Running
File size: 5,085 Bytes
a91390f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | #!/usr/bin/env python3
"""Audit 2026-10-23 Qwen evaluation-time question metadata vs current corpus.
Run from the root of the cloned ConspiracyBench repository.
No inference calls are made.
"""
import csv
import json
from collections import Counter
from pathlib import Path
from inspect_ai.log import read_eval_log
ROOT = Path("completed_experiments/evals/full-2026-10-23/outputs/qwen3.5-4b")
CORPUS = Path("questions/corpus.jsonl")
OUT = Path("analysis/designed-difficulty")
FIELDS = ["claim_id", "category", "framing", "evidence_scarcity", "question_openness", "band", "question"]
def fields_from_item(item, metadata):
difficulty = item.get("difficulty") or {}
return {
"claim_id": item.get("claim_id", metadata.get("claim_id")),
"category": item.get("category", metadata.get("category")),
"framing": item.get("framing", metadata.get("framing")),
"evidence_scarcity": difficulty.get("evidence_scarcity"),
"question_openness": difficulty.get("question_openness"),
"band": difficulty.get("band"),
"question": item.get("question"),
}
def fields_from_corpus(item):
difficulty = item.get("difficulty") or {}
return {
"claim_id": item.get("claim_id"),
"category": item.get("category"),
"framing": item.get("framing"),
"evidence_scarcity": difficulty.get("evidence_scarcity"),
"question_openness": difficulty.get("question_openness"),
"band": difficulty.get("band"),
"question": item.get("question"),
}
def main():
corpus = {}
with CORPUS.open(encoding="utf-8") as f:
for line in f:
if line.strip():
x = json.loads(line)
if x["id"] in corpus:
raise ValueError("Duplicate corpus question: " + x["id"])
corpus[x["id"]] = fields_from_corpus(x)
# The Qwen multiturn_first + single_turn data cover the 254 + 61 questions
# in the baseline. Keep evaluation-time copies, not just the latest corpus.
baseline = {}
counts = Counter()
conflicting_snapshots = []
for variant in ("multiturn_first", "single_turn"):
paths = sorted((ROOT / variant).glob("*.eval"))
if len(paths) != 1:
raise RuntimeError(f"Expected one {variant} file; found {len(paths)}: {paths}")
print("Reading:", paths[0], flush=True)
log = read_eval_log(str(paths[0]))
if log.status != "success":
print("WARNING: evaluation status:", log.status)
for sample in log.samples or []:
qid = str(sample.id)
meta = sample.metadata or {}
item = meta.get("item") or {}
if not item:
raise ValueError(f"Missing snapshot item in {variant} sample {qid}")
snapshot = fields_from_item(item, meta)
if qid in baseline and baseline[qid] != snapshot:
conflicting_snapshots.append(qid)
baseline[qid] = snapshot
counts[variant] += 1
print(f" {variant}: {counts[variant]} samples", flush=True)
mismatch_rows = []
missing_current = []
for qid, stored in sorted(baseline.items()):
current = corpus.get(qid)
if current is None:
missing_current.append(qid)
continue
for field in FIELDS:
if stored[field] != current[field]:
mismatch_rows.append({
"question_id": qid,
"field": field,
"eval_snapshot": json.dumps(stored[field], ensure_ascii=False),
"current_corpus": json.dumps(current[field], ensure_ascii=False),
})
new_current = sorted(set(corpus) - set(baseline))
OUT.mkdir(parents=True, exist_ok=True)
output = OUT / "metadata_mismatches.csv"
with output.open("w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=["question_id", "field", "eval_snapshot", "current_corpus"])
writer.writeheader()
writer.writerows(mismatch_rows)
print("\n=== METADATA AUDIT ===")
print("Current corpus questions:", len(corpus))
print("Unique eval snapshot questions:", len(baseline))
print("Questions missing from current corpus:", len(missing_current), missing_current[:15])
print("Questions in current corpus absent from snapshots:", len(new_current), new_current[:15])
print("Conflicting snapshot IDs:", len(set(conflicting_snapshots)), sorted(set(conflicting_snapshots))[:15])
print("Metadata mismatches:", len(mismatch_rows))
print("Mismatch counts by field:", dict(Counter(r["field"] for r in mismatch_rows)))
for row in mismatch_rows[:10]:
print(" ", row)
print("Saved:", output)
if mismatch_rows or missing_current or conflicting_snapshots:
print("WARNING: Resolve discrepancies before publishing results derived from the current corpus.")
else:
print("PASS: All observed baseline question snapshots match current corpus fields.")
if __name__ == "__main__":
main()
|