File size: 5,085 Bytes
a91390f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
#!/usr/bin/env python3
"""Audit 2026-10-23 Qwen evaluation-time question metadata vs current corpus.
Run from the root of the cloned ConspiracyBench repository.
No inference calls are made.
"""
import csv
import json
from collections import Counter
from pathlib import Path
from inspect_ai.log import read_eval_log

ROOT = Path("completed_experiments/evals/full-2026-10-23/outputs/qwen3.5-4b")
CORPUS = Path("questions/corpus.jsonl")
OUT = Path("analysis/designed-difficulty")
FIELDS = ["claim_id", "category", "framing", "evidence_scarcity", "question_openness", "band", "question"]


def fields_from_item(item, metadata):
    difficulty = item.get("difficulty") or {}
    return {
        "claim_id": item.get("claim_id", metadata.get("claim_id")),
        "category": item.get("category", metadata.get("category")),
        "framing": item.get("framing", metadata.get("framing")),
        "evidence_scarcity": difficulty.get("evidence_scarcity"),
        "question_openness": difficulty.get("question_openness"),
        "band": difficulty.get("band"),
        "question": item.get("question"),
    }


def fields_from_corpus(item):
    difficulty = item.get("difficulty") or {}
    return {
        "claim_id": item.get("claim_id"),
        "category": item.get("category"),
        "framing": item.get("framing"),
        "evidence_scarcity": difficulty.get("evidence_scarcity"),
        "question_openness": difficulty.get("question_openness"),
        "band": difficulty.get("band"),
        "question": item.get("question"),
    }


def main():
    corpus = {}
    with CORPUS.open(encoding="utf-8") as f:
        for line in f:
            if line.strip():
                x = json.loads(line)
                if x["id"] in corpus:
                    raise ValueError("Duplicate corpus question: " + x["id"])
                corpus[x["id"]] = fields_from_corpus(x)

    # The Qwen multiturn_first + single_turn data cover the 254 + 61 questions
    # in the baseline. Keep evaluation-time copies, not just the latest corpus.
    baseline = {}
    counts = Counter()
    conflicting_snapshots = []
    for variant in ("multiturn_first", "single_turn"):
        paths = sorted((ROOT / variant).glob("*.eval"))
        if len(paths) != 1:
            raise RuntimeError(f"Expected one {variant} file; found {len(paths)}: {paths}")
        print("Reading:", paths[0], flush=True)
        log = read_eval_log(str(paths[0]))
        if log.status != "success":
            print("WARNING: evaluation status:", log.status)
        for sample in log.samples or []:
            qid = str(sample.id)
            meta = sample.metadata or {}
            item = meta.get("item") or {}
            if not item:
                raise ValueError(f"Missing snapshot item in {variant} sample {qid}")
            snapshot = fields_from_item(item, meta)
            if qid in baseline and baseline[qid] != snapshot:
                conflicting_snapshots.append(qid)
            baseline[qid] = snapshot
            counts[variant] += 1
        print(f"  {variant}: {counts[variant]} samples", flush=True)

    mismatch_rows = []
    missing_current = []
    for qid, stored in sorted(baseline.items()):
        current = corpus.get(qid)
        if current is None:
            missing_current.append(qid)
            continue
        for field in FIELDS:
            if stored[field] != current[field]:
                mismatch_rows.append({
                    "question_id": qid,
                    "field": field,
                    "eval_snapshot": json.dumps(stored[field], ensure_ascii=False),
                    "current_corpus": json.dumps(current[field], ensure_ascii=False),
                })

    new_current = sorted(set(corpus) - set(baseline))
    OUT.mkdir(parents=True, exist_ok=True)
    output = OUT / "metadata_mismatches.csv"
    with output.open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["question_id", "field", "eval_snapshot", "current_corpus"])
        writer.writeheader()
        writer.writerows(mismatch_rows)

    print("\n=== METADATA AUDIT ===")
    print("Current corpus questions:", len(corpus))
    print("Unique eval snapshot questions:", len(baseline))
    print("Questions missing from current corpus:", len(missing_current), missing_current[:15])
    print("Questions in current corpus absent from snapshots:", len(new_current), new_current[:15])
    print("Conflicting snapshot IDs:", len(set(conflicting_snapshots)), sorted(set(conflicting_snapshots))[:15])
    print("Metadata mismatches:", len(mismatch_rows))
    print("Mismatch counts by field:", dict(Counter(r["field"] for r in mismatch_rows)))
    for row in mismatch_rows[:10]:
        print("  ", row)
    print("Saved:", output)
    if mismatch_rows or missing_current or conflicting_snapshots:
        print("WARNING: Resolve discrepancies before publishing results derived from the current corpus.")
    else:
        print("PASS: All observed baseline question snapshots match current corpus fields.")

if __name__ == "__main__":
    main()