Spaces:
Running
Running
Download analysis/designed-difficulty/audit_metadata.py from daKeshra/ConspiracyBench: direct link, hf CLI and curl.
- Browser
- Download file 5.09 kB
-
https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/analysis/designed-difficulty/audit_metadata.py
- Command line
-
hf download hf://spaces/daKeshra/ConspiracyBench/analysis/designed-difficulty/audit_metadata.py
-
curl -L -o audit_metadata.py https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/analysis/designed-difficulty/audit_metadata.py
5.09 kB
| #!/usr/bin/env python3 | |
| """Audit 2026-10-23 Qwen evaluation-time question metadata vs current corpus. | |
| Run from the root of the cloned ConspiracyBench repository. | |
| No inference calls are made. | |
| """ | |
| import csv | |
| import json | |
| from collections import Counter | |
| from pathlib import Path | |
| from inspect_ai.log import read_eval_log | |
| ROOT = Path("completed_experiments/evals/full-2026-10-23/outputs/qwen3.5-4b") | |
| CORPUS = Path("questions/corpus.jsonl") | |
| OUT = Path("analysis/designed-difficulty") | |
| FIELDS = ["claim_id", "category", "framing", "evidence_scarcity", "question_openness", "band", "question"] | |
| def fields_from_item(item, metadata): | |
| difficulty = item.get("difficulty") or {} | |
| return { | |
| "claim_id": item.get("claim_id", metadata.get("claim_id")), | |
| "category": item.get("category", metadata.get("category")), | |
| "framing": item.get("framing", metadata.get("framing")), | |
| "evidence_scarcity": difficulty.get("evidence_scarcity"), | |
| "question_openness": difficulty.get("question_openness"), | |
| "band": difficulty.get("band"), | |
| "question": item.get("question"), | |
| } | |
| def fields_from_corpus(item): | |
| difficulty = item.get("difficulty") or {} | |
| return { | |
| "claim_id": item.get("claim_id"), | |
| "category": item.get("category"), | |
| "framing": item.get("framing"), | |
| "evidence_scarcity": difficulty.get("evidence_scarcity"), | |
| "question_openness": difficulty.get("question_openness"), | |
| "band": difficulty.get("band"), | |
| "question": item.get("question"), | |
| } | |
| def main(): | |
| corpus = {} | |
| with CORPUS.open(encoding="utf-8") as f: | |
| for line in f: | |
| if line.strip(): | |
| x = json.loads(line) | |
| if x["id"] in corpus: | |
| raise ValueError("Duplicate corpus question: " + x["id"]) | |
| corpus[x["id"]] = fields_from_corpus(x) | |
| # The Qwen multiturn_first + single_turn data cover the 254 + 61 questions | |
| # in the baseline. Keep evaluation-time copies, not just the latest corpus. | |
| baseline = {} | |
| counts = Counter() | |
| conflicting_snapshots = [] | |
| for variant in ("multiturn_first", "single_turn"): | |
| paths = sorted((ROOT / variant).glob("*.eval")) | |
| if len(paths) != 1: | |
| raise RuntimeError(f"Expected one {variant} file; found {len(paths)}: {paths}") | |
| print("Reading:", paths[0], flush=True) | |
| log = read_eval_log(str(paths[0])) | |
| if log.status != "success": | |
| print("WARNING: evaluation status:", log.status) | |
| for sample in log.samples or []: | |
| qid = str(sample.id) | |
| meta = sample.metadata or {} | |
| item = meta.get("item") or {} | |
| if not item: | |
| raise ValueError(f"Missing snapshot item in {variant} sample {qid}") | |
| snapshot = fields_from_item(item, meta) | |
| if qid in baseline and baseline[qid] != snapshot: | |
| conflicting_snapshots.append(qid) | |
| baseline[qid] = snapshot | |
| counts[variant] += 1 | |
| print(f" {variant}: {counts[variant]} samples", flush=True) | |
| mismatch_rows = [] | |
| missing_current = [] | |
| for qid, stored in sorted(baseline.items()): | |
| current = corpus.get(qid) | |
| if current is None: | |
| missing_current.append(qid) | |
| continue | |
| for field in FIELDS: | |
| if stored[field] != current[field]: | |
| mismatch_rows.append({ | |
| "question_id": qid, | |
| "field": field, | |
| "eval_snapshot": json.dumps(stored[field], ensure_ascii=False), | |
| "current_corpus": json.dumps(current[field], ensure_ascii=False), | |
| }) | |
| new_current = sorted(set(corpus) - set(baseline)) | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| output = OUT / "metadata_mismatches.csv" | |
| with output.open("w", encoding="utf-8", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=["question_id", "field", "eval_snapshot", "current_corpus"]) | |
| writer.writeheader() | |
| writer.writerows(mismatch_rows) | |
| print("\n=== METADATA AUDIT ===") | |
| print("Current corpus questions:", len(corpus)) | |
| print("Unique eval snapshot questions:", len(baseline)) | |
| print("Questions missing from current corpus:", len(missing_current), missing_current[:15]) | |
| print("Questions in current corpus absent from snapshots:", len(new_current), new_current[:15]) | |
| print("Conflicting snapshot IDs:", len(set(conflicting_snapshots)), sorted(set(conflicting_snapshots))[:15]) | |
| print("Metadata mismatches:", len(mismatch_rows)) | |
| print("Mismatch counts by field:", dict(Counter(r["field"] for r in mismatch_rows))) | |
| for row in mismatch_rows[:10]: | |
| print(" ", row) | |
| print("Saved:", output) | |
| if mismatch_rows or missing_current or conflicting_snapshots: | |
| print("WARNING: Resolve discrepancies before publishing results derived from the current corpus.") | |
| else: | |
| print("PASS: All observed baseline question snapshots match current corpus fields.") | |
| if __name__ == "__main__": | |
| main() | |