File size: 5,787 Bytes
e35b73e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | #!/usr/bin/env python3
"""
Normalized confusion matrices + accuracy analysis for all four systems
vs ground truth from query_translation_eval.csv.
"""
from __future__ import annotations
import csv, re
from pathlib import Path
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
ROOT = Path(__file__).resolve().parents[1]
# ── Load ground truth ────────────────────────────────────────────────────────
def load_ground_truth() -> list[str]:
gt = []
with (ROOT / "datasets" / "query_translation_eval.csv").open(encoding="utf-8") as f:
for row in csv.DictReader(f):
gt.append(row["answer"].strip().upper())
return gt # 100 entries, "T" or "F"
# ── Load baseline trans_query_eval.csv (end-to-end via baseline UPPAAL model) ──
def load_raw_qna(path: Path) -> list[str]:
preds = []
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
m = re.search(r":\s*([TF])\s*$", line, re.IGNORECASE)
if m:
preds.append(m.group(1).upper())
return preds
# ── Load our e2e results ──────────────────────────────────────────────────────
def load_ours() -> list[str]:
preds = []
with (ROOT / "artifacts" / "e2e_eval" / "e2e_query_eval.csv").open(encoding="utf-8") as f:
for row in csv.DictReader(f):
v = row["verdict"].strip().upper()
preds.append(v if v in ("T", "F") else "?")
return preds
# ── Confusion matrix (normalized by true label) ───────────────────────────────
def confusion(gt: list[str], pred: list[str]):
"""Returns 2x2 numpy array normalized by row (true label).
Rows = actual [T, F], Cols = predicted [T, F]
"""
cm = np.zeros((2, 2), dtype=float)
for g, p in zip(gt, pred):
if g not in ("T", "F") or p not in ("T", "F"):
continue
ri = 0 if g == "T" else 1
ci = 0 if p == "T" else 1
cm[ri, ci] += 1
# normalize by row
row_sums = cm.sum(axis=1, keepdims=True)
row_sums[row_sums == 0] = 1
return cm / row_sums, cm # normalized, raw
def accuracy(gt, pred):
correct = sum(g == p for g, p in zip(gt, pred) if g in ("T","F") and p in ("T","F"))
total = sum(1 for g in gt if g in ("T","F"))
return correct / total if total else 0.0
def precision_recall_f1(gt, pred):
tp = sum(1 for g,p in zip(gt,pred) if g=="T" and p=="T")
fp = sum(1 for g,p in zip(gt,pred) if g=="F" and p=="T")
fn = sum(1 for g,p in zip(gt,pred) if g=="T" and p=="F")
tn = sum(1 for g,p in zip(gt,pred) if g=="F" and p=="F")
prec = tp/(tp+fp) if (tp+fp) else 0.0
rec = tp/(tp+fn) if (tp+fn) else 0.0
f1 = 2*prec*rec/(prec+rec) if (prec+rec) else 0.0
return prec, rec, f1, tp, fp, fn, tn
# ── Main ──────────────────────────────────────────────────────────────────────
gt = load_ground_truth()
systems = {
"Ours\n(pipeline)": load_ours(),
"Claude\n(direct)": load_raw_qna(ROOT / "artifacts/baselines/claude/raw_qna.txt"),
"GPT\n(direct)": load_raw_qna(ROOT / "artifacts/baselines/gpt/raw_qna.txt"),
"Grok\n(direct)": load_raw_qna(ROOT / "artifacts/baselines/grok/raw_qna.txt"),
}
# ── Print stats table ─────────────────────────────────────────────────────────
print(f"\n{'System':<22} {'Acc':>5} {'Prec':>5} {'Rec':>5} {'F1':>5} {'TP':>3} {'TN':>3} {'FP':>3} {'FN':>3}")
print("-" * 70)
stats = {}
for name, pred in systems.items():
acc = accuracy(gt, pred)
prec, rec, f1, tp, fp, fn, tn = precision_recall_f1(gt, pred)
label = name.replace("\n", " ")
print(f"{label:<22} {acc:.3f} {prec:.3f} {rec:.3f} {f1:.3f} {tp:>3} {tn:>3} {fp:>3} {fn:>3}")
stats[name] = (acc, prec, rec, f1)
# ── Plot: 1 row of 4 normalized confusion matrices ───────────────────────────
fig, axes = plt.subplots(1, 4, figsize=(14, 4))
fig.suptitle("Normalized Confusion Matrices — QA Verdict Prediction\n(rows = actual, cols = predicted; normalized by true label)",
fontsize=12, y=1.02)
labels = ["T (True)", "F (False)"]
cmap = "Blues"
for ax, (name, pred) in zip(axes, systems.items()):
cm_norm, cm_raw = confusion(gt, pred)
acc = stats[name][0]
im = ax.imshow(cm_norm, vmin=0, vmax=1, cmap=cmap)
# Cell annotations: normalized value + raw count
for i in range(2):
for j in range(2):
val = cm_norm[i, j]
raw = int(cm_raw[i, j])
color = "white" if val > 0.6 else "black"
ax.text(j, i, f"{val:.2f}\n({raw})", ha="center", va="center",
fontsize=11, color=color, fontweight="bold")
ax.set_xticks([0, 1]); ax.set_yticks([0, 1])
ax.set_xticklabels(labels, fontsize=9)
ax.set_yticklabels(labels, fontsize=9)
ax.set_xlabel("Predicted", fontsize=10)
ax.set_ylabel("Actual", fontsize=10)
ax.set_title(f"{name}\nAcc = {acc:.2f}", fontsize=10)
plt.colorbar(im, ax=axes[-1], fraction=0.046, pad=0.04, label="Proportion")
plt.tight_layout()
out = ROOT / "docs" / "confusion_matrices.png"
plt.savefig(out, dpi=150, bbox_inches="tight")
print(f"\nSaved -> {out}")
plt.show()
|