File size: 2,546 Bytes
9d4467b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | """
Evaluate the reasoning classifier on evaluation CSV.
Reports accuracy, ROC-AUC, and calibration (Brier score).
"""
import argparse
import csv
from typing import List, Dict
import joblib
from sklearn.metrics import accuracy_score, roc_auc_score, brier_score_loss
from backend.models.reasoning_classifier import build_feature_vector, default_model_path
def load_csv(path: str) -> List[Dict[str, str]]:
with open(path, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
return list(reader)
def parse_bool(value: str) -> int:
if isinstance(value, bool):
return int(value)
v = str(value).strip().lower()
return 1 if v in ["1", "true", "yes"] else 0
def build_features_from_row(row: Dict[str, str]) -> List[float]:
return build_feature_vector(
symbolic_score=float(row.get("symbolic_score", 0.0)),
logical_score=float(row.get("logical_score", 0.0)),
avg_consensus=float(row.get("avg_consensus", 0.0)),
hallucination_rate=float(row.get("hallucination_rate", 0.0)),
num_flagged_steps=int(float(row.get("num_flagged_steps", 0))),
ocr_confidence=float(row.get("ocr_confidence", 1.0)),
steps_count=int(float(row.get("steps_count", 0))),
reasoning_length=int(float(row.get("reasoning_length", 0))),
consensus_total_steps=int(float(row.get("consensus_total_steps", 0)))
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--csv", type=str, default="evaluation_results.csv", help="Evaluation CSV")
parser.add_argument("--model", type=str, default="", help="Model path")
parser.add_argument("--split", type=str, default="test", help="Split name to evaluate")
args = parser.parse_args()
rows = load_csv(args.csv)
rows = [r for r in rows if r.get("split") == args.split]
if not rows:
raise ValueError("No rows found for requested split.")
X = [build_features_from_row(r) for r in rows]
y = [parse_bool(r.get("is_correct", 0)) for r in rows]
model_path = args.model or default_model_path()
model = joblib.load(model_path)
y_pred = model.predict(X)
y_proba = model.predict_proba(X)[:, 1]
acc = accuracy_score(y, y_pred)
try:
auc = roc_auc_score(y, y_proba)
except Exception:
auc = 0.0
brier = brier_score_loss(y, y_proba)
print(f"[RESULT] Split: {args.split}")
print(f"Accuracy: {acc:.4f}")
print(f"ROC-AUC: {auc:.4f}")
print(f"Brier: {brier:.4f}")
if __name__ == "__main__":
main()
|