feat(eval): add fallback statistics and failure summaries

evaluation/run_evaluations.py

@@ -223,6 +223,30 @@ def _percentile(values: list[float], percentile: float) -> float:
     return ordered[index]
 
 
+def _summarize_fallback_records(records: list[dict[str, Any]]) -> dict[str, Any]:
+    fallback_count = 0
+    reason_counter = Counter()
+    engine_counter = Counter()
+
+    for record in records:
+        if record.get("used_fallback"):
+            fallback_count += 1
+            reason_counter[str(record.get("fallback_reason") or "unknown")] += 1
+        engine_counter[str(record.get("engine_mode") or "unknown")] += 1
+
+    total = len(records)
+    return {
+        "fallback_count": fallback_count,
+        "fallback_rate": round(fallback_count / total, 4) if total else 0.0,
+        "fallback_reason_breakdown": dict(reason_counter),
+        "engine_mode_breakdown": dict(engine_counter),
+    }
+
+
+def _limit_cases(cases: list[dict[str, Any]], limit: int = 5) -> list[dict[str, Any]]:
+    return cases[:limit]
+
+
 def evaluate_intent_accuracy() -> dict[str, Any]:
     dataset = _load_dataset("intent_accuracy")
     details = []
@@ -353,6 +377,8 @@ def evaluate_latency(repeats: int) -> dict[str, Any]:
     all_total = []
     fallback_total = 0
     total_runs = 0
+    fallback_records = []
+    failure_cases = []
 
     for scenario in dataset:
         runs = []
@@ -378,8 +404,25 @@ def evaluate_latency(repeats: int) -> dict[str, Any]:
                     "engine_mode": telemetry.get("engine_mode"),
                 }
             )
+            fallback_records.append(runs[-1])
 
         total_values = [item["total_latency_ms"] for item in runs]
+        scenario_fallback_rate = sum(1 for item in runs if item["used_fallback"]) / len(runs)
+        if scenario_fallback_rate > 0:
+            failure_cases.append(
+                {
+                    "scenario_id": scenario["id"],
+                    "input": scenario["input"],
+                    "fallback_rate": round(scenario_fallback_rate, 4),
+                    "fallback_reasons": dict(
+                        Counter(
+                            str(item.get("fallback_reason") or "unknown")
+                            for item in runs
+                            if item["used_fallback"]
+                        )
+                    ),
+                }
+            )
         scenario_summaries.append(
             {
                 "id": scenario["id"],
@@ -387,14 +430,19 @@ def evaluate_latency(repeats: int) -> dict[str, Any]:
                 "repeats": repeats,
                 "avg_total_latency_ms": round(statistics.mean(total_values), 2),
                 "p95_total_latency_ms": round(_percentile(total_values, 95), 2),
-                "fallback_rate": round(
-                    sum(1 for item in runs if item["used_fallback"]) / len(runs),
-                    4,
+                "fallback_rate": round(scenario_fallback_rate, 4),
+                "fallback_reason_breakdown": dict(
+                    Counter(
+                        str(item.get("fallback_reason") or "unknown")
+                        for item in runs
+                        if item["used_fallback"]
+                    )
                 ),
                 "runs": runs,
             }
         )
 
+    fallback_summary = _summarize_fallback_records(fallback_records)
     return {
         "task": "latency",
         "scenario_count": len(dataset),
@@ -404,6 +452,10 @@ def evaluate_latency(repeats: int) -> dict[str, Any]:
         "avg_total_latency_ms": round(statistics.mean(all_total), 2) if all_total else 0.0,
         "p95_total_latency_ms": round(_percentile(all_total, 95), 2) if all_total else 0.0,
         "fallback_rate": round(fallback_total / total_runs, 4) if total_runs else 0.0,
+        "fallback_count": fallback_summary["fallback_count"],
+        "fallback_reason_breakdown": fallback_summary["fallback_reason_breakdown"],
+        "engine_mode_breakdown": fallback_summary["engine_mode_breakdown"],
+        "failure_cases": _limit_cases(failure_cases),
         "scenarios": scenario_summaries,
     }
 
@@ -412,6 +464,8 @@ def evaluate_branch_divergence() -> dict[str, Any]:
     dataset = _load_dataset("branch_divergence")
     group_summaries = []
     pair_scores = []
+    fallback_records = []
+    low_divergence_groups = []
 
     for group in dataset:
         branch_results = []
@@ -428,6 +482,15 @@ def evaluate_branch_divergence() -> dict[str, Any]:
                     "telemetry": run_result["final_result"].get("telemetry", {}),
                 }
             )
+            fallback_records.append(
+                {
+                    "used_fallback": bool(
+                        run_result["final_result"].get("telemetry", {}).get("used_fallback", False)
+                    ),
+                    "fallback_reason": run_result["final_result"].get("telemetry", {}).get("fallback_reason"),
+                    "engine_mode": run_result["final_result"].get("telemetry", {}).get("engine_mode"),
+                }
+            )
 
         group_pairs = []
         for left, right in combinations(branch_results, 2):
@@ -457,13 +520,23 @@ def evaluate_branch_divergence() -> dict[str, Any]:
             pair_scores.append(pair_score)
             group_pairs.append(pair_detail)
 
+        avg_pair_divergence = round(
+            statistics.mean([pair["pair_divergence_score"] for pair in group_pairs]),
+            4,
+        ) if group_pairs else 0.0
+        if avg_pair_divergence < 0.2:
+            low_divergence_groups.append(
+                {
+                    "group_id": group["id"],
+                    "avg_pair_divergence": avg_pair_divergence,
+                    "branch_labels": [branch["label"] for branch in branch_results],
+                }
+            )
+
         group_summaries.append(
             {
                 "id": group["id"],
-                "avg_pair_divergence": round(
-                    statistics.mean([pair["pair_divergence_score"] for pair in group_pairs]),
-                    4,
-                ) if group_pairs else 0.0,
+                "avg_pair_divergence": avg_pair_divergence,
                 "branches": [
                     {
                         "label": branch["label"],
@@ -478,6 +551,7 @@ def evaluate_branch_divergence() -> dict[str, Any]:
         )
 
     meaningful_pairs = sum(1 for score in pair_scores if score >= 0.2)
+    fallback_summary = _summarize_fallback_records(fallback_records)
     return {
         "task": "branch_divergence",
         "group_count": len(dataset),
@@ -486,6 +560,11 @@ def evaluate_branch_divergence() -> dict[str, Any]:
             meaningful_pairs / len(pair_scores),
             4,
         ) if pair_scores else 0.0,
+        "fallback_count": fallback_summary["fallback_count"],
+        "fallback_rate": fallback_summary["fallback_rate"],
+        "fallback_reason_breakdown": fallback_summary["fallback_reason_breakdown"],
+        "engine_mode_breakdown": fallback_summary["engine_mode_breakdown"],
+        "failure_cases": _limit_cases(low_divergence_groups),
         "groups": group_summaries,
     }
 
@@ -498,6 +577,69 @@ TASK_RUNNERS = {
 }
 
 
+def _build_failure_summary(results: dict[str, Any]) -> dict[str, Any]:
+    failure_summary: dict[str, Any] = {}
+
+    if "intent" in results:
+        intent_failures = [
+            {
+                "id": detail["id"],
+                "input": detail["input"],
+                "expected_intent": detail["expected_intent"],
+                "predicted_intent": detail["predicted_intent"],
+                "parser_source": detail["parser_source"],
+            }
+            for detail in results["intent"]["details"]
+            if not detail["intent_correct"]
+        ]
+        failure_summary["intent_failures"] = {
+            "count": len(intent_failures),
+            "cases": _limit_cases(intent_failures),
+        }
+
+    if "consistency" in results:
+        consistency_failures = [
+            {
+                "id": detail["id"],
+                "type": "action_guard",
+                "expected_valid": detail["expected_valid"],
+                "predicted_valid": detail["predicted_valid"],
+                "rejection_reason": detail["rejection_reason"],
+            }
+            for detail in results["consistency"]["action_guard_details"]
+            if not detail["correct"]
+        ]
+        consistency_failures.extend(
+            {
+                "id": detail["id"],
+                "type": "state_check",
+                "expected_contradiction": detail["expected_contradiction"],
+                "predicted_contradiction": detail["predicted_contradiction"],
+                "contradictions": detail["contradictions"],
+            }
+            for detail in results["consistency"]["state_check_details"]
+            if not detail["correct"]
+        )
+        failure_summary["consistency_failures"] = {
+            "count": len(consistency_failures),
+            "cases": _limit_cases(consistency_failures),
+        }
+
+    if "latency" in results:
+        failure_summary["latency_failures"] = {
+            "count": len(results["latency"].get("failure_cases", [])),
+            "cases": _limit_cases(results["latency"].get("failure_cases", [])),
+        }
+
+    if "branch" in results:
+        failure_summary["branch_failures"] = {
+            "count": len(results["branch"].get("failure_cases", [])),
+            "cases": _limit_cases(results["branch"].get("failure_cases", [])),
+        }
+
+    return failure_summary
+
+
 def _build_summary(results: dict[str, Any]) -> dict[str, Any]:
     summary = {}
     if "intent" in results:
@@ -507,8 +649,10 @@ def _build_summary(results: dict[str, Any]) -> dict[str, Any]:
     if "latency" in results:
         summary["avg_total_latency_ms"] = results["latency"]["avg_total_latency_ms"]
         summary["latency_fallback_rate"] = results["latency"]["fallback_rate"]
+        summary["latency_fallback_count"] = results["latency"]["fallback_count"]
     if "branch" in results:
         summary["avg_pair_divergence"] = results["branch"]["avg_pair_divergence"]
+        summary["branch_fallback_rate"] = results["branch"]["fallback_rate"]
     return summary
 
 
@@ -541,6 +685,7 @@ def main() -> int:
         "generated_at": datetime.now().isoformat(timespec="seconds"),
         "task": args.task,
         "summary": _build_summary(task_results),
+        "failure_summary": _build_failure_summary(task_results),
         "results": task_results,
     }