feat(core): stabilize benchmark pipeline with accurate latency tracking, retry-empty handling, and refined plots

adapters/llm/openai_provider.py

@@ -147,6 +147,7 @@ CRITICAL RULES:
 6. Use lowercase for SQL keywords (select, from, where, etc.)
 7. Do not add unnecessary parentheses or formatting
 8. Match exact column and table names from the schema (case-sensitive)
+9. NEVER return empty SQL. If unsure, return the simplest valid SQL that answers the question.
 
 IMPORTANT:
 - For counting all rows: Use COUNT(*) not COUNT(column_name)

benchmarks/evaluate_spider_pro.py

@@ -66,31 +66,43 @@ def extract_clean_sql(text: str | None) -> str:
 
 
 def normalize_sql(sql: str) -> str:
-    """Light normalization to make EM stricter-but-fair."""
+    """
+    Conservative normalization for exact-match (EM):
+    - Trim, collapse spaces, drop trailing ';'
+    - Drop trailing 'LIMIT n'
+    - Remove table prefixes only in single-table, no-join queries
+    - Unquote identifiers like `name` or "name"
+    - Uppercase common SQL keywords (string literals unaffected)
+    """
     if not sql:
         return ""
     s = sql.strip()
-    # unify case but keep literals recognizable
-    s = re.sub(r"\s+", " ", s).strip()
-    s = s.rstrip(";")
 
-    # drop table prefixes a.b -> b
-    s = re.sub(r"\b\w+\.(\w+)\b", r"\1", s)
+    # Collapse whitespace early and drop trailing ';'
+    s = re.sub(r"\s+", " ", s).strip().rstrip(";")
+
+    # Drop trailing LIMIT n
+    s = re.sub(r"(?i)\s+LIMIT\s+\d+\s*$", "", s)
+
+    # Remove table prefixes only if single FROM and no JOIN
+    lower = s.lower()
+    if lower.count(" from ") == 1 and " join " not in lower:
+        m = re.search(r"(?i)\bfrom\s+([a-z_][a-z0-9_]*)", s, flags=re.IGNORECASE)
+        if m:
+            table = m.group(1)
+            s = re.sub(rf"\b{re.escape(table)}\.(\w+)\b", r"\1", s)
 
-    # collapse quotes around identifiers
+    # Unquote identifiers: `foo` -> foo, "foo" -> foo (strings '...' remain)
     s = re.sub(r"`([A-Za-z_]\w*)`", r"\1", s)
     s = re.sub(r'"([A-Za-z_]\w*)"', r"\1", s)
 
-    # COUNT(foo) -> COUNT(*), DISTINCT inside COUNT -> COUNT(*)
-    s = re.sub(r"(?i)COUNT\s*\(\s*DISTINCT\s+[^)]+\)", "COUNT(*)", s)
-    s = re.sub(r"(?i)COUNT\s*\(\s*[A-Za-z_]\w*\s*\)", "COUNT(*)", s)
+    # Normalize comma spacing: "a ,  b" -> "a, b"
+    s = re.sub(r"\s*,\s*", ", ", s)
 
-    # strip trailing LIMIT n
-    s = re.sub(r"(?i)\s+LIMIT\s+\d+\s*$", "", s)
-
-    # canonical whitespace + upper keywords for stability
+    # Final whitespace collapse
     s = re.sub(r"\s+", " ", s).strip()
-    # keyword upper (a bit heuristic)
+
+    # Uppercase common keywords (word-boundary safe)
     for kw in [
         "select",
         "from",
@@ -107,6 +119,7 @@ def normalize_sql(sql: str) -> str:
         "desc",
     ]:
         s = re.sub(rf"(?i)\b{kw}\b", kw.upper(), s)
+
     return s
 
 

benchmarks/results_pro/20251109-125209/eval.jsonl

@@ -0,0 +1,5 @@
+{"source": "spider", "db_id": "concert_singer", "query": "How many singers do we have?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 9129, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 6689, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 2430, "summary": "failed", "notes": {"rationale_len": 30}}, {"stage": "safety", "duration_ms": 2, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 2, "summary": "failed", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the total number of singers?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 10009, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 9046, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 960, "summary": "failed", "notes": {"rationale_len": 30}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 2, "summary": "failed", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "Show name, country, age for all singers ordered by age from the oldest to the youngest.", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "", "ok": true, "latency_ms": 0, "em": 0.0, "sm": 0.0, "exec_acc": 0.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "ambiguous", "notes": {"ambiguous": true, "questions_len": 1}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What are the names, countries, and ages for every singer in descending order of age?", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "select Name, Country, Age from singer order by Age desc", "ok": true, "latency_ms": 9560, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 8063, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 1494, "summary": "failed", "notes": {"rationale_len": 85}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 6, "col_count": 3, "sql_length": 55}}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"sql_length": 55, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": false, "mixes_cols": false, "verified": true}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the average, minimum, and maximum age of all singers from France?", "gold_sql": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "pred_sql": "select avg(Age), min(Age), max(Age) from singer where Country = 'France'", "ok": true, "latency_ms": 12661, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 7869, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 2410, "summary": "failed", "notes": {"rationale_len": 67}}, {"stage": "safety", "duration_ms": 3, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 2, "summary": "failed", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}}, {"stage": "verifier", "duration_ms": 1, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}}, {"stage": "repair", "duration_ms": 1179, "summary": "failed", "notes": {"old_sql_len": 72, "new_sql_len": 80}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 3, "sql_length": 80}}, {"stage": "verifier", "duration_ms": 1, "summary": "failed", "notes": {"sql_length": 80, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}}, {"stage": "repair", "duration_ms": 1191, "summary": "failed", "notes": {"old_sql_len": 80, "new_sql_len": 72}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}}, {"stage": "verifier", "duration_ms": 1, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}}, {"stage": "pipeline", "duration_ms": 1, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}

benchmarks/results_pro/20251109-125209/results.csv

@@ -0,0 +1,6 @@
+db_id,query,ok,em,sm,exec_acc,latency_ms
+concert_singer,"How many singers do we have?",✅,1.0,1.0,1.0,9129
+concert_singer,"What is the total number of singers?",✅,1.0,1.0,1.0,10009
+concert_singer,"Show name, country, age for all singers ordered by age from the oldest to the youngest.",✅,0.0,0.0,0.0,0
+concert_singer,"What are the names, countries, and ages for every singer in descending order of age?",✅,0.0,1.0,1.0,9560
+concert_singer,"What is the average, minimum, and maximum age of all singers from France?",✅,0.0,1.0,1.0,12661

benchmarks/results_pro/20251109-125209/summary.json

@@ -0,0 +1,21 @@
+{
+  "timestamp": "2025-11-09T12:52:50",
+  "split": "dev",
+  "config": "configs/sqlite_pipeline.yaml",
+  "total": 5,
+  "success": 5,
+  "success_rate": 1.0,
+  "avg_latency_ms": 8271.8,
+  "p50_latency_ms": 9560.0,
+  "p95_latency_ms": 12130.6,
+  "EM": 0.4,
+  "SM": 0.8,
+  "ExecAcc": 0.8,
+  "detector_avg_ms": 1.0,
+  "planner_avg_ms": 7916.75,
+  "generator_avg_ms": 1823.5,
+  "safety_avg_ms": 1.5,
+  "executor_avg_ms": 1.5,
+  "verifier_avg_ms": 1.0,
+  "repair_avg_ms": 1185.0
+}

benchmarks/results_pro/20251109-125509/eval.jsonl

@@ -0,0 +1,5 @@
+{"source": "spider", "db_id": "concert_singer", "query": "How many singers do we have?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 7007, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 5977, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 1019, "summary": "failed", "notes": {"rationale_len": 30}}, {"stage": "safety", "duration_ms": 3, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 4, "summary": "failed", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the total number of singers?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 5715, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 4893, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 819, "summary": "failed", "notes": {"rationale_len": 30}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "Show name, country, age for all singers ordered by age from the oldest to the youngest.", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "", "ok": true, "latency_ms": 0, "em": 0.0, "sm": 0.0, "exec_acc": 0.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "ambiguous", "notes": {"ambiguous": true, "questions_len": 1}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What are the names, countries, and ages for every singer in descending order of age?", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "select Name, Country, Age from singer order by Age desc", "ok": true, "latency_ms": 7982, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 6702, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 1278, "summary": "failed", "notes": {"rationale_len": 85}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 6, "col_count": 3, "sql_length": 55}}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"sql_length": 55, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": false, "mixes_cols": false, "verified": true}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the average, minimum, and maximum age of all singers from France?", "gold_sql": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "pred_sql": "select avg(Age), min(Age), max(Age) from singer where Country = 'France'", "ok": true, "latency_ms": 11942, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 8334, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 1279, "summary": "failed", "notes": {"rationale_len": 71}}, {"stage": "safety", "duration_ms": 2, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}}, {"stage": "verifier", "duration_ms": 1, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}}, {"stage": "repair", "duration_ms": 1242, "summary": "failed", "notes": {"old_sql_len": 72, "new_sql_len": 80}}, {"stage": "safety", "duration_ms": 2, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 3, "sql_length": 80}}, {"stage": "verifier", "duration_ms": 1, "summary": "failed", "notes": {"sql_length": 80, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}}, {"stage": "repair", "duration_ms": 1075, "summary": "failed", "notes": {"old_sql_len": 80, "new_sql_len": 72}}, {"stage": "safety", "duration_ms": 2, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}}, {"stage": "verifier", "duration_ms": 1, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}}, {"stage": "pipeline", "duration_ms": 1, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}

benchmarks/results_pro/20251109-125509/results.csv

@@ -0,0 +1,6 @@
+db_id,query,ok,em,sm,exec_acc,latency_ms
+concert_singer,"How many singers do we have?",✅,1.0,1.0,1.0,7007
+concert_singer,"What is the total number of singers?",✅,1.0,1.0,1.0,5715
+concert_singer,"Show name, country, age for all singers ordered by age from the oldest to the youngest.",✅,0.0,0.0,0.0,0
+concert_singer,"What are the names, countries, and ages for every singer in descending order of age?",✅,0.0,1.0,1.0,7982
+concert_singer,"What is the average, minimum, and maximum age of all singers from France?",✅,0.0,1.0,1.0,11942

benchmarks/results_pro/20251109-125509/summary.json

@@ -0,0 +1,21 @@
+{
+  "timestamp": "2025-11-09T12:55:42",
+  "split": "dev",
+  "config": "configs/sqlite_pipeline.yaml",
+  "total": 5,
+  "success": 5,
+  "success_rate": 1.0,
+  "avg_latency_ms": 6529.2,
+  "p50_latency_ms": 7007.0,
+  "p95_latency_ms": 11150.0,
+  "EM": 0.4,
+  "SM": 0.8,
+  "ExecAcc": 0.8,
+  "detector_avg_ms": 1.0,
+  "planner_avg_ms": 6476.5,
+  "generator_avg_ms": 1098.75,
+  "safety_avg_ms": 1.83,
+  "executor_avg_ms": 1.5,
+  "verifier_avg_ms": 1.0,
+  "repair_avg_ms": 1158.5
+}