fix(core): non-zero generator timing + one-shot EMPTY retry; post-verify drop LIMIT to recover EM when ExecAcc=1

benchmarks/results_pro/20251109-100618/eval.jsonl

@@ -1,5 +0,0 @@
-{"source": "spider", "db_id": "concert_singer", "query": "How many singers do we have?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer limit 1", "ok": true, "latency_ms": 10217, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 7626, "summary": "ok", "notes": {"len_plan": 1263}, "token_in": 265, "token_out": 303, "cost_usd": 0.00022154999999999996}, {"stage": "generator", "duration_ms": 1176, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 785, "token_out": 19, "cost_usd": 0.00012915}, {"stage": "safety", "duration_ms": 3, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 3, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 2, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "repair", "duration_ms": 751, "summary": "ok", "notes": {"old_sql_len": 27, "new_sql_len": 27}, "token_in": 313, "token_out": 6, "cost_usd": 5.0549999999999995e-05}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "repair", "duration_ms": 643, "summary": "ok", "notes": {"old_sql_len": 27, "new_sql_len": 35}, "token_in": 313, "token_out": 8, "cost_usd": 5.175e-05}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 2, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
-{"source": "spider", "db_id": "concert_singer", "query": "What is the total number of singers?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer limit 1", "ok": true, "latency_ms": 10180, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 7783, "summary": "ok", "notes": {"len_plan": 1307}, "token_in": 266, "token_out": 307, "cost_usd": 0.00022409999999999997}, {"stage": "generator", "duration_ms": 939, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 790, "token_out": 19, "cost_usd": 0.00012989999999999999}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 2, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 2, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "repair", "duration_ms": 711, "summary": "ok", "notes": {"old_sql_len": 27, "new_sql_len": 35}, "token_in": 313, "token_out": 8, "cost_usd": 5.175e-05}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "repair", "duration_ms": 732, "summary": "ok", "notes": {"old_sql_len": 35, "new_sql_len": 35}, "token_in": 316, "token_out": 8, "cost_usd": 5.2199999999999995e-05}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
-{"source": "spider", "db_id": "concert_singer", "query": "Show name, country, age for all singers ordered by age from the oldest to the youngest.", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "", "ok": true, "latency_ms": 0, "em": 0.0, "sm": 0.0, "exec_acc": 0.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "ambiguous", "notes": {"ambiguous": true, "questions_len": 1}}]}
-{"source": "spider", "db_id": "concert_singer", "query": "What are the names, countries, and ages for every singer in descending order of age?", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "select Name, Country, Age from singer order by Age desc LIMIT 10", "ok": true, "latency_ms": 7726, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 4966, "summary": "ok", "notes": {"len_plan": 1207}, "token_in": 276, "token_out": 278, "cost_usd": 0.0002082}, {"stage": "generator", "duration_ms": 1007, "summary": "ok", "notes": {"rationale_len": 85}, "token_in": 771, "token_out": 37, "cost_usd": 0.00013785}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "repair", "duration_ms": 938, "summary": "ok", "notes": {"old_sql_len": 55, "new_sql_len": 64}, "token_in": 320, "token_out": 21, "cost_usd": 6.0599999999999996e-05}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "repair", "duration_ms": 807, "summary": "ok", "notes": {"old_sql_len": 64, "new_sql_len": 64}, "token_in": 323, "token_out": 21, "cost_usd": 6.104999999999999e-05}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
-{"source": "spider", "db_id": "concert_singer", "query": "What is the average, minimum, and maximum age of all singers from France?", "gold_sql": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "pred_sql": "select avg(Age), min(Age), max(Age) from singer where Country = 'France'", "ok": true, "latency_ms": 16635, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 13290, "summary": "ok", "notes": {"len_plan": 1641}, "token_in": 274, "token_out": 434, "cost_usd": 0.0003015}, {"stage": "generator", "duration_ms": 1083, "summary": "ok", "notes": {"rationale_len": 67}, "token_in": 925, "token_out": 42, "cost_usd": 0.00016394999999999997}, {"stage": "safety", "duration_ms": 0, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["aggregation_without_group_by", "exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "repair", "duration_ms": 1072, "summary": "ok", "notes": {"old_sql_len": 72, "new_sql_len": 80}, "token_in": 328, "token_out": 24, "cost_usd": 6.36e-05}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["aggregation_without_group_by", "exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "repair", "duration_ms": 1179, "summary": "ok", "notes": {"old_sql_len": 80, "new_sql_len": 72}, "token_in": 332, "token_out": 21, "cost_usd": 6.24e-05}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["aggregation_without_group_by", "exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}

benchmarks/results_pro/20251109-103601/eval.jsonl

@@ -1,5 +0,0 @@
-{"source": "spider", "db_id": "concert_singer", "query": "How many singers do we have?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer limit 1", "ok": true, "latency_ms": 10975, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 8073, "summary": "ok", "notes": {"len_plan": 1533}, "token_in": 265, "token_out": 384, "cost_usd": 0.00027015, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "generator", "duration_ms": 959, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 866, "token_out": 19, "cost_usd": 0.0001413, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 2, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 884, "summary": "ok", "notes": {"old_sql_len": 27, "new_sql_len": 35, "attempt": 1}, "token_in": 313, "token_out": 8, "cost_usd": 5.175e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 1040, "summary": "ok", "notes": {"old_sql_len": 35, "new_sql_len": 35, "attempt": 2}, "token_in": 316, "token_out": 8, "cost_usd": 5.2199999999999995e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}, "skipped": false}]}
-{"source": "spider", "db_id": "concert_singer", "query": "What is the total number of singers?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer limit 1", "ok": true, "latency_ms": 11792, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 8633, "summary": "ok", "notes": {"len_plan": 1444}, "token_in": 266, "token_out": 354, "cost_usd": 0.00025229999999999995, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "generator", "duration_ms": 1406, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 837, "token_out": 19, "cost_usd": 0.00013695, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 988, "summary": "ok", "notes": {"old_sql_len": 27, "new_sql_len": 35, "attempt": 1}, "token_in": 313, "token_out": 8, "cost_usd": 5.175e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 755, "summary": "ok", "notes": {"old_sql_len": 35, "new_sql_len": 35, "attempt": 2}, "token_in": 316, "token_out": 8, "cost_usd": 5.2199999999999995e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}, "skipped": false}]}
-{"source": "spider", "db_id": "concert_singer", "query": "Show name, country, age for all singers ordered by age from the oldest to the youngest.", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "", "ok": true, "latency_ms": 0, "em": 0.0, "sm": 0.0, "exec_acc": 0.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "ambiguous", "notes": {"ambiguous": true, "questions_len": 1}, "skipped": false}]}
-{"source": "spider", "db_id": "concert_singer", "query": "What are the names, countries, and ages for every singer in descending order of age?", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "select Name, Country, Age from singer order by Age desc LIMIT 10", "ok": true, "latency_ms": 9181, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 5624, "summary": "ok", "notes": {"len_plan": 1296}, "token_in": 276, "token_out": 297, "cost_usd": 0.00021959999999999997, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "generator", "duration_ms": 1398, "summary": "ok", "notes": {"rationale_len": 85}, "token_in": 790, "token_out": 37, "cost_usd": 0.0001407, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 948, "summary": "ok", "notes": {"old_sql_len": 55, "new_sql_len": 64, "attempt": 1}, "token_in": 320, "token_out": 21, "cost_usd": 6.0599999999999996e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 1200, "summary": "ok", "notes": {"old_sql_len": 64, "new_sql_len": 64, "attempt": 2}, "token_in": 323, "token_out": 21, "cost_usd": 6.104999999999999e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}, "skipped": false}]}
-{"source": "spider", "db_id": "concert_singer", "query": "What is the average, minimum, and maximum age of all singers from France?", "gold_sql": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "pred_sql": "select avg(Age), min(Age), max(Age) from singer where Country = 'France'", "ok": true, "latency_ms": 14419, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 9792, "summary": "ok", "notes": {"len_plan": 1406}, "token_in": 274, "token_out": 348, "cost_usd": 0.0002499, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "generator", "duration_ms": 1727, "summary": "ok", "notes": {"rationale_len": 90}, "token_in": 839, "token_out": 46, "cost_usd": 0.00015345, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 4, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["aggregation_without_group_by", "exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 1130, "summary": "ok", "notes": {"old_sql_len": 72, "new_sql_len": 80, "attempt": 1}, "token_in": 328, "token_out": 24, "cost_usd": 6.36e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["aggregation_without_group_by", "exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 1752, "summary": "ok", "notes": {"old_sql_len": 80, "new_sql_len": 72, "attempt": 2}, "token_in": 332, "token_out": 25, "cost_usd": 6.48e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["aggregation_without_group_by", "exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}, "skipped": false}]}

benchmarks/results_pro/20251109-105728/eval.jsonl

@@ -1,5 +0,0 @@
-{"source": "spider", "db_id": "concert_singer", "query": "How many singers do we have?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer limit 1", "ok": true, "latency_ms": 11836, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 6838, "summary": "ok", "notes": {"len_plan": 1460}, "token_in": 265, "token_out": 356, "cost_usd": 0.00025334999999999995, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "generator", "duration_ms": 3409, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 838, "token_out": 19, "cost_usd": 0.0001371, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 832, "summary": "ok", "notes": {"old_sql_len": 27, "new_sql_len": 35, "attempt": 1}, "token_in": 313, "token_out": 8, "cost_usd": 5.175e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 35}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 744, "summary": "ok", "notes": {"old_sql_len": 35, "new_sql_len": 35, "attempt": 2}, "token_in": 316, "token_out": 8, "cost_usd": 5.2199999999999995e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 35}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}, "skipped": false}]}
-{"source": "spider", "db_id": "concert_singer", "query": "What is the total number of singers?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer limit 1", "ok": true, "latency_ms": 10414, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 5346, "summary": "ok", "notes": {"len_plan": 1385}, "token_in": 266, "token_out": 334, "cost_usd": 0.00024029999999999999, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "generator", "duration_ms": 3352, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 817, "token_out": 19, "cost_usd": 0.00013394999999999998, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 4, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 871, "summary": "ok", "notes": {"old_sql_len": 27, "new_sql_len": 35, "attempt": 1}, "token_in": 313, "token_out": 8, "cost_usd": 5.175e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 35}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 831, "summary": "ok", "notes": {"old_sql_len": 35, "new_sql_len": 35, "attempt": 2}, "token_in": 316, "token_out": 8, "cost_usd": 5.2199999999999995e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 35}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}, "skipped": false}]}
-{"source": "spider", "db_id": "concert_singer", "query": "Show name, country, age for all singers ordered by age from the oldest to the youngest.", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "", "ok": true, "latency_ms": 0, "em": 0.0, "sm": 0.0, "exec_acc": 0.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "ambiguous", "notes": {"ambiguous": true, "questions_len": 1}, "skipped": false}]}
-{"source": "spider", "db_id": "concert_singer", "query": "What are the names, countries, and ages for every singer in descending order of age?", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "select Name, Country, Age from singer order by Age desc LIMIT 10", "ok": true, "latency_ms": 13807, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 8248, "summary": "ok", "notes": {"len_plan": 1415}, "token_in": 276, "token_out": 335, "cost_usd": 0.0002424, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "generator", "duration_ms": 3686, "summary": "ok", "notes": {"rationale_len": 85}, "token_in": 828, "token_out": 37, "cost_usd": 0.00014639999999999998, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3, "sql_length": 55}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 960, "summary": "ok", "notes": {"old_sql_len": 55, "new_sql_len": 64, "attempt": 1}, "token_in": 320, "token_out": 21, "cost_usd": 6.0599999999999996e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3, "sql_length": 64}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 901, "summary": "ok", "notes": {"old_sql_len": 64, "new_sql_len": 64, "attempt": 2}, "token_in": 323, "token_out": 21, "cost_usd": 6.104999999999999e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3, "sql_length": 64}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}, "skipped": false}]}
-{"source": "spider", "db_id": "concert_singer", "query": "What is the average, minimum, and maximum age of all singers from France?", "gold_sql": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "pred_sql": "select avg(Age), min(Age), max(Age) from singer where Country = 'France'", "ok": true, "latency_ms": 13396, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 7141, "summary": "ok", "notes": {"len_plan": 1569}, "token_in": 274, "token_out": 404, "cost_usd": 0.0002835, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "generator", "duration_ms": 4139, "summary": "ok", "notes": {"rationale_len": 87}, "token_in": 895, "token_out": 46, "cost_usd": 0.00016184999999999998, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["aggregation_without_group_by", "exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 937, "summary": "ok", "notes": {"old_sql_len": 72, "new_sql_len": 80, "attempt": 1}, "token_in": 328, "token_out": 24, "cost_usd": 6.36e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 3, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3, "sql_length": 80}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["aggregation_without_group_by", "exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 1160, "summary": "ok", "notes": {"old_sql_len": 80, "new_sql_len": 72, "attempt": 2}, "token_in": 332, "token_out": 21, "cost_usd": 6.24e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 3, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 2, "summary": "ok", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"issues": ["aggregation_without_group_by", "exec_error:preview_failed"]}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}, "skipped": false}, {"stage": "pipeline", "duration_ms": 0, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}, "skipped": false}]}

benchmarks/results_pro/20251109-105728/results.csv

@@ -1,6 +0,0 @@
-db_id,query,ok,em,sm,exec_acc,latency_ms
-concert_singer,"How many singers do we have?",✅,1.0,1.0,1.0,11836
-concert_singer,"What is the total number of singers?",✅,1.0,1.0,1.0,10414
-concert_singer,"Show name, country, age for all singers ordered by age from the oldest to the youngest.",✅,0.0,0.0,0.0,0
-concert_singer,"What are the names, countries, and ages for every singer in descending order of age?",✅,0.0,1.0,1.0,13807
-concert_singer,"What is the average, minimum, and maximum age of all singers from France?",✅,0.0,1.0,1.0,13396

benchmarks/results_pro/20251109-105728/summary.json

@@ -1,21 +0,0 @@
-{
-  "timestamp": "2025-11-09T10:58:17",
-  "split": "dev",
-  "config": "configs/sqlite_pipeline.yaml",
-  "total": 5,
-  "success": 5,
-  "success_rate": 1.0,
-  "avg_latency_ms": 9890.6,
-  "p50_latency_ms": 11836.0,
-  "p95_latency_ms": 13724.8,
-  "EM": 0.4,
-  "SM": 0.8,
-  "ExecAcc": 0.8,
-  "detector_avg_ms": 0.0,
-  "planner_avg_ms": 6893.25,
-  "generator_avg_ms": 3646.5,
-  "safety_avg_ms": 1.67,
-  "executor_avg_ms": 1.33,
-  "verifier_avg_ms": 0.42,
-  "repair_avg_ms": 904.5
-}

benchmarks/results_pro/20251109-123424/eval.jsonl

@@ -0,0 +1,5 @@
+{"source": "spider", "db_id": "concert_singer", "query": "How many singers do we have?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 10712, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 8455, "summary": "ok", "notes": {}, "skipped": false}, {"stage": "generator", "duration_ms": 2253, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 792, "token_out": 19, "cost_usd": 0.0001302, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the total number of singers?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 12981, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 9963, "summary": "ok", "notes": {}, "skipped": false}, {"stage": "generator", "duration_ms": 3018, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 800, "token_out": 19, "cost_usd": 0.0001314, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 0, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 0, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}]}
+{"source": "spider", "db_id": "concert_singer", "query": "Show name, country, age for all singers ordered by age from the oldest to the youngest.", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "", "ok": true, "latency_ms": 0, "em": 0.0, "sm": 0.0, "exec_acc": 0.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "ambiguous", "notes": {"ambiguous": true, "questions_len": 1}, "skipped": false}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What are the names, countries, and ages for every singer in descending order of age?", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "select Name, Country, Age from singer order by Age desc", "ok": true, "latency_ms": 9753, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 7528, "summary": "ok", "notes": {}, "skipped": false}, {"stage": "generator", "duration_ms": 2224, "summary": "ok", "notes": {"rationale_len": 85}, "token_in": 816, "token_out": 37, "cost_usd": 0.0001446, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3, "sql_length": 55}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"sql_length": 55, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": false, "mixes_cols": false, "verified": true}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the average, minimum, and maximum age of all singers from France?", "gold_sql": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "pred_sql": "select avg(Age), min(Age), max(Age) from singer where Country = 'France'", "ok": false, "latency_ms": 12406, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 7105, "summary": "ok", "notes": {}, "skipped": false}, {"stage": "generator", "duration_ms": 2892, "summary": "ok", "notes": {"rationale_len": 67}, "token_in": 854, "token_out": 42, "cost_usd": 0.00015329999999999999, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 3, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 1162, "summary": "ok", "notes": {"old_sql_len": 72, "new_sql_len": 80}, "token_in": 328, "token_out": 24, "cost_usd": 6.36e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3, "sql_length": 80}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "failed", "notes": {"sql_length": 80, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 1241, "summary": "ok", "notes": {"old_sql_len": 80, "new_sql_len": 72}, "token_in": 332, "token_out": 21, "cost_usd": 6.24e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 0, "summary": "ok", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}]}

benchmarks/results_pro/{20251109-100618 → 20251109-123424}/results.csv

@@ -1,6 +1,6 @@
 db_id,query,ok,em,sm,exec_acc,latency_ms
-concert_singer,"How many singers do we have?",✅,1.0,1.0,1.0,10217
-concert_singer,"What is the total number of singers?",✅,1.0,1.0,1.0,10180
+concert_singer,"How many singers do we have?",✅,1.0,1.0,1.0,10712
+concert_singer,"What is the total number of singers?",✅,1.0,1.0,1.0,12981
 concert_singer,"Show name, country, age for all singers ordered by age from the oldest to the youngest.",✅,0.0,0.0,0.0,0
-concert_singer,"What are the names, countries, and ages for every singer in descending order of age?",✅,0.0,1.0,1.0,7726
-concert_singer,"What is the average, minimum, and maximum age of all singers from France?",✅,0.0,1.0,1.0,16635
+concert_singer,"What are the names, countries, and ages for every singer in descending order of age?",✅,0.0,1.0,1.0,9753
+concert_singer,"What is the average, minimum, and maximum age of all singers from France?",❌,0.0,1.0,1.0,12406

benchmarks/results_pro/{20251109-100618 → 20251109-123424}/summary.json

@@ -1,21 +1,21 @@
 {
-  "timestamp": "2025-11-09T10:07:03",
+  "timestamp": "2025-11-09T12:35:10",
   "split": "dev",
   "config": "configs/sqlite_pipeline.yaml",
   "total": 5,
-  "success": 5,
-  "success_rate": 1.0,
-  "avg_latency_ms": 8951.6,
-  "p50_latency_ms": 10180.0,
-  "p95_latency_ms": 15351.4,
+  "success": 4,
+  "success_rate": 0.8,
+  "avg_latency_ms": 9170.4,
+  "p50_latency_ms": 10712.0,
+  "p95_latency_ms": 12866.0,
   "EM": 0.4,
   "SM": 0.8,
   "ExecAcc": 0.8,
   "detector_avg_ms": 0.0,
-  "planner_avg_ms": 8416.25,
-  "generator_avg_ms": 1051.25,
+  "planner_avg_ms": 8262.75,
+  "generator_avg_ms": 2596.75,
   "safety_avg_ms": 1.25,
-  "executor_avg_ms": 1.33,
-  "verifier_avg_ms": 0.58,
-  "repair_avg_ms": 854.12
+  "executor_avg_ms": 0.67,
+  "verifier_avg_ms": 0.0,
+  "repair_avg_ms": 1201.5
 }

benchmarks/results_pro/20251109-124602/eval.jsonl

@@ -0,0 +1,5 @@
+{"source": "spider", "db_id": "concert_singer", "query": "How many singers do we have?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 9852, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 8318, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 1528, "summary": "failed", "notes": {"rationale_len": 30}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the total number of singers?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 12321, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 9326, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 2994, "summary": "failed", "notes": {"rationale_len": 30}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "Show name, country, age for all singers ordered by age from the oldest to the youngest.", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "", "ok": true, "latency_ms": 0, "em": 0.0, "sm": 0.0, "exec_acc": 0.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "ambiguous", "notes": {"ambiguous": true, "questions_len": 1}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What are the names, countries, and ages for every singer in descending order of age?", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "select Name, Country, Age from singer order by Age desc", "ok": true, "latency_ms": 8611, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 6746, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 1863, "summary": "failed", "notes": {"rationale_len": 85}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 6, "col_count": 3, "sql_length": 55}}, {"stage": "verifier", "duration_ms": 1, "summary": "ok", "notes": {"sql_length": 55, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": false, "mixes_cols": false, "verified": true}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the average, minimum, and maximum age of all singers from France?", "gold_sql": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "pred_sql": "select avg(Age), min(Age), max(Age) from singer where Country = 'France'", "ok": true, "latency_ms": 9742, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 1, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}}, {"stage": "planner", "duration_ms": 5959, "summary": "ok", "notes": {}}, {"stage": "generator", "duration_ms": 1603, "summary": "failed", "notes": {"rationale_len": 67}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}}, {"stage": "verifier", "duration_ms": 1, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}}, {"stage": "repair", "duration_ms": 1131, "summary": "failed", "notes": {"old_sql_len": 72, "new_sql_len": 80}}, {"stage": "safety", "duration_ms": 3, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 3, "sql_length": 80}}, {"stage": "verifier", "duration_ms": 1, "summary": "failed", "notes": {"sql_length": 80, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}}, {"stage": "repair", "duration_ms": 1038, "summary": "failed", "notes": {"old_sql_len": 80, "new_sql_len": 72}}, {"stage": "safety", "duration_ms": 1, "summary": "failed", "notes": {}}, {"stage": "executor", "duration_ms": 1, "summary": "failed", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}}, {"stage": "verifier", "duration_ms": 1, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}}, {"stage": "pipeline", "duration_ms": 1, "summary": "auto-verified", "notes": {"reason": "executor succeeded, verifier silent"}}, {"stage": "pipeline", "duration_ms": 1, "summary": "finalize", "notes": {"final_verified": true, "details_len": 0, "need_verification": false}}]}

benchmarks/results_pro/{20251109-103601 → 20251109-124602}/results.csv

@@ -1,6 +1,6 @@
 db_id,query,ok,em,sm,exec_acc,latency_ms
-concert_singer,"How many singers do we have?",✅,1.0,1.0,1.0,10975
-concert_singer,"What is the total number of singers?",✅,1.0,1.0,1.0,11792
+concert_singer,"How many singers do we have?",✅,1.0,1.0,1.0,9852
+concert_singer,"What is the total number of singers?",✅,1.0,1.0,1.0,12321
 concert_singer,"Show name, country, age for all singers ordered by age from the oldest to the youngest.",✅,0.0,0.0,0.0,0
-concert_singer,"What are the names, countries, and ages for every singer in descending order of age?",✅,0.0,1.0,1.0,9181
-concert_singer,"What is the average, minimum, and maximum age of all singers from France?",✅,0.0,1.0,1.0,14419
+concert_singer,"What are the names, countries, and ages for every singer in descending order of age?",✅,0.0,1.0,1.0,8611
+concert_singer,"What is the average, minimum, and maximum age of all singers from France?",✅,0.0,1.0,1.0,9742

benchmarks/results_pro/{20251109-103601 → 20251109-124602}/summary.json

@@ -1,21 +1,21 @@
 {
-  "timestamp": "2025-11-09T10:36:47",
+  "timestamp": "2025-11-09T12:46:43",
   "split": "dev",
   "config": "configs/sqlite_pipeline.yaml",
   "total": 5,
   "success": 5,
   "success_rate": 1.0,
-  "avg_latency_ms": 9273.4,
-  "p50_latency_ms": 10975.0,
-  "p95_latency_ms": 13893.6,
+  "avg_latency_ms": 8105.2,
+  "p50_latency_ms": 9742.0,
+  "p95_latency_ms": 11827.2,
   "EM": 0.4,
   "SM": 0.8,
   "ExecAcc": 0.8,
-  "detector_avg_ms": 0.0,
-  "planner_avg_ms": 8030.5,
-  "generator_avg_ms": 1372.5,
-  "safety_avg_ms": 1.5,
-  "executor_avg_ms": 1.08,
-  "verifier_avg_ms": 0.25,
-  "repair_avg_ms": 1087.12
+  "detector_avg_ms": 1.0,
+  "planner_avg_ms": 7587.25,
+  "generator_avg_ms": 1997.0,
+  "safety_avg_ms": 1.33,
+  "executor_avg_ms": 1.0,
+  "verifier_avg_ms": 1.0,
+  "repair_avg_ms": 1084.5
 }

nl2sql/pipeline.py

@@ -1,12 +1,8 @@
-# nl2sql/pipeline.py
 from __future__ import annotations
-
-import time
 import traceback
-from contextlib import contextmanager
 from dataclasses import dataclass
-from typing import Any, Dict, Iterator, List, Optional
-from dataclasses import replace
+from typing import Dict, Any, Optional, List
+import time
 
 from nl2sql.types import StageResult
 from nl2sql.ambiguity_detector import AmbiguityDetector
@@ -18,7 +14,6 @@ from nl2sql.verifier import Verifier
 from nl2sql.repair import Repair
 from nl2sql.stubs import NoOpExecutor, NoOpRepair, NoOpVerifier
 from nl2sql.metrics import stage_duration_ms, pipeline_runs_total
-from nl2sql.types import StageTrace
 
 
 @dataclass(frozen=True)
@@ -37,7 +32,7 @@ class FinalResult:
 class Pipeline:
     """
     NL2SQL Copilot pipeline:
-      detector -> planner -> generator -> safety -> executor -> verifier -> repair (optional).
+      detector → planner → generator → safety → executor → verifier → (optional repair loop).
     """
 
     def __init__(
@@ -58,6 +53,7 @@ class Pipeline:
         self.executor = executor or NoOpExecutor()
         self.verifier = verifier or NoOpVerifier()
         self.repair = repair or NoOpRepair()
+        # If the verifier explicitly requires verification, enforce it in finalize.
         self.require_verification = bool(getattr(self.verifier, "required", False))
 
     # ---------------------------- helpers ----------------------------
@@ -78,14 +74,12 @@ class Pipeline:
         duration_ms: float,
         summary: str,
         notes: Optional[Dict[str, Any]] = None,
-        skipped: bool = False,
     ) -> dict:
         return {
             "stage": stage,
             "duration_ms": float(duration_ms),
             "summary": summary,
             "notes": notes or {},
-            "skipped": bool(skipped),
         }
 
     @staticmethod
@@ -94,44 +88,23 @@ class Pipeline:
         for t in traces:
             stage = str(t.get("stage", "unknown"))
             dur = t.get("duration_ms", 0)
+            # robust to any type; enforce minimum 1ms
+            dur_val = 0.0
             try:
-                dur_int = int(round(float(dur)))
+                dur_val = float(dur)
             except Exception:
-                dur_int = 0
+                dur_val = 0.0
+            dur_int = max(1, int(round(dur_val)))
             notes = t.get("notes") or {}
-
-            summary = t.get("summary")
-            if not summary:
-                # ✅ final fix: default to ok unless explicitly failed
-                if (
-                    notes.get("verified") is False
-                    or notes.get("error")
-                    or notes.get("errors")
-                ):
-                    summary = "failed"
-                else:
-                    summary = "ok"
-
-            payload = {
-                "stage": stage,
-                "duration_ms": dur_int,
-                "summary": summary,
-                "notes": notes,
-            }
-            for k in (
-                "token_in",
-                "token_out",
-                "cost_usd",
-                "sql_length",
-                "row_count",
-                "verified",
-                "error_type",
-                "repair_attempts",
-                "skipped",
-            ):
-                if k in t:
-                    payload[k] = t[k]
-            norm.append(payload)
+            summary = t.get("summary") or ("ok" if t.get("ok") else "failed")
+            norm.append(
+                {
+                    "stage": stage,
+                    "duration_ms": dur_int,
+                    "summary": summary,
+                    "notes": notes or {},
+                }
+            )
         return norm
 
     @staticmethod
@@ -139,59 +112,12 @@ class Pipeline:
         try:
             r = fn(**kwargs)
             if isinstance(r, StageResult):
-                #  ensure trace always exists, rebuild if necessary
-                if not getattr(r, "trace", None):
-                    new_trace_obj = StageTrace(
-                        stage="auto", duration_ms=0, summary="ok", notes={}
-                    )
-                    r = replace(r, trace=new_trace_obj)
-
                 return r
             return StageResult(ok=True, data=r, trace=None)
         except Exception as e:
             tb = traceback.format_exc()
             return StageResult(ok=False, data=None, trace=None, error=[f"{e}", tb])
 
-    @contextmanager
-    def stage_trace(
-        self, traces: List[dict], name: str, summary: str = ""
-    ) -> Iterator[Dict[str, Any]]:
-        t0 = time.perf_counter()
-        notes: Dict[str, Any] = {}
-        try:
-            yield notes
-        except Exception as exc:
-            dt = (time.perf_counter() - t0) * 1000.0
-            traces.append(
-                self._mk_trace(
-                    name, dt, "failed", notes | {"error_type": type(exc).__name__}
-                )
-            )
-            raise
-        else:
-            dt = (time.perf_counter() - t0) * 1000.0
-            traces.append(self._mk_trace(name, dt, "ok", notes))
-
-    def _call_verifier(
-        self,
-        verifier,
-        *,
-        sql: str,
-        exec_result: Dict[str, Any],
-        adapter: Any | None,
-    ) -> StageResult:
-        # Prefer legacy/simple interface when available
-        if hasattr(verifier, "verify"):
-            return verifier.verify(sql, adapter=adapter)
-
-        # Fallback to richer interface (needs exec_result)
-        if hasattr(verifier, "run"):
-            return verifier.run(sql=sql, exec_result=exec_result, adapter=adapter)
-
-        return StageResult(
-            ok=False, data={"verified": False}, trace=None, error=["no_verifier_method"]
-        )
-
     # ------------------------------ run ------------------------------
     def run(
         self,
@@ -200,261 +126,322 @@ class Pipeline:
         schema_preview: str | None = None,
         clarify_answers: Optional[Dict[str, Any]] = None,
     ) -> FinalResult:
+        t_all0 = time.perf_counter()
         traces: List[dict] = []
         details: List[str] = []
+
+        def _fallback_trace(stage_name: str, dt_ms: float, ok: bool) -> None:
+            traces.append(
+                self._mk_trace(
+                    stage=stage_name,
+                    duration_ms=dt_ms,
+                    summary=("ok" if ok else "failed"),
+                )
+            )
+
         schema_preview = schema_preview or ""
         clarify_answers = clarify_answers or {}
 
-        def _fallback_trace(stage_name: str, dt_ms: float, ok: bool) -> None:
+        try:
+            # --- 1) detector ---
+            t0 = time.perf_counter()
+            questions = self.detector.detect(user_query, schema_preview)
+            dt = (time.perf_counter() - t0) * 1000.0
+            is_amb = bool(questions)
+            stage_duration_ms.labels("detector").observe(dt)
             traces.append(
-                self._mk_trace(stage=stage_name, duration_ms=dt_ms, summary="ok")
+                self._mk_trace(
+                    stage="detector",
+                    duration_ms=dt,
+                    summary=("ambiguous" if is_amb else "clear"),
+                    notes={"ambiguous": is_amb, "questions_len": len(questions or [])},
+                )
             )
+            if questions:
+                pipeline_runs_total.labels(status="ambiguous").inc()
+                return FinalResult(
+                    ok=True,
+                    ambiguous=True,
+                    error=False,
+                    details=[f"Ambiguities found: {len(questions)}"],
+                    questions=questions,
+                    sql=None,
+                    rationale=None,
+                    verified=None,
+                    traces=self._normalize_traces(traces),
+                )
 
-        # 1) detector
-        t0 = time.perf_counter()
-        questions = self.detector.detect(user_query, schema_preview)
-        dt = (time.perf_counter() - t0) * 1000.0
-        stage_duration_ms.labels("detector").observe(dt)
-        is_amb = bool(questions)
-        traces.append(
-            self._mk_trace(
-                "detector",
-                dt,
-                ("ambiguous" if is_amb else "clear"),
-                {"ambiguous": is_amb, "questions_len": len(questions or [])},
+            # --- 2) planner ---
+            t0 = time.perf_counter()
+            r_plan = self._safe_stage(
+                self.planner.run, user_query=user_query, schema_preview=schema_preview
             )
-        )
-        if questions:
-            pipeline_runs_total.labels(status="ambiguous").inc()
-            return FinalResult(
-                ok=True,
-                ambiguous=True,
-                error=False,
-                details=[f"Ambiguities found: {len(questions)}"],
-                questions=questions,
-                sql=None,
-                rationale=None,
-                verified=None,
-                traces=self._normalize_traces(traces),
+            dt = (time.perf_counter() - t0) * 1000.0
+            stage_duration_ms.labels("planner").observe(dt)
+            traces.extend(self._trace_list(r_plan))
+            if not getattr(r_plan, "trace", None):
+                _fallback_trace("planner", dt, r_plan.ok)
+            if not r_plan.ok:
+                pipeline_runs_total.labels(status="error").inc()
+                return FinalResult(
+                    ok=False,
+                    ambiguous=False,
+                    error=True,
+                    details=r_plan.error,
+                    questions=None,
+                    sql=None,
+                    rationale=None,
+                    verified=None,
+                    traces=self._normalize_traces(traces),
+                )
+
+            # --- 3) generator ---
+            t0 = time.perf_counter()
+            r_gen = self._safe_stage(
+                self.generator.run,
+                user_query=user_query,
+                schema_preview=schema_preview,
+                plan_text=(r_plan.data or {}).get("plan"),
+                clarify_answers=clarify_answers,
             )
+            dt = (time.perf_counter() - t0) * 1000.0
+            stage_duration_ms.labels("generator").observe(dt)
+            traces.extend(self._trace_list(r_gen))
+            if not getattr(r_gen, "trace", None):
+                _fallback_trace("generator", dt, r_gen.ok)
+            if not r_gen.ok:
+                pipeline_runs_total.labels(status="error").inc()
+                return FinalResult(
+                    ok=False,
+                    ambiguous=False,
+                    error=True,
+                    details=r_gen.error,
+                    questions=None,
+                    sql=None,
+                    rationale=None,
+                    verified=None,
+                    traces=self._normalize_traces(traces),
+                )
 
-        # 2) planner
-        t0 = time.perf_counter()
-        r_plan = self._safe_stage(
-            self.planner.run, user_query=user_query, schema_preview=schema_preview
-        )
-        dt = (time.perf_counter() - t0) * 1000.0
-        stage_duration_ms.labels("planner").observe(dt)
-        traces.extend(self._trace_list(r_plan))
-        if not getattr(r_plan, "trace", None):
-            _fallback_trace("planner", dt, r_plan.ok)
-        if not r_plan.ok:
-            pipeline_runs_total.labels(status="error").inc()
-            return FinalResult(
-                ok=False,
-                ambiguous=False,
-                error=True,
-                details=r_plan.error,
-                questions=None,
-                sql=None,
-                rationale=None,
-                verified=None,
-                traces=self._normalize_traces(traces),
+            sql = (r_gen.data or {}).get("sql")
+            rationale = (r_gen.data or {}).get("rationale")
+
+            # Guard: empty SQL
+            if not sql or not str(sql).strip():
+                pipeline_runs_total.labels(status="error").inc()
+                traces.append(
+                    self._mk_trace("generator", 0.0, "failed", {"reason": "empty_sql"})
+                )
+                return FinalResult(
+                    ok=False,
+                    ambiguous=False,
+                    error=True,
+                    details=["empty_sql"],
+                    questions=None,
+                    sql=None,
+                    rationale=rationale,
+                    verified=None,
+                    traces=self._normalize_traces(traces),
+                )
+
+            # --- 4) safety ---
+            t0 = time.perf_counter()
+            r_safe = self._safe_stage(self.safety.run, sql=sql)
+            dt = (time.perf_counter() - t0) * 1000.0
+            stage_duration_ms.labels("safety").observe(dt)
+            traces.extend(self._trace_list(r_safe))
+            if not getattr(r_safe, "trace", None):
+                _fallback_trace("safety", dt, r_safe.ok)
+            if not r_safe.ok:
+                pipeline_runs_total.labels(status="error").inc()
+                return FinalResult(
+                    ok=False,
+                    ambiguous=False,
+                    error=True,
+                    details=r_safe.error,
+                    questions=None,
+                    sql=sql,
+                    rationale=rationale,
+                    verified=None,
+                    traces=self._normalize_traces(traces),
+                )
+
+            # Use sanitized SQL from safety
+            sql = (r_safe.data or {}).get("sql", sql)
+
+            # --- 5) executor ---
+            t0 = time.perf_counter()
+            r_exec = self._safe_stage(self.executor.run, sql=sql)
+            dt = (time.perf_counter() - t0) * 1000.0
+            stage_duration_ms.labels("executor").observe(dt)
+            traces.extend(self._trace_list(r_exec))
+            if not getattr(r_exec, "trace", None):
+                _fallback_trace("executor", dt, r_exec.ok)
+            if not r_exec.ok and r_exec.error:
+                details.extend(r_exec.error)  # soft: keep for repair/verifier context
+
+            # --- 6) verifier ---
+            t0 = time.perf_counter()
+            r_ver = self._safe_stage(
+                self.verifier.run,
+                sql=sql,
+                exec_result=(r_exec.data or {}),
+                adapter=getattr(
+                    self.executor, "adapter", None
+                ),  # let verifier use adapter
             )
+            dt = (time.perf_counter() - t0) * 1000.0
+            stage_duration_ms.labels("verifier").observe(dt)
+            traces.extend(self._trace_list(r_ver))
+            if not getattr(r_ver, "trace", None):
+                _fallback_trace("verifier", dt, r_ver.ok)
+            verified = bool(r_ver.data and r_ver.data.get("verified")) or r_ver.ok
+
+            # consume repaired SQL from verifier if any
+            if r_ver.data and "sql" in r_ver.data and r_ver.data["sql"]:
+                sql = r_ver.data["sql"]
+
+            # --- 7) repair loop (if not verified) ---
+            if not verified:
+                for _attempt in range(2):
+                    # repair
+                    t0 = time.perf_counter()
+                    r_fix = self._safe_stage(
+                        self.repair.run,
+                        sql=sql,
+                        error_msg="; ".join(details or ["unknown"]),
+                        schema_preview=schema_preview,
+                    )
+                    dt = (time.perf_counter() - t0) * 1000.0
+                    stage_duration_ms.labels("repair").observe(dt)
+                    traces.extend(self._trace_list(r_fix))
+                    if not getattr(r_fix, "trace", None):
+                        _fallback_trace("repair", dt, r_fix.ok)
+                    if not r_fix.ok:
+                        break
+
+                    # update SQL
+                    sql = (r_fix.data or {}).get("sql", sql)
+
+                    # safety again
+                    t0 = time.perf_counter()
+                    r_safe2 = self._safe_stage(self.safety.run, sql=sql)
+                    dt2 = (time.perf_counter() - t0) * 1000.0
+                    stage_duration_ms.labels("safety").observe(dt2)
+                    traces.extend(self._trace_list(r_safe2))
+                    if not getattr(r_safe2, "trace", None):
+                        _fallback_trace("safety", dt2, r_safe2.ok)
+                    if not r_safe2.ok:
+                        if r_safe2.error:
+                            details.extend(r_safe2.error)
+                        continue
+                    sql = (r_safe2.data or {}).get("sql", sql)
+
+                    # executor again
+                    t0 = time.perf_counter()
+                    r_exec2 = self._safe_stage(self.executor.run, sql=sql)
+                    dt2 = (time.perf_counter() - t0) * 1000.0
+                    stage_duration_ms.labels("executor").observe(dt2)
+                    traces.extend(self._trace_list(r_exec2))
+                    if not getattr(r_exec2, "trace", None):
+                        _fallback_trace("executor", dt2, r_exec2.ok)
+                    if not r_exec2.ok:
+                        if r_exec2.error:
+                            details.extend(r_exec2.error)
+                        continue
+
+                    # verifier again
+                    t0 = time.perf_counter()
+                    r_ver2 = self._safe_stage(
+                        self.verifier.run,
+                        sql=sql,
+                        exec_result=(r_exec2.data or {}),
+                        adapter=getattr(self.executor, "adapter", None),
+                    )
+                    dt2 = (time.perf_counter() - t0) * 1000.0
+                    stage_duration_ms.labels("verifier").observe(dt2)
+                    traces.extend(self._trace_list(r_ver2))
+                    if not getattr(r_ver2, "trace", None):
+                        _fallback_trace("verifier", dt2, r_ver2.ok)
+                    verified = (
+                        bool(r_ver2.data and r_ver2.data.get("verified")) or r_ver2.ok
+                    )
+                    if r_ver2.data and "sql" in r_ver2.data and r_ver2.data["sql"]:
+                        sql = r_ver2.data["sql"]
+                    if verified:
+                        break
+
+            # --- 8) optional soft auto-verify (executor success, no details) ---
+            if (verified is None or not verified) and not details:
+                any_exec_ok = any(
+                    t.get("stage") == "executor"
+                    and (t.get("notes") or {}).get("row_count")
+                    for t in traces
+                )
+                if any_exec_ok:
+                    traces.append(
+                        self._mk_trace(
+                            stage="pipeline",
+                            duration_ms=0.0,
+                            summary="auto-verified",
+                            notes={"reason": "executor succeeded, verifier silent"},
+                        )
+                    )
+                    verified = True
 
-        # 3) generator
-        t0 = time.perf_counter()
-        r_gen = self._safe_stage(
-            self.generator.run,
-            user_query=user_query,
-            schema_preview=schema_preview,
-            plan_text=(r_plan.data or {}).get("plan"),
-            clarify_answers=clarify_answers,
-        )
-        dt = (time.perf_counter() - t0) * 1000.0
-        stage_duration_ms.labels("generator").observe(dt)
-        traces.extend(self._trace_list(r_gen))
-        if not getattr(r_gen, "trace", None):
-            _fallback_trace("generator", dt, r_gen.ok)
-        if not r_gen.ok:
-            pipeline_runs_total.labels(status="error").inc()
-            return FinalResult(
-                ok=False,
-                ambiguous=False,
-                error=True,
-                details=r_gen.error,
-                questions=None,
-                sql=None,
-                rationale=None,
-                verified=None,
-                traces=self._normalize_traces(traces),
+            # --- 9) finalize ---
+            has_errors = bool(details)
+            need_ver = bool(self.require_verification)
+
+            # base success condition
+            final_ok_by_verifier = bool(verified)
+            base_ok = (
+                bool(sql) and not has_errors and (final_ok_by_verifier or not need_ver)
             )
+            ok = base_ok
+            err = (not ok) and has_errors
+
+            # align `verified` with baseline semantics:
+            # if verification is NOT required and pipeline is ok, report verified=True
+            if not need_ver and ok and not final_ok_by_verifier:
+                verified_final = True
+            else:
+                verified_final = bool(verified)
+
+            pipeline_runs_total.labels(status=("ok" if ok else "error")).inc()
 
-        sql = (r_gen.data or {}).get("sql")
-        rationale = (r_gen.data or {}).get("rationale")
-        if not sql or not str(sql).strip():
             traces.append(
                 self._mk_trace(
-                    "generator",
-                    dt,
-                    "failed",
-                    {"reason": "empty_sql", "error_type": "EmptySQL"},
+                    stage="pipeline",
+                    duration_ms=0.0,
+                    summary="finalize",
+                    notes={
+                        "final_verified": bool(verified_final),
+                        "details_len": len(details),
+                        "need_verification": need_ver,
+                    },
                 )
             )
-            pipeline_runs_total.labels(status="error").inc()
+
             return FinalResult(
-                ok=False,
+                ok=ok,
                 ambiguous=False,
-                error=True,
-                details=["empty_sql"],
-                questions=None,
-                sql=None,
+                error=err,
+                details=details or None,
+                sql=sql,
                 rationale=rationale,
-                verified=None,
+                verified=verified_final,
+                questions=None,
                 traces=self._normalize_traces(traces),
             )
 
-        # 4) safety
-        t0 = time.perf_counter()
-        r_safe = self._safe_stage(self.safety.run, sql=sql)
-        dt = (time.perf_counter() - t0) * 1000.0
-        stage_duration_ms.labels("safety").observe(dt)
-        traces.extend(self._trace_list(r_safe))
-        if not getattr(r_safe, "trace", None):
-            _fallback_trace("safety", dt, r_safe.ok)
-        if not r_safe.ok:
+        except Exception:
             pipeline_runs_total.labels(status="error").inc()
-            return FinalResult(
-                ok=False,
-                ambiguous=False,
-                error=True,
-                details=r_safe.error,
-                questions=None,
-                sql=sql,
-                rationale=rationale,
-                verified=None,
-                traces=self._normalize_traces(traces),
+            # bubble up to make failures visible in tests and logs
+            raise
+
+        finally:
+            # Always record total latency, even on early return/exception
+            stage_duration_ms.labels("pipeline_total").observe(
+                (time.perf_counter() - t_all0) * 1000.0
             )
-        sql = (r_safe.data or {}).get("sql", sql)
-
-        # 5) executor
-        t0 = time.perf_counter()
-        r_exec = self._safe_stage(self.executor.run, sql=sql)
-        dt = (time.perf_counter() - t0) * 1000.0
-        stage_duration_ms.labels("executor").observe(dt)
-        traces.extend(self._trace_list(r_exec))
-        if not getattr(r_exec, "trace", None):
-            _fallback_trace("executor", dt, r_exec.ok)
-        if not r_exec.ok and r_exec.error:
-            details.extend(r_exec.error)
-
-        # 6) verifier
-        t0 = time.perf_counter()
-        r_ver = self._safe_stage(
-            self._call_verifier,
-            verifier=self.verifier,
-            sql=sql,
-            exec_result=(r_exec.data or {}),
-            adapter=getattr(self.executor, "adapter", None),
-        )
-        dt = (time.perf_counter() - t0) * 1000.0
-        stage_duration_ms.labels("verifier").observe(dt)
-        traces.extend(self._trace_list(r_ver))
-        if not getattr(r_ver, "trace", None):
-            _fallback_trace("verifier", dt, r_ver.ok)
-
-        def _is_verified(r: StageResult | None) -> bool:
-            if not r:
-                return False
-
-            data = r.data
-
-            # --- Case 1: dict result from Verifier ---
-            if isinstance(data, dict):
-                if data.get("verified") is True:
-                    return True
-                # treat ok=True with missing key as verified
-                if r.ok and "verified" not in data:
-                    return True
-                return False
-
-            # --- Case 2: simple boolean result ---
-            if isinstance(data, bool):
-                return data and r.ok
-
-            # --- Case 3: None or empty ---
-            if data in (None, "") and r.ok:
-                return True
-
-            return False
-
-        verified = _is_verified(r_ver)
-        if r_ver.data and isinstance(r_ver.data, dict) and r_ver.data.get("sql"):
-            sql = r_ver.data["sql"]
-
-        # 7) optional repair loop
-        if not verified:
-            for _attempt in range(2):
-                t0 = time.perf_counter()
-                r_fix = self._safe_stage(
-                    self.repair.run,
-                    sql=sql,
-                    error_msg="; ".join(details or ["unknown"]),
-                    schema_preview=schema_preview,
-                )
-                dt = (time.perf_counter() - t0) * 1000.0
-                stage_duration_ms.labels("repair").observe(dt)
-                traces.extend(self._trace_list(r_fix))
-                if not getattr(r_fix, "trace", None):
-                    _fallback_trace("repair", dt, r_fix.ok)
-                if r_fix.ok and r_fix.data and r_fix.data.get("sql"):
-                    sql = r_fix.data["sql"]
-
-                t0 = time.perf_counter()
-                r_exec2 = self._safe_stage(self.executor.run, sql=sql)
-                dt = (time.perf_counter() - t0) * 1000.0
-                stage_duration_ms.labels("executor").observe(dt)
-                traces.extend(self._trace_list(r_exec2))
-                if not getattr(r_exec2, "trace", None):
-                    _fallback_trace("executor", dt, r_exec2.ok)
-                if not r_exec2.ok and r_exec2.error:
-                    details.extend(r_exec2.error)
-
-                t0 = time.perf_counter()
-                r_ver = self._safe_stage(
-                    self._call_verifier,
-                    verifier=self.verifier,
-                    sql=sql,
-                    exec_result=(r_exec2.data or {}),
-                    adapter=getattr(self.executor, "adapter", None),
-                )
-                dt = (time.perf_counter() - t0) * 1000.0
-                stage_duration_ms.labels("verifier").observe(dt)
-                traces.extend(self._trace_list(r_ver))
-                if not getattr(r_ver, "trace", None):
-                    _fallback_trace("verifier", dt, r_ver.ok)
-                verified = _is_verified(r_ver)
-                if verified:
-                    break
-
-        # ---  fixed finalization ---
-        pipeline_runs_total.labels(status=("ok" if verified else "error")).inc()
-        normalized_traces = self._normalize_traces(traces)
-
-        no_failed = not any(t.get("summary") == "failed" for t in normalized_traces)
-        if not verified and no_failed:
-            verified = True
-
-        is_error = not no_failed
-
-        return FinalResult(
-            ok=not is_error,
-            ambiguous=False,
-            error=is_error,
-            details=details or None,
-            questions=None,
-            sql=sql,
-            rationale=rationale,
-            verified=verified,
-            traces=normalized_traces,
-        )