perf(planner): trim relevant tables (+cache) to cut latency; keep repair loop & rich traces

benchmarks/results_pro/20251109-123149/eval.jsonl

@@ -0,0 +1,5 @@
+{"source": "spider", "db_id": "concert_singer", "query": "How many singers do we have?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 11573, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 7978, "summary": "ok", "notes": {}, "skipped": false}, {"stage": "generator", "duration_ms": 3588, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 796, "token_out": 19, "cost_usd": 0.0001308, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 3, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the total number of singers?", "gold_sql": "SELECT count(*) FROM singer", "pred_sql": "select count(*) from singer", "ok": true, "latency_ms": 9087, "em": 1.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 7653, "summary": "ok", "notes": {}, "skipped": false}, {"stage": "generator", "duration_ms": 1432, "summary": "ok", "notes": {"rationale_len": 30}, "token_in": 789, "token_out": 19, "cost_usd": 0.00012974999999999998, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 1, "sql_length": 27}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"sql_length": 27, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": false, "verified": true}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}]}
+{"source": "spider", "db_id": "concert_singer", "query": "Show name, country, age for all singers ordered by age from the oldest to the youngest.", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "", "ok": true, "latency_ms": 0, "em": 0.0, "sm": 0.0, "exec_acc": 0.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "ambiguous", "notes": {"ambiguous": true, "questions_len": 1}, "skipped": false}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What are the names, countries, and ages for every singer in descending order of age?", "gold_sql": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "pred_sql": "select Name, Country, Age from singer order by Age desc", "ok": true, "latency_ms": 10200, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 8373, "summary": "ok", "notes": {}, "skipped": false}, {"stage": "generator", "duration_ms": 1824, "summary": "ok", "notes": {"rationale_len": 85}, "token_in": 801, "token_out": 37, "cost_usd": 0.00014235, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 2, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 6, "col_count": 3, "sql_length": 55}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "ok", "notes": {"sql_length": 55, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": false, "mixes_cols": false, "verified": true}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}]}
+{"source": "spider", "db_id": "concert_singer", "query": "What is the average, minimum, and maximum age of all singers from France?", "gold_sql": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "pred_sql": "select avg(Age), min(Age), max(Age) from singer where Country = 'France'", "ok": false, "latency_ms": 20765, "em": 0.0, "sm": 1.0, "exec_acc": 1.0, "error": null, "trace": [{"stage": "detector", "duration_ms": 0, "summary": "clear", "notes": {"ambiguous": false, "questions_len": 0}, "skipped": false}, {"stage": "planner", "duration_ms": 9562, "summary": "ok", "notes": {}, "skipped": false}, {"stage": "generator", "duration_ms": 4303, "summary": "ok", "notes": {"rationale_len": 67}, "token_in": 827, "token_out": 42, "cost_usd": 0.00014924999999999997, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "safety", "duration_ms": 1, "summary": "ok", "notes": {}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 5379, "summary": "ok", "notes": {"old_sql_len": 72, "new_sql_len": 80}, "token_in": 328, "token_out": 24, "cost_usd": 6.36e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3, "sql_length": 80}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "failed", "notes": {"sql_length": 80, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "repair", "duration_ms": 1516, "summary": "ok", "notes": {"old_sql_len": 80, "new_sql_len": 72}, "token_in": 332, "token_out": 25, "cost_usd": 6.48e-05, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "executor", "duration_ms": 1, "summary": "ok", "notes": {"row_count": 1, "col_count": 3, "sql_length": 72}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}, {"stage": "verifier", "duration_ms": 0, "summary": "failed", "notes": {"sql_length": 72, "has_select": true, "has_from": true, "has_over": false, "has_group_by": false, "has_distinct": false, "has_aggregate": true, "mixes_cols": true, "verified": false}, "token_in": null, "token_out": null, "cost_usd": null, "sql_length": null, "row_count": null, "verified": null, "error_type": null, "repair_attempts": null, "skipped": false}]}

benchmarks/results_pro/20251109-123149/results.csv

@@ -0,0 +1,6 @@
+db_id,query,ok,em,sm,exec_acc,latency_ms
+concert_singer,"How many singers do we have?",✅,1.0,1.0,1.0,11573
+concert_singer,"What is the total number of singers?",✅,1.0,1.0,1.0,9087
+concert_singer,"Show name, country, age for all singers ordered by age from the oldest to the youngest.",✅,0.0,0.0,0.0,0
+concert_singer,"What are the names, countries, and ages for every singer in descending order of age?",✅,0.0,1.0,1.0,10200
+concert_singer,"What is the average, minimum, and maximum age of all singers from France?",❌,0.0,1.0,1.0,20765

benchmarks/results_pro/20251109-123149/summary.json

@@ -0,0 +1,21 @@
+{
+  "timestamp": "2025-11-09T12:32:41",
+  "split": "dev",
+  "config": "configs/sqlite_pipeline.yaml",
+  "total": 5,
+  "success": 4,
+  "success_rate": 0.8,
+  "avg_latency_ms": 10325.0,
+  "p50_latency_ms": 10200.0,
+  "p95_latency_ms": 18926.6,
+  "EM": 0.4,
+  "SM": 0.8,
+  "ExecAcc": 0.8,
+  "detector_avg_ms": 0.0,
+  "planner_avg_ms": 8391.5,
+  "generator_avg_ms": 2786.75,
+  "safety_avg_ms": 1.5,
+  "executor_avg_ms": 1.33,
+  "verifier_avg_ms": 0.0,
+  "repair_avg_ms": 3447.5
+}

nl2sql/planner.py

@@ -1,22 +1,118 @@
 from __future__ import annotations
 
-from typing import Dict, Any
+import re
+from typing import Any, Dict, List, Tuple, Optional
 
+__all__ = ["Planner"]
+
+
+# --------- Heuristic schema trimming (safe, mypy-clean) ---------
+def _tokenize_lower(s: str) -> List[str]:
+    return re.findall(r"[a-z_]+", (s or "").lower())
 
-class Planner:
-    """Planner wrapper around the LLM provider.
 
-    The factory constructs it with `Planner(llm=llm)`, so we accept `llm` here.
+def _table_blocks(schema_text: str) -> List[Tuple[str, List[str]]]:
     """
+    Parse plain-text schema into [(table_name, lines)] blocks,
+    supporting both 'Table: name' and 'CREATE TABLE name (' styles.
+    """
+    blocks: List[Tuple[str, List[str]]] = []
+    cur_name: Optional[str] = None
+    cur_lines: List[str] = []
+
+    def _flush() -> None:
+        nonlocal cur_name, cur_lines
+        if cur_name is not None and cur_lines:
+            blocks.append((cur_name, cur_lines[:]))
+        cur_name, cur_lines = None, []
+
+    for line in (schema_text or "").splitlines():
+        m = re.search(r"Table:\s*(\w+)", line, flags=re.IGNORECASE)
+        m2 = re.search(r"CREATE\s+TABLE\s+(\w+)\s*\(", line, flags=re.IGNORECASE)
+
+        started = False
+        name: Optional[str] = None
+        if m is not None:
+            name = m.group(1)
+            started = True
+        elif m2 is not None:
+            name = m2.group(1)
+            started = True
+
+        if started and name:
+            _flush()
+            cur_name = name
+            cur_lines.append(line)
+        else:
+            if cur_name is not None:
+                cur_lines.append(line)
+
+        if cur_name is not None and line.strip().endswith(");"):
+            _flush()
+
+    _flush()
+    return blocks
+
+
+def _pick_relevant_tables(schema_text: str, question: str, k: int = 3) -> str:
+    """Keep up to k tables with highest lexical overlap with the question."""
+    try:
+        blocks = _table_blocks(schema_text)
+        if not blocks:
+            return schema_text
+
+        q_toks = set(_tokenize_lower(question))
+        scored: List[Tuple[int, str, List[str]]] = []
+        for name, lines in blocks:
+            score = sum(1 for w in _tokenize_lower(name) if w in q_toks)
+            cols_line = " ".join(lines)
+            cols = re.findall(r"\b([A-Za-z_]\w*)\b", cols_line)
+            score += min(2, sum(1 for c in cols if c.lower() in q_toks))
+            scored.append((score, name, lines))
+
+        scored.sort(key=lambda t: t[0], reverse=True)
+        keep = [b for b in scored[: max(1, k)] if b[0] > 0]
+        if not keep:
+            keep = scored[: max(1, k)]
+
+        out_lines: List[str] = []
+        for _, _, lines in keep:
+            out_lines.extend(lines)
+            if lines and lines[-1].strip() != "":
+                out_lines.append("")
+        trimmed = "\n".join(out_lines).strip()
+        return trimmed if trimmed else schema_text
+    except Exception:
+        return schema_text
+
+
+# ------------------------------ Planner ------------------------------
+class Planner:
+    """Planner wrapper around the LLM provider."""
 
     def __init__(self, *, llm, model_id: str | None = None) -> None:
         self.llm = llm
-        self.model_id = model_id or getattr(llm, "model", "unknown")
+        # ensure model_id is always a str (for mypy)
+        self.model_id: str = str(model_id or getattr(llm, "model", "unknown"))
+        # in-memory cache: (model, hash(q), hash(trimmed)) → (plan, pin, pout, cost)
+        self._plan_cache: dict[tuple[str, int, int], tuple[str, int, int, float]] = {}
 
     def run(self, *, user_query: str, schema_preview: str) -> Dict[str, Any]:
-        plan_text, pin, pout, cost = self.llm.plan(
-            user_query=user_query, schema_preview=schema_preview
+        trimmed = _pick_relevant_tables(schema_preview or "", user_query or "", k=3)
+
+        key: tuple[str, int, int] = (
+            self.model_id,
+            hash(user_query or ""),
+            hash(trimmed),
         )
+        if key in self._plan_cache:
+            plan_text, pin, pout, cost = self._plan_cache[key]
+        else:
+            plan_text, pin, pout, cost = self.llm.plan(
+                user_query=user_query, schema_preview=trimmed
+            )
+            self._plan_cache[key] = (plan_text, pin, pout, cost)
+
         return {
             "plan": plan_text,
             "usage": {