[NOTICKET] test: planner eval cases for trend-chart aggregation + column disambiguation

Extends eval/planner to guard the two planner bugs the trend-chart fix addresses:
- New `chart` cases: a trend chart must be aggregated (assertion `chart_aggregated`
= group_by in the IR OR any analyze_* step), not fed raw rows to render_chart.
- New `column_disambiguation` cases: "trend PA" must select PA_Percent, NOT
Plan_PA_Percent — checked by column_id (assertions `selects_col`/`not_selects_col`)
so a wrong column hidden behind alias "pa_percent" is still caught.

New assertions: chart_aggregated, selects_col, not_selects_col, any_tool. Dataset
27 -> 31 cases. Selfcheck extended to cover the new scorer paths (raw->chart fails;
group_by->chart and analyze_trend->chart pass; wrong-aliased-column fails).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

eval/planner/README.md

@@ -31,6 +31,8 @@ Each run writes `results/planner_result_<timestamp>.json` (never overwritten).
 | `count` | scalar count → `count` aggregate (shipped fix) |
 | `ranking` | top/bottom-N entities → `group_by` + `avg` + `order_by` + `limit` (**Bug 1**) |
 | `fuzzy_filter` | partial model ref → `like`, never enumerate from samples (**Bug 2**) |
+| `column_disambiguation` | "trend PA" must select `PA_Percent`, NOT `Plan_PA_Percent` (a wrong-column pick hidden behind alias `pa_percent`) |
+| `chart` | a trend chart must aggregate before `render_chart` (not feed it 9,729 raw rows) + pick the right column |
 | `aggregate`, `descriptive`, `correlation`, `trend`, `merge` | believed-correct baselines |
 | `counter_raw_rows` | "show N records" must stay raw rows (guards Bug 1 fix from over-aggregating) |
 | `counter_exact_filter` | exact filters stay exact (guards Bug 2 fix from over-`like`ing) |

eval/planner/planner_dataset.json

@@ -3,8 +3,12 @@
   "_expect_keys": {
     "has_tool": "some task tool_call uses this tool name",
     "no_tool": "no task uses this tool",
+    "any_tool": "at least one of these tool names is present (e.g. an aggregation step)",
+    "selects_col": "some retrieve_data select references this column NAME (by column_id, not the alias — catches wrong-column-hidden-behind-alias)",
+    "not_selects_col": "NO select references this column NAME (e.g. must not chart Plan_PA_Percent when the user asked for PA_Percent)",
     "select_agg": "some retrieve_data IR select has an agg with this fn (count/sum/avg/min/max/count_distinct)",
     "group_by": "true = some IR has a non-empty group_by; false = NO IR has group_by (raw-row guard)",
+    "chart_aggregated": "true = the chart's data is aggregated (group_by in the IR OR any analyze_* step present), not raw per-record rows",
     "group_by_col": "some IR group_by contains this column NAME",
     "filter_op": "some IR filter uses this op",
     "no_filter_op": "NO IR filter uses this op",
@@ -214,6 +218,38 @@
       "expect": {"select_agg": "count", "has_filter": true},
       "carried_over": true
     },
+    {
+      "id": "disambig_trend_pa_id",
+      "category": "column_disambiguation",
+      "lang": "id",
+      "question": "bagaimana trend PA?",
+      "expect": {"selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"},
+      "carried_over": false
+    },
+    {
+      "id": "disambig_avg_pa_en",
+      "category": "column_disambiguation",
+      "lang": "en",
+      "question": "what is the average PA per section?",
+      "expect": {"selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"},
+      "carried_over": false
+    },
+    {
+      "id": "chart_trend_pa_id",
+      "category": "chart",
+      "lang": "id",
+      "question": "bagaimana visualisasi trend PA?",
+      "expect": {"has_tool": "render_chart", "chart_aggregated": true, "selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"},
+      "carried_over": false
+    },
+    {
+      "id": "chart_avg_pa_by_section_en",
+      "category": "chart",
+      "lang": "en",
+      "question": "show me a bar chart of average PA per section",
+      "expect": {"has_tool": "render_chart", "group_by": true, "selects_col": "PA_Percent"},
+      "carried_over": false
+    },
     {
       "id": "infeasible_churn_id",
       "category": "infeasible",

eval/planner/run_eval.py

@@ -127,6 +127,14 @@ def _filter_ops(f: dict) -> list[str]:
     return [flt.get("op") for ir in f["irs"] for flt in ir.get("filters", []) if isinstance(flt, dict)]
 
 
+def _selected_col_ids(f: dict) -> list[str]:
+    """Every column_id referenced in any retrieve_data select (column or agg).
+    Used to catch the wrong-column bug where the planner selects Plan_PA_Percent
+    but aliases it 'pa_percent' — the alias hides it, the column_id doesn't."""
+    return [s["column_id"] for ir in f["irs"] for s in ir.get("select", [])
+            if isinstance(s, dict) and s.get("column_id")]
+
+
 def evaluate_facts(f: dict, expect: dict, n2id: dict[str, str]) -> list[tuple[str, bool, str]]:
     """Return [(check, passed, detail)] for every assertion. Grouping/agg checks
     honor BOTH the IR and the analyze_aggregate tool."""
@@ -139,12 +147,26 @@ def evaluate_facts(f: dict, expect: dict, n2id: dict[str, str]) -> list[tuple[st
             res.append((f"has_tool={want}", want in tools, f"tools={sorted(tools)}"))
         elif key == "no_tool":
             res.append((f"no_tool={want}", want not in tools, f"tools={sorted(tools)}"))
+        elif key == "any_tool":  # at least one of these tools present
+            res.append((f"any_tool={want}", any(t in tools for t in want), f"tools={sorted(tools)}"))
+        elif key == "selects_col":  # a select references this column (by id, not the alias)
+            col_id = n2id.get(want, want)
+            ids = _selected_col_ids(f)
+            res.append((f"selects_col={want}", col_id in ids, f"selected={ids}"))
+        elif key == "not_selects_col":  # this column must NOT be selected (wrong-column guard)
+            col_id = n2id.get(want, want)
+            ids = _selected_col_ids(f)
+            res.append((f"not_selects_col={want}", col_id not in ids, f"selected={ids}"))
         elif key == "select_agg":
             syn = _AGG_SYN.get(want, {want})
             got = _all_agg_fns(f)
             res.append((f"select_agg={want}", any(g in syn for g in got), f"aggs={got}"))
         elif key == "group_by":
             res.append(("group_by" if want else "no_group_by", grouped == want, f"grouped={grouped}"))
+        elif key == "chart_aggregated":  # chart data is aggregated somehow (group_by IR OR any analyze_* step), not raw rows
+            analyze = [t for t in tools if t.startswith("analyze_")]
+            ok = grouped or bool(analyze)
+            res.append(("chart_aggregated" if want else "chart_raw", ok == want, f"grouped={grouped} analyze={analyze}"))
         elif key == "group_by_col":
             col_id = n2id.get(want, want)
             a2id = _alias_to_id(f)
@@ -384,7 +406,35 @@ def _selfcheck() -> None:
     assert ev(ok_ir, {"no_filter_op": "in"}), "non-enumerated filter should pass"
 
     assert ev(NS(tasks=[], infeasible_reason="no churn data"), {"infeasible": True})
-    print("selfcheck OK — scorer distinguishes good vs buggy plans (IR + analyze_aggregate paths)")
+
+    # column disambiguation: selecting Plan_PA_Percent aliased "pa_percent" must FAIL
+    disambig = {"selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"}
+    wrong_col = plan([[("retrieve_data", {"ir": {"select": [
+        {"kind": "column", "column_id": "c_plan_pa_percent", "alias": "pa_percent"}]}})]])
+    assert not ev(wrong_col, disambig), "wrong column (Plan_PA_Percent aliased pa_percent) should fail"
+    right_col = plan([[("retrieve_data", {"ir": {"select": [
+        {"kind": "column", "column_id": "c_pa_percent", "alias": "pa_percent"}]}})]])
+    assert ev(right_col, disambig), "right column (PA_Percent) should pass"
+
+    # trend chart must be aggregated (group_by in IR OR an analyze_* step), not raw
+    chart_exp = {"has_tool": "render_chart", "chart_aggregated": True}
+    raw_chart = plan([
+        [("retrieve_data", {"ir": {"select": [{"kind": "column", "column_id": "c_pa_percent"}]}})],
+        [("render_chart", {})]])
+    assert not ev(raw_chart, chart_exp), "raw retrieve -> chart (no aggregation) should fail"
+    # aggregate in the retrieve IR (group_by date) -> chart : valid, no analyze_* needed
+    grp_chart = plan([
+        [("retrieve_data", {"ir": {"select": [{"kind": "agg", "fn": "avg", "column_id": "c_pa_percent"}],
+                                    "group_by": ["c_from_date"]}})],
+        [("render_chart", {})]])
+    assert ev(grp_chart, chart_exp), "retrieve(group_by date) -> chart should pass"
+    # or analyze_trend -> chart : also valid
+    trend_chart = plan([
+        [("retrieve_data", {"ir": {"select": [{"kind": "column", "column_id": "c_pa_percent"}]}})],
+        [("analyze_trend", {})], [("render_chart", {})]])
+    assert ev(trend_chart, chart_exp), "retrieve -> trend -> chart should pass"
+
+    print("selfcheck OK — scorer distinguishes good vs buggy plans (agg paths + column + chart)")
 
 
 if __name__ == "__main__":