{
  "_about": "Golden planner dataset (E-planner eval — runs against the LIVE planner LLM, PlannerService.plan). Each case is a question + `expect` = a set of IR/plan ASSERTIONS (rule-compliance scoring, NOT exact IR match — many IRs are valid, we only pin the properties that matter). Grounded in `PA Data Dummy.xlsx` via eval/planner/catalog_fixture.py. `id` is a stable per-case handle so timestamped runs (results/planner_result_<ts>.json) diff case-by-case. `carried_over`=true marks behavior believed CORRECT today (regression guard); the rest target the 3 known planner bugs. Balanced ID/EN because users code-switch.",
  "_expect_keys": {
    "has_tool": "some task tool_call uses this tool name",
    "no_tool": "no task uses this tool",
    "any_tool": "at least one of these tool names is present (e.g. an aggregation step)",
    "selects_col": "some retrieve_data select references this column NAME (by column_id, not the alias — catches wrong-column-hidden-behind-alias)",
    "not_selects_col": "NO select references this column NAME (e.g. must not chart Plan_PA_Percent when the user asked for PA_Percent)",
    "select_agg": "some retrieve_data IR select has an agg with this fn (count/sum/avg/min/max/count_distinct)",
    "group_by": "true = some IR has a non-empty group_by; false = NO IR has group_by (raw-row guard)",
    "chart_aggregated": "true = the chart's data is aggregated (group_by in the IR OR any analyze_* step present), not raw per-record rows",
    "group_by_col": "some IR group_by contains this column NAME",
    "filter_op": "some IR filter uses this op",
    "no_filter_op": "NO IR filter uses this op",
    "has_filter": "true = at least one IR has a filter",
    "order_dir": "some IR order_by uses this dir (asc/desc)",
    "limit": "some IR has exactly this limit",
    "infeasible": "true = plan has no tasks / an infeasible_reason (measure not in catalog)"
  },
  "cases": [
    {
      "id": "count_zero_pa",
      "category": "count",
      "lang": "en",
      "question": "how many records have PA_Percent = 0?",
      "expect": {"select_agg": "count", "has_filter": true, "group_by": false},
      "carried_over": true
    },
    {
      "id": "count_mttr_gt20_id",
      "category": "count",
      "lang": "id",
      "question": "berapa banyak record dengan MTTR di atas 20?",
      "expect": {"select_agg": "count", "has_filter": true},
      "carried_over": true
    },
    {
      "id": "count_section_hauler",
      "category": "count",
      "lang": "en",
      "question": "how many rows are in section OB HAULER?",
      "expect": {"select_agg": "count", "has_filter": true},
      "carried_over": true
    },
    {
      "id": "rank_units_worst_pa_id",
      "category": "ranking",
      "lang": "id",
      "question": "5 unit dengan PA terburuk?",
      "expect": {"group_by": true, "group_by_col": "Equipment_Number", "select_agg": "avg", "order_dir": "asc", "limit": 5},
      "carried_over": false
    },
    {
      "id": "rank_models_top_mttr_id",
      "category": "ranking",
      "lang": "id",
      "question": "top 3 model dengan MTTR tertinggi?",
      "expect": {"group_by": true, "group_by_col": "Model_Unit", "select_agg": "avg", "order_dir": "desc", "limit": 3},
      "carried_over": true
    },
    {
      "id": "rank_sections_lowest_pa_en",
      "category": "ranking",
      "lang": "en",
      "question": "which section has the lowest average PA?",
      "expect": {"group_by": true, "group_by_col": "Section", "select_agg": "avg"},
      "carried_over": true
    },
    {
      "id": "rank_units_most_breakdown_id",
      "category": "ranking",
      "lang": "id",
      "question": "unit mana yang paling sering breakdown?",
      "expect": {"group_by": true, "group_by_col": "Equipment_Number", "order_dir": "desc"},
      "carried_over": false
    },
    {
      "id": "rank_units_worst_pa_en",
      "category": "ranking",
      "lang": "en",
      "question": "list the 10 worst units by availability",
      "expect": {"group_by": true, "group_by_col": "Equipment_Number", "select_agg": "avg", "order_dir": "asc", "limit": 10},
      "carried_over": false
    },
    {
      "id": "fuzzy_model_777_id",
      "category": "fuzzy_filter",
      "lang": "id",
      "question": "berapa banyak model 777?",
      "expect": {"select_agg": "count", "no_filter_op": "in"},
      "carried_over": false
    },
    {
      "id": "fuzzy_model_hd785_id",
      "category": "fuzzy_filter",
      "lang": "id",
      "question": "berapa banyak unit HD785?",
      "expect": {"select_agg": "count", "no_filter_op": "in"},
      "carried_over": true
    },
    {
      "id": "fuzzy_model_ex_en",
      "category": "fuzzy_filter",
      "lang": "en",
      "question": "how many EX excavator units are there?",
      "expect": {"no_filter_op": "in"},
      "carried_over": false
    },
    {
      "id": "agg_pa_per_section_id",
      "category": "aggregate",
      "lang": "id",
      "question": "berapa rata-rata PA per section?",
      "expect": {"group_by": true, "group_by_col": "Section", "select_agg": "avg"},
      "carried_over": true
    },
    {
      "id": "agg_mttr_per_model_en",
      "category": "aggregate",
      "lang": "en",
      "question": "what is the average MTTR per model unit?",
      "expect": {"group_by": true, "group_by_col": "Model_Unit", "select_agg": "avg"},
      "carried_over": true
    },
    {
      "id": "agg_downhour_per_group_id",
      "category": "aggregate",
      "lang": "id",
      "question": "total down hour per equipment group?",
      "expect": {"group_by": true, "group_by_col": "Equipment_Group_ID", "select_agg": "sum"},
      "carried_over": true
    },
    {
      "id": "desc_mttr_stats_id",
      "category": "descriptive",
      "lang": "id",
      "question": "berikan ringkasan statistik MTTR",
      "expect": {"has_tool": "analyze_descriptive"},
      "carried_over": true
    },
    {
      "id": "desc_pa_stats_en",
      "category": "descriptive",
      "lang": "en",
      "question": "give me the summary statistics for PA_Percent",
      "expect": {"has_tool": "analyze_descriptive"},
      "carried_over": true
    },
    {
      "id": "corr_mttr_pa_id",
      "category": "correlation",
      "lang": "id",
      "question": "apakah ada korelasi antara MTTR dan PA?",
      "expect": {"has_tool": "analyze_correlation"},
      "carried_over": true
    },
    {
      "id": "corr_freq_pa_en",
      "category": "correlation",
      "lang": "en",
      "question": "is breakdown frequency correlated with availability?",
      "expect": {"has_tool": "analyze_correlation"},
      "carried_over": true
    },
    {
      "id": "trend_pa_daily_id",
      "category": "trend",
      "lang": "id",
      "question": "bagaimana trend PA harian?",
      "expect": {"has_tool": "analyze_trend"},
      "carried_over": true
    },
    {
      "id": "trend_downhour_en",
      "category": "trend",
      "lang": "en",
      "question": "show the trend of total down hours over time",
      "expect": {"has_tool": "analyze_trend"},
      "carried_over": true
    },
    {
      "id": "merge_worst_pa_and_mttr_id",
      "category": "merge",
      "lang": "id",
      "question": "model mana yang PA-nya paling buruk sekaligus MTTR-nya paling tinggi?",
      "expect": {"group_by": true, "group_by_col": "Model_Unit"},
      "carried_over": true
    },
    {
      "id": "raw_rows_low_pa_id",
      "category": "counter_raw_rows",
      "lang": "id",
      "question": "tampilkan 10 record dengan PA di bawah 50",
      "expect": {"group_by": false, "has_filter": true, "limit": 10},
      "carried_over": true
    },
    {
      "id": "raw_rows_head_en",
      "category": "counter_raw_rows",
      "lang": "en",
      "question": "show me the first 5 rows of the data",
      "expect": {"group_by": false},
      "carried_over": true
    },
    {
      "id": "exact_model_777d_id",
      "category": "counter_exact_filter",
      "lang": "id",
      "question": "berapa banyak record untuk model 777D?",
      "expect": {"select_agg": "count", "has_filter": true},
      "carried_over": true
    },
    {
      "id": "exact_section_loader_en",
      "category": "counter_exact_filter",
      "lang": "en",
      "question": "how many records are in the OB LOADER section?",
      "expect": {"select_agg": "count", "has_filter": true},
      "carried_over": true
    },
    {
      "id": "disambig_trend_pa_id",
      "category": "column_disambiguation",
      "lang": "id",
      "question": "bagaimana trend PA?",
      "expect": {"selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"},
      "carried_over": false
    },
    {
      "id": "disambig_avg_pa_en",
      "category": "column_disambiguation",
      "lang": "en",
      "question": "what is the average PA per section?",
      "expect": {"selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"},
      "carried_over": false
    },
    {
      "id": "chart_trend_pa_id",
      "category": "chart",
      "lang": "id",
      "question": "bagaimana visualisasi trend PA?",
      "expect": {"has_tool": "render_chart", "chart_aggregated": true, "selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"},
      "carried_over": false
    },
    {
      "id": "chart_avg_pa_by_section_en",
      "category": "chart",
      "lang": "en",
      "question": "show me a bar chart of average PA per section",
      "expect": {"has_tool": "render_chart", "group_by": true, "selects_col": "PA_Percent"},
      "carried_over": false
    },
    {
      "id": "infeasible_churn_id",
      "category": "infeasible",
      "lang": "id",
      "question": "berapa churn rate pelanggan?",
      "expect": {"infeasible": true},
      "carried_over": true
    },
    {
      "id": "infeasible_profit_en",
      "category": "infeasible",
      "lang": "en",
      "question": "what is the monthly profit margin?",
      "expect": {"infeasible": true},
      "carried_over": true
    }
  ]
}