| { |
| "_about": "Golden planner dataset (E-planner eval — runs against the LIVE planner LLM, PlannerService.plan). Each case is a question + `expect` = a set of IR/plan ASSERTIONS (rule-compliance scoring, NOT exact IR match — many IRs are valid, we only pin the properties that matter). Grounded in `PA Data Dummy.xlsx` via eval/planner/catalog_fixture.py. `id` is a stable per-case handle so timestamped runs (results/planner_result_<ts>.json) diff case-by-case. `carried_over`=true marks behavior believed CORRECT today (regression guard); the rest target the 3 known planner bugs. Balanced ID/EN because users code-switch.", |
| "_expect_keys": { |
| "has_tool": "some task tool_call uses this tool name", |
| "no_tool": "no task uses this tool", |
| "any_tool": "at least one of these tool names is present (e.g. an aggregation step)", |
| "selects_col": "some retrieve_data select references this column NAME (by column_id, not the alias — catches wrong-column-hidden-behind-alias)", |
| "not_selects_col": "NO select references this column NAME (e.g. must not chart Plan_PA_Percent when the user asked for PA_Percent)", |
| "select_agg": "some retrieve_data IR select has an agg with this fn (count/sum/avg/min/max/count_distinct)", |
| "group_by": "true = some IR has a non-empty group_by; false = NO IR has group_by (raw-row guard)", |
| "chart_aggregated": "true = the chart's data is aggregated (group_by in the IR OR any analyze_* step present), not raw per-record rows", |
| "group_by_col": "some IR group_by contains this column NAME", |
| "filter_op": "some IR filter uses this op", |
| "no_filter_op": "NO IR filter uses this op", |
| "has_filter": "true = at least one IR has a filter", |
| "order_dir": "some IR order_by uses this dir (asc/desc)", |
| "limit": "some IR has exactly this limit", |
| "infeasible": "true = plan has no tasks / an infeasible_reason (measure not in catalog)" |
| }, |
| "cases": [ |
| { |
| "id": "count_zero_pa", |
| "category": "count", |
| "lang": "en", |
| "question": "how many records have PA_Percent = 0?", |
| "expect": {"select_agg": "count", "has_filter": true, "group_by": false}, |
| "carried_over": true |
| }, |
| { |
| "id": "count_mttr_gt20_id", |
| "category": "count", |
| "lang": "id", |
| "question": "berapa banyak record dengan MTTR di atas 20?", |
| "expect": {"select_agg": "count", "has_filter": true}, |
| "carried_over": true |
| }, |
| { |
| "id": "count_section_hauler", |
| "category": "count", |
| "lang": "en", |
| "question": "how many rows are in section OB HAULER?", |
| "expect": {"select_agg": "count", "has_filter": true}, |
| "carried_over": true |
| }, |
| { |
| "id": "rank_units_worst_pa_id", |
| "category": "ranking", |
| "lang": "id", |
| "question": "5 unit dengan PA terburuk?", |
| "expect": {"group_by": true, "group_by_col": "Equipment_Number", "select_agg": "avg", "order_dir": "asc", "limit": 5}, |
| "carried_over": false |
| }, |
| { |
| "id": "rank_models_top_mttr_id", |
| "category": "ranking", |
| "lang": "id", |
| "question": "top 3 model dengan MTTR tertinggi?", |
| "expect": {"group_by": true, "group_by_col": "Model_Unit", "select_agg": "avg", "order_dir": "desc", "limit": 3}, |
| "carried_over": true |
| }, |
| { |
| "id": "rank_sections_lowest_pa_en", |
| "category": "ranking", |
| "lang": "en", |
| "question": "which section has the lowest average PA?", |
| "expect": {"group_by": true, "group_by_col": "Section", "select_agg": "avg"}, |
| "carried_over": true |
| }, |
| { |
| "id": "rank_units_most_breakdown_id", |
| "category": "ranking", |
| "lang": "id", |
| "question": "unit mana yang paling sering breakdown?", |
| "expect": {"group_by": true, "group_by_col": "Equipment_Number", "order_dir": "desc"}, |
| "carried_over": false |
| }, |
| { |
| "id": "rank_units_worst_pa_en", |
| "category": "ranking", |
| "lang": "en", |
| "question": "list the 10 worst units by availability", |
| "expect": {"group_by": true, "group_by_col": "Equipment_Number", "select_agg": "avg", "order_dir": "asc", "limit": 10}, |
| "carried_over": false |
| }, |
| { |
| "id": "fuzzy_model_777_id", |
| "category": "fuzzy_filter", |
| "lang": "id", |
| "question": "berapa banyak model 777?", |
| "expect": {"select_agg": "count", "no_filter_op": "in"}, |
| "carried_over": false |
| }, |
| { |
| "id": "fuzzy_model_hd785_id", |
| "category": "fuzzy_filter", |
| "lang": "id", |
| "question": "berapa banyak unit HD785?", |
| "expect": {"select_agg": "count", "no_filter_op": "in"}, |
| "carried_over": true |
| }, |
| { |
| "id": "fuzzy_model_ex_en", |
| "category": "fuzzy_filter", |
| "lang": "en", |
| "question": "how many EX excavator units are there?", |
| "expect": {"no_filter_op": "in"}, |
| "carried_over": false |
| }, |
| { |
| "id": "agg_pa_per_section_id", |
| "category": "aggregate", |
| "lang": "id", |
| "question": "berapa rata-rata PA per section?", |
| "expect": {"group_by": true, "group_by_col": "Section", "select_agg": "avg"}, |
| "carried_over": true |
| }, |
| { |
| "id": "agg_mttr_per_model_en", |
| "category": "aggregate", |
| "lang": "en", |
| "question": "what is the average MTTR per model unit?", |
| "expect": {"group_by": true, "group_by_col": "Model_Unit", "select_agg": "avg"}, |
| "carried_over": true |
| }, |
| { |
| "id": "agg_downhour_per_group_id", |
| "category": "aggregate", |
| "lang": "id", |
| "question": "total down hour per equipment group?", |
| "expect": {"group_by": true, "group_by_col": "Equipment_Group_ID", "select_agg": "sum"}, |
| "carried_over": true |
| }, |
| { |
| "id": "desc_mttr_stats_id", |
| "category": "descriptive", |
| "lang": "id", |
| "question": "berikan ringkasan statistik MTTR", |
| "expect": {"has_tool": "analyze_descriptive"}, |
| "carried_over": true |
| }, |
| { |
| "id": "desc_pa_stats_en", |
| "category": "descriptive", |
| "lang": "en", |
| "question": "give me the summary statistics for PA_Percent", |
| "expect": {"has_tool": "analyze_descriptive"}, |
| "carried_over": true |
| }, |
| { |
| "id": "corr_mttr_pa_id", |
| "category": "correlation", |
| "lang": "id", |
| "question": "apakah ada korelasi antara MTTR dan PA?", |
| "expect": {"has_tool": "analyze_correlation"}, |
| "carried_over": true |
| }, |
| { |
| "id": "corr_freq_pa_en", |
| "category": "correlation", |
| "lang": "en", |
| "question": "is breakdown frequency correlated with availability?", |
| "expect": {"has_tool": "analyze_correlation"}, |
| "carried_over": true |
| }, |
| { |
| "id": "trend_pa_daily_id", |
| "category": "trend", |
| "lang": "id", |
| "question": "bagaimana trend PA harian?", |
| "expect": {"has_tool": "analyze_trend"}, |
| "carried_over": true |
| }, |
| { |
| "id": "trend_downhour_en", |
| "category": "trend", |
| "lang": "en", |
| "question": "show the trend of total down hours over time", |
| "expect": {"has_tool": "analyze_trend"}, |
| "carried_over": true |
| }, |
| { |
| "id": "merge_worst_pa_and_mttr_id", |
| "category": "merge", |
| "lang": "id", |
| "question": "model mana yang PA-nya paling buruk sekaligus MTTR-nya paling tinggi?", |
| "expect": {"group_by": true, "group_by_col": "Model_Unit"}, |
| "carried_over": true |
| }, |
| { |
| "id": "raw_rows_low_pa_id", |
| "category": "counter_raw_rows", |
| "lang": "id", |
| "question": "tampilkan 10 record dengan PA di bawah 50", |
| "expect": {"group_by": false, "has_filter": true, "limit": 10}, |
| "carried_over": true |
| }, |
| { |
| "id": "raw_rows_head_en", |
| "category": "counter_raw_rows", |
| "lang": "en", |
| "question": "show me the first 5 rows of the data", |
| "expect": {"group_by": false}, |
| "carried_over": true |
| }, |
| { |
| "id": "exact_model_777d_id", |
| "category": "counter_exact_filter", |
| "lang": "id", |
| "question": "berapa banyak record untuk model 777D?", |
| "expect": {"select_agg": "count", "has_filter": true}, |
| "carried_over": true |
| }, |
| { |
| "id": "exact_section_loader_en", |
| "category": "counter_exact_filter", |
| "lang": "en", |
| "question": "how many records are in the OB LOADER section?", |
| "expect": {"select_agg": "count", "has_filter": true}, |
| "carried_over": true |
| }, |
| { |
| "id": "disambig_trend_pa_id", |
| "category": "column_disambiguation", |
| "lang": "id", |
| "question": "bagaimana trend PA?", |
| "expect": {"selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"}, |
| "carried_over": false |
| }, |
| { |
| "id": "disambig_avg_pa_en", |
| "category": "column_disambiguation", |
| "lang": "en", |
| "question": "what is the average PA per section?", |
| "expect": {"selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"}, |
| "carried_over": false |
| }, |
| { |
| "id": "chart_trend_pa_id", |
| "category": "chart", |
| "lang": "id", |
| "question": "bagaimana visualisasi trend PA?", |
| "expect": {"has_tool": "render_chart", "chart_aggregated": true, "selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"}, |
| "carried_over": false |
| }, |
| { |
| "id": "chart_avg_pa_by_section_en", |
| "category": "chart", |
| "lang": "en", |
| "question": "show me a bar chart of average PA per section", |
| "expect": {"has_tool": "render_chart", "group_by": true, "selects_col": "PA_Percent"}, |
| "carried_over": false |
| }, |
| { |
| "id": "infeasible_churn_id", |
| "category": "infeasible", |
| "lang": "id", |
| "question": "berapa churn rate pelanggan?", |
| "expect": {"infeasible": true}, |
| "carried_over": true |
| }, |
| { |
| "id": "infeasible_profit_en", |
| "category": "infeasible", |
| "lang": "en", |
| "question": "what is the monthly profit margin?", |
| "expect": {"infeasible": true}, |
| "carried_over": true |
| } |
| ] |
| } |
|
|
