File size: 10,004 Bytes
f282b15 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 | {
"_about": "Golden planner dataset (E-planner eval — runs against the LIVE planner LLM, PlannerService.plan). Each case is a question + `expect` = a set of IR/plan ASSERTIONS (rule-compliance scoring, NOT exact IR match — many IRs are valid, we only pin the properties that matter). Grounded in `PA Data Dummy.xlsx` via eval/planner/catalog_fixture.py. `id` is a stable per-case handle so timestamped runs (results/planner_result_<ts>.json) diff case-by-case. `carried_over`=true marks behavior believed CORRECT today (regression guard); the rest target the 3 known planner bugs. Balanced ID/EN because users code-switch.",
"_expect_keys": {
"has_tool": "some task tool_call uses this tool name",
"no_tool": "no task uses this tool",
"any_tool": "at least one of these tool names is present (e.g. an aggregation step)",
"selects_col": "some retrieve_data select references this column NAME (by column_id, not the alias — catches wrong-column-hidden-behind-alias)",
"not_selects_col": "NO select references this column NAME (e.g. must not chart Plan_PA_Percent when the user asked for PA_Percent)",
"select_agg": "some retrieve_data IR select has an agg with this fn (count/sum/avg/min/max/count_distinct)",
"group_by": "true = some IR has a non-empty group_by; false = NO IR has group_by (raw-row guard)",
"chart_aggregated": "true = the chart's data is aggregated (group_by in the IR OR any analyze_* step present), not raw per-record rows",
"group_by_col": "some IR group_by contains this column NAME",
"filter_op": "some IR filter uses this op",
"no_filter_op": "NO IR filter uses this op",
"has_filter": "true = at least one IR has a filter",
"order_dir": "some IR order_by uses this dir (asc/desc)",
"limit": "some IR has exactly this limit",
"infeasible": "true = plan has no tasks / an infeasible_reason (measure not in catalog)"
},
"cases": [
{
"id": "count_zero_pa",
"category": "count",
"lang": "en",
"question": "how many records have PA_Percent = 0?",
"expect": {"select_agg": "count", "has_filter": true, "group_by": false},
"carried_over": true
},
{
"id": "count_mttr_gt20_id",
"category": "count",
"lang": "id",
"question": "berapa banyak record dengan MTTR di atas 20?",
"expect": {"select_agg": "count", "has_filter": true},
"carried_over": true
},
{
"id": "count_section_hauler",
"category": "count",
"lang": "en",
"question": "how many rows are in section OB HAULER?",
"expect": {"select_agg": "count", "has_filter": true},
"carried_over": true
},
{
"id": "rank_units_worst_pa_id",
"category": "ranking",
"lang": "id",
"question": "5 unit dengan PA terburuk?",
"expect": {"group_by": true, "group_by_col": "Equipment_Number", "select_agg": "avg", "order_dir": "asc", "limit": 5},
"carried_over": false
},
{
"id": "rank_models_top_mttr_id",
"category": "ranking",
"lang": "id",
"question": "top 3 model dengan MTTR tertinggi?",
"expect": {"group_by": true, "group_by_col": "Model_Unit", "select_agg": "avg", "order_dir": "desc", "limit": 3},
"carried_over": true
},
{
"id": "rank_sections_lowest_pa_en",
"category": "ranking",
"lang": "en",
"question": "which section has the lowest average PA?",
"expect": {"group_by": true, "group_by_col": "Section", "select_agg": "avg"},
"carried_over": true
},
{
"id": "rank_units_most_breakdown_id",
"category": "ranking",
"lang": "id",
"question": "unit mana yang paling sering breakdown?",
"expect": {"group_by": true, "group_by_col": "Equipment_Number", "order_dir": "desc"},
"carried_over": false
},
{
"id": "rank_units_worst_pa_en",
"category": "ranking",
"lang": "en",
"question": "list the 10 worst units by availability",
"expect": {"group_by": true, "group_by_col": "Equipment_Number", "select_agg": "avg", "order_dir": "asc", "limit": 10},
"carried_over": false
},
{
"id": "fuzzy_model_777_id",
"category": "fuzzy_filter",
"lang": "id",
"question": "berapa banyak model 777?",
"expect": {"select_agg": "count", "no_filter_op": "in"},
"carried_over": false
},
{
"id": "fuzzy_model_hd785_id",
"category": "fuzzy_filter",
"lang": "id",
"question": "berapa banyak unit HD785?",
"expect": {"select_agg": "count", "no_filter_op": "in"},
"carried_over": true
},
{
"id": "fuzzy_model_ex_en",
"category": "fuzzy_filter",
"lang": "en",
"question": "how many EX excavator units are there?",
"expect": {"no_filter_op": "in"},
"carried_over": false
},
{
"id": "agg_pa_per_section_id",
"category": "aggregate",
"lang": "id",
"question": "berapa rata-rata PA per section?",
"expect": {"group_by": true, "group_by_col": "Section", "select_agg": "avg"},
"carried_over": true
},
{
"id": "agg_mttr_per_model_en",
"category": "aggregate",
"lang": "en",
"question": "what is the average MTTR per model unit?",
"expect": {"group_by": true, "group_by_col": "Model_Unit", "select_agg": "avg"},
"carried_over": true
},
{
"id": "agg_downhour_per_group_id",
"category": "aggregate",
"lang": "id",
"question": "total down hour per equipment group?",
"expect": {"group_by": true, "group_by_col": "Equipment_Group_ID", "select_agg": "sum"},
"carried_over": true
},
{
"id": "desc_mttr_stats_id",
"category": "descriptive",
"lang": "id",
"question": "berikan ringkasan statistik MTTR",
"expect": {"has_tool": "analyze_descriptive"},
"carried_over": true
},
{
"id": "desc_pa_stats_en",
"category": "descriptive",
"lang": "en",
"question": "give me the summary statistics for PA_Percent",
"expect": {"has_tool": "analyze_descriptive"},
"carried_over": true
},
{
"id": "corr_mttr_pa_id",
"category": "correlation",
"lang": "id",
"question": "apakah ada korelasi antara MTTR dan PA?",
"expect": {"has_tool": "analyze_correlation"},
"carried_over": true
},
{
"id": "corr_freq_pa_en",
"category": "correlation",
"lang": "en",
"question": "is breakdown frequency correlated with availability?",
"expect": {"has_tool": "analyze_correlation"},
"carried_over": true
},
{
"id": "trend_pa_daily_id",
"category": "trend",
"lang": "id",
"question": "bagaimana trend PA harian?",
"expect": {"has_tool": "analyze_trend"},
"carried_over": true
},
{
"id": "trend_downhour_en",
"category": "trend",
"lang": "en",
"question": "show the trend of total down hours over time",
"expect": {"has_tool": "analyze_trend"},
"carried_over": true
},
{
"id": "merge_worst_pa_and_mttr_id",
"category": "merge",
"lang": "id",
"question": "model mana yang PA-nya paling buruk sekaligus MTTR-nya paling tinggi?",
"expect": {"group_by": true, "group_by_col": "Model_Unit"},
"carried_over": true
},
{
"id": "raw_rows_low_pa_id",
"category": "counter_raw_rows",
"lang": "id",
"question": "tampilkan 10 record dengan PA di bawah 50",
"expect": {"group_by": false, "has_filter": true, "limit": 10},
"carried_over": true
},
{
"id": "raw_rows_head_en",
"category": "counter_raw_rows",
"lang": "en",
"question": "show me the first 5 rows of the data",
"expect": {"group_by": false},
"carried_over": true
},
{
"id": "exact_model_777d_id",
"category": "counter_exact_filter",
"lang": "id",
"question": "berapa banyak record untuk model 777D?",
"expect": {"select_agg": "count", "has_filter": true},
"carried_over": true
},
{
"id": "exact_section_loader_en",
"category": "counter_exact_filter",
"lang": "en",
"question": "how many records are in the OB LOADER section?",
"expect": {"select_agg": "count", "has_filter": true},
"carried_over": true
},
{
"id": "disambig_trend_pa_id",
"category": "column_disambiguation",
"lang": "id",
"question": "bagaimana trend PA?",
"expect": {"selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"},
"carried_over": false
},
{
"id": "disambig_avg_pa_en",
"category": "column_disambiguation",
"lang": "en",
"question": "what is the average PA per section?",
"expect": {"selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"},
"carried_over": false
},
{
"id": "chart_trend_pa_id",
"category": "chart",
"lang": "id",
"question": "bagaimana visualisasi trend PA?",
"expect": {"has_tool": "render_chart", "chart_aggregated": true, "selects_col": "PA_Percent", "not_selects_col": "Plan_PA_Percent"},
"carried_over": false
},
{
"id": "chart_avg_pa_by_section_en",
"category": "chart",
"lang": "en",
"question": "show me a bar chart of average PA per section",
"expect": {"has_tool": "render_chart", "group_by": true, "selects_col": "PA_Percent"},
"carried_over": false
},
{
"id": "infeasible_churn_id",
"category": "infeasible",
"lang": "id",
"question": "berapa churn rate pelanggan?",
"expect": {"infeasible": true},
"carried_over": true
},
{
"id": "infeasible_profit_en",
"category": "infeasible",
"lang": "en",
"question": "what is the monthly profit margin?",
"expect": {"infeasible": true},
"carried_over": true
}
]
}
|