File size: 12,137 Bytes
3743cfe 17e31c6 3743cfe 17e31c6 3743cfe 17e31c6 3743cfe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | {
"_about": "Golden dataset for the Help skill (`src/agents/handlers/help.HelpAgent`). Unlike intent/readiness this calls the LIVE model: each case declares an analysis state + report-readiness + chat history, the runner streams HelpAgent.astream for real, and asserts RULES the reply must obey (not text similarity — help replies are free prose with no single correct wording). Metric is COMPLIANCE (% of rule assertions that hold), reported separately for held-out vs carried_over cases.",
"_groups": "language (reply matches the user's language — the button-path bug), report_guard (never suggest a report when report_ready.ready=false; do suggest it when true — this also IS the 'no action leakage' check, since generate_report is the only gated action), orientation (quality of the suggested starter questions — MANUAL review, not auto-scored).",
"_asserts": "language_match {expected} — detect the reply's language (help._detect_reply_language over the OUTPUT) must equal expected. must_not_contain_any {patterns} — none of the (case-insensitive) patterns appear. must_contain_any {patterns} — at least one appears.",
"_carried_over": "carried_over:true rows MIRROR an example in config/prompts/help.md (the row `id` IS the help.md `<!-- id: ... -->`). They are the regression guard: if the prompt is refactored, the demonstrated rule must still hold. What is mirrored is the INPUT spec + the assertion — NOT the example's reply text (temperature>0 makes exact match invalid). Held-out rows (carried_over:false) are NOT in the prompt; their compliance is the real generalization signal. If help.md's Examples change, keep these ids in sync (manual, like intent).",
"_missing_codes": "report_ready.missing uses codes mapped to the real strings is_report_ready emits (imported in run_eval): analysis -> _MISSING_ANALYSIS, delta -> _MISSING_DELTA. Kept as codes so the dataset survives wording changes.",
"schema": {
"id": "stable handle; for carried_over rows this equals the help.md example id",
"group": "language | report_guard | orientation",
"carried_over": "bool — mirrors a help.md example",
"manual_review": "bool — run but exclude from the auto compliance rate (read output_text)",
"state": "{ analysis_title, objective, business_questions[], report_id }",
"report_ready": "{ ready: bool, missing: [analysis|delta] }",
"history": "[{ role: human|ai, content }] — drives language on the button path",
"message": "the human turn; null = button path (HelpAgent falls back to a per-language trigger)",
"asserts": "[{ type, ...spec }] — the rules the reply must obey",
"note": "human-readable description"
},
"cases": [
{
"id": "lang_01", "group": "language", "carried_over": false, "manual_review": false,
"state": { "analysis_title": "Analisis penjualan", "objective": "memahami performa penjualan bulanan", "business_questions": ["produk mana yang paling laku?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [{ "role": "human", "content": "aku baru upload datanya, terus aku harus ngapain?" }],
"message": null,
"asserts": [{ "type": "language_match", "expected": "Indonesian" }],
"note": "REGRESSION of the button-path bug: Indonesian conversation, message=null. Reply must be Indonesian, not English."
},
{
"id": "lang_02", "group": "language", "carried_over": false, "manual_review": false,
"state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [{ "role": "human", "content": "okay I uploaded my data, what do I do next?" }],
"message": null,
"asserts": [{ "type": "language_match", "expected": "English" }],
"note": "English conversation, button path — reply must stay English."
},
{
"id": "lang_03", "group": "language", "carried_over": false, "manual_review": false,
"state": { "analysis_title": "Analisis churn", "objective": "menurunkan churn pelanggan", "business_questions": ["segmen mana yang paling banyak churn?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [],
"message": "gimana caranya mulai analisis ini ya?",
"asserts": [{ "type": "language_match", "expected": "Indonesian" }],
"note": "Intent path: the real Indonesian user turn drives the language."
},
{
"id": "lang_04", "group": "language", "carried_over": false, "manual_review": false,
"state": { "analysis_title": "Retention analysis", "objective": "understand user retention", "business_questions": ["what drives repeat usage?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [],
"message": null,
"asserts": [{ "type": "language_match", "expected": "English" }],
"note": "Fresh analysis, no chat yet, button path — with no turn to read, the user-authored goal (English objective + business_questions, required at onboarding) drives the language."
},
{
"id": "lang_06", "group": "language", "carried_over": false, "manual_review": false,
"state": { "analysis_title": "Analisis retensi", "objective": "memahami retensi pengguna", "business_questions": ["apa yang mendorong penggunaan berulang?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [],
"message": null,
"asserts": [{ "type": "language_match", "expected": "Indonesian" }],
"note": "Same fresh-analysis path as lang_04 but the goal is Indonesian — the goal signal must yield Indonesian (not the hard fallback, which only fires when the goal is empty too)."
},
{
"id": "lang_05", "group": "language", "carried_over": false, "manual_review": false,
"state": { "analysis_title": "Analisis penjualan", "objective": "memahami tren penjualan", "business_questions": ["bagaimana tren bulanan?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [
{ "role": "human", "content": "apa saja yang bisa aku tanyakan tentang data ini?" },
{ "role": "ai", "content": "You can start by asking which products sell the most." }
],
"message": null,
"asserts": [{ "type": "language_match", "expected": "Indonesian" }],
"note": "Last AI turn is English but the human turn is Indonesian — mirror the human, reply Indonesian."
},
{
"id": "help_ex_guard_delta", "group": "report_guard", "carried_over": true, "manual_review": false,
"state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": "rep-1" },
"report_ready": { "ready": false, "missing": ["delta"] },
"history": [{ "role": "human", "content": "what should I do next?" }],
"message": null,
"asserts": [{ "type": "must_not_contain_any", "patterns": ["/report", "generate the report", "generate your report", "create the report"] }],
"note": "MIRRORS help.md example help_ex_guard_delta. A report exists and nothing new since — must NOT tell the user to generate a report; steer them to run a fresh analysis first."
},
{
"id": "help_ex_guard_ready", "group": "report_guard", "carried_over": true, "manual_review": false,
"state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": null },
"report_ready": { "ready": true, "missing": [] },
"history": [{ "role": "human", "content": "what should I do next?" }],
"message": null,
"asserts": [
{ "type": "must_contain_any", "patterns": ["Generate", "generate", "report"] },
{ "type": "must_not_contain_any", "patterns": ["/report"] }
],
"note": "MIRRORS help.md example help_ex_guard_ready. Enough analysis done — SHOULD nudge toward the report by pointing at the Generate button; must NOT offer /report (removed in v6)."
},
{
"id": "guard_03", "group": "report_guard", "carried_over": false, "manual_review": false,
"state": { "analysis_title": "Retention analysis", "objective": "improve 30-day retention", "business_questions": ["which cohort retains best?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [{ "role": "human", "content": "can I get a report now?" }],
"message": null,
"asserts": [{ "type": "must_not_contain_any", "patterns": ["/report", "generate the report", "generate your report", "you can generate"] }],
"note": "No analysis run yet, user asks for a report directly — must NOT offer to generate; redirect to running an analysis first."
},
{
"id": "guard_04", "group": "report_guard", "carried_over": false, "manual_review": false,
"state": { "analysis_title": "Analisis penjualan", "objective": "memahami performa penjualan", "business_questions": ["produk mana yang paling laku?"], "report_id": null },
"report_ready": { "ready": true, "missing": [] },
"history": [{ "role": "human", "content": "selanjutnya aku ngapain?" }],
"message": null,
"asserts": [
{ "type": "must_contain_any", "patterns": ["Generate", "generate", "laporan", "report"] },
{ "type": "must_not_contain_any", "patterns": ["/report"] },
{ "type": "language_match", "expected": "Indonesian" }
],
"note": "Ready + Indonesian conversation — should nudge toward the report (Generate button) AND stay in Indonesian; must NOT offer /report (removed in v6)."
},
{
"id": "guard_05", "group": "report_guard", "carried_over": false, "manual_review": false,
"state": { "analysis_title": "Analisis churn", "objective": "menurunkan churn", "business_questions": ["segmen mana yang paling churn?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [{ "role": "human", "content": "aku mau bikin laporan dong" }],
"message": null,
"asserts": [
{ "type": "must_not_contain_any", "patterns": ["/report", "silakan buat laporan", "kamu bisa membuat laporan", "generate your report"] },
{ "type": "language_match", "expected": "Indonesian" }
],
"note": "Indonesian, not ready, user asks for a report — must NOT offer it and must reply in Indonesian."
},
{
"id": "help_ex_orient", "group": "orientation", "carried_over": true, "manual_review": true,
"state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [],
"message": null,
"asserts": [],
"note": "MIRRORS help.md example help_ex_orient. MANUAL: are the 2-3 starter questions concrete, descriptive-first, and tied to the objective? Read output_text."
},
{
"id": "orient_02", "group": "orientation", "carried_over": false, "manual_review": true,
"state": { "analysis_title": "Retention analysis", "objective": "improve 30-day retention", "business_questions": ["which acquisition channel retains best?"], "report_id": null },
"report_ready": { "ready": false, "missing": ["analysis"] },
"history": [
{ "role": "human", "content": "which channel brings the most signups?" },
{ "role": "ai", "content": "Organic search brought the most signups last month (1,240)." }
],
"message": null,
"asserts": [],
"note": "MANUAL: one question already answered — does help build on it with a NEW follow-up (retention by channel), not re-suggest the answered question? Read output_text."
}
]
}
|