{
  "_about": "Golden dataset for the Help skill (`src/agents/handlers/help.HelpAgent`). Unlike intent/readiness this calls the LIVE model: each case declares an analysis state + report-readiness + chat history, the runner streams HelpAgent.astream for real, and asserts RULES the reply must obey (not text similarity — help replies are free prose with no single correct wording). Metric is COMPLIANCE (% of rule assertions that hold), reported separately for held-out vs carried_over cases.",
  "_groups": "language (reply matches the user's language — the button-path bug), report_guard (never suggest a report when report_ready.ready=false; do suggest it when true — this also IS the 'no action leakage' check, since generate_report is the only gated action), orientation (quality of the suggested starter questions — MANUAL review, not auto-scored).",
  "_asserts": "language_match {expected} — detect the reply's language (help._detect_reply_language over the OUTPUT) must equal expected. must_not_contain_any {patterns} — none of the (case-insensitive) patterns appear. must_contain_any {patterns} — at least one appears.",
  "_carried_over": "carried_over:true rows MIRROR an example in config/prompts/help.md (the row `id` IS the help.md `<!-- id: ... -->`). They are the regression guard: if the prompt is refactored, the demonstrated rule must still hold. What is mirrored is the INPUT spec + the assertion — NOT the example's reply text (temperature>0 makes exact match invalid). Held-out rows (carried_over:false) are NOT in the prompt; their compliance is the real generalization signal. If help.md's Examples change, keep these ids in sync (manual, like intent).",
  "_missing_codes": "report_ready.missing uses codes mapped to the real strings is_report_ready emits (imported in run_eval): analysis -> _MISSING_ANALYSIS, delta -> _MISSING_DELTA. Kept as codes so the dataset survives wording changes.",
  "schema": {
    "id": "stable handle; for carried_over rows this equals the help.md example id",
    "group": "language | report_guard | orientation",
    "carried_over": "bool — mirrors a help.md example",
    "manual_review": "bool — run but exclude from the auto compliance rate (read output_text)",
    "state": "{ analysis_title, objective, business_questions[], report_id }",
    "report_ready": "{ ready: bool, missing: [analysis|delta] }",
    "history": "[{ role: human|ai, content }] — drives language on the button path",
    "message": "the human turn; null = button path (HelpAgent falls back to a per-language trigger)",
    "asserts": "[{ type, ...spec }] — the rules the reply must obey",
    "note": "human-readable description"
  },
  "cases": [
    {
      "id": "lang_01", "group": "language", "carried_over": false, "manual_review": false,
      "state": { "analysis_title": "Analisis penjualan", "objective": "memahami performa penjualan bulanan", "business_questions": ["produk mana yang paling laku?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [{ "role": "human", "content": "aku baru upload datanya, terus aku harus ngapain?" }],
      "message": null,
      "asserts": [{ "type": "language_match", "expected": "Indonesian" }],
      "note": "REGRESSION of the button-path bug: Indonesian conversation, message=null. Reply must be Indonesian, not English."
    },
    {
      "id": "lang_02", "group": "language", "carried_over": false, "manual_review": false,
      "state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [{ "role": "human", "content": "okay I uploaded my data, what do I do next?" }],
      "message": null,
      "asserts": [{ "type": "language_match", "expected": "English" }],
      "note": "English conversation, button path — reply must stay English."
    },
    {
      "id": "lang_03", "group": "language", "carried_over": false, "manual_review": false,
      "state": { "analysis_title": "Analisis churn", "objective": "menurunkan churn pelanggan", "business_questions": ["segmen mana yang paling banyak churn?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [],
      "message": "gimana caranya mulai analisis ini ya?",
      "asserts": [{ "type": "language_match", "expected": "Indonesian" }],
      "note": "Intent path: the real Indonesian user turn drives the language."
    },
    {
      "id": "lang_04", "group": "language", "carried_over": false, "manual_review": false,
      "state": { "analysis_title": "Retention analysis", "objective": "understand user retention", "business_questions": ["what drives repeat usage?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [],
      "message": null,
      "asserts": [{ "type": "language_match", "expected": "English" }],
      "note": "Fresh analysis, no chat yet, button path — with no turn to read, the user-authored goal (English objective + business_questions, required at onboarding) drives the language."
    },
    {
      "id": "lang_06", "group": "language", "carried_over": false, "manual_review": false,
      "state": { "analysis_title": "Analisis retensi", "objective": "memahami retensi pengguna", "business_questions": ["apa yang mendorong penggunaan berulang?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [],
      "message": null,
      "asserts": [{ "type": "language_match", "expected": "Indonesian" }],
      "note": "Same fresh-analysis path as lang_04 but the goal is Indonesian — the goal signal must yield Indonesian (not the hard fallback, which only fires when the goal is empty too)."
    },
    {
      "id": "lang_05", "group": "language", "carried_over": false, "manual_review": false,
      "state": { "analysis_title": "Analisis penjualan", "objective": "memahami tren penjualan", "business_questions": ["bagaimana tren bulanan?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [
        { "role": "human", "content": "apa saja yang bisa aku tanyakan tentang data ini?" },
        { "role": "ai", "content": "You can start by asking which products sell the most." }
      ],
      "message": null,
      "asserts": [{ "type": "language_match", "expected": "Indonesian" }],
      "note": "Last AI turn is English but the human turn is Indonesian — mirror the human, reply Indonesian."
    },
    {
      "id": "help_ex_guard_delta", "group": "report_guard", "carried_over": true, "manual_review": false,
      "state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": "rep-1" },
      "report_ready": { "ready": false, "missing": ["delta"] },
      "history": [{ "role": "human", "content": "what should I do next?" }],
      "message": null,
      "asserts": [{ "type": "must_not_contain_any", "patterns": ["/report", "generate the report", "generate your report", "create the report"] }],
      "note": "MIRRORS help.md example help_ex_guard_delta. A report exists and nothing new since — must NOT tell the user to generate a report; steer them to run a fresh analysis first."
    },
    {
      "id": "help_ex_guard_ready", "group": "report_guard", "carried_over": true, "manual_review": false,
      "state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": null },
      "report_ready": { "ready": true, "missing": [] },
      "history": [{ "role": "human", "content": "what should I do next?" }],
      "message": null,
      "asserts": [{ "type": "must_contain_any", "patterns": ["/report", "report"] }],
      "note": "MIRRORS help.md example help_ex_guard_ready. Enough analysis done — SHOULD nudge toward the report (mention /report or the report option)."
    },
    {
      "id": "guard_03", "group": "report_guard", "carried_over": false, "manual_review": false,
      "state": { "analysis_title": "Retention analysis", "objective": "improve 30-day retention", "business_questions": ["which cohort retains best?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [{ "role": "human", "content": "can I get a report now?" }],
      "message": null,
      "asserts": [{ "type": "must_not_contain_any", "patterns": ["/report", "generate the report", "generate your report", "you can generate"] }],
      "note": "No analysis run yet, user asks for a report directly — must NOT offer to generate; redirect to running an analysis first."
    },
    {
      "id": "guard_04", "group": "report_guard", "carried_over": false, "manual_review": false,
      "state": { "analysis_title": "Analisis penjualan", "objective": "memahami performa penjualan", "business_questions": ["produk mana yang paling laku?"], "report_id": null },
      "report_ready": { "ready": true, "missing": [] },
      "history": [{ "role": "human", "content": "selanjutnya aku ngapain?" }],
      "message": null,
      "asserts": [
        { "type": "must_contain_any", "patterns": ["/report", "laporan", "report"] },
        { "type": "language_match", "expected": "Indonesian" }
      ],
      "note": "Ready + Indonesian conversation — should nudge toward the report AND stay in Indonesian (two rules at once)."
    },
    {
      "id": "guard_05", "group": "report_guard", "carried_over": false, "manual_review": false,
      "state": { "analysis_title": "Analisis churn", "objective": "menurunkan churn", "business_questions": ["segmen mana yang paling churn?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [{ "role": "human", "content": "aku mau bikin laporan dong" }],
      "message": null,
      "asserts": [
        { "type": "must_not_contain_any", "patterns": ["/report", "silakan buat laporan", "kamu bisa membuat laporan", "generate your report"] },
        { "type": "language_match", "expected": "Indonesian" }
      ],
      "note": "Indonesian, not ready, user asks for a report — must NOT offer it and must reply in Indonesian."
    },
    {
      "id": "help_ex_orient", "group": "orientation", "carried_over": true, "manual_review": true,
      "state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [],
      "message": null,
      "asserts": [],
      "note": "MIRRORS help.md example help_ex_orient. MANUAL: are the 2-3 starter questions concrete, descriptive-first, and tied to the objective? Read output_text."
    },
    {
      "id": "orient_02", "group": "orientation", "carried_over": false, "manual_review": true,
      "state": { "analysis_title": "Retention analysis", "objective": "improve 30-day retention", "business_questions": ["which acquisition channel retains best?"], "report_id": null },
      "report_ready": { "ready": false, "missing": ["analysis"] },
      "history": [
        { "role": "human", "content": "which channel brings the most signups?" },
        { "role": "ai", "content": "Organic search brought the most signups last month (1,240)." }
      ],
      "message": null,
      "asserts": [],
      "note": "MANUAL: one question already answered — does help build on it with a NEW follow-up (retention by channel), not re-suggest the answered question? Read output_text."
    }
  ]
}