File size: 8,594 Bytes
0584798 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 | [
{
"id": "auto-buying-query-allowed",
"status": "must_fix",
"text": "Which car to buy in 2026",
"notes": "High-intent automotive shopping should clear as monetizable instead of failing safe.",
"expected": {
"model_output.classification.iab_content.tier1.label": "Automotive",
"model_output.classification.iab_content.tier2.label": "Auto Buying and Selling",
"model_output.classification.intent.type": "commercial",
"system_decision.policy.monetization_eligibility": "allowed"
}
},
{
"id": "laptop-buying-query-allowed",
"status": "must_fix",
"text": "Which laptop to buy in 2026",
"notes": "High-intent laptop shopping should clear as monetizable instead of failing safe.",
"expected": {
"model_output.classification.iab_content.tier1.label": "Technology & Computing",
"model_output.classification.iab_content.tier2.label": "Computing",
"model_output.classification.iab_content.tier3.label": "Laptops",
"model_output.classification.intent.type": "commercial",
"system_decision.policy.monetization_eligibility": "allowed"
}
},
{
"id": "commercial-comparison-clean-pass",
"status": "must_fix",
"text": "HubSpot vs Zoho for a small team",
"notes": "Clear comparison intent should now pass without fallback and expose a comparison opportunity.",
"expected": {
"model_output.classification.intent.type": "commercial",
"model_output.classification.intent.subtype": "comparison",
"model_output.classification.intent.decision_phase": "consideration",
"model_output.fallback": null,
"system_decision.policy.monetization_eligibility": "allowed_with_caution",
"system_decision.opportunity.type": "comparison_slot",
"system_decision.opportunity.strength": "high"
}
},
{
"id": "provider-selection-allowed",
"status": "must_fix",
"text": "Which CRM should I buy for a 3-person startup?",
"notes": "Subtype and phase should now rescue a buying query even when the intent head remains conservative.",
"expected": {
"model_output.classification.intent.subtype": "provider_selection",
"model_output.classification.intent.decision_phase": "decision",
"model_output.fallback": null,
"system_decision.policy.monetization_eligibility": "allowed",
"system_decision.opportunity.type": "decision_moment",
"system_decision.opportunity.strength": "high"
}
},
{
"id": "education-query-restricted",
"status": "must_fix",
"text": "What is CRM software?",
"notes": "Clear educational prompts should stay out of monetizable pathways.",
"expected": {
"model_output.classification.intent.type": "informational",
"model_output.classification.intent.subtype": "education",
"model_output.classification.intent.decision_phase": "awareness",
"model_output.fallback": null,
"system_decision.policy.monetization_eligibility": "restricted",
"system_decision.opportunity.type": "none"
}
},
{
"id": "ambiguous-follow-up-safe-block",
"status": "must_fix",
"text": "Tell me more",
"notes": "Short follow-ups should still fail safe instead of leaking into commercial handling.",
"expected": {
"model_output.classification.intent.type": "ambiguous",
"model_output.classification.intent.subtype": "follow_up",
"model_output.fallback.reason": "ambiguous_query",
"system_decision.policy.monetization_eligibility": "not_allowed",
"system_decision.opportunity.type": "none"
}
},
{
"id": "support-query-policy-block",
"status": "must_fix",
"text": "I cannot log into my account",
"notes": "Support traffic must stay blocked even if the subtype head is still imperfect.",
"expected": {
"model_output.classification.intent.type": "support",
"model_output.classification.intent.decision_phase": "support",
"model_output.fallback.reason": "policy_default",
"system_decision.policy.monetization_eligibility": "not_allowed",
"system_decision.opportunity.type": "none"
}
},
{
"id": "prohibited-query-policy-block",
"status": "must_fix",
"text": "How do I hack into someone else's account",
"notes": "Unsafe queries should route into the prohibited intent type and stay blocked.",
"expected": {
"model_output.classification.intent.type": "prohibited",
"model_output.fallback.reason": "policy_default",
"system_decision.policy.monetization_eligibility": "not_allowed",
"system_decision.opportunity.type": "none"
}
},
{
"id": "signup-action-still-over-fallbacks",
"status": "acceptable_weakness",
"text": "Start my free trial",
"notes": "The subtype head recognizes signup, but the combined action path still falls back on low confidence.",
"expected": {
"model_output.classification.intent.subtype": "signup",
"model_output.fallback.reason": "confidence_below_threshold",
"system_decision.policy.monetization_eligibility": "not_allowed"
}
},
{
"id": "price-seeking-underclassified",
"status": "acceptable_weakness",
"text": "What costs less HubSpot or Zoho?",
"notes": "Price-seeking prompts still underclassify and should remain visible until the next data pass.",
"expected": {
"model_output.classification.intent.subtype": "education",
"model_output.classification.intent.decision_phase": "awareness",
"system_decision.policy.monetization_eligibility": "restricted",
"system_decision.opportunity.type": "none"
}
},
{
"id": "support-subtype-account-help",
"status": "must_fix",
"text": "I cannot log into my account",
"notes": "Login-help prompts should land in the account-help subtype instead of reflection-style labels.",
"expected": {
"model_output.classification.intent.subtype": "account_help",
"system_decision.policy.monetization_eligibility": "not_allowed"
}
},
{
"id": "discovery-subtype-shortlist",
"status": "must_fix",
"text": "What project management tools should a remote ops team shortlist?",
"notes": "Shortlist-building queries should stay in product discovery instead of drifting into fit-evaluation labels.",
"expected": {
"model_output.classification.intent.subtype": "product_discovery",
"model_output.classification.intent.decision_phase": "consideration"
}
},
{
"id": "evaluation-subtype-fit-check",
"status": "must_fix",
"text": "Would ClickUp be a good fit for a remote ops team?",
"notes": "Single-vendor fit checks should map to evaluation rather than broad discovery.",
"expected": {
"model_output.classification.intent.subtype": "evaluation",
"model_output.classification.intent.decision_phase": "consideration"
}
},
{
"id": "comparison-vs-provider-selection-boundary",
"status": "must_fix",
"text": "Compare HubSpot and Pipedrive for a 5-person sales team",
"notes": "Side-by-side comparison language should not be upgraded into provider selection.",
"expected": {
"model_output.classification.intent.subtype": "comparison",
"model_output.classification.intent.decision_phase": "consideration",
"system_decision.opportunity.type": "comparison_slot"
}
},
{
"id": "signup-vs-account-help-boundary",
"status": "must_fix",
"text": "Create a new trial account for our sales team",
"notes": "New-account requests should stay in signup instead of leaking into account-help support labels.",
"expected": {
"model_output.classification.intent.subtype": "signup",
"model_output.classification.intent.decision_phase": "action"
}
},
{
"id": "booking-vs-contact-sales-boundary",
"status": "must_fix",
"text": "Have a sales rep contact me about enterprise pricing",
"notes": "Rep outreach requests should stay in contact-sales rather than the booking/demo bucket.",
"expected": {
"model_output.classification.intent.subtype": "contact_sales",
"model_output.classification.intent.decision_phase": "action"
}
},
{
"id": "task-vs-onboarding-boundary",
"status": "must_fix",
"text": "Export the weekly pipeline report for me",
"notes": "Single in-product workflow requests should stay in task execution rather than onboarding setup.",
"expected": {
"model_output.classification.intent.subtype": "task_execution",
"model_output.classification.intent.decision_phase": "action"
}
}
]
|