Tighten package-sensitive legacy fallback paths

backend/app/knowledgebase/core.py

@@ -38612,6 +38612,9 @@ class UnifiedKnowledgebaseCore:
                     return str(catalog_row.get("display_name") or model_text)
                 return str(model_text or "")
 
+            def _fallback_choice_key(model_text: str) -> str:
+                return _compact_model(_fallback_display(model_text) or model_text)
+
             def _fallback_recommendation_label(model_text: str, *, include_manufacturer: bool = False) -> str:
                 display = _fallback_display(model_text)
                 if not include_manufacturer:
@@ -38769,8 +38772,35 @@ class UnifiedKnowledgebaseCore:
                 token in query.normalized_message
                 for token in ("which replacement path is cleaner", "cleaner path", "cleaner for")
             )
+            asks_package_confirmation = any(
+                token in query.normalized_message
+                for token in (
+                    "exact variant",
+                    "exact sku",
+                    "exact package",
+                    "sku/package",
+                    "variant/package",
+                    "package still matters",
+                    "variant still matters",
+                    "sku still matters",
+                )
+            )
             suppress_family_branding = bool(match.get("_family_collapsed") or analysis.get("_family_safe") or family_safe_note)
             has_only_lifecycle_fallback = fallback_source or ((not primary_rows) and (not backup_rows) and bool(lifecycle_fallback_texts or legacy_5g or legacy_4g))
+            lifecycle_fallback_primary = lifecycle_fallback_texts[0] if lifecycle_fallback_texts else ""
+            lifecycle_fallback_primary_key = _fallback_choice_key(lifecycle_fallback_primary)
+            legacy_5g_key = _fallback_choice_key(legacy_5g)
+            legacy_4g_key = _fallback_choice_key(legacy_4g)
+            show_legacy_5g_recommendation = bool(legacy_5g) and (
+                asks_5g_replacement or (not lifecycle_fallback_primary and not primary_rows and not same_manufacturer_backup_rows)
+            )
+            if show_legacy_5g_recommendation and legacy_5g_key and legacy_5g_key == lifecycle_fallback_primary_key:
+                show_legacy_5g_recommendation = False
+            show_legacy_4g_recommendation = bool(legacy_4g)
+            if show_legacy_4g_recommendation and legacy_4g_key and legacy_4g_key == lifecycle_fallback_primary_key:
+                show_legacy_4g_recommendation = False
+            if show_legacy_4g_recommendation and legacy_4g_key and legacy_4g_key == legacy_5g_key:
+                show_legacy_4g_recommendation = False
             resolved_subject_label = _norm(
                 match.get("_canonical_resolved_label")
                 or _as_dict(analysis.get("product")).get("_canonical_resolved_label")
@@ -38803,9 +38833,24 @@ class UnifiedKnowledgebaseCore:
                         f"If that normalization is wrong, stop here and send the exact device label before using these recommendations.",
                         "",
                     ]
-                )
+            )
             if family_safe_note:
                 lines.extend([family_safe_note, ""])
+            if asks_package_confirmation:
+                if suppress_family_branding or has_only_lifecycle_fallback:
+                    lines.extend(
+                        [
+                            "Exact variant/package still matters: yes. This answer stays family-safe, so confirm the final SKU/package before quote lock.",
+                            "",
+                        ]
+                    )
+                else:
+                    lines.extend(
+                        [
+                            "Exact variant/package still matters: not for the replacement lane itself, but confirm the final SKU/package before quoting feature-sensitive details.",
+                            "",
+                        ]
+                    )
             if (not bool(query.current_only)) or any(
                 token in query.normalized_message
                 for token in ("legacy", "older", "old", "historical", "similar")
@@ -38977,13 +39022,13 @@ class UnifiedKnowledgebaseCore:
                 lines.append("Fallback note: the lifecycle rows below are sourced migration hints, not workbook-ready replacement lanes, so feature fit still needs validation before quoting.")
                 lines.append("")
                 lines.append("Fallback recommendations:")
-                if lifecycle_fallback_texts:
+                if lifecycle_fallback_primary:
                     lines.append(
-                        f"- Best sourced current path: `{_fallback_display(lifecycle_fallback_texts[0])}`. {_fallback_lineage_note(lifecycle_fallback_texts[0], include_manufacturer=False)}. Tradeoff: {_fallback_tradeoff(lifecycle_fallback_texts[0])}."
+                        f"- Best sourced current path: `{_fallback_display(lifecycle_fallback_primary)}`. {_fallback_lineage_note(lifecycle_fallback_primary, include_manufacturer=False)}. Tradeoff: {_fallback_tradeoff(lifecycle_fallback_primary)}."
                     )
-                if legacy_5g:
+                if show_legacy_5g_recommendation:
                     lines.append(f"- Best sourced 5G path: `{_fallback_display(legacy_5g)}`. {_fallback_lineage_note(legacy_5g, include_manufacturer=False)}. Tradeoff: {_fallback_tradeoff(legacy_5g)}.")
-                if legacy_4g:
+                if show_legacy_4g_recommendation:
                     lines.append(f"- Best sourced 4G bridge: `{_fallback_display(legacy_4g)}`. {_fallback_lineage_note(legacy_4g, include_manufacturer=False)}. Tradeoff: {_fallback_tradeoff(legacy_4g, bridge_mode=True)}.")
                 lines.append("")
             lines.extend(
@@ -39054,7 +39099,7 @@ class UnifiedKnowledgebaseCore:
                     lines.append(
                         f"| {lane_label} | {_md_cell(_display_name(row))} | {_md_cell(combined_note)} |"
                     )
-            else:
+            elif not (show_sourced_fallback_rows and (lifecycle_fallback_primary or show_legacy_5g_recommendation or show_legacy_4g_recommendation)):
                 lines.append(
                     f"| {'Sourced same-manufacturer backup' if fallback_source else 'Same-manufacturer backup'} | None listed | {_md_cell(_same_brand_empty_note())} |"
                 )
@@ -39071,19 +39116,19 @@ class UnifiedKnowledgebaseCore:
                 lines.append(
                     f"| {'Sourced cross-vendor backup' if fallback_source else 'Cross-vendor backup'} | None listed | {_md_cell(_backup_empty_note())} |"
                 )
-            if show_sourced_fallback_rows and lifecycle_fallback_texts:
+            if show_sourced_fallback_rows and lifecycle_fallback_primary:
                 lines.append(
-                    f"| Sourced lifecycle fallback | {_md_cell(_fallback_display(lifecycle_fallback_texts[0]))} | {_md_cell('Lifecycle mapping fallback only; ' + _fallback_lineage_note(lifecycle_fallback_texts[0], include_manufacturer=False) + '; tradeoff: ' + _fallback_tradeoff(lifecycle_fallback_texts[0]))} |"
+                    f"| Sourced lifecycle fallback | {_md_cell(_fallback_display(lifecycle_fallback_primary))} | {_md_cell('Lifecycle mapping fallback only; ' + _fallback_lineage_note(lifecycle_fallback_primary, include_manufacturer=False) + '; tradeoff: ' + _fallback_tradeoff(lifecycle_fallback_primary))} |"
                 )
-            elif show_sourced_fallback_rows and (legacy_5g or legacy_4g):
-                if legacy_5g and not emitted_requested_5g_path:
+            elif show_sourced_fallback_rows and (show_legacy_5g_recommendation or show_legacy_4g_recommendation):
+                if show_legacy_5g_recommendation and not emitted_requested_5g_path:
                     legacy_5g_label = "Sourced lifecycle 5G fallback"
                     if asks_5g_replacement:
                         legacy_5g_label = "Requested 5G same-brand path" if _fallback_is_same_vendor_5g(legacy_5g) else "Requested 5G path"
                     lines.append(
                         f"| {legacy_5g_label} | {_md_cell(_fallback_recommendation_label(legacy_5g, include_manufacturer=False))} | {_md_cell('Lifecycle mapping fallback only; ' + _fallback_lineage_note(legacy_5g, include_manufacturer=False) + '; tradeoff: ' + _fallback_tradeoff(legacy_5g))} |"
                     )
-                if legacy_4g and not requested_5g_priority:
+                if show_legacy_4g_recommendation and not requested_5g_priority:
                     lines.append(
                         f"| Sourced lifecycle 4G bridge | {_md_cell(_fallback_display(legacy_4g))} | {_md_cell('Lifecycle mapping fallback only; ' + _fallback_lineage_note(legacy_4g, include_manufacturer=False) + '; tradeoff: ' + _fallback_tradeoff(legacy_4g, bridge_mode=True))} |"
                     )

backend/app/test_unified_kb_core.py

@@ -11677,10 +11677,76 @@ def test_unified_kb_mbr1400_fallback_replacement_prefers_current_fallback_before
     assistant = str(out.get("assistant") or "")
     assert out["meta"]["retrieval_mode"] == "deterministic_router_workbook_replacements"
     assert "1. `Same-brand path` -> `E300`" in assistant
-    assert "Best sourced 5G path: `E400`" in assistant
+    assert "Best sourced 5G path: `E400`" not in assistant
+    assert "Fallback recommendations:" in assistant
     assert "1. `Same-brand path` -> `E400`" not in assistant
     assert "| Sourced same-manufacturer path | E300 |" in assistant
     assert "| Same manufacturer | None workbook-ready |" not in assistant
+    assert "| Sourced same-manufacturer backup | None listed |" not in assistant
+
+
+def test_unified_kb_br1_mini_package_sensitive_fallback_answers_variant_question_cleanly(tmp_path: Path, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    core = _build_core_with_loaded_workbook(tmp_path, router_core=_synthetic_workbook_router_core(tmp_path))
+    workbook_core = core._rapid_router_intelligence_core()
+    assert workbook_core is not None
+    monkeypatch.setattr(
+        core,
+        "_router_workbook_resolve_replacement_analysis",
+        lambda _workbook_core, manufacturer_text, product_text: {
+            "ok": True,
+            "resolution_mode": "family_alias_provisional",
+            "analysis": {
+                "match": {
+                    "product_key": "prod_peplink_br1_mini_family",
+                    "product_id": "BR1MINI",
+                    "display_name": "BR1 Mini",
+                    "manufacturer_group": "Peplink",
+                    "status_bucket": "End of Life",
+                    "_requested_label": "BR1 Mini",
+                    "_family_collapsed": True,
+                },
+                "replacements": {
+                    "primary_candidates": [],
+                    "primary_replacement": None,
+                    "same_manufacturer_backup_replacements": [],
+                    "backup_replacements": [],
+                    "historical_only_replacements": [],
+                    "review_blocked_count": 0,
+                    "candidate_count": 0,
+                    "no_replacement": False,
+                },
+                "_replacement_source_mode": "lifecycle_fallback",
+                "_replacement_subject_lifecycle": {
+                    "events": [
+                        {"recommended_replacement_text": "MAX BR1 Mini 5G"},
+                    ]
+                },
+                "_replacement_legacy_lifecycle": {
+                    "rep5g": "MAX BR1 Mini 5G",
+                    "alt4g": "BR1 Mini LTEA",
+                },
+                "review_required": True,
+                "manual_review_reasons": [],
+                "_family_safe_note": "Requested `BR1 Mini` matched a workbook family row; exact variant still needs confirmation.",
+            },
+        },
+    )
+
+    out = core.handle_message(
+        "Give the same-brand and backup upgrade path for BR1 Mini, and say if the exact variant/package still matters.",
+        {},
+        mode="router_lifecycle",
+        audience="auto",
+        show_citations=True,
+    )
+
+    assistant = str(out.get("assistant") or "")
+    assert out["meta"]["retrieval_mode"] == "deterministic_router_workbook_replacements"
+    assert "Exact variant/package still matters: yes." in assistant
+    assert "Best sourced current path:" in assistant
+    assert "Best sourced 5G path:" not in assistant
+    assert "Best sourced 4G bridge:" in assistant
+    assert "| Sourced same-manufacturer backup | None listed |" not in assistant
 
 
 def test_unified_kb_rut950_replacement_drops_na_backup_and_uses_primary_only_note(tmp_path: Path, monkeypatch) -> None:  # type: ignore[no-untyped-def]

docs/testing/router_regression_packs/README.md

@@ -161,3 +161,42 @@ Scored outputs will land next to the chosen `--out-csv` path:
 How to use the expectations file:
 - Treat `router_compare_family_edge_expected.json` as the behavior snapshot for these compares.
 - If the scorer passes but the answer still collapses models, drifts to sparse alias rows, or hides behind generic snippet noise instead of a family-safe compare or conservative clarification, treat that as a real regression.
+
+### Router Package-Sensitive Legacy Edge Pack
+
+Files:
+- `router_package_sensitive_legacy_edge_pack.csv`
+- `router_package_sensitive_legacy_edge_expected.json`
+
+Purpose:
+- Track legacy or package-sensitive replacement prompts where the app can drift from workbook-backed current paths, overstate bundle-level certainty, or fall back into generic lifecycle/details wording.
+- Re-run them against hosted canary without perturbing the broader release-gate packs.
+
+Current legacy/package-sensitive targets:
+- `BR1 Mini`
+- `MBR1400`
+- `CBA850`
+- `CBA250`
+- `AER1650`
+- `C1101-4P`
+
+Recommended hosted canary run:
+
+```bash
+cd backend
+python3 scripts/run_router_canary_ab_eval_shard.py \
+  --in-csv ../docs/testing/router_regression_packs/router_package_sensitive_legacy_edge_pack.csv \
+  --out-csv /tmp/router_package_sensitive_legacy_edge_results.csv \
+  --base-url https://crazycrazypete-masters-four-tab-openai-canary.hf.space \
+  --auth-env-file ../frontend/.env.e2e \
+  --expected-git-sha <live-canary-sha> \
+  --score-after
+```
+
+Scored outputs will land next to the chosen `--out-csv` path:
+- `/tmp/router_package_sensitive_legacy_edge_results_scored.csv`
+- `/tmp/router_package_sensitive_legacy_edge_results_scored_summary.json`
+
+How to use the expectations file:
+- Treat `router_package_sensitive_legacy_edge_expected.json` as the behavior snapshot for this package-sensitive legacy wave.
+- A conservative family-safe answer is acceptable when the workbook still requires exact package confirmation, but it should not invent bundle-level certainty or hide the current workbook-backed path.

docs/testing/router_regression_packs/router_package_sensitive_legacy_edge_expected.json

@@ -0,0 +1,74 @@
+{
+  "package_legacy_001": {
+    "models": ["BR1 Mini"],
+    "expected_behavior": "package_sensitive_same_brand_replacement_with_variant_caution",
+    "must_not": [
+      "pretend one BR1 Mini package is universally correct",
+      "generic lifecycle-only answer with no replacement guidance"
+    ],
+    "notes": [
+      "A current workbook-backed path is acceptable if package caution stays visible.",
+      "A conservative clarify-first answer is acceptable if the workbook is still too variant-sensitive."
+    ]
+  },
+  "package_legacy_002": {
+    "models": ["MBR1400"],
+    "expected_behavior": "primary_same_brand_fallback_e300_before_5g_lane",
+    "must_not": [
+      "E400 outranking E300 as the primary same-brand path",
+      "contradictory same-brand-none-workbook-ready wording"
+    ],
+    "notes": [
+      "This is a replacement-ordering guardrail.",
+      "Backup path can still exist, but E300 should stay primary."
+    ]
+  },
+  "package_legacy_003": {
+    "models": ["CBA850"],
+    "expected_behavior": "current_family_safe_same_brand_upgrade_with_package_caution",
+    "must_not": [
+      "false exact-bundle certainty",
+      "historical-only replacement presented as the current path"
+    ],
+    "notes": [
+      "Family-safe replacement guidance is acceptable here.",
+      "The answer should say exact package/SKU can still matter."
+    ]
+  },
+  "package_legacy_004": {
+    "models": ["CBA250"],
+    "expected_behavior": "current_path_s700_not_historical_r980",
+    "must_not": [
+      "R980 displacing S700 as the current workbook-backed target",
+      "unclear current-vs-historical ordering"
+    ],
+    "notes": [
+      "The answer should keep the current target explicit.",
+      "Historical paths may still be mentioned if clearly labeled."
+    ]
+  },
+  "package_legacy_005": {
+    "models": ["AER1650"],
+    "expected_behavior": "replacement_lane_surfaces_e300_primary",
+    "must_not": [
+      "generic lifecycle/details answer with no replacement path",
+      "E400-only drift"
+    ],
+    "notes": [
+      "This stays a replacement-lane routing guardrail.",
+      "The answer should remain workbook-backed."
+    ]
+  },
+  "package_legacy_006": {
+    "models": ["C1101-4P"],
+    "expected_behavior": "same_brand_primary_row_c1121_4p",
+    "must_not": [
+      "generic details fallback",
+      "cross-vendor replacement suggestion"
+    ],
+    "notes": [
+      "The answer should keep the same-brand workbook primary row visible.",
+      "Current-path framing should be explicit."
+    ]
+  }
+}

docs/testing/router_regression_packs/router_package_sensitive_legacy_edge_pack.csv

@@ -0,0 +1,7 @@
+exported_at,global_index,shard_id,shard_case_index,case_id,family,subfamily,difficulty,mode_hint,audience,setup_kind,run_readiness,setup_summary,api_endpoint,api_method,api_payload_template_json,conversation_steps_json,prompt,judge_focus,notes,run_status,http_status,request_id,latency_ms,response_assistant,response_sources_json,response_files_json,response_meta_json,response_state_json,response_error,judge_model,judge_fact_score,judge_instruction_score,judge_coverage_score,judge_readability_score,judge_safety_score,judge_overall_score,judge_grade,judge_pass,judge_issues_json,judge_rationale
+2026-04-02T02:12:00+00:00,1,package_sensitive_legacy_pack,1,package_legacy_001,router_lifecycle,package_sensitive_replacement,moderate,auto,auto,none,ready,Direct single-turn case. No pre-seeded KB state required.,/api/knowledgebase/message,POST,"{""audience"": ""auto"", ""message"": ""Give the same-brand and backup upgrade path for BR1 Mini, and say if the exact variant/package still matters."", ""mode"": null, ""request_id"": ""router-package-pack-001"", ""show_citations"": true, ""state"": {}}","[""Give the same-brand and backup upgrade path for BR1 Mini, and say if the exact variant/package still matters.""]","Give the same-brand and backup upgrade path for BR1 Mini, and say if the exact variant/package still matters.","Verify the answer stays conservative for BR1 Mini package-sensitive ambiguity, surfaces current workbook-backed replacement guidance if available, and explicitly says whether exact package confirmation still matters.","Package-sensitive Peplink replacement probe.",,,,,,,,,,,,,,,,,,
+2026-04-02T02:12:00+00:00,2,package_sensitive_legacy_pack,2,package_legacy_002,router_lifecycle,package_sensitive_replacement,moderate,auto,auto,none,ready,Direct single-turn case. No pre-seeded KB state required.,/api/knowledgebase/message,POST,"{""audience"": ""auto"", ""message"": ""What is the primary same-brand replacement and backup path for MBR1400, workbook-only?"", ""mode"": null, ""request_id"": ""router-package-pack-002"", ""show_citations"": true, ""state"": {}}","[""What is the primary same-brand replacement and backup path for MBR1400, workbook-only?""]","What is the primary same-brand replacement and backup path for MBR1400, workbook-only?","Verify the answer keeps E300 as the primary same-brand fallback path, avoids contradictory E400 drift, and stays workbook-backed.","Legacy Cradlepoint replacement-ordering probe.",,,,,,,,,,,,,,,,,,
+2026-04-02T02:12:00+00:00,3,package_sensitive_legacy_pack,3,package_legacy_003,router_lifecycle,package_sensitive_replacement,moderate,auto,auto,none,ready,Direct single-turn case. No pre-seeded KB state required.,/api/knowledgebase/message,POST,"{""audience"": ""auto"", ""message"": ""Give the current same-brand upgrade path for CBA850, and tell me if package/SKU confirmation still matters."", ""mode"": null, ""request_id"": ""router-package-pack-003"", ""show_citations"": true, ""state"": {}}","[""Give the current same-brand upgrade path for CBA850, and tell me if package/SKU confirmation still matters.""]","Give the current same-brand upgrade path for CBA850, and tell me if package/SKU confirmation still matters.","Verify the answer surfaces the current same-brand CBA850 family path conservatively and keeps package/SKU caution visible instead of pretending one exact bundle is universal.","Package-sensitive CBA850 family replacement probe.",,,,,,,,,,,,,,,,,,
+2026-04-02T02:12:00+00:00,4,package_sensitive_legacy_pack,4,package_legacy_004,router_lifecycle,package_sensitive_replacement,moderate,auto,auto,none,ready,Direct single-turn case. No pre-seeded KB state required.,/api/knowledgebase/message,POST,"{""audience"": ""auto"", ""message"": ""Give the replacement path for CBA250 and tell me whether the current workbook-backed target is still current."", ""mode"": null, ""request_id"": ""router-package-pack-004"", ""show_citations"": true, ""state"": {}}","[""Give the replacement path for CBA250 and tell me whether the current workbook-backed target is still current.""]","Give the replacement path for CBA250 and tell me whether the current workbook-backed target is still current.","Verify the answer keeps S700 as the current path, distinguishes it from historical paths, and does not let R980 displace the current workbook-backed recommendation.","Current-vs-historical replacement ordering probe.",,,,,,,,,,,,,,,,,,
+2026-04-02T02:12:00+00:00,5,package_sensitive_legacy_pack,5,package_legacy_005,router_lifecycle,package_sensitive_replacement,moderate,auto,auto,none,ready,Direct single-turn case. No pre-seeded KB state required.,/api/knowledgebase/message,POST,"{""audience"": ""auto"", ""message"": ""Give the same-brand and backup upgrade path for AER1650, workbook-only."", ""mode"": null, ""request_id"": ""router-package-pack-005"", ""show_citations"": true, ""state"": {}}","[""Give the same-brand and backup upgrade path for AER1650, workbook-only.""]","Give the same-brand and backup upgrade path for AER1650, workbook-only.","Verify the answer stays in the replacement lane and surfaces the workbook-backed E300 primary path instead of falling back into generic lifecycle/details copy.","Legacy AirLink replacement routing probe.",,,,,,,,,,,,,,,,,,
+2026-04-02T02:12:00+00:00,6,package_sensitive_legacy_pack,6,package_legacy_006,router_lifecycle,package_sensitive_replacement,moderate,auto,auto,none,ready,Direct single-turn case. No pre-seeded KB state required.,/api/knowledgebase/message,POST,"{""audience"": ""auto"", ""message"": ""What is the same-brand upgrade path for C1101-4P, and is the workbook primary row current?"", ""mode"": null, ""request_id"": ""router-package-pack-006"", ""show_citations"": true, ""state"": {}}","[""What is the same-brand upgrade path for C1101-4P, and is the workbook primary row current?""]","What is the same-brand upgrade path for C1101-4P, and is the workbook primary row current?","Verify the answer surfaces the workbook-backed C1121-4P path directly, stays same-brand, and avoids collapsing into generic details/lifecycle fallback.","Legacy Cisco replacement-lane probe.",,,,,,,,,,,,,,,,,,