feat: conversational profile updates in free-form chat (D-021)

After fact-find onboarding completes, users often share new profile facts
in ordinary conversation — "I just turned 40", "we had a baby", "I was
diagnosed with diabetes". Vanilla retrieval didn't know to update the
session profile from these utterances.

Implementation:
- backend/profile_extractor.py — lightweight NIM extractor (Llama 3.3 70B)
runs on each free-form user message. Returns validated dict of
{field_name: new_value} for high-confidence updates only. Conservative
validation: bad types / enums / bounds get dropped silently.
- handle_turn() in orchestrator.py — after fact-find branch exits and
free_form_session is set, extract → apply to session.profile → re-upsert
the profile chunk in Chroma so THIS turn's retrieval reflects new state.
- Health conditions are MERGED (additive, deduped) — existing conditions
are preserved; only new ones get appended.
- TurnResult.profile_updates / ChatResponse.profile_updates surface the
field changes to the frontend so it can refresh the completeness panel.

Failure isolation: extractor exceptions NEVER block the chat — fall back
to no-update silently.

Unit-tested validator: out-of-bounds age dropped, invalid enums dropped,
unknown fields dropped, valid inputs preserved.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/main.py

@@ -111,6 +111,14 @@ class ChatResponse(BaseModel):
     faithfulness_passed: bool = True
     faithfulness_reasons: list[str] = Field(default_factory=list)
     blocked: bool = False
+    profile_updates: dict = Field(
+        default_factory=dict,
+        description=(
+            "Any profile fields auto-extracted from the user's free-form message "
+            "this turn (age, dependents, health_conditions, etc.). Frontend can "
+            "flash an acknowledgment + refresh the completeness panel."
+        ),
+    )
 
 
 class TTSRequest(BaseModel):
@@ -307,6 +315,7 @@ async def chat(req: ChatRequest):
         faithfulness_passed=turn.faithfulness_passed,
         faithfulness_reasons=turn.faithfulness_reasons,
         blocked=turn.blocked,
+        profile_updates=turn.profile_updates,
     )
 
 

backend/orchestrator.py

@@ -117,6 +117,7 @@ class TurnResult:
     faithfulness_passed: bool = True
     faithfulness_reasons: list[str] = field(default_factory=list)
     blocked: bool = False
+    profile_updates: dict = field(default_factory=dict)
 
 
 async def handle_turn(
@@ -204,6 +205,52 @@ async def handle_turn(
         session.set_awaiting(None)
         session.free_form_session = True
 
+    # 1c. CONVERSATIONAL PROFILE UPDATES (free-form mode)
+    # In free-form chat the user often shares new profile facts ("I just turned 40",
+    # "we had a baby", "I was diagnosed with diabetes"). Run a lightweight LLM
+    # extractor, apply high-confidence updates to session.profile, and re-upsert
+    # the profile chunk so THIS turn's retrieval reflects the new state.
+    profile_updates_applied: dict = {}
+    try:
+        from backend.profile_extractor import extract_profile_updates
+        extracted = await extract_profile_updates(user_text, session.profile)
+        if extracted:
+            for field_name, new_value in extracted.items():
+                if field_name == "health_conditions":
+                    existing = list(session.profile.health_conditions or [])
+                    existing_lower = {c.lower() for c in existing if c}
+                    merged = list(existing)
+                    for cond in new_value:
+                        if cond.lower() not in existing_lower:
+                            merged.append(cond)
+                            existing_lower.add(cond.lower())
+                    session.update_profile_field("health_conditions", merged)
+                    profile_updates_applied["health_conditions"] = merged
+                else:
+                    session.update_profile_field(field_name, new_value)
+                    profile_updates_applied[field_name] = new_value
+            # Re-upsert profile chunk so retrieval sees fresh profile THIS turn
+            try:
+                from backend.profile_rag import upsert_profile_chunk
+                profile_dict_for_chunk = {
+                    "age": session.profile.age,
+                    "dependents": session.profile.dependents,
+                    "income_band": session.profile.income_band,
+                    "existing_cover_inr": session.profile.existing_cover_inr,
+                    "primary_goal": session.profile.primary_goal,
+                    "location_tier": session.profile.location_tier,
+                    "parents_to_insure": session.profile.parents_to_insure,
+                    "parents_age_max": session.profile.parents_age_max,
+                    "parents_has_ped": session.profile.parents_has_ped,
+                    "budget_band": session.profile.budget_band,
+                    "health_conditions": session.profile.health_conditions,
+                }
+                await upsert_profile_chunk(session_id or "anonymous", profile_dict_for_chunk)
+            except Exception:
+                pass  # chunk upsert failure must not block the chat
+    except Exception:
+        pass  # extraction failure must never block the chat
+
     # 2. Retrieve — pass session_id so the user's profile chunk (stored in
     # Chroma at POST /api/profile time) gets boosted to the top of the
     # context. Without session_id this path is dormant and the brain never
@@ -353,4 +400,5 @@ async def handle_turn(
         faithfulness_passed=verdict.passed,
         faithfulness_reasons=verdict.reasons,
         blocked=blocked,
+        profile_updates=profile_updates_applied,
     )

backend/profile_extractor.py

@@ -0,0 +1,191 @@
+"""Conversational profile updates in free-form chat.
+
+After fact-find onboarding completes, users often share new profile facts
+in ordinary conversation — "I just turned 40", "we had a baby last month",
+"I was diagnosed with diabetes". Vanilla retrieval doesn't know to update
+the session profile from these utterances.
+
+This module runs a lightweight LLM extractor on each free-form user message
+to pull out any concrete profile updates the user just revealed. High-
+confidence updates get applied to session.profile + re-upserted as the
+profile chunk in Chroma, so subsequent retrieval (and the brain's reply
+to THIS same turn) reflect the new state.
+
+Design choices:
+- Cheap-tier NIM model (Llama 3.3 70B) — extraction doesn't need the frontier.
+- Conservative validation: drop any field that fails type/enum/bounds checks.
+- Health conditions are MERGED with existing list (additive, deduped).
+- Extraction failure NEVER blocks the chat — falls back to no-update silently.
+- Enum values match backend/needs_finder.py::Profile exactly (under_5L / first_buy / ...).
+"""
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from backend.needs_finder import Profile
+
+_EXTRACTOR_SYSTEM = """You extract profile updates from a single user message.
+
+Output a JSON object containing ONLY fields the user EXPLICITLY revealed in this single message. Use null/omit for unmentioned fields.
+
+Fields and allowed values:
+- age: integer (years, 1-120)
+- dependents: one of "self", "self+spouse", "self+spouse+kids", "self+parents", "self+spouse+kids+parents"
+- income_band: one of "under_5L", "5L-10L", "10L-25L", "25L+"
+- existing_cover_inr: integer (current sum insured in rupees, e.g. 500000 for 5 lakh)
+- primary_goal: one of "first_buy", "upgrade", "compare_specific", "tax_planning"
+- location_tier: one of "metro", "tier1", "tier2", "tier3"
+- parents_to_insure: boolean
+- parents_age_max: integer (30-120, oldest parent's age in years)
+- parents_has_ped: boolean (true if any parent has pre-existing disease)
+- budget_band: one of "under_15k", "15k_30k", "30k_60k", "60k+"
+- health_conditions: list of NEW condition strings the user just mentioned (additive — do not echo old ones)
+
+Rules:
+1. Only extract what the user EXPLICITLY stated in this message. Never infer.
+2. Be conservative — when ambiguous, omit the field. Wrong updates are worse than missed ones.
+3. If the user said nothing new about their profile, return an empty object: {}
+4. Output the JSON object only. No prose. No code fences. No <think> blocks."""
+
+
+_EXTRACTOR_USER_TEMPLATE = """User just said:
+\"\"\"
+{user_text}
+\"\"\"
+
+Current known profile (for context, do NOT echo unchanged fields back):
+{profile_summary}
+
+Return JSON of any NEW profile facts revealed in the user's message above."""
+
+
+async def extract_profile_updates(
+    user_text: str,
+    current_profile: Profile,
+) -> dict[str, Any]:
+    """Return validated dict of {field_name: new_value}.
+
+    Empty dict means nothing extractable. Never raises.
+    """
+    if not user_text or not user_text.strip():
+        return {}
+
+    from backend.providers.nvidia_nim_llm import NvidiaNimLLM
+    from backend.providers.base import ChatMessage
+
+    summary_parts = []
+    for k, v in current_profile.__dict__.items():
+        if v in (None, "", []) or k in ("asked", "free_form_session"):
+            continue
+        summary_parts.append(f"{k}={v}")
+    profile_summary = ", ".join(summary_parts) or "(empty)"
+
+    messages = [
+        ChatMessage(role="system", content=_EXTRACTOR_SYSTEM),
+        ChatMessage(
+            role="user",
+            content=_EXTRACTOR_USER_TEMPLATE.format(
+                user_text=user_text[:1500],
+                profile_summary=profile_summary[:500],
+            ),
+        ),
+    ]
+
+    try:
+        llm = NvidiaNimLLM(model="meta/llama-3.3-70b-instruct")
+        result = await llm.chat(messages=messages, temperature=0.0, max_tokens=300)
+        raw = (result.text or "").strip()
+    except Exception as e:
+        logging.warning("profile_extractor LLM call failed: %s: %s", type(e).__name__, e)
+        return {}
+
+    # Strip code fences / think blocks if model added them despite instructions
+    if "<think>" in raw and "</think>" in raw:
+        raw = raw.split("</think>", 1)[1].strip()
+    if raw.startswith("```"):
+        lines = [l for l in raw.split("\n") if not l.startswith("```")]
+        raw = "\n".join(lines).strip()
+    if not raw.startswith("{"):
+        return {}
+
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        return {}
+
+    if not isinstance(parsed, dict):
+        return {}
+
+    return _validate(parsed)
+
+
+_ALLOWED_FIELDS: dict[str, type] = {
+    "age": int,
+    "dependents": str,
+    "income_band": str,
+    "existing_cover_inr": int,
+    "primary_goal": str,
+    "location_tier": str,
+    "parents_to_insure": bool,
+    "parents_age_max": int,
+    "parents_has_ped": bool,
+    "budget_band": str,
+    "health_conditions": list,
+}
+
+
+_ENUM_VALUES: dict[str, set[str]] = {
+    "dependents": {"self", "self+spouse", "self+spouse+kids", "self+parents", "self+spouse+kids+parents"},
+    "income_band": {"under_5L", "5L-10L", "10L-25L", "25L+"},
+    "primary_goal": {"first_buy", "upgrade", "compare_specific", "tax_planning"},
+    "location_tier": {"metro", "tier1", "tier2", "tier3"},
+    "budget_band": {"under_15k", "15k_30k", "30k_60k", "60k+"},
+}
+
+
+def _validate(updates: dict) -> dict:
+    """Coerce types, enforce enums, drop anything that fails."""
+    clean: dict[str, Any] = {}
+    for k, v in updates.items():
+        if k not in _ALLOWED_FIELDS or v is None:
+            continue
+
+        expected = _ALLOWED_FIELDS[k]
+        try:
+            if expected is int:
+                if isinstance(v, bool):
+                    continue
+                v = int(v)
+            elif expected is bool:
+                if not isinstance(v, bool):
+                    continue
+            elif expected is str:
+                if not isinstance(v, str):
+                    continue
+                v = v.strip()
+                if not v:
+                    continue
+            elif expected is list:
+                if not isinstance(v, list):
+                    continue
+                v = [str(x).strip().lower() for x in v if x and isinstance(x, (str, int))]
+                v = [c for c in v if c]
+                if not v:
+                    continue
+        except (TypeError, ValueError):
+            continue
+
+        if k in _ENUM_VALUES and v not in _ENUM_VALUES[k]:
+            continue
+        if k == "age" and not (1 <= v <= 120):
+            continue
+        if k == "parents_age_max" and not (30 <= v <= 120):
+            continue
+        if k == "existing_cover_inr" and v < 0:
+            continue
+
+        clean[k] = v
+
+    return clean