Fix chat agent web search, greeting resume, and lead capture gaps

Competitor comparison tables defaulted straight to "Requires confirmation"
because the agentic prompt never told the model to actually use its
search_the_web tool for the comparison — only that the tool existed.
Now the model is instructed to search before falling back, and to label
web-sourced cells as unverified public info rather than a settled claim.

The widget's opening greeting silently disappeared on any repeat open
within the same browser (existing localStorage session skipped the
greeting fetch entirely, with no history shown either). Added a session-
resume endpoint so reopening replays prior messages, or shows the
greeting if nothing was exchanged yet.

Chat-agent-sourced leads were linked into Contacts the same way as
Smartlead reply-imports, which the enrichment "Fetch" gate explicitly
excludes — and no extraction logic existed at all for company name, only
visitor name. Chat-agent leads now get form-style (enrichable) contacts,
and company name is extracted from the conversation the same way name
already was.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

backend/app/chat_agent_chat.py

@@ -163,6 +163,40 @@ def extract_visitor_name(recent_turns: List[Dict[str, str]]) -> str:
         return ""
 
 
+def extract_visitor_company(recent_turns: List[Dict[str, str]]) -> str:
+    """
+    Same pattern as extract_visitor_name — small, cheap, non-streaming call, only invoked once we
+    have an email but no company yet, to pull the visitor's company name out of the last few chat
+    turns (e.g. the model's own lead-capture follow-up asking for it, per build_system_prompt's
+    LEAD CAPTURE instructions). Returns "" if no company was actually stated (never invents one).
+    """
+    convo = "\n".join(f"{t['role']}: {t['content']}" for t in recent_turns[-6:])
+    try:
+        resp = _openai_client().chat.completions.create(
+            model=CHAT_MODEL_OPENAI,
+            messages=[
+                {
+                    "role": "system",
+                    "content": (
+                        "Extract ONLY the human visitor's company/organization name if they "
+                        "explicitly stated it in this chat transcript. Reply with just the company "
+                        "name and nothing else. If no company was stated, reply with exactly: none"
+                    ),
+                },
+                {"role": "user", "content": convo},
+            ],
+            temperature=0,
+            max_tokens=20,
+        )
+        text = (resp.choices[0].message.content or "").strip()
+        if not text or text.lower() == "none":
+            return ""
+        return text[:200]
+    except Exception as e:
+        logger.warning("Visitor company extraction failed: %s", e)
+        return ""
+
+
 def build_system_prompt(
     *,
     skills_md: str,
@@ -285,14 +319,27 @@ def build_agentic_system_prompt(
         "search for the competitor's name or the general topic) before concluding you don't have "
         "enough — a single narrow miss doesn't mean the knowledge base has nothing relevant.",
         "- When you do reply to a comparison request, default to a table (columns like "
-        "'Evaluation area | Our approach | Alternative approach | What to validate') even when you "
-        "have no confirmed facts about the other product — mark cells you can't confirm as "
-        "'Requires confirmation' rather than skipping the comparison entirely or giving only a "
-        "vague capability summary. Never state a specific claim about the other product unless it "
-        "came from an actual search_knowledge_base result. This table is not an exception to the "
+        "'Evaluation area | Our approach | Alternative approach | What to validate')."
+        + (
+            " Before writing anything in the Alternative approach column, call search_the_web "
+            "(once is usually enough) for the named competitor/alternative — do not default "
+            "straight to 'Requires confirmation' without having tried it. When a cell is based on "
+            "a search_the_web result, phrase it as a short neutral fact prefixed with 'Per public "
+            "web sources (unverified):' so it reads as external, unverified information rather "
+            "than a settled claim about a company you don't have inside knowledge of. Only write "
+            "'Requires confirmation' for a specific cell if search_the_web also came back empty, "
+            "vague, or unclear on that particular point."
+            if web_search_enabled
+            else " Mark cells you can't confirm as 'Requires confirmation' rather than skipping "
+            "the comparison entirely or giving only a vague capability summary."
+        )
+        + " Never state a specific claim about the other product unless it came from an actual "
+        "search_knowledge_base"
+        + (" or search_the_web" if web_search_enabled else "")
+        + " result. This table is not an exception to the "
         "pricing rule below — if a row would compare pricing, cost, or discounts, write 'Not "
         "disclosed via chat' in the Our approach cell rather than any figure, even one that came "
-        "back from search_knowledge_base. End with a recommendation for the single most useful "
+        "back from a tool. End with a recommendation for the single most useful "
         "next validation step (e.g. a structured demo using the same use case).",
         "- If the knowledge base still doesn't cover something after searching, and it's not a "
         "comparison request (see the rule above for those), your ENTIRE answer is: say so "
@@ -319,12 +366,14 @@ def build_agentic_system_prompt(
             "- You also have a search_the_web tool for live PUBLIC web search, to use only when "
             "search_knowledge_base has already been tried and didn't have enough for a detailed, "
             "useful answer — e.g. background on a named competitor, industry/technology context, "
-            "or an integration/partner ecosystem question. Never use it to look up anything about "
+            "or an integration/partner ecosystem question. This is the primary tool for filling in "
+            "a competitor comparison table (see the comparison-request rule above) — don't skip it "
+            "and fall back to a placeholder without trying. Never use it to look up anything about "
             "our own company, our own pricing, our own confidential/internal information, or "
             "personal information about a private individual. Its results are external and "
             "unverified — they're still subject to every rule above (never disclose confidential "
             "info, never insult or speculate about competitors, never state pricing for anyone), "
-            "so present them as requiring confirmation, not as settled fact."
+            "so always present them as unverified public information, not settled fact."
         )
 
     if skills_md.strip():

backend/app/chat_agent_routes.py

@@ -26,6 +26,7 @@ from .chat_agent_chat import (
     build_agentic_system_prompt,
     build_system_prompt,
     extract_email,
+    extract_visitor_company,
     extract_visitor_name,
     run_eval_assertion,
     stream_agentic_reply_openai,
@@ -57,6 +58,7 @@ from .database import (
     ChatAgentIndexLog,
     ChatAgentMessage,
     ChatAgentSkill,
+    Contact,
     CrmLead,
     db_commit_with_retry,
     get_db,
@@ -673,6 +675,7 @@ def _capture_or_update_lead(db: Session, cfg: ChatAgentConfig, convo: ChatAgentC
             email=email,
             first_name=first_name,
             last_name=last_name,
+            company_name=convo.visitor_company or "",
             last_reply_subject="Website chat",
             last_reply_body=last_body,
             last_reply_at=datetime.utcnow(),
@@ -693,6 +696,8 @@ def _capture_or_update_lead(db: Session, cfg: ChatAgentConfig, convo: ChatAgentC
             lead.first_name = first_name
         if last_name and not lead.last_name:
             lead.last_name = last_name
+        if convo.visitor_company and not lead.company_name:
+            lead.company_name = convo.visitor_company
         lead.last_reply_body = last_body
         lead.last_reply_at = datetime.utcnow()
         rw = dict(lead.raw_webhook) if isinstance(lead.raw_webhook, dict) else {}
@@ -705,6 +710,16 @@ def _capture_or_update_lead(db: Session, cfg: ChatAgentConfig, convo: ChatAgentC
         lead.updated_at = datetime.utcnow()
         if not lead.contact_id:
             link_lead_to_contact(db, lead)
+        elif convo.visitor_company:
+            # Contact may already exist (e.g. matched by email to a prior record) — keep its
+            # company in sync now that we know it, same as _merge_lead_fields_into_contact does
+            # for a brand-new link.
+            contact = db.query(Contact).filter(Contact.id == lead.contact_id).first()
+            if contact and not contact.company:
+                contact.company = convo.visitor_company
+                rd = dict(contact.raw_data or {})
+                rd["Company Name"] = convo.visitor_company
+                contact.raw_data = rd
 
     convo.lead_id = lead.id
     db_commit_with_retry(db)
@@ -746,6 +761,48 @@ def public_create_session(public_id: str, body: ChatAgentSessionRequest, db: Ses
     )
 
 
+@router.options("/api/public/chat-agent/{public_id}/session/{session_id}")
+async def public_session_lookup_options():
+    return Response(headers=_CORS_HEADERS)
+
+
+@router.get("/api/public/chat-agent/{public_id}/session/{session_id}")
+def public_get_session(public_id: str, session_id: str, db: Session = Depends(get_db)):
+    """Resume an existing widget session (e.g. reopening the widget after a page reload, where
+    the client remembers session_id in localStorage). Returns prior messages so the widget can
+    replay the conversation instead of showing a blank panel; the greeting is included too, for
+    the edge case where a session was created but the panel was closed before anything was ever
+    exchanged (no messages yet, so nothing to replay)."""
+    try:
+        cfg = _get_enabled_config(db, public_id)
+    except HTTPException as e:
+        # Cross-origin callers need CORS headers on error responses too, or the browser reports
+        # a generic network failure instead of a readable 404 — the widget's fetch call handles
+        # either the same way (falls back to creating a fresh session), but a real status lets
+        # that fallback be reliable rather than accidental.
+        return JSONResponse({"detail": e.detail}, status_code=e.status_code, headers=_CORS_HEADERS)
+    convo = (
+        db.query(ChatAgentConversation)
+        .filter(ChatAgentConversation.tenant_id == cfg.tenant_id, ChatAgentConversation.session_id == session_id)
+        .first()
+    )
+    if not convo:
+        return JSONResponse({"detail": "Session not found"}, status_code=404, headers=_CORS_HEADERS)
+    rows = (
+        db.query(ChatAgentMessage)
+        .filter(ChatAgentMessage.conversation_id == convo.id)
+        .order_by(ChatAgentMessage.created_at.asc(), ChatAgentMessage.id.asc())
+        .all()
+    )
+    return JSONResponse(
+        {
+            "greeting": cfg.greeting_message or "",
+            "messages": [{"role": m.role, "content": m.content or ""} for m in rows],
+        },
+        headers=_CORS_HEADERS,
+    )
+
+
 @router.options("/api/public/chat-agent/{public_id}/message")
 async def public_message_options():
     return Response(headers=_CORS_HEADERS)
@@ -770,11 +827,22 @@ async def public_send_message(public_id: str, body: ChatAgentMessageRequest, db:
     found_email = extract_email(user_text)
     if found_email and not convo.visitor_email:
         convo.visitor_email = found_email
-    if convo.visitor_email and not convo.visitor_name:
+    if convo.visitor_email and (not convo.visitor_name or not convo.visitor_company):
         recent = _recent_messages_for_prompt(db, convo.id)
-        name = extract_visitor_name(recent)
-        if name:
-            convo.visitor_name = name
+        if not convo.visitor_name:
+            name = extract_visitor_name(recent)
+            if name:
+                convo.visitor_name = name
+        if not convo.visitor_company:
+            company = extract_visitor_company(recent)
+            if company:
+                convo.visitor_company = company
+                # A lead may already exist for this conversation (company arriving a turn or two
+                # after name/email did) — keep it in sync rather than only setting it at creation.
+                if convo.lead_id:
+                    existing_lead = db.query(CrmLead).filter(CrmLead.id == convo.lead_id).first()
+                    if existing_lead and not existing_lead.company_name:
+                        existing_lead.company_name = company
     db_commit_with_retry(db)
 
     tenant_id = cfg.tenant_id
@@ -1059,22 +1127,48 @@ _WIDGET_JS_TEMPLATE = r"""(function () {
     flushText();
   }
 
+  function createFreshSession() {
+    return fetch(API + '/api/public/chat-agent/' + PUBLIC_ID + '/session', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ page_url: window.location.href })
+    }).then(function (r) { return r.json(); }).then(function (d) {
+      sessionId = d.session_id;
+      try { window.localStorage.setItem(STORAGE_KEY, sessionId); } catch (e) {}
+      if (d.greeting) addMsg('assistant', d.greeting);
+    });
+  }
+
   function ensureSession() {
     if (sessionReady) return sessionReady;
     sessionReady = (function () {
       try {
         sessionId = window.localStorage.getItem(STORAGE_KEY);
       } catch (e) {}
-      if (sessionId) return Promise.resolve();
-      return fetch(API + '/api/public/chat-agent/' + PUBLIC_ID + '/session', {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ page_url: window.location.href })
-      }).then(function (r) { return r.json(); }).then(function (d) {
-        sessionId = d.session_id;
-        try { window.localStorage.setItem(STORAGE_KEY, sessionId); } catch (e) {}
-        if (d.greeting) addMsg('assistant', d.greeting);
-      });
+      if (!sessionId) return createFreshSession();
+      // Reopening the widget with a session remembered from an earlier page load — replay
+      // whatever was actually exchanged so far instead of showing a blank panel. If the
+      // session no longer exists server-side (expired/deleted), fall back to a fresh one.
+      return fetch(API + '/api/public/chat-agent/' + PUBLIC_ID + '/session/' + sessionId)
+        .then(function (r) {
+          if (!r.ok) throw new Error('session lookup failed');
+          return r.json();
+        })
+        .then(function (d) {
+          if (d.messages && d.messages.length) {
+            d.messages.forEach(function (m) {
+              var el = addMsg(m.role, '');
+              renderRichContent(el, m.content || '');
+            });
+          } else if (d.greeting) {
+            addMsg('assistant', d.greeting);
+          }
+        })
+        .catch(function () {
+          try { window.localStorage.removeItem(STORAGE_KEY); } catch (e) {}
+          sessionId = null;
+          return createFreshSession();
+        });
     })();
     return sessionReady;
   }

backend/app/crm_lead_contacts.py

@@ -1,4 +1,4 @@
-"""Link CRM leads to Contacts rows (campaign replies, CTA forms, manual move)."""
+"""Link CRM leads to Contacts rows (campaign replies, CTA forms, chat agent, manual move)."""
 
 from __future__ import annotations
 
@@ -12,13 +12,27 @@ from .database import Contact, CrmLead
 SMARTLEAD_IMPORT_FILE_ID = "smartlead-imports"
 MANUAL_CONTACT_FILE_ID = "manual-contacts"
 CTA_FORM_SOURCE = "cta_form"
+CHAT_AGENT_SOURCE = "chat_agent"
+SMARTLEAD_REPLY_SOURCE = "smartlead_reply"
 
+# Leads a human directly typed into a form-like interface (CTA form, chat widget) rather than an
+# inbound reply to an outbound campaign — these get enrichable, manually-added-style Contact
+# records instead of lower-trust reply-import ones.
+_FORM_STYLE_SOURCES = {CTA_FORM_SOURCE, CHAT_AGENT_SOURCE}
 
-def is_cta_form_lead(lead: CrmLead) -> bool:
+
+def _lead_origin_source(lead: CrmLead) -> str:
     rw = lead.raw_webhook if isinstance(lead.raw_webhook, dict) else {}
-    if rw.get("source") == CTA_FORM_SOURCE:
-        return True
-    return (lead.smartlead_lead_id or "").startswith("cta-")
+    source = rw.get("source")
+    if source in _FORM_STYLE_SOURCES:
+        return source
+    if (lead.smartlead_lead_id or "").startswith("cta-"):
+        return CTA_FORM_SOURCE
+    return SMARTLEAD_REPLY_SOURCE
+
+
+def is_form_style_lead(lead: CrmLead) -> bool:
+    return _lead_origin_source(lead) in _FORM_STYLE_SOURCES
 
 
 def _merge_lead_fields_into_contact(contact: Contact, lead: CrmLead) -> None:
@@ -43,8 +57,8 @@ def _merge_lead_fields_into_contact(contact: Contact, lead: CrmLead) -> None:
     contact.raw_data = rd
 
 
-def _contact_raw_for_lead(lead: CrmLead, *, is_cta: bool) -> dict:
-    if is_cta:
+def _contact_raw_for_lead(lead: CrmLead, *, origin_source: str) -> dict:
+    if origin_source in _FORM_STYLE_SOURCES:
         rw = lead.raw_webhook if isinstance(lead.raw_webhook, dict) else {}
         return {
             "First Name": lead.first_name or "",
@@ -52,13 +66,16 @@ def _contact_raw_for_lead(lead: CrmLead, *, is_cta: bool) -> dict:
             "Email": (lead.email or "").strip(),
             "Company Name": lead.company_name or "",
             "Title": lead.title or "",
-            "source": CTA_FORM_SOURCE,
+            "source": origin_source,
             "form_id": rw.get("form_id"),
             "form_public_id": rw.get("form_public_id"),
             "form_name": rw.get("form_name"),
+            "conversation_id": rw.get("conversation_id"),
+            "session_id": rw.get("session_id"),
+            "page_url": rw.get("page_url"),
             "last_reply_subject": lead.last_reply_subject or "",
             "last_reply_body": lead.last_reply_body or "",
-            "cta_form_submission": rw,
+            "cta_form_submission": rw if origin_source == CTA_FORM_SOURCE else None,
         }
     return {
         "Company Name": lead.company_name or "",
@@ -73,7 +90,8 @@ def _contact_raw_for_lead(lead: CrmLead, *, is_cta: bool) -> dict:
 
 
 def link_lead_to_contact(db: Session, lead: CrmLead) -> dict:
-    """Create or link a Contact for this lead. CTA leads use manual-style contacts (enrichable)."""
+    """Create or link a Contact for this lead. Form-style leads (CTA form, chat agent) use
+    manual-style contacts (enrichable)."""
     if lead.contact_id:
         return {"ok": True, "contact_id": lead.contact_id, "message": "Already linked to Contacts"}
 
@@ -81,7 +99,8 @@ def link_lead_to_contact(db: Session, lead: CrmLead) -> dict:
     if not email:
         return {"ok": False, "error": "Lead has no email"}
 
-    is_cta = is_cta_form_lead(lead)
+    origin_source = _lead_origin_source(lead)
+    is_form_style = origin_source in _FORM_STYLE_SOURCES
     existing = (
         db.query(Contact)
         .filter(
@@ -92,21 +111,21 @@ def link_lead_to_contact(db: Session, lead: CrmLead) -> dict:
     )
     if existing:
         lead.contact_id = existing.id
-        if is_cta:
+        if is_form_style:
             _merge_lead_fields_into_contact(existing, lead)
         return {"ok": True, "contact_id": existing.id, "message": "Linked to existing contact"}
 
     contact = Contact(
         tenant_id=lead.tenant_id,
-        file_id=MANUAL_CONTACT_FILE_ID if is_cta else SMARTLEAD_IMPORT_FILE_ID,
+        file_id=MANUAL_CONTACT_FILE_ID if is_form_style else SMARTLEAD_IMPORT_FILE_ID,
         row_index=lead.id,
         first_name=lead.first_name or "",
         last_name=lead.last_name or "",
         email=email,
         company=lead.company_name or "",
         title=lead.title or "",
-        source=CTA_FORM_SOURCE if is_cta else "smartlead",
-        raw_data=_contact_raw_for_lead(lead, is_cta=is_cta),
+        source=origin_source if is_form_style else "smartlead",
+        raw_data=_contact_raw_for_lead(lead, origin_source=origin_source),
     )
     db.add(contact)
     db.flush()

backend/app/database.py

@@ -609,6 +609,7 @@ class ChatAgentConversation(Base):
     session_id = Column(String, unique=True, index=True, nullable=False)
     visitor_name = Column(String, default="")
     visitor_email = Column(String, default="", index=True)
+    visitor_company = Column(String, default="")
     lead_id = Column(Integer, ForeignKey("crm_leads.id"), nullable=True, index=True)
     page_url = Column(String, default="")
     started_at = Column(DateTime, default=datetime.utcnow)
@@ -811,6 +812,12 @@ def run_migrations(connection_engine):
             if "generation_cancel_requested" not in ufcols:
                 conn.execute(text("ALTER TABLE uploaded_files ADD COLUMN generation_cancel_requested INTEGER DEFAULT 0"))
 
+        insp = inspect(connection_engine)
+        if insp.has_table("chat_agent_conversations"):
+            caccols = [c["name"] for c in insp.get_columns("chat_agent_conversations")]
+            if "visitor_company" not in caccols:
+                conn.execute(text("ALTER TABLE chat_agent_conversations ADD COLUMN visitor_company TEXT DEFAULT ''"))
+
         insp = inspect(connection_engine)
         if insp.has_table("unipile_accounts"):
             uacols = [c["name"] for c in insp.get_columns("unipile_accounts")]

backend/app/main.py

@@ -4219,7 +4219,7 @@ async def enrich_contact(contact_id: int, t: TenantContext = Depends(get_tenant_
     if not contact:
         raise HTTPException(status_code=404, detail="Contact not found")
     src = (contact.source or "").strip().lower()
-    if src not in ("manual", "cta_form") and contact.file_id != MANUAL_CONTACT_FILE_ID:
+    if src not in ("manual", "cta_form", "chat_agent") and contact.file_id != MANUAL_CONTACT_FILE_ID:
         raise HTTPException(
             status_code=400,
             detail="Enrich is only available for manually added or form-submission contacts",