Update app.py

app.py

@@ -1,3 +1,4 @@
+# app.py
 import os, re, json, traceback, pathlib
 from functools import lru_cache
 
@@ -61,35 +62,28 @@ USE_HOSTED_COHERE = bool(COHERE_API_KEY and _HAS_COHERE)
 # Larger output (Cohere + HF fallback)
 MAX_NEW_TOKENS = int(os.getenv("MAX_NEW_TOKENS", "2048"))
 
-# ---------- System Master (two-phase, LLM-only behavior) ----------
+# ---------- System Master (two-phase + normal chat) ----------
 SYSTEM_MASTER = """
 SYSTEM ROLE (fixed, always on)
 You are ClarityOps, a medical analytics system that interacts only via this chat.
 
+Operating modes:
+- Normal Chat: answer general questions naturally.
+- Scenario Mode (two phases, no assumptions):
+  Phase 1: Ask up to 5 concise clarification questions, grouped by category (Prioritization, Capacity, Cost, Clinical, Recommendations). Only ask for items still missing from the scenario + uploaded data. Then STOP and WAIT.
+  Phase 2: After answers are provided, produce the final structured analysis in the required format. If any critical input remains missing, output EXACTLY: INSUFFICIENT_DATA and list the missing fields.
+
 Absolute rules:
-- Use ONLY information provided in this conversation (scenario text + uploaded files).
-- Never invent data. If something required is missing after clarifications, output the literal token: INSUFFICIENT_DATA.
-- Always run in TWO PHASES when the user provides a medical scenario (case study / program design / evaluation):
-  Phase 1: Ask up to 5 concise clarification questions, grouped by category (Prioritization, Capacity, Cost, Clinical, Recommendations). Then STOP and WAIT.
-  Phase 2: After answers are provided, produce the final structured analysis exactly in the required format.
-
-Core behavior:
-- Read and synthesize any user-uploaded files (e.g., CSV/XLSX/PDF) relevant to the scenario.
-- Prefer analytics/longitudinal recommendations (risk targeting, follow-up, clustering) over generic ops advice.
-- Show all calculations explicitly for capacity and costs (e.g., “6 teams × 8 clients/day × 60 days = 2,880”).
-- Use correct clinical units and plausible ranges.
-- Include a brief “Provenance” section mapping each key output to scenario text, files, and/or clarified answers.
-
-Medical guardrails (always apply):
-- Units: BP in mmHg, A1c in %, BMI in kg/m², Total Cholesterol in mmol/L (or as provided), Percentages in %.
-- Plausible ranges: A1c 3–20 %, SBP 60–250 mmHg, DBP 30–150 mmHg, BMI 10–70 kg/m², Total Chol 2–12 mmol/L.
-- Privacy: avoid PHI; aggregate only; apply small-cell suppression where cohort < 10 (describe at a higher level).
-- When data includes mixed or ambiguous indicators, ask to confirm preferred indicators (e.g., obesity/metabolic syndrome vs self-reported diabetes).
-
-Formatting hard rules (only for scenarios):
-- Phase 1 output MUST include the header line: “Clarification Questions”
-- Phase 2 output MUST include the header line: “Structured Analysis”
-- Phase 2 MUST follow this exact section order:
+- Use ONLY information in this conversation (scenario text + uploaded files + user answers).
+- Never invent data or assume defaults without explicit user confirmation.
+- Prefer analytics/longitudinal insights (risk targeting, follow-up, clustering) over generic ops advice.
+- Show calculations for capacity and costs. Use correct clinical units/ranges.
+- Add a short Provenance mapping each key output to its source (scenario text, files, answers).
+
+Formatting hard rules for Scenario Mode:
+- Phase 1 header: “Clarification Questions”
+- Phase 2 header: “Structured Analysis”
+- Phase 2 section order:
   1. Prioritization
   2. Capacity
   3. Cost
@@ -114,52 +108,14 @@ def is_identity_query(message, history):
         r"\bwho\s+am\s+i\s+chatting\s+with\b",
     ]
     def match(t):
-        return any(re.search(p, (t or "").strip().lower()) for p in patterns)
+        t = (t or "").strip().lower()
+        return any(re.search(p, t) for p in patterns)
     if match(message): return True
     if history:
         last_user = history[-1][0] if isinstance(history[-1], (list, tuple)) else None
         if match(last_user): return True
     return False
 
-GREETING_RE = re.compile(
-    r'^\s*(hi|hello|hey|yo|good\s*(morning|afternoon|evening)|howdy|sup)[\s!.\)]*$', re.I
-)
-
-def is_smalltalk(msg: str) -> bool:
-    if not msg: return True
-    if len(msg.strip()) < 6: return True
-    if GREETING_RE.match(msg.strip()): return True
-    # single short sentence, no punctuation complexity, no digits
-    if len(msg.split()) < 10 and not re.search(r'[\d,:;]|(case|scenario|study|objective|dataset|csv|program)', msg, re.I):
-        return True
-    return False
-
-SCENARIO_MARKERS = [
-    "background", "case study", "objective", "objectives", "available data", "data inputs",
-    "evaluation questions", "expected output", "structured analysis", "methods", "assumptions"
-]
-MEDICAL_TERMS = [
-    "diabetes", "a1c", "metabolic syndrome", "obesity", "blood pressure", "cholesterol",
-    "screening", "clinic", "patients", "prevalence", "capacity", "cost per client",
-    "program cost", "longitudinal", "outcomes", "cohort", "settlements", "indigenous", "métis"
-]
-
-def is_scenario_like(msg: str, artifacts, uploads_present: bool) -> bool:
-    if not msg: return False
-    low = msg.lower()
-    # length + markers
-    has_len = len(low) > 400 or len(low.split()) > 120
-    has_marker = any(m in low for m in SCENARIO_MARKERS)
-    med_hits = sum(1 for t in MEDICAL_TERMS if t in low)
-    has_medical = med_hits >= 2
-    csv_present = any((a.get("kind") == "csv") for a in (artifacts or []))
-    # Declare scenario if: (length & marker & medical) OR (uploads with csv and medical) OR explicit "scenario"/"case study"
-    explicit = ("scenario" in low) or ("case study" in low)
-    if explicit: return True
-    if (has_len and has_marker and has_medical): return True
-    if (uploads_present and csv_present and has_medical): return True
-    return False
-
 def _iter_user_assistant(history):
     for item in (history or []):
         if isinstance(item, (list, tuple)):
@@ -180,7 +136,7 @@ def _history_to_prompt(message, history):
     parts.append("Assistant:")
     return "\n".join(parts)
 
-# ---------- Cohere first ----------
+# ---------- LLM invocation ----------
 def cohere_chat(message, history):
     if not USE_HOSTED_COHERE:
         return None
@@ -200,7 +156,6 @@ def cohere_chat(message, history):
     except Exception:
         return None
 
-# ---------- Local model (HF) ----------
 @lru_cache(maxsize=1)
 def load_local_model():
     if not HF_TOKEN:
@@ -270,6 +225,102 @@ def _load_snapshot(path=SNAPSHOT_PATH):
 init_retriever()
 _session_rag = SessionRAG()
 
+# ---------- Scenario detection & gap analysis ----------
+def detect_scenario_type(text: str, artifacts):
+    """
+    Very simple keyword detector for now. Returns "diabetes_screening" or None.
+    """
+    t = (text or "").lower()
+    joined = " ".join((a.get("name","") + " " + " ".join(a.get("columns", []))) for a in (artifacts or []))
+    tt = f"{t} {joined}".lower()
+    diabetes_terms = [
+        "mobile diabetes screening", "mdsi", "a1c", "metabolic syndrome",
+        "metis settlement", "obesity", "pre-diabetes", "screening program"
+    ]
+    if any(k in tt for k in diabetes_terms):
+        return "diabetes_screening"
+    return None
+
+def build_data_summary(artifacts):
+    """
+    Human-readable summary of uploaded data coverage (CSV columns & sample rows count).
+    """
+    lines = []
+    for a in (artifacts or []):
+        if a.get("kind") == "csv":
+            cols = ", ".join(map(str, a.get("columns", []))) or "<no columns found>"
+            lines.append(f"- **{a.get('name','(csv)')}** · columns: {cols} · sampled_rows: {a.get('n_rows_sampled',0)}")
+    return "\n".join(lines) if lines else "_No structured CSVs detected._"
+
+def _has_cols(artifacts, name_hint, required_cols):
+    """
+    Check if any CSV artifact whose filename contains name_hint has all required columns.
+    """
+    for a in (artifacts or []):
+        if a.get("kind") != "csv":
+            continue
+        if name_hint and name_hint not in a.get("name","").lower():
+            continue
+        cols = set(map(lambda s: s.strip().lower(), a.get("columns", [])))
+        if all(rc.lower() in cols for rc in required_cols):
+            return True
+    return False
+
+def analyze_gaps(scenario_text: str, artifacts):
+    """
+    Returns: (missing_critical: list[str], missing_nice: list[str], scenario_note: str)
+    Only checks what's applicable for the detected scenario.
+    """
+    stype = detect_scenario_type(scenario_text, artifacts)
+    crit_missing, nice_missing = [], []
+    note = ""
+
+    if stype == "diabetes_screening":
+        note = "Detected scenario: **Mobile Diabetes Screening in rural communities**."
+
+        # Check for prioritization data coverage
+        if not (_has_cols(artifacts, "population", ["settlement","population"]) or
+                _has_cols(artifacts, "metis", ["settlement","population"]) or
+                _has_cols(artifacts, "", ["settlement","population"])):
+            crit_missing.append("Population by settlement (CSV with columns like: settlement, population)")
+
+        if not (_has_cols(artifacts, "health", ["settlement","diabetes_prevalence"]) or
+                _has_cols(artifacts, "", ["settlement","diabetes_prevalence"])):
+            crit_missing.append("Diabetes prevalence by settlement (e.g., settlement, diabetes_prevalence)")
+
+        # Risk factors
+        if not (_has_cols(artifacts, "health", ["obesity"]) or _has_cols(artifacts, "", ["obesity"])):
+            nice_missing.append("Obesity prevalence by settlement (e.g., obesity %)")
+        if not (_has_cols(artifacts, "health", ["metabolic_syndrome"]) or _has_cols(artifacts, "", ["metabolic_syndrome"])):
+            nice_missing.append("Metabolic syndrome prevalence by settlement (%)")
+
+        # Capacity assumptions (teams/day)
+        txt = scenario_text.lower()
+        if "teams" not in txt and "mobile clinic" not in txt:
+            crit_missing.append("Number of mobile teams and work schedule (days/week, duration)")
+        if "clients/day" not in txt and "per day" not in txt:
+            crit_missing.append("Throughput per team (clients per day)")
+
+        # Cost
+        if not (_has_cols(artifacts, "program_cost", ["startup_cost_per_client"]) or "startup cost" in txt):
+            crit_missing.append("Startup cost per client")
+        if not (_has_cols(artifacts, "program_cost", ["ongoing_cost_per_client"]) and "ongoing cost" in txt):
+            # either file column or explicit in text is okay
+            crit_missing.append("Ongoing cost per client")
+
+        # Longitudinal outcomes (not always critical for Phase 2, but preferred)
+        if not (_has_cols(artifacts, "longitudinal", ["a1c"]) or "a1c" in txt):
+            nice_missing.append("Longitudinal A1c change for repeat participants")
+        if not (_has_cols(artifacts, "longitudinal", ["systolic_bp"]) or "blood pressure" in txt):
+            nice_missing.append("Longitudinal systolic/diastolic BP change")
+        if not (_has_cols(artifacts, "longitudinal", ["bmi"]) or "bmi" in txt):
+            nice_missing.append("Longitudinal BMI change")
+        if not (_has_cols(artifacts, "longitudinal", ["cholesterol"]) or "cholesterol" in txt):
+            nice_missing.append("Longitudinal total cholesterol change")
+
+    return crit_missing, nice_missing, note
+
+# ---------- Executive pre-compute (optional) ----------
 def _mdsi_block():
     base_capacity = capacity_projection(18, 48, 6)
     cons_capacity = capacity_projection(12, 48, 6)
@@ -282,12 +333,13 @@ def _mdsi_block():
         "outcomes_summary": outcomes
     }, indent=2)
 
-# ---------- Core chat logic (auto scenario detection) ----------
-def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answers=False):
+# ---------- Core chat logic (auto scenario; no assumptions) ----------
+def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answers=False, force_phase=None):
     """
     awaiting_answers:
-      - False: If message looks like a medical scenario -> Phase 1; else general chat
-      - True:  We expect the user's answers to Phase 1 -> produce Phase 2
+      - False: not waiting for Phase 2 answers
+      - True:  expecting answers to clarifications for Scenario Mode
+    force_phase: None | "clarify" | "analyze" (internal, used by upload handler)
     """
     try:
         log_event("user_message", None, {"sizes": {"chars": len(user_msg or "")}})
@@ -303,7 +355,7 @@ def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answe
             ans = "I am ClarityOps, your strategic decision making AI partner."
             return history + [(user_msg, ans)], awaiting_answers
 
-        # Ingest uploads first (so detector can use artifacts)
+        # Ingest uploads if paths present (also handled in upload event; safe to repeat)
         artifacts = []
         if uploaded_files_paths:
             ing = extract_text_from_files(uploaded_files_paths)
@@ -315,87 +367,109 @@ def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answe
                 _session_rag.register_artifacts(artifacts)
             log_event("uploads_added", None, {"chunks": len(chunks), "artifacts": len(artifacts)})
 
-        # Column helper (explicit)
-        if re.search(r"\b(columns?|headers?)\b", (safe_in or "").lower()):
-            cols = _session_rag.get_latest_csv_columns()
-            if cols:
-                return history + [(user_msg, "Here are the column names from your most recent CSV upload:\n\n- " + "\n- ".join(cols))], awaiting_answers
-
-        # Decide mode
-        uploads_present = bool(uploaded_files_paths)
-        scenario_mode = (not awaiting_answers) and is_scenario_like(safe_in or "", artifacts, uploads_present)
-        smalltalk = is_smalltalk(safe_in or "")
-
-        # Prepare retrieval/preamble only if needed
-        session_snips = ""
-        system_preamble = ""
-        phase_directive = ""
-
-        if awaiting_answers:
-            # We are in Phase 2 (user answered Phase 1); force scenario flow
-            scenario_mode = True
-
-        if scenario_mode:
-            # Session retrieval to enrich the system preamble
-            session_snips = "\n---\n".join(_session_rag.retrieve(
-                "diabetes screening Indigenous Métis mobile program cost throughput outcomes logistics",
-                k=6
-            ))
-            snapshot = _load_snapshot()
-            policy_context = retrieve_context(
-                "mobile diabetes screening Indigenous community outreach cultural safety data governance outcomes"
+        # Session retrieval context
+        session_snips = "\n---\n".join(_session_rag.retrieve(
+            "diabetes screening Indigenous Métis mobile program cost throughput outcomes logistics",
+            k=6
+        ))
+
+        snapshot = _load_snapshot()
+        policy_context = retrieve_context(
+            "mobile diabetes screening Indigenous community outreach cultural safety data governance outcomes"
+        )
+        computed = compute_operational_numbers(snapshot)
+
+        user_lower = (safe_in or "").lower()
+        mdsi_extra = _mdsi_block() if ("diabetes" in user_lower or "mdsi" in user_lower or "mobile screening" in user_lower) else ""
+
+        scenario_block = safe_in if len((safe_in or "")) > 0 else ""
+        system_preamble = build_system_preamble(
+            snapshot=snapshot,
+            policy_context=policy_context,
+            computed_numbers=computed,
+            scenario_text=scenario_block + (f"\n\nExecutive Pre-Computed Blocks:\n{mdsi_extra}" if mdsi_extra else ""),
+            session_snips=session_snips
+        )
+
+        # Decide mode (normal vs scenario)
+        stype = detect_scenario_type(safe_in, _session_rag.artifacts)
+        in_scenario = bool(stype)
+
+        # Gap analysis
+        crit_missing, nice_missing, det_note = analyze_gaps(safe_in, _session_rag.artifacts)
+
+        # Determine phase directive
+        if force_phase == "clarify":
+            awaiting = True
+            directive = (
+                "\n\n[INSTRUCTION TO MODEL]\n"
+                "Produce **Phase 1** only:\n"
+                "- Header: 'Clarification Questions'\n"
+                "- Ask ONLY for the items listed as missing (critical first, then optional). Group by category.\n"
+                "- Then STOP and WAIT.\n"
             )
-            computed = compute_operational_numbers(snapshot)
-            user_lower = (safe_in or "").lower()
-            mdsi_extra = _mdsi_block() if ("diabetes" in user_lower or "mdsi" in user_lower or "mobile screening" in user_lower) else ""
-            scenario_block = safe_in if len((safe_in or "")) > 0 else ""
-            system_preamble = build_system_preamble(
-                snapshot=snapshot,
-                policy_context=policy_context,
-                computed_numbers=computed,
-                scenario_text=scenario_block + (f"\n\nExecutive Pre-Computed Blocks:\n{mdsi_extra}" if mdsi_extra else ""),
-                session_snips=session_snips
+        elif force_phase == "analyze":
+            awaiting = False
+            if crit_missing:
+                # hard block
+                return history + [(user_msg,
+                    "INSUFFICIENT_DATA\n\nMissing critical inputs:\n- " + "\n- ".join(crit_missing)
+                )], False
+            directive = (
+                "\n\n[INSTRUCTION TO MODEL]\n"
+                "Produce **Phase 2** only:\n"
+                "- Header: 'Structured Analysis'\n"
+                "- Follow the exact section order (Prioritization, Capacity, Cost, Clinical Benefits, ClarityOps Top 3 Recommendations).\n"
+                "- Use uploaded files + the user's latest answers as authoritative. Show calculations, units, and a brief Provenance.\n"
             )
-            if not awaiting_answers:
-                phase_directive = (
-                    "\n\n[INSTRUCTION TO MODEL]\n"
-                    "Produce **Phase 1** only: output a header 'Clarification Questions' and ask up to 5 concise, grouped questions "
-                    "(Prioritization, Capacity, Cost, Clinical, Recommendations). Then STOP and WAIT.\n"
-                )
+        else:
+            # Auto-decide
+            if in_scenario:
+                if not awaiting_answers:
+                    # entering Phase 1 if there are any missing fields; if nothing missing, we can go to Phase 2 immediately
+                    if crit_missing:
+                        awaiting = True
+                        directive = (
+                            "\n\n[INSTRUCTION TO MODEL]\n"
+                            "Produce **Phase 1** only:\n"
+                            "- Header: 'Clarification Questions'\n"
+                            "- Ask ONLY for the items listed as missing (critical first, then optional). Group by category.\n"
+                            "- Then STOP and WAIT.\n"
+                        )
+                    else:
+                        awaiting = False
+                        directive = (
+                            "\n\n[INSTRUCTION TO MODEL]\n"
+                            "Produce **Phase 2** only:\n"
+                            "- Header: 'Structured Analysis'\n"
+                            "- Follow the exact section order (Prioritization, Capacity, Cost, Clinical Benefits, ClarityOps Top 3 Recommendations).\n"
+                            "- Use uploaded files + the user's latest answers as authoritative. Show calculations, units, and a brief Provenance.\n"
+                        )
+                else:
+                    # expecting answers; attempt Phase 2 but block if still missing critical
+                    if crit_missing:
+                        return history + [(user_msg,
+                            "INSUFFICIENT_DATA\n\nMissing critical inputs:\n- " + "\n- ".join(crit_missing)
+                        )], True
+                    awaiting = False
+                    directive = (
+                        "\n\n[INSTRUCTION TO MODEL]\n"
+                        "Produce **Phase 2** only:\n"
+                        "- Header: 'Structured Analysis'\n"
+                        "- Follow the exact section order (Prioritization, Capacity, Cost, Clinical Benefits, ClarityOps Top 3 Recommendations).\n"
+                        "- Use uploaded files + the user's latest answers as authoritative. Show calculations, units, and a brief Provenance.\n"
+                    )
             else:
-                phase_directive = (
-                    "\n\n[INSTRUCTION TO MODEL]\n"
-                    "Produce **Phase 2** only: output a header 'Structured Analysis' and follow the exact section order "
-                    "(Prioritization, Capacity, Cost, Clinical Benefits, ClarityOps Top 3 Recommendations). "
-                    "Use uploaded files + the user's latest answers as authoritative. Show calculations, units, and a brief Provenance.\n"
-                )
+                # Normal chat mode
+                awaiting = awaiting_answers
+                directive = "\n\n[INSTRUCTION TO MODEL]\nAnswer normally as a helpful assistant.\n"
 
-        # Build final message to model
-        if scenario_mode:
-            augmented_user = SYSTEM_MASTER + "\n\n" + system_preamble + "\n\nUser message:\n" + (safe_in or "") + phase_directive
-        else:
-            # General chat path: NO phase directive, NO heavy preamble; still keep SYSTEM_MASTER safety/medical guardrails
-            augmented_user = (
-                "System: You are ClarityOps, a helpful medical & operations assistant. "
-                "Answer normally and concisely. If the user pastes a long, structured medical scenario, you will switch to the two-phase flow; "
-                "but this message does not qualify.\n\n"
-                f"User: {safe_in}\nAssistant:"
-            )
+        augmented_user = SYSTEM_MASTER + "\n\n" + system_preamble + "\n\nUser message:\n" + safe_in + directive
 
         # Call LLM
         out = cohere_chat(augmented_user, history)
         if not out:
             model, tokenizer = load_local_model()
-            # For local fallback we still use chat template with SYSTEM_MASTER included
-            def build_inputs(tokenizer, message, history):
-                msgs = [{"role": "system", "content": SYSTEM_MASTER}]
-                for u, a in _iter_user_assistant(history):
-                    if u: msgs.append({"role": "user", "content": u})
-                    if a: msgs.append({"role": "assistant", "content": a})
-                msgs.append({"role": "user", "content": message})
-                return tokenizer.apply_chat_template(
-                    msgs, tokenize=True, add_generation_prompt=True, return_tensors="pt"
-                )
             inputs = build_inputs(tokenizer, augmented_user, history)
             out = local_generate(model, tokenizer, inputs, max_new_tokens=MAX_NEW_TOKENS)
 
@@ -411,20 +485,19 @@ def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answe
         if blocked_out:
             safe_out = refusal_reply(reason_out)
 
-        # Flip phase state based on headers (only if we were in scenario mode)
-        new_awaiting = awaiting_answers
-        if scenario_mode:
-            low = safe_out.lower()
-            if not awaiting_answers and "clarification questions" in low:
+        # Flip phase state based on headers (scenario only)
+        new_awaiting = awaiting
+        if in_scenario:
+            low = (safe_out or "").lower()
+            if "clarification questions" in low:
                 new_awaiting = True
-            elif awaiting_answers and "structured analysis" in low:
+            elif "structured analysis" in low:
                 new_awaiting = False
 
         log_event("assistant_reply", None, {
             **hash_summary("prompt", augmented_user if not PERSIST_CONTENT else ""),
             **hash_summary("reply", safe_out if not PERSIST_CONTENT else ""),
-            "awaiting_next_phase": new_awaiting,
-            "scenario_mode": scenario_mode
+            "awaiting_next_phase": new_awaiting
         })
 
         return history + [(user_msg, safe_out)], new_awaiting
@@ -445,7 +518,7 @@ custom_css = """
 html, body, .gradio-container { height: 100vh; }
 .gradio-container { background: var(--brand-bg); display: flex; flex-direction: column; }
 
-/* HERO (landing) */
+/* HERO (initial Google-like screen) */
 #hero-wrap { height: 70vh; display: grid; place-items: center; }
 #hero { text-align: center; }
 #hero h2 { color: #0f172a; font-weight: 800; font-size: 32px; margin-bottom: 22px; }
@@ -462,7 +535,7 @@ textarea, input, .gr-input { border-radius: 12px !important; }
 
 # ---------- UI ----------
 with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
-    # --- HERO (initial Google-like screen) ---
+    # --- HERO (landing) ---
     with gr.Column(elem_id="hero-wrap", visible=True) as hero_wrap:
         with gr.Column(elem_id="hero"):
             gr.HTML("<h2>What can I help with?</h2>")
@@ -474,9 +547,9 @@ with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
                     elem_classes="hero-box"
                 )
                 hero_send = gr.Button("➤", scale=0)
-            gr.Markdown('<div class="hint">ClarityOps will first ask up to 5 clarifications for long medical scenarios, then produce a structured analysis.</div>')
+            gr.Markdown('<div class="hint">ClarityOps will parse uploads, compare to your scenario, ask only for what’s missing (no assumptions), then produce a structured analysis.</div>')
 
-    # --- MAIN APP (hidden until first message) ---
+    # --- MAIN APP ---
     with gr.Column(elem_id="chat-container", visible=False) as app_wrap:
         chat = gr.Chatbot(label="", show_label=False, height="64vh")
         with gr.Row():
@@ -497,22 +570,60 @@ with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
     # ---- State
     state_history = gr.State(value=[])
     state_uploaded = gr.State(value=[])
-    state_awaiting = gr.State(value=False)  # False -> Phase 1 next if scenario; True -> awaiting answers for Phase 2
+    state_awaiting = gr.State(value=False)  # False -> not waiting; True -> expecting answers to clarifications
 
-    # ---- Uploads
-    def _store_uploads(files, current):
-        paths = []
+    # ---- Upload handler (immediate ingest + gap summary to chat)
+    def _on_upload(files, history, uploaded_paths):
+        new_paths = []
         for f in (files or []):
-            paths.append(getattr(f, "name", None) or f)
-        return (current or []) + paths
-
-    uploads.change(fn=_store_uploads, inputs=[uploads, state_uploaded], outputs=state_uploaded)
+            new_paths.append(getattr(f, "name", None) or f)
+        all_paths = (uploaded_paths or []) + new_paths
+
+        # Ingest now
+        ing = extract_text_from_files(new_paths)
+        chunks = ing.get("chunks", []) if isinstance(ing, dict) else (ing or [])
+        arts = ing.get("artifacts", []) if isinstance(ing, dict) else []
+        if chunks:
+            _session_rag.add_docs(chunks)
+        if arts:
+            _session_rag.register_artifacts(arts)
+
+        # Build coverage & gap view using the last user scenario message if any
+        last_user_msg = ""
+        for u, a in _iter_user_assistant(history):
+            if u:
+                last_user_msg = u  # take the latest user utterance
+
+        crit_missing, nice_missing, note = analyze_gaps(last_user_msg, _session_rag.artifacts)
+        coverage = build_data_summary(_session_rag.artifacts)
+
+        # Compose bot message
+        parts = ["**Data Intake Summary**"]
+        if note: parts.append(note)
+        parts.append("**Files parsed & coverage:**\n" + (coverage or "_No files parsed._"))
+        if crit_missing:
+            parts.append("**Missing (critical):**\n- " + "\n- ".join(crit_missing))
+        if nice_missing:
+            parts.append("**Missing (optional but useful):**\n- " + "\n- ".join(nice_missing))
+        parts.append("\nIf you can, provide the missing details now. Otherwise, say “proceed” and I’ll continue (but Phase 2 will block if critical items remain).")
+        bot_msg = "\n\n".join(parts)
+
+        new_hist = (history or []) + [("", bot_msg)]
+        # If there are critical gaps AND we are in scenario context already, set awaiting=True (Phase 1)
+        awaiting = bool(crit_missing and detect_scenario_type(last_user_msg, _session_rag.artifacts))
+
+        return all_paths, new_hist, awaiting
+
+    uploads.change(
+        _on_upload,
+        inputs=[uploads, state_history, state_uploaded],
+        outputs=[state_uploaded, state_history, state_awaiting]
+    )
 
     # ---- Core send (used by both hero input and chat input)
     def _on_send(user_msg, history, up_paths, awaiting):
         try:
             if not user_msg or not user_msg.strip():
-                # no toggle on empty
                 return history, "", history, awaiting
             new_history, new_awaiting = clarityops_reply(
                 user_msg.strip(), history or [], None, up_paths or [], awaiting_answers=awaiting
@@ -557,7 +668,11 @@ with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
                concurrency_limit=2, queue=True)
 
     def _on_clear():
-        # reset to fresh hero screen
+        # fresh session (clears RAG too)
+        try:
+            _session_rag.clear()
+        except Exception:
+            pass
         return (
             [], "", [], False,
             gr.update(visible=True),   # show hero
@@ -570,3 +685,4 @@ with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
 if __name__ == "__main__":
     port = int(os.environ.get("PORT", "7860"))
     demo.launch(server_name="0.0.0.0", server_port=port, show_api=False, max_threads=8)
+