Update app.py

app.py

@@ -1,16 +1,17 @@
 # app.py
 import os, re, json, traceback, pathlib
 from functools import lru_cache
+from typing import List, Dict, Any, Tuple
 
 import gradio as gr
 import torch
-import regex as re2  # pip install regex
+import regex as re2  # robust control-char sanitizer
 
 from settings import SNAPSHOT_PATH, PERSIST_CONTENT
 from audit_log import log_event, hash_summary
 from privacy import redact_text
 
-# ---------- Environment / cache (Spaces-safe, writable) ----------
+# ---------- Writable caches (HF Spaces-safe) ----------
 HOME = pathlib.Path.home()
 HF_HOME = str(HOME / ".cache" / "huggingface")
 HF_HUB_CACHE = str(HOME / ".cache" / "huggingface" / "hub")
@@ -59,37 +60,28 @@ HF_TOKEN = os.getenv("HUGGINGFACE_HUB_TOKEN") or os.getenv("HF_TOKEN")
 COHERE_API_KEY = os.getenv("COHERE_API_KEY")
 USE_HOSTED_COHERE = bool(COHERE_API_KEY and _HAS_COHERE)
 
-# Larger output (Cohere + HF fallback)
+# Larger output budget for Phase 2
 MAX_NEW_TOKENS = int(os.getenv("MAX_NEW_TOKENS", "2048"))
 
-# ---------- System Master (two-phase + normal chat) ----------
+# ---------- System Master (Phase 2) ----------
 SYSTEM_MASTER = """
-SYSTEM ROLE (fixed, always on)
+SYSTEM ROLE
 You are ClarityOps, a medical analytics system that interacts only via this chat.
 
-Operating modes:
-- Normal Chat: answer general questions naturally.
-- Scenario Mode (two phases, no assumptions):
-  Phase 1: Ask up to 5 concise clarification questions, grouped by category (Prioritization, Capacity, Cost, Clinical, Recommendations). Only ask for items still missing from the scenario + uploaded data. Then STOP and WAIT.
-  Phase 2: After answers are provided, produce the final structured analysis in the required format. If any critical input remains missing, output EXACTLY: INSUFFICIENT_DATA and list the missing fields.
-
 Absolute rules:
-- Use ONLY information in this conversation (scenario text + uploaded files + user answers).
-- Never invent data or assume defaults without explicit user confirmation.
-- Prefer analytics/longitudinal insights (risk targeting, follow-up, clustering) over generic ops advice.
-- Show calculations for capacity and costs. Use correct clinical units/ranges.
-- Add a short Provenance mapping each key output to its source (scenario text, files, answers).
-
-Formatting hard rules for Scenario Mode:
-- Phase 1 header: “Clarification Questions”
-- Phase 2 header: “Structured Analysis”
-- Phase 2 section order:
+- Use ONLY information provided in this conversation (scenario text + uploaded files + user answers).
+- Never invent data. If something required is missing after clarifications, write the literal token: INSUFFICIENT_DATA.
+- Produce clear calculations (show multipliers and totals), follow medical units, and keep privacy safeguards (aggregate; suppress cohorts <10).
+
+Formatting hard rules for Phase 2:
+- Start with the header: “Structured Analysis”
+- Follow this section order:
   1. Prioritization
   2. Capacity
   3. Cost
   4. Clinical Benefits
   5. ClarityOps Top 3 Recommendations
-  (Include a short Provenance block at the end.)
+- End with a brief “Provenance” mapping outputs to scenario text, uploaded files, and answers.
 """.strip()
 
 # ---------- Helpers ----------
@@ -107,9 +99,7 @@ def is_identity_query(message, history):
         r"\bdescribe\s+yourself\b", r"\band\s+you\s*\?\b", r"\byour\s+name\b",
         r"\bwho\s+am\s+i\s+chatting\s+with\b",
     ]
-    def match(t):
-        t = (t or "").strip().lower()
-        return any(re.search(p, t) for p in patterns)
+    def match(t): return any(re.search(p, (t or "").strip().lower()) for p in patterns)
     if match(message): return True
     if history:
         last_user = history[-1][0] if isinstance(history[-1], (list, tuple)) else None
@@ -124,25 +114,29 @@ def _iter_user_assistant(history):
             yield u, a
 
 def _sanitize_text(s: str) -> str:
-    if not isinstance(s, str): return s
+    if not isinstance(s, str):
+        return s
     return re2.sub(r'[\p{C}--[\n\t]]+', '', s)
 
-def _history_to_prompt(message, history):
-    parts = [f"System: {SYSTEM_MASTER}"]
-    for u, a in _iter_user_assistant(history):
-        if u: parts.append(f"User: {u}")
-        if a: parts.append(f"Assistant: {a}")
-    parts.append(f"User: {message}")
-    parts.append("Assistant:")
-    return "\n".join(parts)
+def is_scenario_triggered(text: str, uploaded_files_paths) -> bool:
+    t = (text or "").lower()
+    has_keyword = "scenario" in t
+    has_files = bool(uploaded_files_paths)
+    return has_keyword or has_files
 
-# ---------- LLM invocation ----------
+# ---------- Cohere first ----------
 def cohere_chat(message, history):
     if not USE_HOSTED_COHERE:
         return None
     try:
         client = cohere.Client(api_key=COHERE_API_KEY)
-        prompt = _history_to_prompt(message, history)
+        # Build a simple conversational prompt (history included)
+        parts = []
+        for u, a in _iter_user_assistant(history):
+            if u: parts.append(f"User: {u}")
+            if a: parts.append(f"Assistant: {a}")
+        parts.append(f"User: {message}")
+        prompt = "\n".join(parts) + "\nAssistant:"
         resp = client.chat(
             model="command-r7b-12-2024",
             message=prompt,
@@ -156,6 +150,7 @@ def cohere_chat(message, history):
     except Exception:
         return None
 
+# ---------- Local model (HF) ----------
 @lru_cache(maxsize=1)
 def load_local_model():
     if not HF_TOKEN:
@@ -207,7 +202,7 @@ def local_generate(model, tokenizer, input_ids, max_new_tokens=MAX_NEW_TOKENS):
     gen_only = out[0, input_ids.shape[-1]:]
     return tokenizer.decode(gen_only, skip_special_tokens=True).strip()
 
-# ---------- Snapshot, retriever, RAG ----------
+# ---------- Snapshot & retrieval ----------
 def _load_snapshot(path=SNAPSHOT_PATH):
     try:
         with open(path, "r", encoding="utf-8") as f:
@@ -225,102 +220,7 @@ def _load_snapshot(path=SNAPSHOT_PATH):
 init_retriever()
 _session_rag = SessionRAG()
 
-# ---------- Scenario detection & gap analysis ----------
-def detect_scenario_type(text: str, artifacts):
-    """
-    Very simple keyword detector for now. Returns "diabetes_screening" or None.
-    """
-    t = (text or "").lower()
-    joined = " ".join((a.get("name","") + " " + " ".join(a.get("columns", []))) for a in (artifacts or []))
-    tt = f"{t} {joined}".lower()
-    diabetes_terms = [
-        "mobile diabetes screening", "mdsi", "a1c", "metabolic syndrome",
-        "metis settlement", "obesity", "pre-diabetes", "screening program"
-    ]
-    if any(k in tt for k in diabetes_terms):
-        return "diabetes_screening"
-    return None
-
-def build_data_summary(artifacts):
-    """
-    Human-readable summary of uploaded data coverage (CSV columns & sample rows count).
-    """
-    lines = []
-    for a in (artifacts or []):
-        if a.get("kind") == "csv":
-            cols = ", ".join(map(str, a.get("columns", []))) or "<no columns found>"
-            lines.append(f"- **{a.get('name','(csv)')}** · columns: {cols} · sampled_rows: {a.get('n_rows_sampled',0)}")
-    return "\n".join(lines) if lines else "_No structured CSVs detected._"
-
-def _has_cols(artifacts, name_hint, required_cols):
-    """
-    Check if any CSV artifact whose filename contains name_hint has all required columns.
-    """
-    for a in (artifacts or []):
-        if a.get("kind") != "csv":
-            continue
-        if name_hint and name_hint not in a.get("name","").lower():
-            continue
-        cols = set(map(lambda s: s.strip().lower(), a.get("columns", [])))
-        if all(rc.lower() in cols for rc in required_cols):
-            return True
-    return False
-
-def analyze_gaps(scenario_text: str, artifacts):
-    """
-    Returns: (missing_critical: list[str], missing_nice: list[str], scenario_note: str)
-    Only checks what's applicable for the detected scenario.
-    """
-    stype = detect_scenario_type(scenario_text, artifacts)
-    crit_missing, nice_missing = [], []
-    note = ""
-
-    if stype == "diabetes_screening":
-        note = "Detected scenario: **Mobile Diabetes Screening in rural communities**."
-
-        # Check for prioritization data coverage
-        if not (_has_cols(artifacts, "population", ["settlement","population"]) or
-                _has_cols(artifacts, "metis", ["settlement","population"]) or
-                _has_cols(artifacts, "", ["settlement","population"])):
-            crit_missing.append("Population by settlement (CSV with columns like: settlement, population)")
-
-        if not (_has_cols(artifacts, "health", ["settlement","diabetes_prevalence"]) or
-                _has_cols(artifacts, "", ["settlement","diabetes_prevalence"])):
-            crit_missing.append("Diabetes prevalence by settlement (e.g., settlement, diabetes_prevalence)")
-
-        # Risk factors
-        if not (_has_cols(artifacts, "health", ["obesity"]) or _has_cols(artifacts, "", ["obesity"])):
-            nice_missing.append("Obesity prevalence by settlement (e.g., obesity %)")
-        if not (_has_cols(artifacts, "health", ["metabolic_syndrome"]) or _has_cols(artifacts, "", ["metabolic_syndrome"])):
-            nice_missing.append("Metabolic syndrome prevalence by settlement (%)")
-
-        # Capacity assumptions (teams/day)
-        txt = scenario_text.lower()
-        if "teams" not in txt and "mobile clinic" not in txt:
-            crit_missing.append("Number of mobile teams and work schedule (days/week, duration)")
-        if "clients/day" not in txt and "per day" not in txt:
-            crit_missing.append("Throughput per team (clients per day)")
-
-        # Cost
-        if not (_has_cols(artifacts, "program_cost", ["startup_cost_per_client"]) or "startup cost" in txt):
-            crit_missing.append("Startup cost per client")
-        if not (_has_cols(artifacts, "program_cost", ["ongoing_cost_per_client"]) and "ongoing cost" in txt):
-            # either file column or explicit in text is okay
-            crit_missing.append("Ongoing cost per client")
-
-        # Longitudinal outcomes (not always critical for Phase 2, but preferred)
-        if not (_has_cols(artifacts, "longitudinal", ["a1c"]) or "a1c" in txt):
-            nice_missing.append("Longitudinal A1c change for repeat participants")
-        if not (_has_cols(artifacts, "longitudinal", ["systolic_bp"]) or "blood pressure" in txt):
-            nice_missing.append("Longitudinal systolic/diastolic BP change")
-        if not (_has_cols(artifacts, "longitudinal", ["bmi"]) or "bmi" in txt):
-            nice_missing.append("Longitudinal BMI change")
-        if not (_has_cols(artifacts, "longitudinal", ["cholesterol"]) or "cholesterol" in txt):
-            nice_missing.append("Longitudinal total cholesterol change")
-
-    return crit_missing, nice_missing, note
-
-# ---------- Executive pre-compute (optional) ----------
+# ---------- Executive pre-compute (MDSi block) ----------
 def _mdsi_block():
     base_capacity = capacity_projection(18, 48, 6)
     cons_capacity = capacity_projection(12, 48, 6)
@@ -333,13 +233,126 @@ def _mdsi_block():
         "outcomes_summary": outcomes
     }, indent=2)
 
-# ---------- Core chat logic (auto scenario; no assumptions) ----------
-def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answers=False, force_phase=None):
+# ---------- Dynamic Phase 1 question generator ----------
+def _extract_present_domains(artifacts: List[Dict[str, Any]]) -> Dict[str, bool]:
+    """
+    Inspect artifact names/columns to see which domains are present.
+    Returns flags for: population, cost, clinical, capacity/logistics.
+    """
+    flags = dict(population=False, cost=False, clinical=False, capacity=False)
+    for a in artifacts or []:
+        name = (a.get("name") or "").lower()
+        cols = [c.lower() for c in (a.get("columns") or [])]
+        if any(k in name for k in ["population", "census", "membership"]) or any(
+            k in ",".join(cols) for k in ["population", "census", "residence", "settlement", "age"]
+        ):
+            flags["population"] = True
+        if any(k in name for k in ["cost", "finance", "budget"]) or any(
+            k in ",".join(cols) for k in ["cost", "startup", "ongoing", "per_client", "per-visit"]
+        ):
+            flags["cost"] = True
+        if any(k in name for k in ["a1c", "outcome", "bp", "chol"]) or any(
+            k in ",".join(cols) for k in ["a1c", "bmi", "bp", "chol", "outcome"]
+        ):
+            flags["clinical"] = True
+        if any(k in name for k in ["ops", "capacity", "throughput", "volume"]) or any(
+            k in ",".join(cols) for k in ["clients_per_day", "teams", "visits", "throughput"]
+        ):
+            flags["capacity"] = True
+    return flags
+
+def _domain_from_text(text: str) -> Dict[str, bool]:
+    t = (text or "").lower()
+    return {
+        "population": any(k in t for k in ["population", "census", "settlement", "membership"]),
+        "cost": any(k in t for k in ["cost", "budget", "startup", "per client", "per-client", "ongoing"]),
+        "clinical": any(k in t for k in ["a1c", "bmi", "blood pressure", "bp", "cholesterol", "outcome"]),
+        "capacity": any(k in t for k in ["capacity", "throughput", "clients per day", "teams", "screen", "volume"]),
+    }
+
+def _is_mdsi_diabetes(text: str) -> bool:
+    t = (text or "").lower()
+    return any(k in t for k in ["mdsi", "mobile diabetes", "diabetes", "metabolic", "a1c", "metis"])
+
+def build_dynamic_clarifications(scenario_text: str, artifacts: List[Dict[str, Any]]) -> str:
+    """
+    Build up to 5 grouped clarification questions based on what's MISSING.
+    Groups: Prioritization, Capacity, Cost, Clinical, Recommendations.
+    Only ask for domains not covered by uploads/scenario text.
+    """
+    flags_from_files = _extract_present_domains(artifacts)
+    flags_from_text = _domain_from_text(scenario_text)
+    missing = {
+        k: not (flags_from_files.get(k) or flags_from_text.get(k))
+        for k in ["population", "capacity", "cost", "clinical"]
+    }
+
+    qs: List[Tuple[str, str]] = []
+    is_mdsi = _is_mdsi_diabetes(scenario_text)
+
+    # Prioritization
+    if missing["population"]:
+        if is_mdsi:
+            qs.append(("Prioritization",
+                       "Confirm prioritization inputs: settlement membership living on-settlement (latest), obesity/metabolic syndrome prevalence, and any access-to-care constraints to weigh."))
+        else:
+            qs.append(("Prioritization",
+                       "Which population/risk indicators should drive prioritization (size, prevalence, access, equity factors)?"))
+
+    # Capacity
+    if missing["capacity"]:
+        if is_mdsi:
+            qs.append(("Capacity",
+                       "What is the realistic per-team screening rate (clients/day) and operating schedule (days/week, weeks/3-month window)?"))
+        else:
+            qs.append(("Capacity",
+                       "What per-team throughput and operating schedule should be used for capacity calculations?"))
+
+    # Cost
+    if missing["cost"]:
+        if is_mdsi:
+            qs.append(("Cost",
+                       "Provide startup cost per client and ongoing cost per client/visit (or total program costs) to price scenarios like 1,200 screens."))
+        else:
+            qs.append(("Cost",
+                       "Provide fixed setup costs and variable cost per client to model total program spend."))
+
+    # Clinical
+    if missing["clinical"]:
+        if is_mdsi:
+            qs.append(("Clinical",
+                       "What longitudinal deltas should we expect (e.g., ΔA1c, ΔBP, ΔBMI, lipids) from repeat screenings, and over what interval?"))
+        else:
+            qs.append(("Clinical",
+                       "Which clinical indicators and expected effect sizes should be tracked for outcomes?"))
+
+    # Recommendations – always ask one targeted planning question last
+    if is_mdsi:
+        qs.append(("Recommendations",
+                   "Are there community constraints (events/seasonality/cultural protocols) that should shape routing and visit cadence?"))
+    else:
+        qs.append(("Recommendations",
+                   "Any operational constraints (scheduling, staffing, partnerships) we should incorporate into deployment modeling?"))
+
+    # Cap at 5 groups
+    qs = qs[:5]
+
+    # Assemble markdown
+    out = ["**Clarification Questions**"]
+    current_group = None
+    for grp, q in qs:
+        if grp != current_group:
+            out.append(f"\n**{grp}:**")
+            current_group = grp
+        out.append(f"- {q}")
+    return "\n".join(out)
+
+# ---------- Core chat logic (auto scenario, dynamic Phase 1) ----------
+def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answers=False):
     """
     awaiting_answers:
-      - False: not waiting for Phase 2 answers
-      - True:  expecting answers to clarifications for Scenario Mode
-    force_phase: None | "clarify" | "analyze" (internal, used by upload handler)
+      - False: If scenario triggered -> Phase 1 (dynamic questions). Else normal chat.
+      - True:  If scenario triggered -> Phase 2 (structured analysis). Else normal chat.
     """
     try:
         log_event("user_message", None, {"sizes": {"chars": len(user_msg or "")}})
@@ -355,7 +368,7 @@ def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answe
             ans = "I am ClarityOps, your strategic decision making AI partner."
             return history + [(user_msg, ans)], awaiting_answers
 
-        # Ingest uploads if paths present (also handled in upload event; safe to repeat)
+        # Ingest uploads FIRST (files alone can trigger scenario mode)
         artifacts = []
         if uploaded_files_paths:
             ing = extract_text_from_files(uploaded_files_paths)
@@ -367,7 +380,54 @@ def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answe
                 _session_rag.register_artifacts(artifacts)
             log_event("uploads_added", None, {"chunks": len(chunks), "artifacts": len(artifacts)})
 
-        # Session retrieval context
+        # CSV columns helper (works in both modes)
+        if re.search(r"\b(columns?|headers?)\b", (safe_in or "").lower()):
+            cols = _session_rag.get_latest_csv_columns()
+            if cols:
+                return history + [(user_msg, "Here are the column names from your most recent CSV upload:\n\n- " + "\n- ".join(cols))], awaiting_answers
+
+        # Decide mode
+        scenario_mode = is_scenario_triggered(safe_in, uploaded_files_paths)
+
+        if not scenario_mode:
+            # ---------- Normal conversational chat ----------
+            out = cohere_chat(safe_in, history) if USE_HOSTED_COHERE else None
+            if not out:
+                # Small system nudge for normal chat
+                model, tokenizer = load_local_model()
+                tiny = [{"role": "system", "content": "You are a helpful assistant."}]
+                for u, a in _iter_user_assistant(history):
+                    if u: tiny.append({"role": "user", "content": u})
+                    if a: tiny.append({"role": "assistant", "content": a})
+                tiny.append({"role": "user", "content": safe_in})
+                inputs = tokenizer.apply_chat_template(tiny, tokenize=True, add_generation_prompt=True, return_tensors="pt")
+                out = local_generate(model, tokenizer, inputs, max_new_tokens=MAX_NEW_TOKENS)
+
+            out = _sanitize_text(out or "")
+            safe_out, blocked_out, reason_out = safety_filter(out, mode="output")
+            if blocked_out:
+                safe_out = refusal_reply(reason_out)
+            log_event("assistant_reply", None, {
+                **hash_summary("prompt", safe_in if not PERSIST_CONTENT else ""),
+                **hash_summary("reply", safe_out if not PERSIST_CONTENT else ""),
+                "mode": "normal_chat",
+            })
+            return history + [(user_msg, safe_out)], awaiting_answers
+
+        # ---------- Scenario Mode ----------
+        if not awaiting_answers:
+            # PHASE 1: generate dynamic questions here (no assumptions)
+            phase1 = build_dynamic_clarifications(scenario_text=safe_in, artifacts=artifacts or _session_rag.artifacts)
+            phase1 = _sanitize_text(phase1)
+            log_event("assistant_reply", None, {
+                **hash_summary("prompt", safe_in if not PERSIST_CONTENT else ""),
+                **hash_summary("reply", phase1 if not PERSIST_CONTENT else ""),
+                "mode": "scenario_phase1",
+                "awaiting_next_phase": True
+            })
+            return history + [(user_msg, phase1)], True
+
+        # PHASE 2: build rich system preamble + feed to LLM
         session_snips = "\n---\n".join(_session_rag.retrieve(
             "diabetes screening Indigenous Métis mobile program cost throughput outcomes logistics",
             k=6
@@ -382,125 +442,62 @@ def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answe
         user_lower = (safe_in or "").lower()
         mdsi_extra = _mdsi_block() if ("diabetes" in user_lower or "mdsi" in user_lower or "mobile screening" in user_lower) else ""
 
+        # Summarize artifacts for the model (concise, structured)
+        arts = _session_rag.artifacts or []
+        if arts:
+            arts_summ = []
+            for a in arts:
+                nm = a.get("name") or "<unnamed>"
+                cols = ", ".join(a.get("columns") or [])[:600]
+                rows = a.get("n_rows_sampled") or 0
+                arts_summ.append(f"- {nm}: columns[{cols}] sample_rows={rows}")
+            artifact_block = "Uploaded Data Files (summarized):\n" + "\n".join(arts_summ)
+        else:
+            artifact_block = "Uploaded Data Files (summarized):\n- <none>"
+
+        # Build system preamble
         scenario_block = safe_in if len((safe_in or "")) > 0 else ""
         system_preamble = build_system_preamble(
             snapshot=snapshot,
             policy_context=policy_context,
             computed_numbers=computed,
-            scenario_text=scenario_block + (f"\n\nExecutive Pre-Computed Blocks:\n{mdsi_extra}" if mdsi_extra else ""),
+            scenario_text=scenario_block + f"\n\n{artifact_block}" + (f"\n\nExecutive Pre-Computed Blocks:\n{mdsi_extra}" if mdsi_extra else ""),
             session_snips=session_snips
         )
 
-        # Decide mode (normal vs scenario)
-        stype = detect_scenario_type(safe_in, _session_rag.artifacts)
-        in_scenario = bool(stype)
-
-        # Gap analysis
-        crit_missing, nice_missing, det_note = analyze_gaps(safe_in, _session_rag.artifacts)
-
-        # Determine phase directive
-        if force_phase == "clarify":
-            awaiting = True
-            directive = (
-                "\n\n[INSTRUCTION TO MODEL]\n"
-                "Produce **Phase 1** only:\n"
-                "- Header: 'Clarification Questions'\n"
-                "- Ask ONLY for the items listed as missing (critical first, then optional). Group by category.\n"
-                "- Then STOP and WAIT.\n"
-            )
-        elif force_phase == "analyze":
-            awaiting = False
-            if crit_missing:
-                # hard block
-                return history + [(user_msg,
-                    "INSUFFICIENT_DATA\n\nMissing critical inputs:\n- " + "\n- ".join(crit_missing)
-                )], False
-            directive = (
-                "\n\n[INSTRUCTION TO MODEL]\n"
-                "Produce **Phase 2** only:\n"
-                "- Header: 'Structured Analysis'\n"
-                "- Follow the exact section order (Prioritization, Capacity, Cost, Clinical Benefits, ClarityOps Top 3 Recommendations).\n"
-                "- Use uploaded files + the user's latest answers as authoritative. Show calculations, units, and a brief Provenance.\n"
-            )
-        else:
-            # Auto-decide
-            if in_scenario:
-                if not awaiting_answers:
-                    # entering Phase 1 if there are any missing fields; if nothing missing, we can go to Phase 2 immediately
-                    if crit_missing:
-                        awaiting = True
-                        directive = (
-                            "\n\n[INSTRUCTION TO MODEL]\n"
-                            "Produce **Phase 1** only:\n"
-                            "- Header: 'Clarification Questions'\n"
-                            "- Ask ONLY for the items listed as missing (critical first, then optional). Group by category.\n"
-                            "- Then STOP and WAIT.\n"
-                        )
-                    else:
-                        awaiting = False
-                        directive = (
-                            "\n\n[INSTRUCTION TO MODEL]\n"
-                            "Produce **Phase 2** only:\n"
-                            "- Header: 'Structured Analysis'\n"
-                            "- Follow the exact section order (Prioritization, Capacity, Cost, Clinical Benefits, ClarityOps Top 3 Recommendations).\n"
-                            "- Use uploaded files + the user's latest answers as authoritative. Show calculations, units, and a brief Provenance.\n"
-                        )
-                else:
-                    # expecting answers; attempt Phase 2 but block if still missing critical
-                    if crit_missing:
-                        return history + [(user_msg,
-                            "INSUFFICIENT_DATA\n\nMissing critical inputs:\n- " + "\n- ".join(crit_missing)
-                        )], True
-                    awaiting = False
-                    directive = (
-                        "\n\n[INSTRUCTION TO MODEL]\n"
-                        "Produce **Phase 2** only:\n"
-                        "- Header: 'Structured Analysis'\n"
-                        "- Follow the exact section order (Prioritization, Capacity, Cost, Clinical Benefits, ClarityOps Top 3 Recommendations).\n"
-                        "- Use uploaded files + the user's latest answers as authoritative. Show calculations, units, and a brief Provenance.\n"
-                    )
-            else:
-                # Normal chat mode
-                awaiting = awaiting_answers
-                directive = "\n\n[INSTRUCTION TO MODEL]\nAnswer normally as a helpful assistant.\n"
-
-        augmented_user = SYSTEM_MASTER + "\n\n" + system_preamble + "\n\nUser message:\n" + safe_in + directive
-
-        # Call LLM
+        directive = (
+            "\n\n[INSTRUCTION TO MODEL]\n"
+            "Produce **Phase 2** only now: start with 'Structured Analysis' and follow the exact section order "
+            "(Prioritization, Capacity, Cost, Clinical Benefits, ClarityOps Top 3 Recommendations). "
+            "Use uploaded files + the user's latest answers as authoritative. Show calculations, units, and a brief Provenance.\n"
+        )
+
+        augmented_user = SYSTEM_MASTER + "\n\n" + system_preamble + "\n\nUser scenario & answers:\n" + safe_in + directive
+
         out = cohere_chat(augmented_user, history)
         if not out:
             model, tokenizer = load_local_model()
             inputs = build_inputs(tokenizer, augmented_user, history)
             out = local_generate(model, tokenizer, inputs, max_new_tokens=MAX_NEW_TOKENS)
 
-        # Clean + sanitize
         if isinstance(out, str):
             for tag in ("Assistant:", "System:", "User:"):
                 if out.startswith(tag):
                     out = out[len(tag):].strip()
-        out = _sanitize_text(out)
+        out = _sanitize_text(out or "")
 
-        # Safety (output)
         safe_out, blocked_out, reason_out = safety_filter(out, mode="output")
         if blocked_out:
             safe_out = refusal_reply(reason_out)
 
-        # Flip phase state based on headers (scenario only)
-        new_awaiting = awaiting
-        if in_scenario:
-            low = (safe_out or "").lower()
-            if "clarification questions" in low:
-                new_awaiting = True
-            elif "structured analysis" in low:
-                new_awaiting = False
-
         log_event("assistant_reply", None, {
             **hash_summary("prompt", augmented_user if not PERSIST_CONTENT else ""),
             **hash_summary("reply", safe_out if not PERSIST_CONTENT else ""),
-            "awaiting_next_phase": new_awaiting
+            "mode": "scenario_phase2",
+            "awaiting_next_phase": False
         })
 
-        return history + [(user_msg, safe_out)], new_awaiting
+        return history + [(user_msg, safe_out)], False
 
     except Exception as e:
         err = f"Error: {e}"
@@ -514,11 +511,10 @@ def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answe
 theme = gr.themes.Soft(primary_hue="teal", neutral_hue="slate", radius_size=gr.themes.sizes.radius_lg)
 custom_css = """
 :root { --brand-bg: #e6f7f8; --brand-accent: #0d9488; --brand-text: #0f172a; --brand-text-light: #ffffff; }
-
 html, body, .gradio-container { height: 100vh; }
 .gradio-container { background: var(--brand-bg); display: flex; flex-direction: column; }
 
-/* HERO (initial Google-like screen) */
+/* HERO (landing) */
 #hero-wrap { height: 70vh; display: grid; place-items: center; }
 #hero { text-align: center; }
 #hero h2 { color: #0f172a; font-weight: 800; font-size: 32px; margin-bottom: 22px; }
@@ -528,28 +524,28 @@ html, body, .gradio-container { height: 100vh; }
 
 /* CHAT */
 #chat-container { position: relative; }
-.message.user, .message.bot { background: var(--brand-accent) !important; color: var(--brand-text-light) !important; border-radius: 12px !important; padding: 8px 12px !important; }
 .chatbot header, .chatbot .label, .chatbot .label-wrap { display: none !important; }
+.message.user, .message.bot { background: var(--brand-accent) !important; color: var(--brand-text-light) !important; border-radius: 12px !important; padding: 8px 12px !important; }
 textarea, input, .gr-input { border-radius: 12px !important; }
 """
 
 # ---------- UI ----------
 with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
-    # --- HERO (landing) ---
+    # --- HERO (initial Google-like screen) ---
     with gr.Column(elem_id="hero-wrap", visible=True) as hero_wrap:
         with gr.Column(elem_id="hero"):
             gr.HTML("<h2>What can I help with?</h2>")
             with gr.Row(elem_classes="search-row"):
                 hero_msg = gr.Textbox(
-                    placeholder="Ask anything (paste scenarios here; you can attach files after)...",
+                    placeholder="Ask anything (type 'scenario' and/or attach files for Scenario Mode)…",
                     show_label=False,
                     lines=1,
                     elem_classes="hero-box"
                 )
                 hero_send = gr.Button("➤", scale=0)
-            gr.Markdown('<div class="hint">ClarityOps will parse uploads, compare to your scenario, ask only for what’s missing (no assumptions), then produce a structured analysis.</div>')
+            gr.Markdown('<div class="hint">Scenario Mode triggers when you type the word <b>scenario</b> or upload files. Phase&nbsp;1 asks dynamic clarifications; Phase&nbsp;2 returns a structured analysis.</div>')
 
-    # --- MAIN APP ---
+    # --- MAIN APP (hidden until first message) ---
     with gr.Column(elem_id="chat-container", visible=False) as app_wrap:
         chat = gr.Chatbot(label="", show_label=False, height="64vh")
         with gr.Row():
@@ -561,7 +557,7 @@ with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
             msg = gr.Textbox(
                 label="",
                 show_label=False,
-                placeholder="Continue here. Paste scenario details, add files below.",
+                placeholder="Continue here. Paste scenario details (include the word 'scenario' to trigger), add files below.",
                 scale=10
             )
             send = gr.Button("Send", scale=1)
@@ -570,55 +566,16 @@ with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
     # ---- State
     state_history = gr.State(value=[])
     state_uploaded = gr.State(value=[])
-    state_awaiting = gr.State(value=False)  # False -> not waiting; True -> expecting answers to clarifications
+    state_awaiting = gr.State(value=False)  # False -> Phase 1 next; True -> Phase 2 next (awaiting answers)
 
-    # ---- Upload handler (immediate ingest + gap summary to chat)
-    def _on_upload(files, history, uploaded_paths):
-        new_paths = []
+    # ---- Uploads
+    def _store_uploads(files, current):
+        paths = []
         for f in (files or []):
-            new_paths.append(getattr(f, "name", None) or f)
-        all_paths = (uploaded_paths or []) + new_paths
-
-        # Ingest now
-        ing = extract_text_from_files(new_paths)
-        chunks = ing.get("chunks", []) if isinstance(ing, dict) else (ing or [])
-        arts = ing.get("artifacts", []) if isinstance(ing, dict) else []
-        if chunks:
-            _session_rag.add_docs(chunks)
-        if arts:
-            _session_rag.register_artifacts(arts)
+            paths.append(getattr(f, "name", None) or f)
+        return (current or []) + paths
 
-        # Build coverage & gap view using the last user scenario message if any
-        last_user_msg = ""
-        for u, a in _iter_user_assistant(history):
-            if u:
-                last_user_msg = u  # take the latest user utterance
-
-        crit_missing, nice_missing, note = analyze_gaps(last_user_msg, _session_rag.artifacts)
-        coverage = build_data_summary(_session_rag.artifacts)
-
-        # Compose bot message
-        parts = ["**Data Intake Summary**"]
-        if note: parts.append(note)
-        parts.append("**Files parsed & coverage:**\n" + (coverage or "_No files parsed._"))
-        if crit_missing:
-            parts.append("**Missing (critical):**\n- " + "\n- ".join(crit_missing))
-        if nice_missing:
-            parts.append("**Missing (optional but useful):**\n- " + "\n- ".join(nice_missing))
-        parts.append("\nIf you can, provide the missing details now. Otherwise, say “proceed” and I’ll continue (but Phase 2 will block if critical items remain).")
-        bot_msg = "\n\n".join(parts)
-
-        new_hist = (history or []) + [("", bot_msg)]
-        # If there are critical gaps AND we are in scenario context already, set awaiting=True (Phase 1)
-        awaiting = bool(crit_missing and detect_scenario_type(last_user_msg, _session_rag.artifacts))
-
-        return all_paths, new_hist, awaiting
-
-    uploads.change(
-        _on_upload,
-        inputs=[uploads, state_history, state_uploaded],
-        outputs=[state_uploaded, state_history, state_awaiting]
-    )
+    uploads.change(fn=_store_uploads, inputs=[uploads, state_uploaded], outputs=state_uploaded)
 
     # ---- Core send (used by both hero input and chat input)
     def _on_send(user_msg, history, up_paths, awaiting):
@@ -668,11 +625,7 @@ with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
                concurrency_limit=2, queue=True)
 
     def _on_clear():
-        # fresh session (clears RAG too)
-        try:
-            _session_rag.clear()
-        except Exception:
-            pass
+        # Reset to fresh hero screen
         return (
             [], "", [], False,
             gr.update(visible=True),   # show hero
@@ -685,4 +638,3 @@ with gr.Blocks(theme=theme, css=custom_css, analytics_enabled=False) as demo:
 if __name__ == "__main__":
     port = int(os.environ.get("PORT", "7860"))
     demo.launch(server_name="0.0.0.0", server_port=port, show_api=False, max_threads=8)
-