Update app.py

app.py

@@ -1,10 +1,3 @@
-# app.py
-# Universal AI Data Analyst with:
-# - Unchanged analysis & assessment logic
-# - Fixed Gradio event wiring (uses gr.State for history)
-# - Triple-quoted progress strings (no unterminated literals)
-# - Sleek full-width UI and Voice-to-Text (browser Web Speech API)
-# - Optional HIPAA flags (fallback defaults if not present in settings.py)
 from __future__ import annotations
 
 import io
@@ -40,682 +33,9 @@ from audit_log import log_event
 from privacy import safety_filter, refusal_reply
 from llm_router import cohere_chat, _co_client, cohere_embed
 
-# ---------------------- Helpers (analysis logic unchanged) ----------------------
-def load_markdown_text(filepath: str) -> str:
-    try:
-        with open(filepath, "r", encoding="utf-8") as f:
-            return f.read()
-    except FileNotFoundError:
-        return f"**Error:** Document `{os.path.basename(filepath)}` not found."
-
-def _sanitize_text(s: str) -> str:
-    if not isinstance(s, str):
-        return s
-    # Remove control characters (except newline and tab)
-    return re2.sub(r"[\p{C}--[\n\t]]+", "", s)
-
-# Conservative PHI redaction patterns (only applied if PHI_MODE & REDACT_BEFORE_LLM are enabled)
-PHI_PATTERNS = [
-    (re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), "[REDACTED_SSN]"),
-    (re.compile(r"\b\d{9}\b"), "[REDACTED_MRN]"),
-    (re.compile(r"\b\d{3}[-.\s]?\d{3}[-.\s]?\d{4}\b"), "[REDACTED_PHONE]"),
-    (re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"), "[REDACTED_EMAIL]"),
-    (re.compile(r"\b(19|20)\d{2}-\d{2}-\d{2}\b"), "[REDACTED_DOB]"),
-    (re.compile(r"\b\d{2}/\d{2}/(19|20)\d{2}\b"), "[REDACTED_DOB]"),
-    (re.compile(r"\b\d{5}(-\d{4})?\b"), "[REDACTED_ZIP]"),
-]
-
-def redact_phi(text: str) -> str:
-    if not isinstance(text, str):
-        return text
-    t = text
-    for pat, repl in PHI_PATTERNS:
-        t = pat.sub(repl, t)
-    return t
-
-def safe_log(event_name: str, meta: dict | None = None):
-    # Avoid logging raw PHI or payloads
-    try:
-        meta = (meta or {}).copy()
-        meta.pop("raw", None)
-        log_event(event_name, None, meta)
-    except Exception:
-        # Never raise from logging
-        pass
-
-
-# ---------------------- JSON Validation ----------------------
-
-class JSONValidationError(Exception):
-    """Raised when script output fails JSON validation."""
-    pass
-
-
-def validate_json_output(raw_output: str) -> Dict[str, Any]:
-    """
-    Validates and parses JSON output from the analysis script.
-    
-    This creates the "hard boundary" between calculation and communication
-    as described in the ClarityOps architecture. The function:
-    1. Strips whitespace and handles empty output
-    2. Attempts to parse as JSON
-    3. Validates the structure is a dictionary (not array or primitive)
-    4. Checks for error indicators in the output
-    5. Returns validated Python dict for report generation
-    
-    Args:
-        raw_output: Raw string captured from script stdout
-        
-    Returns:
-        Validated dictionary containing analysis findings
-        
-    Raises:
-        JSONValidationError: If output is empty, malformed, or contains errors
-    """
-    # Strip whitespace
-    cleaned_output = raw_output.strip()
-    
-    # Check for empty output
-    if not cleaned_output:
-        raise JSONValidationError(
-            "Analysis script produced no output. The script must print a JSON object to stdout."
-        )
-    
-    # Handle multiple JSON objects (take the last complete one)
-    # This handles cases where debug prints precede the final JSON
-    json_candidates = []
-    brace_count = 0
-    current_start = None
-    
-    for i, char in enumerate(cleaned_output):
-        if char == '{':
-            if brace_count == 0:
-                current_start = i
-            brace_count += 1
-        elif char == '}':
-            brace_count -= 1
-            if brace_count == 0 and current_start is not None:
-                json_candidates.append(cleaned_output[current_start:i+1])
-                current_start = None
-    
-    # If no valid JSON structure found, try parsing the whole output
-    if not json_candidates:
-        json_to_parse = cleaned_output
-    else:
-        # Use the last JSON object (most likely the final output)
-        json_to_parse = json_candidates[-1]
-    
-    # Attempt JSON parsing
-    try:
-        parsed = json.loads(json_to_parse)
-    except json.JSONDecodeError as e:
-        # Provide helpful error message with context
-        error_context = cleaned_output[:500] + ("..." if len(cleaned_output) > 500 else "")
-        raise JSONValidationError(
-            f"Analysis script produced invalid JSON. Parse error: {e.msg} at position {e.pos}.\n\n"
-            f"Raw output (first 500 chars):\n```\n{error_context}\n```"
-        )
-    
-    # Validate structure is a dictionary
-    if not isinstance(parsed, dict):
-        raise JSONValidationError(
-            f"Analysis output must be a JSON object (dictionary), not {type(parsed).__name__}. "
-            f"Ensure your script prints a dictionary with json.dumps()."
-        )
-    
-    # Check for error indicators in the output
-    if "error" in parsed:
-        error_msg = parsed.get("error", "Unknown error")
-        raise JSONValidationError(
-            f"Analysis script reported an error: {error_msg}"
-        )
-    
-    # Validate output is not empty dict
-    if not parsed:
-        raise JSONValidationError(
-            "Analysis script produced an empty JSON object. "
-            "Ensure your script populates the output dictionary with findings."
-        )
-    
-    # Log successful validation (without sensitive data)
-    safe_log("json_validation_success", {"keys": list(parsed.keys()), "key_count": len(parsed)})
-    
-    return parsed
-
-
-def format_validated_json_for_report(validated_data: Dict[str, Any]) -> str:
-    """
-    Formats validated JSON data for the report generator.
-    
-    Converts the validated Python dictionary back to a formatted JSON string
-    for the LLM to interpret. This ensures consistent formatting and handles
-    any edge cases in serialization.
-    
-    Args:
-        validated_data: Validated dictionary from validate_json_output()
-        
-    Returns:
-        Formatted JSON string ready for report generation
-    """
-    try:
-        return json.dumps(validated_data, indent=2, default=str, ensure_ascii=False)
-    except (TypeError, ValueError) as e:
-        # Fallback to string representation if JSON serialization fails
-        safe_log("json_format_warning", {"error": str(e)})
-        return json.dumps({"raw_data": str(validated_data)}, indent=2)
-
-
-# ---------------------- Analysis Script Generation ----------------------
-
-def _create_python_script(user_scenario: str, schema_context: str) -> str:
-    EXPERT_ANALYTICAL_GUIDELINES = """
---- EXPERT ANALYTICAL GUIDELINES ---
-When writing your script, you MUST follow these expert business rules:
-1.  **Linking Datasets Rule:** If you need to connect facilities to health zones when the 'zone' column is not in the facility list,
-    you must first identify the high-priority zone from the beds data, then find the major city (by facility count) in the facility list,
-    and *then* assess that city's capacity. Do not try to filter the facility list by a 'zone' column if it does not exist in the schema.
-2.  **Prioritization Rule:** To prioritize locations, you MUST combine the most recent population data with specific high-risk health indicators
-    to create a multi-factor risk score.
-3.  **Capacity Calculation Rule:** For capacity over a 3-month window, assume **60 working days**.
-4.  **Cost Calculation Rule:** Sum 'Startup cost' and 'Ongoing cost' per person before multiplying.
-"""
-    prompt_for_coder = f"""\
-You are an expert Python data scientist. Your job is to write a script to extract the data needed to answer the user's request.
-You have dataframes in a list `dfs`.
-
-{EXPERT_ANALYTICAL_GUIDELINES}
-
---- DATA SCHEMA ---
-{schema_context}
---- END DATA SCHEMA ---
-
-CRITICAL RULES:
-1.  **DO NOT READ FILES:** You MUST NOT include `pd.read_csv`. The data is ALREADY loaded in the `dfs` variable. You MUST use this variable. Failure to do so will cause a fatal error.
-2.  **JSON OUTPUT ONLY:** Your script's ONLY output must be a single JSON object printed to stdout containing the raw data findings.
-3.  **BE PRECISE:** Use the exact, case-sensitive column names from the schema and robustly clean strings (`re.sub()`) before converting to numbers.
-4.  **JSON SERIALIZATION:** Before adding data to your final dictionary for JSON conversion, you MUST convert any pandas-specific types (like `int64`) to standard Python types using `.item()` for single values or `.tolist()` for lists.
-5.  **SINGLE JSON OUTPUT:** Print exactly ONE JSON object at the end of your script. Do not print debug statements or multiple JSON objects.
-6.  **VALID JSON STRUCTURE:** The output MUST be a dictionary/object, not an array or primitive value.
-
---- USER'S SCENARIO ---
-{user_scenario}
-
---- PYTHON SCRIPT ---
-Now, write the complete Python script that performs the analysis and prints a single, serializable JSON object.
-```python
-"""
-    generated_text = cohere_chat(prompt_for_coder)
-    match = re2.search(r"```python\n(.*?)```", generated_text, re2.DOTALL)
-    if match:
-        return match.group(1).strip()
-    return "print(json.dumps({'error': 'Failed to generate a valid Python script.'}))"
-
-
-def _generate_long_report(prompt: str) -> str:
-    try:
-        client = _co_client()
-        if not client:
-            return "Error: Cohere client not initialized."
-        response = client.chat(
-            model=COHERE_MODEL_PRIMARY,
-            message=prompt,
-            max_tokens=4096,
-        )
-        return response.text
-    except Exception as e:
-        safe_log("cohere_chat_error", {"err": str(e)})
-        return f"Error during final report generation: {e}"
-
-
-def _generate_final_report(user_scenario: str, validated_json_str: str) -> str:
-    prompt_for_writer = f"""\
-You are an expert management consultant and data analyst.
-A data science script has run to extract key findings. You have the user's original request and the validated JSON data.
-
-Your task is to synthesize these validated findings into a single, comprehensive, and professional report that directly answers all of the user's questions with detailed justifications.
-
---- USER'S ORIGINAL SCENARIO & DELIVERABLES ---
-{user_scenario}
---- END SCENARIO ---
-
---- VALIDATED DATA FINDINGS (JSON) ---
-{validated_json_str}
---- END VALIDATED DATA ---
-
-Now, write the final, polished report. The report MUST:
-1.  Follow the "Expected Output Format" requested by the user.
-2.  Use tables, bullet points, and DETAILED narrative justifications for each recommendation.
-3.  Synthesize the validated data into actionable insights. Do not just copy the raw numbers; interpret them.
-4.  Ensure you fully address ALL evaluation questions, especially the final recommendations.
-"""
-    return _generate_long_report(prompt_for_writer)
-
-
-def _append_msg(h: List[Dict[str, str]], r: str, c: str) -> List[Dict[str, str]]:
-    return (h or []) + [{"role": r, "content": c}]
-
-
-def ping_cohere() -> str:
-    try:
-        cli = _co_client()
-        if not cli:
-            return "Cohere client not initialized."
-        vecs = cohere_embed(["hello", "world"])
-        return f"Cohere OK ✅ (model={COHERE_MODEL_PRIMARY})" if vecs else "Cohere reachable."
-    except Exception as e:
-        return f"Cohere ping failed: {e}"
-
-
-def handle(user_msg: str, files: list, yield_update) -> str:
-    try:
-        # Safety filter on incoming message
-        safe_in, blocked_in, reason_in = safety_filter(user_msg, mode="input")
-        if blocked_in:
-            return refusal_reply(reason_in)
-
-        # Optional PHI redaction for prompts sent to an external LLM
-        redacted_in = safe_in
-        if PHI_MODE and REDACT_BEFORE_LLM:
-            redacted_in = redact_phi(safe_in)
-
-        file_paths: List[str] = [getattr(f, "name", None) or f for f in (files or [])]
-
-        if file_paths:
-            # CSV analysis path
-            dataframes, schema_parts = [], []
-            for i, p in enumerate(file_paths):
-                if p.endswith(".csv"):
-                    try:
-                        df = pd.read_csv(p)
-                    except UnicodeDecodeError:
-                        df = pd.read_csv(p, encoding="latin1")
-                    dataframes.append(df)
-                    schema_parts.append(
-                        f"DataFrame `dfs[{i}]` (`{os.path.basename(p)}`):\n{df.head().to_markdown()}\n"
-                    )
-
-            if not dataframes:
-                return "Please upload at least one CSV file."
-
-            schema_context = "\n".join(schema_parts)
-
-            # If external PHI is not allowed, use redacted prompt; otherwise use original
-            prompt_for_code = redacted_in if (PHI_MODE and not ALLOW_EXTERNAL_PHI) else safe_in
-
-            yield_update("""```
-🧠 Generating aligned analysis script...
-```""")
-            analysis_script = _create_python_script(prompt_for_code, schema_context)
-
-            yield_update("""```
-⚙️ Executing script to extract raw data...
-```""")
-            execution_namespace = {"dfs": dataframes, "pd": pd, "re": re, "json": json}
-            output_buffer = io.StringIO()
-
-            try:
-                with redirect_stdout(output_buffer):
-                    exec(analysis_script, execution_namespace)
-                raw_data_output = output_buffer.getvalue()
-            except Exception as e:
-                return (
-                    f"An error occurred executing the script: {e}\n\nGenerated Script:\n"
-                    f"```python\n{analysis_script}\n```"
-                )
-
-            # JSON Validation - creates hard boundary between calculation and communication
-            yield_update("""```
-🔍 Validating JSON output...
-```""")
-            try:
-                validated_data = validate_json_output(raw_data_output)
-                validated_json_str = format_validated_json_for_report(validated_data)
-                safe_log("json_validation_passed", {"output_keys": list(validated_data.keys())})
-            except JSONValidationError as e:
-                safe_log("json_validation_failed", {"error": str(e)})
-                return (
-                    f"**JSON Validation Failed**\n\n{e}\n\n"
-                    f"Generated Script:\n```python\n{analysis_script}\n```"
-                )
-
-            yield_update("""```
-✍️ Synthesizing final comprehensive report...
-```""")
-            writer_input = redacted_in if (PHI_MODE and not ALLOW_EXTERNAL_PHI) else safe_in
-            final_report = _generate_final_report(writer_input, validated_json_str)
-            return _sanitize_text(final_report)
-        else:
-            # Pure chat path
-            chat_input = redacted_in if (PHI_MODE and not ALLOW_EXTERNAL_PHI) else safe_in
-            prompt = f"{GENERAL_CONVERSATION_PROMPT}\n\nUser: {chat_input}\nAssistant:"
-            return _sanitize_text(cohere_chat(prompt) or "How can I help further?")
-
-    except Exception as e:
-        tb = traceback.format_exc()
-        safe_log("app_error", {"err": str(e)})
-        return "A critical error occurred. Please contact your administrator." if PHI_MODE else f"A critical error occurred: {e}"
-
-
-PRIVACY_POLICY_TEXT = load_markdown_text("privacy_policy.md")
-TERMS_OF_SERVICE_TEXT = load_markdown_text("terms_of_service.md")
-
-
-# ---------------------- Sleek UI assets (CSS/JS only) ----------------------
-
-SLEEK_CSS = """
-/* Full-bleed, modern look */
-:root, body, #root, .gradio-container { height: 100%; }
-.gradio-container { padding: 0 !important; }
-.block { padding: 0 !important; }
-
-/* Header */
-.header {
-  padding: 20px 28px;
-  background: linear-gradient(135deg, #0e1726, #1d2a44 60%, #243a5e);
-  color: #fff;
-  display: flex; align-items: center; justify-content: space-between;
-  gap: 16px;
-}
-.header h1 { margin: 0; font-size: 22px; letter-spacing: 0.3px; font-weight: 600; }
-.header .badge { font-size: 12px; opacity: 0.9; background:#ffffff22; padding:6px 10px; border-radius: 999px; }
-
-/* Main layout */
-.main {
-  display: grid;
-  grid-template-columns: 420px 1fr;
-  gap: 16px;
-  padding: 16px;
-  height: calc(100vh - 72px);
-  box-sizing: border-box;
-}
-.left, .right {
-  background: #0b1020;
-  color: #e9edf3;
-  border-radius: 16px;
-  border: 1px solid #1c2642;
-}
-.left { padding: 16px; display: flex; flex-direction: column; gap: 12px; }
-.right { padding: 0; display: flex; flex-direction: column; }
-
-/* Panels */
-.panel-title { font-size: 14px; font-weight: 600; color: #aeb8cc; margin-bottom: 6px; }
-.helper { font-size: 12px; color: #97a3bb; margin-bottom: 8px; }
-
-/* Sticky actions */
-.actions {
-  display: flex; gap: 8px; align-items: center; justify-content: stretch;
-}
-.actions .gr-button { flex: 1; }
-
-/* Tabs full height */
-.right .tabs { height: 100%; display: flex; flex-direction: column; }
-.right .tabitem { flex: 1; display: flex; flex-direction: column; }
-#chatbot_container { flex: 1; }
-#chatbot_container .gr-chatbot { height: 100%; }
-
-/* Tiny separators */
-.hr { height: 1px; background: #16203b; margin: 10px 0; }
-
-/* Voice hint */
-.voice-hint { font-size: 12px; color:#9fb0cc; margin-top: 4px; }
-"""
-
-VOICE_STT_HTML = """
-<script>
-let __rs_rec = null;
-function rs_toggle_stt(elemId){
-  const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
-  if (!SpeechRecognition){
-    alert("This browser does not support Speech Recognition. Try Chrome or Edge.");
-    return;
-  }
-  if (__rs_rec){ __rs_rec.stop(); __rs_rec = null; return; }
-  __rs_rec = new SpeechRecognition();
-  __rs_rec.lang = "en-US";
-  __rs_rec.interimResults = true;
-  __rs_rec.continuous = true;
-
-  const box = document.querySelector(`#${elemId} textarea`);
-  if (!box){ alert("Prompt box not found."); return; }
-  let base = box.value || "";
-
-  __rs_rec.onresult = (ev) => {
-    let t = "";
-    for (let i = ev.resultIndex; i < ev.results.length; i++){
-      t += ev.results[i].transcript;
-    }
-    box.value = (base + " " + t).trim();
-    box.dispatchEvent(new Event("input", { bubbles: true }));
-  };
-  __rs_rec.onend = () => { __rs_rec = null; };
-  __rs_rec.start();
-}
-</script>
-"""
-
-
-# ---------------------- Sleek UI (with fixed State wiring) ----------------------
-
-with gr.Blocks(theme=gr.themes.Soft(), css=SLEEK_CSS, fill_width=True) as demo:
-    # Persistent in-memory history component (fixes list/_id error)
-    assessment_history = gr.State([])
-
-    # Header
-    with gr.Row(elem_classes=["header"]):
-        gr.Markdown("<h1>Clarity Ops Augemented Decision Support</h1>")
-        pill = "PHI Mode ON · history off" if (PHI_MODE and not PERSIST_HISTORY) else \
-               "PHI Mode ON" if PHI_MODE else "PHI Mode OFF"
-        gr.Markdown(f"<span class='badge'>{pill}</span>")
-
-    # Main layout
-    with gr.Row(elem_classes=["main"]):
-        # Left panel
-        with gr.Column(elem_classes=["left"]):
-            gr.Markdown("<div class='panel-title'>New Assessment</div>")
-            gr.Markdown("<div class='helper'>Upload CSVs for analysis, or enter a prompt. Voice works in modern browsers.</div>")
-            files_input = gr.Files(
-                label="Upload Data Files (.csv)",
-                file_count="multiple",
-                type="filepath",
-                file_types=[".csv"],
-            )
-            prompt_input = gr.Textbox(
-                label="Prompt",
-                placeholder="Paste your scenario or question here...",
-                lines=12,
-                elem_id="prompt_box",
-                autofocus=True,
-            )
-
-            with gr.Row(elem_classes=["actions"]):
-                send_btn = gr.Button("▶️ Run Analysis", variant="primary")
-                clear_btn = gr.Button("🧹 Clear")
-                voice_btn = gr.Button("🎙️ Voice")
-
-            gr.Markdown("<div class='voice-hint'>Click Voice to start/stop dictation into the prompt box.</div>")
-            ping_btn = gr.Button("🔌 Ping Cohere")
-            ping_out = gr.Markdown()
-
-            gr.Markdown("<div class='hr'></div>")
-            if PHI_MODE:
-                gr.Markdown(
-                    "⚠️ **PHI Mode:** History persistence is disabled by default. Avoid unnecessary identifiers."
-                )
-
-            with gr.Accordion("Privacy & Terms", open=False):
-                gr.Markdown(PRIVACY_POLICY_TEXT)
-                gr.Markdown("<div class='hr'></div>")
-                gr.Markdown(TERMS_OF_SERVICE_TEXT)
-
-        # Right panel
-        with gr.Column(elem_classes=["right"]):
-            with gr.Tabs(elem_classes=["tabs"]):
-                with gr.TabItem("Current Assessment", id=0, elem_classes=["tabitem"]):
-                    with gr.Column(elem_id="chatbot_container"):
-                        chat_history_output = gr.Chatbot(label="Analysis Output", type="messages", container=False, autoscroll=True)
-                with gr.TabItem("Assessment History", id=1, elem_classes=["tabitem"]):
-                    gr.Markdown("### Review Past Assessments")
-                    history_dropdown = gr.Dropdown(label="Select an assessment to review", choices=[])
-                    history_display = gr.Markdown(label="Selected Assessment Details")
-
-    # Inject voice-to-text helper
-    gr.HTML(VOICE_STT_HTML)
-
-    # --------- Event logic (unchanged analysis flow) ----------
-
-    def run_analysis_wrapper(prompt, files, chat_history_list, history_state_list):
-        if not prompt:
-            gr.Warning("Please enter a prompt.")
-            yield chat_history_list, history_state_list, gr.update()
-            return
-
-        # Append user's message
-        chat_with_user_msg = _append_msg(chat_history_list, "user", prompt)
-
-        # Optional progress callback (not streaming in this UI)
-        def dummy_update(message: str):
-            pass
-
-        # Thinking bubble
-        thinking_message = _append_msg(
-            chat_with_user_msg,
-            "assistant",
-            """```
-🧠 Generating and executing analysis... Please wait.
-```""",
-        )
-        yield thinking_message, history_state_list, gr.update()
-
-        # Run analysis/chat
-        ai_response_text = handle(prompt, files, dummy_update)
-
-        # Append final assistant response
-        final_chat = _append_msg(chat_with_user_msg, "assistant", ai_response_text)
-        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-
-        # Capture filenames (if any)
-        file_names: List[str] = []
-        if files:
-            file_names = [
-                os.path.basename(f.name if hasattr(f, "name") else f) for f in files
-            ]
-
-        # Build history record
-        new_entry = {
-            "id": timestamp,
-            "prompt": prompt,
-            "files": file_names,
-            "response": ai_response_text,
-            "chat_history": final_chat,
-        }
-
-        # Respect PHI/history flags
-        if PERSIST_HISTORY and (not PHI_MODE or (PHI_MODE and HISTORY_TTL_DAYS > 0)):
-            updated_history: List[Dict[str, Any]] = (history_state_list or []) + [new_entry]
-        else:
-            updated_history = history_state_list or []
-
-        history_labels = [f"{item['id']} - {item['prompt'][:40]}..." for item in updated_history]
-
-        yield final_chat, updated_history, gr.update(choices=history_labels)
-
-    def view_history(selection: str, history_state_list: List[Dict[str, Any]]) -> str:
-        if not selection or not history_state_list:
-            return ""
-        try:
-            selected_id = selection.split(" - ", 1)[0]
-        except Exception:
-            selected_id = selection
-
-        selected_assessment = next(
-            (item for item in history_state_list if item.get("id") == selected_id), None
-        )
-        if not selected_assessment:
-            return "Could not find the selected assessment."
-
-        file_list = selected_assessment.get("files", [])
-        file_list_md = "\n- ".join(file_list) if file_list else "*(no files uploaded)*"
-
-        chat_entries = selected_assessment.get("chat_history", [])
-        chat_md_lines = []
-        for msg in chat_entries:
-            role = msg.get("role", "").capitalize()
-            content = msg.get("content", "")
-            chat_md_lines.append(f"**{role}:** {content}")
-        chat_md = "\n\n".join(chat_md_lines)
-
-        return f"""### Assessment from: {selected_assessment['id']}
-**Files Used:**
-- {file_list_md}
----
-**Original Prompt:**
-> {selected_assessment['prompt']}
----
-**AI Generated Response:**
-{selected_assessment['response']}
----
-**Chat Transcript:**
-{chat_md}
-"""
-
-    # Wire events (using proper gr.State component for history)
-    send_btn.click(
-        run_analysis_wrapper,
-        inputs=[prompt_input, files_input, chat_history_output, assessment_history],
-        outputs=[chat_history_output, assessment_history, history_dropdown],
-    )
-    history_dropdown.change(
-        view_history,
-        inputs=[history_dropdown, assessment_history],
-        outputs=[history_display],
-    )
-    clear_btn.click(
-        lambda: (None, None, []),
-        outputs=[prompt_input, files_input, chat_history_output],
-    )
-    ping_btn.click(ping_cohere, outputs=[ping_out])
-    voice_btn.click(None, [], [], js="rs_toggle_stt('prompt_box')")
-
-
-if __name__ == "__main__":
-    if not os.getenv("COHERE_API_KEY"):
-        print("🔴 COHERE_API_KEY environment variable not set. Application may not function correctly.")
-    demo.launch(server_name="0.0.0.0", server_port=int(os.getenv("PORT", "7860")))from __future__ import annotations
-
-import io
-import json
-import os
-import traceback
-from contextlib import redirect_stdout
-from datetime import datetime
-from typing import Any, Dict, List
-
-import gradio as gr
-import pandas as pd
-import regex as re2
-import re
-from langchain_cohere import ChatCohere  # noqa: F401
-from settings import (
-    GENERAL_CONVERSATION_PROMPT,
-    COHERE_MODEL_PRIMARY,
-    COHERE_TIMEOUT_S,   # noqa: F401
-    USE_OPEN_FALLBACKS  # noqa: F401
-)
-# Try to import optional HIPAA flags; fall back to safe defaults if not defined.
-try:
-    from settings import PHI_MODE, PERSIST_HISTORY, HISTORY_TTL_DAYS, REDACT_BEFORE_LLM, ALLOW_EXTERNAL_PHI
-except Exception:
-    PHI_MODE = False
-    PERSIST_HISTORY = True
-    HISTORY_TTL_DAYS = 365
-    REDACT_BEFORE_LLM = False
-    ALLOW_EXTERNAL_PHI = True
 
-from audit_log import log_event
-from privacy import safety_filter, refusal_reply
-from llm_router import cohere_chat, _co_client, cohere_embed
+# ---------------------- Helpers ----------------------
 
-# ---------------------- Helpers (analysis logic unchanged) ----------------------
 def load_markdown_text(filepath: str) -> str:
     try:
         with open(filepath, "r", encoding="utf-8") as f:
@@ -723,13 +43,14 @@ def load_markdown_text(filepath: str) -> str:
     except FileNotFoundError:
         return f"**Error:** Document `{os.path.basename(filepath)}` not found."
 
+
 def _sanitize_text(s: str) -> str:
     if not isinstance(s, str):
         return s
-    # Remove control characters (except newline and tab)
     return re2.sub(r"[\p{C}--[\n\t]]+", "", s)
 
-# Conservative PHI redaction patterns (only applied if PHI_MODE & REDACT_BEFORE_LLM are enabled)
+
+# Conservative PHI redaction patterns
 PHI_PATTERNS = [
     (re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), "[REDACTED_SSN]"),
     (re.compile(r"\b\d{9}\b"), "[REDACTED_MRN]"),
@@ -740,6 +61,7 @@ PHI_PATTERNS = [
     (re.compile(r"\b\d{5}(-\d{4})?\b"), "[REDACTED_ZIP]"),
 ]
 
+
 def redact_phi(text: str) -> str:
     if not isinstance(text, str):
         return text
@@ -748,14 +70,13 @@ def redact_phi(text: str) -> str:
         t = pat.sub(repl, t)
     return t
 
+
 def safe_log(event_name: str, meta: dict | None = None):
-    # Avoid logging raw PHI or payloads
     try:
         meta = (meta or {}).copy()
         meta.pop("raw", None)
         log_event(event_name, None, meta)
     except Exception:
-        # Never raise from logging
         pass
 
 
@@ -769,35 +90,16 @@ class JSONValidationError(Exception):
 def validate_json_output(raw_output: str) -> Dict[str, Any]:
     """
     Validates and parses JSON output from the analysis script.
-    
-    This creates the "hard boundary" between calculation and communication
-    as described in the ClarityOps architecture. The function:
-    1. Strips whitespace and handles empty output
-    2. Attempts to parse as JSON
-    3. Validates the structure is a dictionary (not array or primitive)
-    4. Checks for error indicators in the output
-    5. Returns validated Python dict for report generation
-    
-    Args:
-        raw_output: Raw string captured from script stdout
-        
-    Returns:
-        Validated dictionary containing analysis findings
-        
-    Raises:
-        JSONValidationError: If output is empty, malformed, or contains errors
+    Creates the hard boundary between calculation and communication.
     """
-    # Strip whitespace
     cleaned_output = raw_output.strip()
     
-    # Check for empty output
     if not cleaned_output:
         raise JSONValidationError(
             "Analysis script produced no output. The script must print a JSON object to stdout."
         )
     
     # Handle multiple JSON objects (take the last complete one)
-    # This handles cases where debug prints precede the final JSON
     json_candidates = []
     brace_count = 0
     current_start = None
@@ -813,69 +115,45 @@ def validate_json_output(raw_output: str) -> Dict[str, Any]:
                 json_candidates.append(cleaned_output[current_start:i+1])
                 current_start = None
     
-    # If no valid JSON structure found, try parsing the whole output
     if not json_candidates:
         json_to_parse = cleaned_output
     else:
-        # Use the last JSON object (most likely the final output)
         json_to_parse = json_candidates[-1]
     
-    # Attempt JSON parsing
     try:
         parsed = json.loads(json_to_parse)
     except json.JSONDecodeError as e:
-        # Provide helpful error message with context
         error_context = cleaned_output[:500] + ("..." if len(cleaned_output) > 500 else "")
         raise JSONValidationError(
             f"Analysis script produced invalid JSON. Parse error: {e.msg} at position {e.pos}.\n\n"
             f"Raw output (first 500 chars):\n```\n{error_context}\n```"
         )
     
-    # Validate structure is a dictionary
     if not isinstance(parsed, dict):
         raise JSONValidationError(
             f"Analysis output must be a JSON object (dictionary), not {type(parsed).__name__}. "
             f"Ensure your script prints a dictionary with json.dumps()."
         )
     
-    # Check for error indicators in the output
     if "error" in parsed:
         error_msg = parsed.get("error", "Unknown error")
-        raise JSONValidationError(
-            f"Analysis script reported an error: {error_msg}"
-        )
+        raise JSONValidationError(f"Analysis script reported an error: {error_msg}")
     
-    # Validate output is not empty dict
     if not parsed:
         raise JSONValidationError(
             "Analysis script produced an empty JSON object. "
             "Ensure your script populates the output dictionary with findings."
         )
     
-    # Log successful validation (without sensitive data)
     safe_log("json_validation_success", {"keys": list(parsed.keys()), "key_count": len(parsed)})
-    
     return parsed
 
 
 def format_validated_json_for_report(validated_data: Dict[str, Any]) -> str:
-    """
-    Formats validated JSON data for the report generator.
-    
-    Converts the validated Python dictionary back to a formatted JSON string
-    for the LLM to interpret. This ensures consistent formatting and handles
-    any edge cases in serialization.
-    
-    Args:
-        validated_data: Validated dictionary from validate_json_output()
-        
-    Returns:
-        Formatted JSON string ready for report generation
-    """
+    """Formats validated JSON data for the report generator."""
     try:
         return json.dumps(validated_data, indent=2, default=str, ensure_ascii=False)
     except (TypeError, ValueError) as e:
-        # Fallback to string representation if JSON serialization fails
         safe_log("json_format_warning", {"error": str(e)})
         return json.dumps({"raw_data": str(validated_data)}, indent=2)
 
@@ -983,12 +261,10 @@ def ping_cohere() -> str:
 
 def handle(user_msg: str, files: list, yield_update) -> str:
     try:
-        # Safety filter on incoming message
         safe_in, blocked_in, reason_in = safety_filter(user_msg, mode="input")
         if blocked_in:
             return refusal_reply(reason_in)
 
-        # Optional PHI redaction for prompts sent to an external LLM
         redacted_in = safe_in
         if PHI_MODE and REDACT_BEFORE_LLM:
             redacted_in = redact_phi(safe_in)
@@ -996,7 +272,6 @@ def handle(user_msg: str, files: list, yield_update) -> str:
         file_paths: List[str] = [getattr(f, "name", None) or f for f in (files or [])]
 
         if file_paths:
-            # CSV analysis path
             dataframes, schema_parts = [], []
             for i, p in enumerate(file_paths):
                 if p.endswith(".csv"):
@@ -1013,18 +288,12 @@ def handle(user_msg: str, files: list, yield_update) -> str:
                 return "Please upload at least one CSV file."
 
             schema_context = "\n".join(schema_parts)
-
-            # If external PHI is not allowed, use redacted prompt; otherwise use original
             prompt_for_code = redacted_in if (PHI_MODE and not ALLOW_EXTERNAL_PHI) else safe_in
 
-            yield_update("""```
-🧠 Generating aligned analysis script...
-```""")
+            yield_update("```\n🧠 Generating aligned analysis script...\n```")
             analysis_script = _create_python_script(prompt_for_code, schema_context)
 
-            yield_update("""```
-⚙️ Executing script to extract raw data...
-```""")
+            yield_update("```\n⚙️ Executing script to extract raw data...\n```")
             execution_namespace = {"dfs": dataframes, "pd": pd, "re": re, "json": json}
             output_buffer = io.StringIO()
 
@@ -1039,9 +308,7 @@ def handle(user_msg: str, files: list, yield_update) -> str:
                 )
 
             # JSON Validation - creates hard boundary between calculation and communication
-            yield_update("""```
-🔍 Validating JSON output...
-```""")
+            yield_update("```\n🔍 Validating JSON output...\n```")
             try:
                 validated_data = validate_json_output(raw_data_output)
                 validated_json_str = format_validated_json_for_report(validated_data)
@@ -1053,20 +320,16 @@ def handle(user_msg: str, files: list, yield_update) -> str:
                     f"Generated Script:\n```python\n{analysis_script}\n```"
                 )
 
-            yield_update("""```
-✍️ Synthesizing final comprehensive report...
-```""")
+            yield_update("```\n✍️ Synthesizing final comprehensive report...\n```")
             writer_input = redacted_in if (PHI_MODE and not ALLOW_EXTERNAL_PHI) else safe_in
             final_report = _generate_final_report(writer_input, validated_json_str)
             return _sanitize_text(final_report)
         else:
-            # Pure chat path
             chat_input = redacted_in if (PHI_MODE and not ALLOW_EXTERNAL_PHI) else safe_in
             prompt = f"{GENERAL_CONVERSATION_PROMPT}\n\nUser: {chat_input}\nAssistant:"
             return _sanitize_text(cohere_chat(prompt) or "How can I help further?")
 
     except Exception as e:
-        tb = traceback.format_exc()
         safe_log("app_error", {"err": str(e)})
         return "A critical error occurred. Please contact your administrator." if PHI_MODE else f"A critical error occurred: {e}"
 
@@ -1075,15 +338,13 @@ PRIVACY_POLICY_TEXT = load_markdown_text("privacy_policy.md")
 TERMS_OF_SERVICE_TEXT = load_markdown_text("terms_of_service.md")
 
 
-# ---------------------- Sleek UI assets (CSS/JS only) ----------------------
+# ---------------------- UI Assets ----------------------
 
 SLEEK_CSS = """
-/* Full-bleed, modern look */
 :root, body, #root, .gradio-container { height: 100%; }
 .gradio-container { padding: 0 !important; }
 .block { padding: 0 !important; }
 
-/* Header */
 .header {
   padding: 20px 28px;
   background: linear-gradient(135deg, #0e1726, #1d2a44 60%, #243a5e);
@@ -1094,7 +355,6 @@ SLEEK_CSS = """
 .header h1 { margin: 0; font-size: 22px; letter-spacing: 0.3px; font-weight: 600; }
 .header .badge { font-size: 12px; opacity: 0.9; background:#ffffff22; padding:6px 10px; border-radius: 999px; }
 
-/* Main layout */
 .main {
   display: grid;
   grid-template-columns: 420px 1fr;
@@ -1112,26 +372,20 @@ SLEEK_CSS = """
 .left { padding: 16px; display: flex; flex-direction: column; gap: 12px; }
 .right { padding: 0; display: flex; flex-direction: column; }
 
-/* Panels */
 .panel-title { font-size: 14px; font-weight: 600; color: #aeb8cc; margin-bottom: 6px; }
 .helper { font-size: 12px; color: #97a3bb; margin-bottom: 8px; }
 
-/* Sticky actions */
 .actions {
   display: flex; gap: 8px; align-items: center; justify-content: stretch;
 }
 .actions .gr-button { flex: 1; }
 
-/* Tabs full height */
 .right .tabs { height: 100%; display: flex; flex-direction: column; }
 .right .tabitem { flex: 1; display: flex; flex-direction: column; }
 #chatbot_container { flex: 1; }
 #chatbot_container .gr-chatbot { height: 100%; }
 
-/* Tiny separators */
 .hr { height: 1px; background: #16203b; margin: 10px 0; }
-
-/* Voice hint */
 .voice-hint { font-size: 12px; color:#9fb0cc; margin-top: 4px; }
 """
 
@@ -1169,22 +423,18 @@ function rs_toggle_stt(elemId){
 """
 
 
-# ---------------------- Sleek UI (with fixed State wiring) ----------------------
+# ---------------------- Gradio UI ----------------------
 
 with gr.Blocks(theme=gr.themes.Soft(), css=SLEEK_CSS, fill_width=True) as demo:
-    # Persistent in-memory history component (fixes list/_id error)
     assessment_history = gr.State([])
 
-    # Header
     with gr.Row(elem_classes=["header"]):
-        gr.Markdown("<h1>Clarity Ops Augemented Decision Support</h1>")
+        gr.Markdown("<h1>Clarity Ops Augmented Decision Support</h1>")
         pill = "PHI Mode ON · history off" if (PHI_MODE and not PERSIST_HISTORY) else \
                "PHI Mode ON" if PHI_MODE else "PHI Mode OFF"
         gr.Markdown(f"<span class='badge'>{pill}</span>")
 
-    # Main layout
     with gr.Row(elem_classes=["main"]):
-        # Left panel
         with gr.Column(elem_classes=["left"]):
             gr.Markdown("<div class='panel-title'>New Assessment</div>")
             gr.Markdown("<div class='helper'>Upload CSVs for analysis, or enter a prompt. Voice works in modern browsers.</div>")
@@ -1222,7 +472,6 @@ with gr.Blocks(theme=gr.themes.Soft(), css=SLEEK_CSS, fill_width=True) as demo:
                 gr.Markdown("<div class='hr'></div>")
                 gr.Markdown(TERMS_OF_SERVICE_TEXT)
 
-        # Right panel
         with gr.Column(elem_classes=["right"]):
             with gr.Tabs(elem_classes=["tabs"]):
                 with gr.TabItem("Current Assessment", id=0, elem_classes=["tabitem"]):
@@ -1233,49 +482,37 @@ with gr.Blocks(theme=gr.themes.Soft(), css=SLEEK_CSS, fill_width=True) as demo:
                     history_dropdown = gr.Dropdown(label="Select an assessment to review", choices=[])
                     history_display = gr.Markdown(label="Selected Assessment Details")
 
-    # Inject voice-to-text helper
     gr.HTML(VOICE_STT_HTML)
 
-    # --------- Event logic (unchanged analysis flow) ----------
-
     def run_analysis_wrapper(prompt, files, chat_history_list, history_state_list):
         if not prompt:
             gr.Warning("Please enter a prompt.")
             yield chat_history_list, history_state_list, gr.update()
             return
 
-        # Append user's message
         chat_with_user_msg = _append_msg(chat_history_list, "user", prompt)
 
-        # Optional progress callback (not streaming in this UI)
         def dummy_update(message: str):
             pass
 
-        # Thinking bubble
         thinking_message = _append_msg(
             chat_with_user_msg,
             "assistant",
-            """```
-🧠 Generating and executing analysis... Please wait.
-```""",
+            "```\n🧠 Generating and executing analysis... Please wait.\n```",
         )
         yield thinking_message, history_state_list, gr.update()
 
-        # Run analysis/chat
         ai_response_text = handle(prompt, files, dummy_update)
 
-        # Append final assistant response
         final_chat = _append_msg(chat_with_user_msg, "assistant", ai_response_text)
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
-        # Capture filenames (if any)
         file_names: List[str] = []
         if files:
             file_names = [
                 os.path.basename(f.name if hasattr(f, "name") else f) for f in files
             ]
 
-        # Build history record
         new_entry = {
             "id": timestamp,
             "prompt": prompt,
@@ -1284,7 +521,6 @@ with gr.Blocks(theme=gr.themes.Soft(), css=SLEEK_CSS, fill_width=True) as demo:
             "chat_history": final_chat,
         }
 
-        # Respect PHI/history flags
         if PERSIST_HISTORY and (not PHI_MODE or (PHI_MODE and HISTORY_TTL_DAYS > 0)):
             updated_history: List[Dict[str, Any]] = (history_state_list or []) + [new_entry]
         else:
@@ -1333,7 +569,6 @@ with gr.Blocks(theme=gr.themes.Soft(), css=SLEEK_CSS, fill_width=True) as demo:
 {chat_md}
 """
 
-    # Wire events (using proper gr.State component for history)
     send_btn.click(
         run_analysis_wrapper,
         inputs=[prompt_input, files_input, chat_history_output, assessment_history],