chat_interface.py

"""
Chat-based Demo Builder Interface
Clean, conversational interface for creating ThoughtSpot demos
"""


import warnings
warnings.filterwarnings('ignore', message='.*tuples.*format.*chatbot.*deprecated.*')

import gradio as gr
import os
import sys
import json
import time
import glob
from datetime import datetime
from dotenv import load_dotenv
from demo_builder_class import DemoBuilder

APP_ENV = os.getenv("APP_ENV", "production").lower()
IS_TEST = APP_ENV == "test"
from supabase_client import load_gradio_settings, get_admin_setting, inject_admin_settings_to_env
from main_research import MultiLLMResearcher, Website
from demo_personas import (
    build_company_analysis_prompt,
    build_industry_research_prompt,
    VERTICALS,
    FUNCTIONS,
    MATRIX_OVERRIDES,
    VERTICAL_LINES,
    DEMO_FUNCTIONS,
    get_use_case_config,
    parse_use_case
)
from llm_config import (
    DEFAULT_LLM_MODEL,
    UI_MODEL_CHOICES,
    get_openai_api_key,
    map_llm_display_to_provider,
)

load_dotenv(override=True)


# ==========================================================================
# TS ENVIRONMENT HELPERS
# ==========================================================================

def get_ts_environments() -> list:
    """Return list of environment labels from TS_ENV_N_LABEL/URL/.env entries."""
    envs = []
    i = 1
    while True:
        label = os.getenv(f'TS_ENV_{i}_LABEL', '').strip()
        url = os.getenv(f'TS_ENV_{i}_URL', '').strip()
        if not label or not url:
            break
        envs.append(label)
        i += 1
    return envs or ['(no environments configured)']


def get_ts_env_url(label: str) -> str:
    """Return the URL for a given environment label."""
    i = 1
    while True:
        env_label = os.getenv(f'TS_ENV_{i}_LABEL', '').strip()
        if not env_label:
            break
        if env_label == label:
            return os.getenv(f'TS_ENV_{i}_URL', '').strip().rstrip('/')
        i += 1
    return ''


def get_ts_env_auth_key(label: str) -> str:
    """Return the actual auth key value for a given environment label.
    TS_ENV_N_KEY_VAR holds the trusted auth key directly.
    """
    i = 1
    while True:
        env_label = os.getenv(f'TS_ENV_{i}_LABEL', '').strip()
        if not env_label:
            break
        if env_label == label:
            return os.getenv(f'TS_ENV_{i}_KEY_VAR', '').strip()
        i += 1
    return ''


# ==========================================================================
# SETTINGS SCHEMA - Single source of truth for all settings
# To add a new setting: add ONE entry here, then create the UI component
# Format: (component_key, storage_key, default_value, converter_fn)
# ==========================================================================
SETTINGS_SCHEMA = [
    # ── Panel defaults (mirrors App tab right panel) ──────────────────────────
    ('default_ai_model', 'default_llm', DEFAULT_LLM_MODEL, str),
    ('default_ts_env', 'default_ts_env', 'secloud - primary', str),
    ('liveboard_name', 'liveboard_name', '', str),
    ('default_data_size', 'default_data_size', 'Medium', str),
    ('geo_scope', 'geo_scope', 'USA Only', str),
    ('tag_name', 'tag_name', '', str),
    ('column_naming_style', 'column_naming_style', 'Regular Case', str),
    ('object_naming_prefix', 'object_naming_prefix', '', str),
    ('share_with', 'share_with', '', str),
    # ── Optional run input defaults ───────────────────────────────────────────
    ('use_default_inputs', 'use_default_inputs', False, bool),
    ('default_vertical', 'default_vertical', '', str),
    ('default_line', 'default_line', '', str),
    ('default_function', 'default_function', '', str),
    ('default_company_url', 'default_company_url', '', str),
    # ── Other settings ────────────────────────────────────────────────────────
    ('validation_mode', 'validation_mode', 'Off', str),
    # Legacy hidden fields — kept for backward compat, not shown in UI
    ('default_use_case', 'default_use_case', 'Sales Analytics', str),
    ('fact_table_size', 'fact_table_size', '1000', str),
    ('dim_table_size', 'dim_table_size', '100', str),
    ('use_existing_model', 'use_existing_model', False, bool),
    ('existing_model_guid', 'existing_model_guid', '', str),
    # Advanced AI Settings (collapsed accordion)
    ('temperature_slider', 'temperature', 0.3, float),
    ('max_tokens', 'max_tokens', 4000, int),
    ('batch_size', 'batch_size', 5000, int),
    ('thread_count', 'thread_count', 4, int),
    # Database Connection Settings (collapsed accordion)
    ('sf_account', 'snowflake_account', '', str),
    ('sf_user', 'snowflake_user', '', str),
    ('sf_role', 'snowflake_role', 'ACCOUNTADMIN', str),
    ('default_warehouse', 'default_warehouse', 'COMPUTE_WH', str),
    ('default_database', 'default_database', 'DEMO_DB', str),
    ('default_schema', 'default_schema', 'PUBLIC', str),
    # ts_instance_url removed — replaced by TS Environment dropdown on front page
    ('ts_username', 'thoughtspot_username', '', str),
    ('data_adjuster_url', 'data_adjuster_url', '', str),
    # Status (special - not saved, just displayed)
    ('settings_status', None, '', str),
]


def get_settings_defaults():
    """Return list of default values in schema order"""
    return [default for _, _, default, _ in SETTINGS_SCHEMA]


def load_settings_values(settings_dict: dict, user_email: str = "") -> list:
    """Load settings from dict, returning values in schema order"""
    values = []
    for component_key, storage_key, default, converter in SETTINGS_SCHEMA:
        if storage_key is None:  # Special case: settings_status
            values.append(f"✅ Settings loaded for {user_email}" if user_email else "")
        else:
            raw_value = settings_dict.get(storage_key, default)
            try:
                if storage_key == 'default_llm' and raw_value not in UI_MODEL_CHOICES:
                    raw_value = default
                if converter == bool:
                    # Special handling: bool("False") returns True (non-empty string)
                    # We need to check the actual string value
                    if isinstance(raw_value, bool):
                        values.append(raw_value)
                    elif isinstance(raw_value, str):
                        values.append(raw_value.lower() in ('true', '1', 'yes'))
                    else:
                        values.append(bool(raw_value) if raw_value else default)
                else:
                    values.append(converter(raw_value) if raw_value else default)
            except (ValueError, TypeError):
                values.append(default)
    return values


def build_settings_save_dict(values: list) -> dict:
    """Build dict for saving from values in schema order"""
    save_dict = {}
    for i, (component_key, storage_key, default, converter) in enumerate(SETTINGS_SCHEMA):
        if storage_key is None:  # Skip settings_status
            continue
        if i < len(values):
            value = values[i]
            # Convert to string for storage (Supabase stores as text)
            save_dict[storage_key] = str(value) if value is not None else str(default)
    return save_dict


def require_authenticated_email(request: gr.Request = None, user_email: str = None) -> str:
    """Require an authenticated user identity for settings operations."""
    request_user = getattr(request, 'username', None) if request else None
    resolved_user = request_user or user_email
    if resolved_user and str(resolved_user).strip():
        return str(resolved_user).strip().lower()

    # Local-dev no-auth mode is explicit and still requires a concrete user key.
    no_auth = os.getenv('DEMOPREP_NO_AUTH', 'false').lower() in ('true', '1', 'yes')
    if no_auth:
        dev_user_email = os.getenv('DEMOPREP_DEV_USER_EMAIL', '').strip().lower()
        if dev_user_email:
            return dev_user_email
        raise ValueError(
            "DEMOPREP_NO_AUTH=true requires DEMOPREP_DEV_USER_EMAIL to be set. "
            "Set both values and retry."
        )

    raise ValueError(
        "Authenticated username is required for settings operations. "
        "Please sign in and retry."
    )


def resolve_app_url_for_invite(request: gr.Request = None) -> str:
    """Resolve the public app URL for copy/paste onboarding invites."""
    configured_url = (
        os.getenv("DEMOPREP_APP_URL", "").strip()
        or os.getenv("PUBLIC_APP_URL", "").strip()
        or os.getenv("SPACE_HOST", "").strip()
    )
    if configured_url:
        if configured_url.startswith("http"):
            return configured_url.rstrip("/")
        return f"https://{configured_url.strip('/')}"

    try:
        headers = getattr(request, "headers", {}) if request else {}
        referer = headers.get("referer") or headers.get("origin") or ""
        if referer:
            return referer.split("?")[0].rstrip("/")
    except Exception:
        pass

    return "https://thoughtspot-dp-test-demoprep.hf.space"


def build_initial_chat_message(company: str, use_case: str) -> str:
    """Build the pre-filled chat message from current settings."""
    if company and use_case:
        return f"{company}, {use_case}"
    return ""


def safe_print(*args, **kwargs):
    """Print that silently handles broken pipes (background processes)"""
    try:
        print(*args, **kwargs)
    except BrokenPipeError:
        pass  # Ignore - happens when running in background


class ChatDemoInterface:
    """
    New chat-based interface for demo creation
    """
    
    def __init__(self, user_email: str = None):
        self.user_email = user_email  # Must be set by login system
        self.demo_builder = None
        self.conversation_history = []
        self.settings = self.load_default_settings()
        self.ai_feedback_log = []
        self.ddl_code = ""
        self.population_code = ""
        self.research_results = None
        # Vertical × Function use case system
        self.vertical = None
        self.function = None
        self.use_case_config = None
        # Generic use case handling
        self.is_generic_use_case = False
        self.generic_use_case_context = ""
        self.pending_generic_company = None
        self.pending_generic_use_case = None
        # New tab content
        self.live_progress_log = []  # Real-time deployment progress
        self.phase_log = []          # High-level pipeline status (shown in right panel)
        self.demo_pack_content = ""    # Generated demo pack markdown
        self.spotter_story_ai = ""     # Pure AI-generated Spotter Viz story
        self.spotter_story_matrix = "" # Matrix/ThoughtSpot-recommended Spotter Viz story
        self.deployment_completion = None  # Final model/liveboard links shown in the right panel
        # Per-session loggers (NOT module-level singletons — avoids cross-session contamination)
        self._session_logger = None
        self._prompt_logger = None
        self._demo_bundle = None

    def clear_deployment_completion(self) -> None:
        self.deployment_completion = None

    def record_deployment_completion(self, results: dict, database: str, schema_name: str, use_case: str) -> None:
        """Store final ThoughtSpot artifact links for the UI completion panel."""
        if not isinstance(results, dict):
            self.deployment_completion = None
            return

        ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/')
        model_guid = results.get('model_guid') or ''
        liveboard_guid = results.get('liveboard_guid') or results.get('liveboard_id') or ''
        liveboard_url = results.get('liveboard_url') or (
            f"{ts_url}/#/pinboard/{liveboard_guid}" if ts_url and liveboard_guid else ''
        )
        model_url = f"{ts_url}/#/data/tables/{model_guid}" if ts_url and model_guid else ''
        backup = bool(results.get('backup_liveboard')) or results.get('liveboard_creation_path') == 'spotter_tml_backup'
        warnings = results.get('warnings') or []
        errors = results.get('errors') or []

        if backup:
            status = "Backup liveboard created"
            note = (
                "MCP/Spotter answer generation was unavailable, so DemoPrep created a clearly marked "
                "Spotter/TML backup liveboard. The model and data are still available."
            )
        elif results.get('success') and liveboard_url:
            status = "MCP liveboard created"
            note = "The liveboard was created through the MCP path and enhanced after creation."
        elif model_url:
            status = "Model created"
            note = "The model was created, but a liveboard link was not returned. Use the model link to continue in ThoughtSpot."
        else:
            status = "Deployment finished"
            note = "Review the pipeline status for details."

        self.deployment_completion = {
            "status": status,
            "note": note,
            "code_version": results.get("code_version") or "",
            "ts_environment": results.get("ts_environment") or ts_url,
            "ts_username": results.get("ts_username") or "",
            "liveboard_creation_path": results.get("liveboard_creation_path") or ("spotter_tml_backup" if backup else "mcp" if liveboard_url else "none"),
            "fallback_reason": results.get("fallback_reason") or "",
            "model_url": model_url,
            "liveboard_url": liveboard_url,
            "model_guid": model_guid,
            "liveboard_guid": liveboard_guid,
            "connection": results.get('connection') or '',
            "schema": f"{database}.{schema_name}" if database and schema_name else schema_name,
            "use_case": use_case,
            "warnings": warnings,
            "errors": errors,
            "success": bool(results.get('success')),
        }

    def render_deployment_completion_html(self):
        """Render final artifact links for the completion panel."""
        if not self.deployment_completion:
            return gr.update(value="", visible=False)

        import html as _html
        item = self.deployment_completion
        status = _html.escape(str(item.get("status") or "Deployment complete"))
        note = _html.escape(str(item.get("note") or ""))
        model_url = str(item.get("model_url") or "")
        liveboard_url = str(item.get("liveboard_url") or "")
        schema = _html.escape(str(item.get("schema") or ""))
        connection = _html.escape(str(item.get("connection") or ""))
        ts_environment = _html.escape(str(item.get("ts_environment") or ""))
        ts_username = _html.escape(str(item.get("ts_username") or ""))
        code_version = _html.escape(str(item.get("code_version") or ""))
        liveboard_path = _html.escape(str(item.get("liveboard_creation_path") or "unknown"))
        fallback_reason = _html.escape(str(item.get("fallback_reason") or ""))
        warnings = item.get("warnings") or []
        errors = item.get("errors") or []
        is_mcp = item.get("status") == "MCP liveboard created"
        border = "#22c55e" if is_mcp and item.get("success") and not item.get("errors") else "#f59e0b"
        badge_bg = "#dcfce7" if is_mcp else "#fef3c7"
        badge_color = "#166534" if is_mcp else "#92400e"

        def _link_button(label, url):
            if not url:
                return f"<span style='color:#6b7280;font-size:13px;'>{_html.escape(label)} unavailable</span>"
            safe_url = _html.escape(url, quote=True)
            return (
                f"<a href='{safe_url}' target='_blank' rel='noopener noreferrer' "
                "style='display:inline-block;padding:9px 12px;margin:4px 6px 4px 0;"
                "border-radius:6px;background:#2563eb;color:white;text-decoration:none;"
                "font-weight:600;font-size:13px;'>"
                f"{_html.escape(label)}</a>"
            )

        warning_html = ""
        if warnings:
            warning_items = "".join(f"<li>{_html.escape(str(w))}</li>" for w in warnings[:3])
            warning_html = f"<ul style='margin:8px 0 0 18px;color:#92400e;font-size:12px;'>{warning_items}</ul>"
        error_html = ""
        if errors:
            error_items = "".join(f"<li>{_html.escape(str(e))}</li>" for e in errors[:2])
            error_html = f"<ul style='margin:8px 0 0 18px;color:#991b1b;font-size:12px;'>{error_items}</ul>"

        html_value = f"""
<div style="border:1px solid {border};border-left:5px solid {border};border-radius:8px;padding:12px;margin:10px 0;background:#fff;">
  <div style="display:flex;align-items:center;justify-content:space-between;gap:8px;">
    <div style="font-weight:700;color:#111827;">Deployment Links</div>
    <div style="padding:3px 8px;border-radius:999px;background:{badge_bg};color:{badge_color};font-size:12px;font-weight:700;">{status}</div>
  </div>
  <div style="margin-top:8px;color:#374151;font-size:13px;line-height:1.35;">{note}</div>
  <div style="margin-top:10px;">
    {_link_button("Open Liveboard", liveboard_url)}
    {_link_button("Open Model", model_url)}
  </div>
  <div style="margin-top:8px;color:#6b7280;font-size:12px;line-height:1.4;">
    <div><strong>Code:</strong> {code_version or "unknown"}</div>
    <div><strong>Environment:</strong> {ts_environment or "unknown"}</div>
    <div><strong>ThoughtSpot user:</strong> {ts_username or "unknown"}</div>
    <div><strong>Liveboard path:</strong> {liveboard_path}{f" &middot; <strong>Fallback reason:</strong> {fallback_reason}" if fallback_reason else ""}</div>
    <div><strong>Schema:</strong> {schema or "n/a"}</div>
    <div><strong>Connection:</strong> {connection or "n/a"}</div>
  </div>
  {warning_html}
  {error_html}
</div>
"""
        return gr.update(value=html_value, visible=True)

    def _get_effective_user_email(self) -> str:
        """Resolve and cache effective user identity for settings access."""
        self.user_email = require_authenticated_email(user_email=self.user_email)
        return self.user_email

    def _temporary_password_block_message(self) -> str:
        """Return a blocking message when a temp-password user tries to run the app."""
        try:
            from supabase_client import UserManager
            user_email = self._get_effective_user_email()
            um = UserManager()
            if um.enabled and um.must_change_password(user_email):
                return (
                    "🔒 **Password change required**\n\n"
                    "You are signed in with a temporary password. "
                    "Open **Settings → Change Password**, set your own password, "
                    "then come back and run DemoPrep."
                )
        except Exception as e:
            print(f"[Auth] Unable to check temporary-password status: {e}")
        return ""
    
    def load_default_settings(self):
        """Load settings from Supabase or defaults"""
        # Fallback defaults (ONLY used if settings not found)
        defaults = {
            'company': '',
            'use_case': '',
            'model': DEFAULT_LLM_MODEL,
            'fact_table_size': '5000',
            'dim_table_size': '100',
            'stage': 'initialization',
            'tag_name': None,
            'validation_mode': 'Off',  # Off = auto-run, On = pause at checkpoints
            'geo_scope': 'USA Only',  # USA Only or International
        }
        
        try:
            # Try to load from Supabase
            user_email = self._get_effective_user_email()
            if user_email:
                settings = load_gradio_settings(user_email)
                # Only override if values are meaningful (not generic placeholders)
                company = settings.get('default_company_url', '').strip()
                use_case = settings.get('default_use_case', '').strip()
                
                if company and company not in ['your company', 'yourcompany', '']:
                    defaults['company'] = company
                if use_case and use_case not in ['analytics', '']:
                    defaults['use_case'] = use_case
                if settings.get('default_llm'):
                    saved_model = settings.get('default_llm')
                    defaults['model'] = saved_model if saved_model in UI_MODEL_CHOICES else DEFAULT_LLM_MODEL
                if settings.get('fact_table_size'):
                    defaults['fact_table_size'] = settings.get('fact_table_size')
                if settings.get('dim_table_size'):
                    defaults['dim_table_size'] = settings.get('dim_table_size')
                if settings.get('tag_name'):
                    defaults['tag_name'] = settings.get('tag_name')
                if settings.get('geo_scope'):
                    defaults['geo_scope'] = settings.get('geo_scope')
                if settings.get('validation_mode'):
                    defaults['validation_mode'] = settings.get('validation_mode')
        except Exception as e:
            print(f"Could not load settings from Supabase: {e}")
        
        print(f"DEBUG: Final settings - company: {defaults['company']}, use_case: {defaults['use_case']}")
        return defaults
    
    def format_welcome_message(self, company, use_case):
        """Create the initial welcome message"""
        return """## Welcome to ThoughtSpot Demo Builder

I'll research a company, build a Snowflake schema, generate realistic data, and deploy a ThoughtSpot model and liveboard — all from a single prompt.

**How to start:**
- **Defined** — pick a vertical and function from the dropdowns below, add a company URL if you have one, and hit **→ GO**.
- **Custom** — describe what you want in your own words in the chat box.

<details>
<summary>📋 Example use cases</summary>

| Company | Vertical | Function | Story |
|---------|----------|----------|-------|
| Target.com | Retail | Sales | ASP decline, regional variance, holiday surge |
| Walmart.com | Retail | Supply Chain | Stockout risk, OTIF, days on hand |
| Chase.com | Banking | Marketing | Funnel drop-off, channel CTR, cost per acquisition |
| Salesforce.com | Software | Sales | ARR by segment, pipeline coverage, win rate |
| Caterpillar.com | Manufacturing | Supply Chain | Inventory levels, supplier performance |

</details>

> 💡 Select your ThoughtSpot environment in the right panel before starting."""
    
    def validate_required_settings(self) -> list:
        """
        Check that required admin and user settings are configured.
        Returns list of missing settings. Empty list = all good.
        """
        missing = []
        
        # Check admin settings (from environment, injected from Supabase)
        admin_checks = {
            'SNOWFLAKE_ACCOUNT': get_admin_setting('SNOWFLAKE_ACCOUNT', required=False),
            'SNOWFLAKE_KP_USER': get_admin_setting('SNOWFLAKE_KP_USER', required=False),
            'SNOWFLAKE_KP_PK': get_admin_setting('SNOWFLAKE_KP_PK', required=False),
            'SNOWFLAKE_ROLE': get_admin_setting('SNOWFLAKE_ROLE', required=False),
            'SNOWFLAKE_WAREHOUSE': get_admin_setting('SNOWFLAKE_WAREHOUSE', required=False),
            'SNOWFLAKE_DATABASE': get_admin_setting('SNOWFLAKE_DATABASE', required=False),
        }
        
        # LLM key now comes from environment only.
        has_llm = bool(get_openai_api_key(required=False))
        
        for key, val in admin_checks.items():
            if not val:
                missing.append(key)
        
        if not has_llm:
            missing.append('OPENAI_API_KEY')
        
        return missing

    def _create_run_loggers(self, force: bool = False):
        """Create a run-scoped session logger and prompt logger."""
        if self._session_logger is not None and not force:
            return self._session_logger

        from session_logger import build_session_id, create_session_logger
        from prompt_logger import PromptLogger

        tag = self.settings.get('tag_name', '')
        session_id = build_session_id(tag)
        self._session_logger = create_session_logger(session_id, user_email=getattr(self, 'user_email', None))
        self._prompt_logger = PromptLogger(session_id=session_id)

        model_setting = self.settings.get('model', DEFAULT_LLM_MODEL)
        try:
            provider_name, model_name = map_llm_display_to_provider(model_setting)
            resolved_model = f"{provider_name}/{model_name}"
        except Exception:
            resolved_model = str(model_setting or DEFAULT_LLM_MODEL)

        self._session_logger.log(
            'pipeline',
            'run started',
            test_tag=tag,
            model=resolved_model,
            model_setting=model_setting,
            payload=self._snapshot_run_payload(),
        )
        return self._session_logger

    def _snapshot_run_payload(self):
        """
        One JSON-safe record of everything that defines this run.

        Logged into session_logs meta on every run so Run History and ad-hoc
        queries can always answer "what was run": the raw GO form inputs
        (App tab), the resolved use case, and a secret-redacted snapshot of
        the controller settings. Secrets never leave the process — see
        sanitize_payload() in session_logger.py.
        """
        from session_logger import sanitize_payload
        return sanitize_payload({
            'interface': getattr(self, '_run_source', 'chat'),
            'company': getattr(self, 'pending_generic_company', '') or self.settings.get('company', ''),
            'use_case': getattr(self, 'pending_generic_use_case', '') or self.settings.get('use_case', ''),
            'vertical': getattr(self, 'vertical', None),
            'line': getattr(self, 'line', None),
            'function': getattr(self, 'function', None),
            'is_custom': getattr(self, 'is_generic_use_case', False),
            'additional_context': getattr(self, 'generic_use_case_context', '') or '',
            'form': getattr(self, '_run_payload', None),
            'settings': dict(self.settings),
        })
    
    def process_chat_message(self, message, chat_history, current_stage, current_model, company, use_case):
        """
        Process user message and return updated chat history and state (with streaming)
        Returns: (chat_history, current_stage, current_model, company, use_case, next_textbox_value)
        """
        # Chat-tab runs enter at 'initialization' (App tab jumps straight to
        # 'awaiting_context' after stashing its form payload) — reset any
        # leftover App-tab run metadata so the logged payload matches this run.
        if current_stage == 'initialization':
            self._run_source = 'chat'
            self._run_payload = None

        # Pipeline starts from awaiting_context; always give it a fresh run log.
        self._create_run_loggers(force=(current_stage == 'awaiting_context'))
        _slog = self._session_logger
        _slog.log(current_stage or 'init', f"user message received: {message[:120]}")

        # Add user message to history
        chat_history.append((message, None))

        password_block = self._temporary_password_block_message()
        if password_block and current_stage in {'initialization', 'awaiting_context'}:
            chat_history[-1] = (message, password_block)
            yield chat_history, current_stage, current_model, company, use_case, ""
            return

        # If data_adjuster_url is saved in settings and we're at init, inject it as the message
        # so the user lands directly in Data Adjuster without having to paste the URL manually
        da_url = self.settings.get('data_adjuster_url', '').strip()
        if da_url and current_stage == 'initialization' and 'pinboard/' in da_url:
            message = da_url
            chat_history[-1] = (da_url, None)

        # Validate required settings before proceeding
        missing = self.validate_required_settings()
        if missing and current_stage == 'initialization':
            missing_str = ", ".join(missing)
            error_msg = (
                f"**⚠️ Missing required settings:**\n\n"
                f"`{missing_str}`\n\n"
                f"Please configure these values. "
                f"LLM key must be set in `.env` as `OPENAI_API_KEY`."
            )
            chat_history[-1] = (message, error_msg)
            yield chat_history, current_stage, current_model, company, use_case, ""
            return
        
        # Check for special commands
        if message.strip().lower().startswith('/over'):
            # Override command - extract new values
            response = self.handle_override(message)
            chat_history[-1] = (message, response)
            yield chat_history, current_stage, current_model, company, use_case, ""
            return
        
        message_lower = message.lower()
        
        # Pre-clean any URLs in the message (fix typos like double dots)
        import re
        cleaned_message = re.sub(r'\.{2,}', '.', message)
        
        # Stage-based processing
        if current_stage == 'initialization':
            # Check if user pasted a ThoughtSpot liveboard URL → jump straight to data adjuster
            lb_guid_match = re.search(
                r'pinboard/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})',
                message, re.I
            )
            if lb_guid_match:
                liveboard_guid = lb_guid_match.group(1)
                chat_history[-1] = (message, "🔍 **Resolving liveboard context...**")
                yield chat_history, current_stage, current_model, company, use_case, ""

                try:
                    from smart_data_adjuster import SmartDataAdjuster, load_context_from_liveboard
                    from thoughtspot_deployer import ThoughtSpotDeployer
                    from supabase_client import get_admin_setting

                    ts_url = (self.settings.get('thoughtspot_url') or '').strip()
                    ts_secret = (self.settings.get('thoughtspot_trusted_auth_key') or '').strip()
                    if not ts_url or not ts_secret:
                        raise ValueError("ThoughtSpot environment not set — select a TS environment from the dropdown")
                    ts_user = self._get_effective_user_email()

                    ts_client = ThoughtSpotDeployer(base_url=ts_url, username=ts_user, secret_key=ts_secret)
                    ts_client.authenticate()

                    ctx = load_context_from_liveboard(liveboard_guid, ts_client)

                    llm_model = self.settings.get('model', DEFAULT_LLM_MODEL)
                    adjuster = SmartDataAdjuster(
                        database=ctx['database'],
                        schema=ctx['schema'],
                        liveboard_guid=liveboard_guid,
                        llm_model=llm_model,
                        ts_url=ts_url,
                        ts_secret=ts_secret,
                        username=ts_user,
                        prompt_logger=self._prompt_logger,
                    )
                    adjuster.connect()

                    if not adjuster.load_liveboard_context():
                        raise ValueError("Liveboard has no answer-based visualizations to adjust.")

                    self._adjuster = adjuster
                    current_stage = 'outlier_adjustment'

                    viz_list = "\n".join(
                        f"  [{i+1}] {v['name']}"
                        for i, v in enumerate(adjuster.visualizations)
                    )
                    response = f"""✅ **Liveboard context loaded — ready for data adjustments**

**Liveboard:** {ctx['liveboard_name']}
**Model:** {ctx['model_name']}
**Snowflake:** `{ctx['database']}`.`{ctx['schema']}`

**Visualizations:**
{viz_list}

Tell me what to change — e.g. *"increase webcam revenue by 20%"* or *"make Acme Corp 50B"*.
Type **done** when finished."""

                except Exception as e:
                    response = f"❌ **Could not load liveboard context**\n\n`{e}`"

                chat_history[-1] = (message, response)
                yield chat_history, current_stage, current_model, company, use_case, ""
                return

            # Check if user just provided a standalone URL (e.g., "Comscore.com")
            standalone_url = re.search(r'^([a-zA-Z0-9-]+\.[a-zA-Z]{2,})$', cleaned_message.strip())
            if standalone_url:
                # User provided just a URL - ask them to include the use case too
                detected_company = standalone_url.group(1)
                # Build dynamic use case list from VERTICALS × FUNCTIONS
                uc_opts = [f"- {v} {f}" for v in VERTICALS.keys() for f in FUNCTIONS.keys()]
                uc_opts_str = "\n".join(uc_opts)
                response = f"""I see you want to use **{detected_company}** - great choice!

Now I need to know what kind of demo you want. Please tell me both together:

```
I'm creating a demo for company: {detected_company} use case: Retail Sales
```

**Configured use cases** (with KPIs, outliers, Spotter questions):
{uc_opts_str}
- Or any custom use case you want!

What use case would you like for {detected_company}?"""
                chat_history[-1] = (message, response)
                yield chat_history, current_stage, current_model, detected_company, use_case, ""
                return
            
            # Check if user is providing company and use case
            if "creating a demo for" in message_lower or "create a demo for" in message_lower:
                # Extract company and use case (use cleaned message for better matching)
                extracted_company = self.extract_company_from_message(cleaned_message)
                extracted_use_case = self.extract_use_case_from_message(cleaned_message)
                
                if extracted_company:
                    company = extracted_company
                if extracted_use_case:
                    # Parse use case into vertical × function using new system
                    self.vertical, self.function = parse_use_case(extracted_use_case)
                    self.use_case_config = get_use_case_config(
                        self.vertical or "Generic", 
                        self.function or "Generic"
                    )
                    
                    # Determine if this is a known or generic use case
                    is_known = self.vertical and self.function and not self.use_case_config.get('is_generic')
                    use_case_display = self.use_case_config.get('use_case_name', extracted_use_case)
                    
                    # Store company/use case for context prompt
                    if is_known:
                        # Matched a configured vertical × function combination
                        self.is_generic_use_case = False
                        self.pending_generic_company = company
                        self.pending_generic_use_case = use_case_display
                        use_case_type_note = f"\n\n*Matched: **{self.vertical}** × **{self.function}** — using configured KPIs, outlier patterns, and Spotter questions.*"
                    elif self.vertical or self.function:
                        # Partial match - have vertical OR function but not both
                        self.is_generic_use_case = True
                        self.pending_generic_company = company
                        self.pending_generic_use_case = extracted_use_case
                        matched = self.vertical or self.function
                        use_case_type_note = f"\n\n*Partial match: **{matched}** recognized — AI will fill in the gaps based on research.*"
                    else:
                        # Fully generic/custom use case
                        self.is_generic_use_case = True
                        self.pending_generic_company = company
                        self.pending_generic_use_case = extracted_use_case
                        use_case_type_note = "\n\n*Custom use case — AI will research the industry to build a relevant schema and KPIs.*"
                    
                    # ALWAYS ask for additional context (for both generic and standard use cases)
                    context_prompt = f"""✅ **Demo Configuration**

I am creating a demo for **{company}** with use case: **{self.pending_generic_use_case}**{use_case_type_note}

**Default Schema:**
- 1 fact table (transactions/events)
- 3-4 dimension tables (customers, products, dates, etc.)

**Want to customize?** You can add requirements like:
- "I need 2 fact tables: SALES and INVENTORY"
- "Include a RETURNS table"
- "Focus on employee retention metrics"

**Type your requirements, or say "proceed" to use defaults.**"""
                        
                    chat_history[-1] = (message, context_prompt)
                    current_stage = 'awaiting_context'
                    yield chat_history, current_stage, current_model, company, use_case, "proceed"
                    return
                
                # Validate both are provided before proceeding
                if not extracted_company or not extracted_use_case:
                    # Friendly message based on what's missing
                    if extracted_company and not extracted_use_case:
                        # Got company, need use case
                        error_msg = f"""Great! I see you want to create a demo for **{extracted_company}**.

Do you have a use case in mind? For example:
- Sales Analytics
- Supply Chain
- Customer Analytics
- Or any custom use case you'd like!

**Just tell me like this:**
```
I'm creating a demo for company: {extracted_company} use case: Supply Chain
```

What would you like to analyze?"""
                    elif extracted_use_case and not extracted_company:
                        # Got use case, need company
                        error_msg = f"""Perfect! I see you want to analyze **{extracted_use_case}**.

Which company should I research? (Must be a real website)

**Examples:** Nike.com, Target.com, Walmart.com

**Just tell me like this:**
```
I'm creating a demo for company: Nike.com use case: {extracted_use_case}
```

What company URL should I use?"""
                    else:
                        # Got neither - shouldn't happen with our pattern matching, but just in case
                        error_msg = """I need both a company and a use case to get started!

**Example:**
```
I'm creating a demo for company: Nike.com use case: Supply Chain
```

What company and use case would you like?"""
                    
                    chat_history[-1] = (message, error_msg)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return
                
                # Update stage to research
                current_stage = 'research'
                
                # Show confirmation and starting message
                confirmation_msg = f"""✅ **Got it!**

**Company:** {company}
**Use Case:** {use_case}

🔍 **Starting Research...**

I'm now researching {company}'s business model and {use_case} requirements.
This may take 1-2 minutes. Watch the AI Feedback tab for progress!"""
                
                chat_history[-1] = (message, confirmation_msg)
                yield chat_history, current_stage, current_model, company, use_case, ""
                
                # Start research with streaming
                last_response = ""
                for response in self.run_research_streaming(company, use_case):
                    chat_history[-1] = (message, response)
                    last_response = response
                    yield chat_history, current_stage, current_model, company, use_case, ""
                
                # Stay in research stage - will move to create_ddl when user approves
                # After research completes, pre-fill "yes" if it ended with the approval question
                with open('/tmp/demoprep_debug.log', 'a') as f:
                    f.write(f"RESEARCH COMPLETE, last_response contains 'Ready to create'? {'Ready to create the DDL now?' in last_response}\n")
                if "Ready to create the DDL now?" in last_response or "Would you like to use the cached results?" in last_response:
                    with open('/tmp/demoprep_debug.log', 'a') as f:
                        f.write(f"PRE-FILLING yes for DDL\n")
                    yield chat_history, current_stage, current_model, company, use_case, "yes"
                else:
                    yield chat_history, current_stage, current_model, company, use_case, ""
                
                return

            # --- Catch-all: try to extract company + use case from any free-form message ---
            # Handles: "Nike.com, Retail Sales" / "Salesforce - Software Sales" / etc.
            extracted_company = self.extract_company_from_message(cleaned_message)
            extracted_use_case = self.extract_use_case_from_message(cleaned_message)

            if extracted_company and extracted_use_case:
                company = extracted_company
                self.vertical, self.function = parse_use_case(extracted_use_case)
                self.use_case_config = get_use_case_config(
                    self.vertical or "Generic", self.function or "Generic"
                )
                is_known = self.vertical and self.function and not self.use_case_config.get('is_generic')
                use_case_display = self.use_case_config.get('use_case_name', extracted_use_case)
                self.is_generic_use_case = not is_known
                self.pending_generic_company = company
                self.pending_generic_use_case = use_case_display

                if is_known:
                    note = f"\n\n*Matched: **{self.vertical}** × **{self.function}** — KPIs, outliers, and Spotter questions ready.*"
                elif self.vertical or self.function:
                    note = f"\n\n*Partial match: **{self.vertical or self.function}** recognized — AI will fill in the gaps.*"
                else:
                    note = "\n\n*Custom use case — AI will research and build from scratch.*"

                context_prompt = f"""✅ **Demo Configuration**

**Company:** {company}
**Use Case:** {use_case_display}{note}

**Want to add any requirements?** (or just say "proceed")
- "Include a RETURNS table"
- "Focus on enterprise accounts only"
- "I need 2 fact tables: Sales and Inventory"

*Type your requirements, or say **"proceed"** to use defaults.*"""

                chat_history[-1] = (message, context_prompt)
                current_stage = 'awaiting_context'
                yield chat_history, current_stage, current_model, company, use_case_display, "proceed"
                return

            elif extracted_company and not extracted_use_case:
                uc_opts = "\n".join([f"- {v} {f}" for v in VERTICALS.keys() for f in FUNCTIONS.keys()])
                response = f"""Got it — **{extracted_company}**!

What use case are we building? A few options:
{uc_opts}
- Or describe any custom scenario!"""
                chat_history[-1] = (message, response)
                yield chat_history, 'awaiting_use_case', current_model, extracted_company, use_case, ""
                return

            else:
                # Nothing useful extracted — show a brief prompt
                response = """I need a **company** and **use case** to get started.

Try something like:
- *"Nike.com, Retail Sales"*
- *"Salesforce.com — Software pipeline analytics"*
- *"Walmart.com, Supply Chain"*"""
                chat_history[-1] = (message, response)
                yield chat_history, current_stage, current_model, company, use_case, ""
                return

        elif current_stage == 'awaiting_use_case':
            # User is responding to "what use case?" — treat the entire message as the use case,
            # do NOT re-run company extraction (that's what caused "NJ.Products" bugs)
            extracted_use_case = self.extract_use_case_from_message(message) or message.strip()
            self.vertical, self.function = parse_use_case(extracted_use_case)
            self.use_case_config = get_use_case_config(
                self.vertical or "Generic", self.function or "Generic"
            )
            is_known = self.vertical and self.function and not self.use_case_config.get('is_generic')
            use_case_display = self.use_case_config.get('use_case_name', extracted_use_case)
            self.is_generic_use_case = not is_known
            self.pending_generic_company = company
            self.pending_generic_use_case = use_case_display

            if is_known:
                note = f"\n\n*Matched: **{self.vertical}** × **{self.function}** — KPIs, outliers, and Spotter questions ready.*"
            elif self.vertical or self.function:
                note = f"\n\n*Partial match: **{self.vertical or self.function}** recognized — AI will fill in the gaps.*"
            else:
                note = "\n\n*Custom use case — AI will research and build from scratch.*"

            context_prompt = f"""✅ **Demo Configuration**

**Company:** {company}
**Use Case:** {use_case_display}{note}

**Want to add any requirements?** (or just say "proceed")
- "Include a RETURNS table"
- "Focus on enterprise accounts only"
- "I need 2 fact tables: Sales and Inventory"

*Type your requirements, or say **"proceed"** to use defaults.*"""

            chat_history[-1] = (message, context_prompt)
            current_stage = 'awaiting_context'
            yield chat_history, current_stage, current_model, company, use_case_display, "proceed"
            return

        elif current_stage == 'awaiting_context':
            # User is providing context for ANY use case (both generic and established)
            if message_lower.strip() in ['proceed', 'continue', 'no', 'skip']:
                # User wants to proceed without additional context
                self.generic_use_case_context = ""
                context_note = "Proceeding with standard configuration."
            else:
                # User provided additional context - store it
                self.generic_use_case_context = message.strip()
                context_note = f"Using additional context:\n> {message.strip()}"
            
            # Get stored company and use case
            company = self.pending_generic_company
            use_case = self.pending_generic_use_case
            use_case_type = "generic" if self.is_generic_use_case else "established"
            
            # Check validation_mode setting. Local launcher runs without Gradio
            # auth, so fall back to the controller's in-memory settings when
            # user-scoped Supabase settings are unavailable.
            try:
                from supabase_client import load_gradio_settings
                settings = load_gradio_settings(self._get_effective_user_email())
            except Exception as settings_error:
                print(f"[Settings] Using in-memory settings for validation_mode: {settings_error}", flush=True)
                settings = self.settings
            validation_mode = settings.get('validation_mode', 'Off')
            
            if validation_mode == 'Off':
                # AUTO-RUN MODE: Run entire pipeline without any more prompts
                _slog = self._session_logger
                current_stage = 'research'
                self.phase_log.append(f"🚀 Starting pipeline — {company} · {use_case}")
                self.phase_log.append("🔍 Phase 1: Research")

                auto_run_msg = f"""✅ **Starting Auto-Run Mode**

**Company:** {company}
**Use Case:** {use_case} ({use_case_type})
{context_note}

🚀 **Running complete pipeline...**
- Research → Blueprint → Schema/data → Snowflake → ThoughtSpot

Watch the AI Feedback tab for real-time progress!"""
                
                chat_history[-1] = (message, auto_run_msg)
                yield chat_history, current_stage, current_model, company, use_case, ""
                
                # Run research - yield every response to keep Gradio alive
                last_research_response = ""
                for response in self.run_research_streaming(company, use_case, self.generic_use_case_context):
                    last_research_response = response
                    chat_history[-1] = (message, response)
                    yield chat_history, current_stage, current_model, company, use_case, ""

                if str(last_research_response).lstrip().startswith("❌ Research failed"):
                    self.phase_log.append("❌ Research failed")
                    if _slog:
                        _slog.log("run", "run failed", error="research failed", failed_stage="research")
                    current_stage = 'research'
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return
                
                # Show research complete, move to schema stage
                current_stage = 'create_ddl'
                self.phase_log.append("✅ Research complete")
                self.phase_log.append("📐 Phase 2: Schema creation")
                chat_history[-1] = (message, "✅ **Research Complete!**\n\n📝 **Creating schema...**")
                yield chat_history, current_stage, current_model, company, use_case, ""

                # Auto-create DDL
                ddl_response, ddl_code = self.run_ddl_creation()

                # Check if DDL creation failed
                if not ddl_code or ddl_code.strip() == "":
                    if _slog:
                        _slog.log("run", "run failed", error="DDL creation returned no code", failed_stage="ddl")
                    chat_history[-1] = (message, f"{ddl_response}\n\n❌ **Schema creation failed — check Pipeline Status for details.**")
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return

                current_stage = 'deploy'
                self.phase_log.append("✅ Dataset schema created")
                self.phase_log.append("🏗️ Phase 3: Snowflake deploy")
                chat_history[-1] = (message, f"✅ Dataset schema created\n\n🚀 **Deploying to Snowflake...**")
                yield chat_history, current_stage, current_model, company, use_case, ""

                # Auto-deploy
                with open('/tmp/demoprep_debug.log', 'a') as f:
                    f.write(f"AUTO-RUN: Starting deployment streaming\n")
                try:
                    final_result = None
                    deploy_update_count = 0
                    for progress_update in self.run_deployment_streaming():
                        if isinstance(progress_update, tuple):
                            final_result = progress_update
                        elif isinstance(progress_update, dict):
                            current_stage = progress_update.get('stage', current_stage)
                            chat_history[-1] = (message, progress_update['response'])
                            yield chat_history, current_stage, current_model, company, use_case, ""
                        else:
                            deploy_update_count += 1
                            # Yield every update — run_deployment_streaming already sleeps 2s between yields
                            chat_history[-1] = (message, progress_update)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                    
                    with open('/tmp/demoprep_debug.log', 'a') as f:
                        f.write(f"AUTO-RUN: Deployment loop EXITED, final_result={type(final_result)}, len={len(final_result) if final_result else 0}\n")
                    
                    if final_result:
                        with open('/tmp/demoprep_debug.log', 'a') as f:
                            f.write(f"AUTO-RUN: final_result[1]={final_result[1] if len(final_result) > 1 else 'N/A'}\n")
                        
                        # Check for auto_ts to continue to ThoughtSpot deployment
                        if len(final_result) == 3 and final_result[1] == "auto_ts":
                            deploy_response, _, auto_schema = final_result
                            with open('/tmp/demoprep_debug.log', 'a') as f:
                                f.write(f"AUTO-RUN: auto_ts detected, schema={auto_schema}\n")
                            self.phase_log.append("✅ Snowflake deploy complete")
                            self.phase_log.append("🔷 Phase 4: ThoughtSpot")
                            chat_history[-1] = (message, deploy_response)
                            yield chat_history, current_stage, current_model, company, use_case, ""

                            # Run ThoughtSpot deployment with detailed logging
                            with open('/tmp/demoprep_debug.log', 'a') as f:
                                f.write(f"AUTO-RUN: Creating TS generator...\n")
                            ts_generator = self._run_thoughtspot_deployment(auto_schema, company, use_case)
                            with open('/tmp/demoprep_debug.log', 'a') as f:
                                f.write(f"AUTO-RUN: Generator created, starting iteration...\n")
                            ts_update_count = 0
                            ts_final_stage = current_stage
                            ts_final_response = ""
                            for ts_update in ts_generator:
                                ts_update_count += 1
                                with open('/tmp/demoprep_debug.log', 'a') as f:
                                    update_preview = str(ts_update)[:100] if ts_update else "None"
                                    f.write(f"AUTO-RUN: TS update #{ts_update_count}: {update_preview}\n")
                                if isinstance(ts_update, dict):
                                    current_stage = ts_update.get('stage', current_stage)
                                    ts_final_stage = current_stage
                                    ts_final_response = str(ts_update.get('response', ''))
                                    chat_history[-1] = (message, ts_update['response'])
                                else:
                                    ts_final_response = str(ts_update)
                                    chat_history[-1] = (message, ts_update)
                                yield chat_history, current_stage, current_model, company, use_case, ""
                            with open('/tmp/demoprep_debug.log', 'a') as f:
                                f.write(f"AUTO-RUN: TS deployment loop complete, {ts_update_count} updates\n")
                            ts_failed = (
                                ts_final_stage != 'outlier_adjustment'
                                and (
                                    "ThoughtSpot Deployment Failed" in ts_final_response
                                    or "ThoughtSpot Deployment Error" in ts_final_response
                                    or "Partial Success" in ts_final_response
                                )
                            )
                            if ts_failed:
                                self.phase_log.append("❌ ThoughtSpot failed")
                                if _slog:
                                    _slog.log("run", "run failed", error="ThoughtSpot deployment failed", failed_stage="thoughtspot")
                                current_stage = 'deploy'
                            else:
                                self.phase_log.append("✅ ThoughtSpot complete")
                                self.phase_log.append("🎉 Pipeline done!")
                                if _slog:
                                    _slog.log("run", "run completed")
                                current_stage = 'complete'
                            yield chat_history, current_stage, current_model, company, use_case, ""
                        else:
                            # Not auto_ts, just show the result
                            if len(final_result) >= 2:
                                final_response = final_result[0]
                                next_msg = final_result[1] if len(final_result) > 1 else ""
                            else:
                                final_response = str(final_result)
                                next_msg = ""
                            if _slog:
                                if next_msg == "thoughtspot":
                                    _slog.log(
                                        "run",
                                        "run waiting for user",
                                        checkpoint="thoughtspot",
                                        reason="validation mode requested manual ThoughtSpot start",
                                    )
                                else:
                                    _slog.log(
                                        "run",
                                        "run interrupted",
                                        error="Snowflake deploy completed but auto ThoughtSpot handoff was not returned",
                                        failed_stage="thoughtspot_handoff",
                                        next_message=next_msg,
                                    )
                            chat_history[-1] = (message, final_response)
                            yield chat_history, current_stage, current_model, company, use_case, next_msg
                    else:
                        if _slog:
                            _slog.log(
                                "run",
                                "run interrupted",
                                error="Snowflake deploy stream ended without a final handoff result",
                                failed_stage="thoughtspot_handoff",
                            )
                        self.phase_log.append("⚠️ Pipeline interrupted after Snowflake deploy")
                        chat_history[-1] = (
                            message,
                            "⚠️ **Pipeline interrupted after Snowflake deploy.**\n\n"
                            "Snowflake may have been created, but the app did not receive the ThoughtSpot handoff signal. "
                            "Please retry ThoughtSpot deployment or check the run logs.",
                        )
                        yield chat_history, current_stage, current_model, company, use_case, ""
                except Exception as e:
                    import traceback
                    full_trace = traceback.format_exc()
                    with open('/tmp/demoprep_debug.log', 'a') as f:
                        f.write(f"AUTO-RUN: EXCEPTION: {str(e)}\n{full_trace}\n")
                    # Log real error to Supabase for investigation
                    if _slog:
                        _slog.log(
                            current_stage or 'pipeline',
                            "unhandled pipeline exception",
                            error=str(e),
                            traceback=full_trace,
                        )
                    # Show clean message to user — real error goes to Live Progress + logs
                    self.log_feedback(f"[ERROR] Pipeline exception: {str(e)}")
                    self.log_feedback(full_trace)
                    error_msg = "**❌ Something went wrong during the pipeline.** Our team has been notified. Check the Live Progress tab for details, or try again."
                    chat_history[-1] = (message, error_msg)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                
                return
            else:
                # VALIDATION MODE ON: Follow normal flow with pauses
                current_stage = 'research'
                
                confirmation_msg = f"""✅ **Got it!**

**Company:** {company}
**Use Case:** {use_case} ({use_case_type})
{context_note}

🔍 **Starting Research...**

I'm now researching {company}'s business model and {use_case} requirements.
This may take 1-2 minutes. Watch the AI Feedback tab for progress!"""
                
                chat_history[-1] = (message, confirmation_msg)
                yield chat_history, current_stage, current_model, company, use_case, ""
                
                # Start research with streaming
                last_response = ""
                for response in self.run_research_streaming(company, use_case, self.generic_use_case_context):
                    chat_history[-1] = (message, response)
                    last_response = response
                    yield chat_history, current_stage, current_model, company, use_case, ""
                
                # Pre-fill "yes" if ready for next step
                if "Ready to create the DDL now?" in last_response or "Would you like to use the cached results?" in last_response:
                    yield chat_history, current_stage, current_model, company, use_case, "yes"
                else:
                    yield chat_history, current_stage, current_model, company, use_case, ""
                
                return
        
        elif current_stage == 'research':
            # Check if we're waiting for cache response
            if hasattr(self, '_cache_available') and self._cache_available:
                if 'yes' in message_lower:
                    # User wants to use cache
                    chat_history[-1] = (message, "Loading cached research...")
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    
                    success = self._load_cached_research(self._cached_research_path, company, use_case)
                    if success:
                        company_name = self.demo_builder.extract_company_name()
                        response = f"""✅ **Research Loaded from Cache!**

**Company:** {company_name}
**Use Case:** {use_case}

**Cached results successfully loaded!**
- Company analysis retrieved
- Industry research retrieved
- Ready to proceed

"""
                        chat_history[-1] = (message, response)
                        current_stage = 'research'
                        self._cache_available = False
                        yield chat_history, current_stage, current_model, company, use_case, "yes"  # Pre-fill "yes"
                        return
                    else:
                        response = "❌ Failed to load cache, running fresh research..."
                        chat_history[-1] = (message, response)
                        yield chat_history, current_stage, current_model, company, use_case, ""
                        self._cache_available = False
                        # Fall through to run fresh research
                
                elif 'no' in message_lower:
                    # User wants fresh research
                    chat_history[-1] = (message, "Running fresh research...")
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    self._cache_available = False
                    # Fall through to run fresh research
                else:
                    # Invalid response, ask again
                    response = "Please type 'yes' to use cached results or 'no' to run fresh research."
                    chat_history[-1] = (message, response)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return
                
                # If we get here and cache was declined, run fresh research
                if not hasattr(self, '_cache_available') or not self._cache_available:
                    last_response = ""
                    for response in self.run_research_streaming(company, use_case):
                        chat_history[-1] = (message, response)
                        last_response = response
                        yield chat_history, current_stage, current_model, company, use_case, ""
                    
                    # Stay in research stage - will move to create_ddl when user approves
                    # Pre-fill "yes" after research
                    if "Ready to create the DDL now?" in last_response:
                        yield chat_history, current_stage, current_model, company, use_case, "yes"
                    else:
                        yield chat_history, current_stage, current_model, company, use_case, ""
                    return
            
            # Normal research stage handling (no cache prompt active)
            # This is only reached when validation_mode = On (otherwise auto-run handles it)
            if 'yes' in message_lower or 'proceed' in message_lower or 'continue' in message_lower:
                current_stage = 'create_ddl'
                response, ddl_code = self.run_ddl_creation()
                chat_history[-1] = (message, response)
                # With validation_mode On, we show DDL and wait for approval
                yield chat_history, current_stage, current_model, company, use_case, "yes"
                return
            elif 'no' in message_lower or 'redo' in message_lower:
                chat_history[-1] = (message, "Restarting research...")
                yield chat_history, current_stage, current_model, company, use_case, ""
                self._cache_available = False  # Clear cache flag
                last_response = ""
                for response in self.run_research_streaming(company, use_case):
                    chat_history[-1] = (message, response)
                    last_response = response
                    yield chat_history, current_stage, current_model, company, use_case, ""
                
                # Stay in research stage, pre-fill "yes"
                if "Ready to create the DDL now?" in last_response:
                    yield chat_history, current_stage, current_model, company, use_case, "yes"
                else:
                    yield chat_history, current_stage, current_model, company, use_case, ""
                return
        
        elif current_stage == 'create_ddl':
            # Waiting for DDL approval
            with open('/tmp/demoprep_debug.log', 'a') as f:
                f.write(f"ENTERED create_ddl stage, message={message_lower[:50]}\n")
            if 'yes' in message_lower or 'approve' in message_lower or 'proceed' in message_lower:
                with open('/tmp/demoprep_debug.log', 'a') as f:
                    f.write(f"DDL APPROVED, moving to deploy\n")
                current_stage = 'deploy'
                
                # Stream deployment progress directly to chat
                try:
                    final_result = None
                    for progress_update in self.run_deployment_streaming():
                        if isinstance(progress_update, tuple):
                            final_result = progress_update
                        elif isinstance(progress_update, dict):
                            current_stage = progress_update.get('stage', current_stage)
                            chat_history[-1] = (message, f"**DDL Approved - Deploying...**\n\n{progress_update['response']}")
                            yield chat_history, current_stage, current_model, company, use_case, ""
                        else:
                            chat_history[-1] = (message, f"**DDL Approved - Deploying...**\n\n{progress_update}")
                            yield chat_history, current_stage, current_model, company, use_case, ""
                    
                    # Handle final result
                    def debug_log(msg):
                        with open('/tmp/demoprep_debug.log', 'a') as f:
                            import datetime
                            f.write(f"[{datetime.datetime.now()}] {msg}\n")
                            f.flush()
                        self.log_feedback(msg)
                    
                    if final_result:
                        debug_log(f"DEBUG create_ddl: final_result len={len(final_result)}, [1]='{final_result[1] if len(final_result) > 1 else 'N/A'}'")
                        if len(final_result) == 3 and final_result[1] == "auto_ts":
                            # Auto-continue to ThoughtSpot deployment
                            deploy_response, _, auto_schema = final_result
                            debug_log(f"DEBUG create_ddl: auto_ts, schema={auto_schema}")
                            chat_history[-1] = (message, deploy_response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            
                            debug_log("DEBUG create_ddl: About to call _run_thoughtspot_deployment")
                            # Run ThoughtSpot deployment (mirrors 'thoughtspot' handler)
                            ts_update_count = 0
                            for ts_update in self._run_thoughtspot_deployment(auto_schema, company, use_case):
                                ts_update_count += 1
                                debug_log(f"DEBUG create_ddl: ts_update #{ts_update_count} type={type(ts_update)}")
                                if isinstance(ts_update, dict):
                                    # Final result
                                    current_stage = ts_update.get('stage', current_stage)
                                    chat_history[-1] = (message, ts_update['response'])
                                else:
                                    # Progress update
                                    chat_history[-1] = (message, ts_update)
                                yield chat_history, current_stage, current_model, company, use_case, ""
                            debug_log(f"DEBUG create_ddl: TS deployment loop complete, {ts_update_count} updates")
                            current_stage = 'complete'
                            yield chat_history, current_stage, current_model, company, use_case, ""
                        else:
                            deploy_response, next_msg = final_result
                            chat_history[-1] = (message, deploy_response)
                            yield chat_history, current_stage, current_model, company, use_case, next_msg
                except Exception as e:
                    import traceback
                    error_msg = f"**Deployment Error:** {str(e)}\n\n```\n{traceback.format_exc()}\n```"
                    with open('/tmp/demoprep_debug.log', 'a') as f:
                        f.write(f"DEBUG create_ddl: Exception: {str(e)}\n{traceback.format_exc()}\n")
                    chat_history[-1] = (message, error_msg)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                return
            elif 'no' in message_lower or 'redo' in message_lower:
                response, ddl_code = self.run_ddl_creation()
                chat_history[-1] = (message, response)
                yield chat_history, current_stage, current_model, company, use_case, ""
                return
        
        elif current_stage == 'populate':
            # Handle "deploy" command when LegitData is ready
            if 'deploy' in message_lower:
                current_stage = 'deploy'
                try:
                    # Stream deployment progress directly to chat
                    final_result = None
                    for progress_update in self.run_deployment_streaming():
                        if isinstance(progress_update, tuple):
                            final_result = progress_update
                        elif isinstance(progress_update, dict):
                            current_stage = progress_update.get('stage', current_stage)
                            chat_history[-1] = (message, progress_update['response'])
                            yield chat_history, current_stage, current_model, company, use_case, ""
                        else:
                            chat_history[-1] = (message, progress_update)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                    
                    # Handle final result
                    if final_result:
                        if len(final_result) == 3 and final_result[1] == "auto_ts":
                            # Auto-continue to ThoughtSpot deployment
                            deploy_response, _, auto_schema = final_result
                            chat_history[-1] = (message, deploy_response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            
                            # Run ThoughtSpot deployment (mirrors 'thoughtspot' handler)
                            for ts_update in self._run_thoughtspot_deployment(auto_schema, company, use_case):
                                if isinstance(ts_update, dict):
                                    # Final result
                                    current_stage = ts_update.get('stage', current_stage)
                                    chat_history[-1] = (message, ts_update['response'])
                                else:
                                    # Progress update
                                    chat_history[-1] = (message, ts_update)
                                yield chat_history, current_stage, current_model, company, use_case, ""
                            current_stage = 'complete'
                            yield chat_history, current_stage, current_model, company, use_case, ""
                        else:
                            deploy_response, next_msg = final_result
                            chat_history[-1] = (message, deploy_response)
                            yield chat_history, current_stage, current_model, company, use_case, next_msg
                except Exception as e:
                    # Deployment failed - show error message
                    error_msg = f"**Deployment Error:** {str(e)}"
                    chat_history[-1] = (message, error_msg)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                return
            # Handle population retry if needed
            elif 'yes' in message_lower or 'retry' in message_lower:
                response, pop_code = self.run_population()
                chat_history[-1] = (message, response)
                yield chat_history, current_stage, current_model, company, use_case, ""
                # Auto-proceed to deployment if successful
                if "Complete" in response:
                    current_stage = 'deploy'
                    try:
                        # Stream deployment progress
                        final_result = None
                        for progress_update in self.run_deployment_streaming():
                            if isinstance(progress_update, tuple):
                                final_result = progress_update
                            elif isinstance(progress_update, dict):
                                current_stage = progress_update.get('stage', current_stage)
                                chat_history.append((None, progress_update['response']))
                                yield chat_history, current_stage, current_model, company, use_case, ""
                            else:
                                chat_history.append((None, progress_update))
                                yield chat_history, current_stage, current_model, company, use_case, ""
                        
                        if final_result:
                            if len(final_result) == 3 and final_result[1] == "auto_ts":
                                # Auto-continue to ThoughtSpot deployment
                                deploy_response, _, auto_schema = final_result
                                chat_history.append((None, deploy_response))
                                yield chat_history, current_stage, current_model, company, use_case, ""
                                
                                # Run ThoughtSpot deployment (mirrors 'thoughtspot' handler)
                                for ts_update in self._run_thoughtspot_deployment(auto_schema, company, use_case):
                                    if isinstance(ts_update, dict):
                                        # Final result
                                        current_stage = ts_update.get('stage', current_stage)
                                        chat_history.append((None, ts_update['response']))
                                    else:
                                        # Progress update
                                        chat_history.append((None, ts_update))
                                    yield chat_history, current_stage, current_model, company, use_case, ""
                                current_stage = 'complete'
                                yield chat_history, current_stage, current_model, company, use_case, ""
                            else:
                                deploy_response, next_msg = final_result
                                chat_history.append((None, deploy_response))
                                yield chat_history, current_stage, current_model, company, use_case, next_msg
                    except Exception as e:
                        error_msg = str(e)
                        chat_history.append((None, error_msg))
                        yield chat_history, current_stage, current_model, company, use_case, ""
                return
        
        elif current_stage == 'deploy':
            # Handle post-deployment commands (thoughtspot, truncate)
            if 'thoughtspot' in message_lower:
                # User wants to create ThoughtSpot objects - use helper method
                schema_name = getattr(self, '_deployed_schema_name', getattr(self, '_last_schema_name', 'UNKNOWN'))
                
                for ts_update in self._run_thoughtspot_deployment(schema_name, company, use_case):
                    if isinstance(ts_update, dict):
                        # Final result
                        current_stage = ts_update.get('stage', current_stage)
                        chat_history[-1] = (message, ts_update['response'])
                    else:
                        # Progress update
                        chat_history[-1] = (message, ts_update)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                return
            
            # Handle deployment errors (usually population failures)
            if hasattr(self, '_last_population_error'):
                # Handle '1' or 'retry' - retry with same code
                if 'retry' in message_lower or message_lower.strip() == '1':
                    # Retry with same code
                    chat_history[-1] = (message, "Retrying population...")
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    
                    try:
                        # Check required attributes exist
                        if not hasattr(self.demo_builder, 'data_population_results') or not self.demo_builder.data_population_results:
                            response = "❌ **Error:** Population code not found. Please run population again first."
                            chat_history[-1] = (message, response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            return
                        
                        if not hasattr(self, '_last_schema_name') or not self._last_schema_name:
                            response = "❌ **Error:** Schema name not found. Please run deployment again first."
                            chat_history[-1] = (message, response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            return
                        
                        from demo_prep import execute_population_script
                        is_template = getattr(self.demo_builder, 'population_code_source', 'llm') == 'template'
                        success, msg = execute_population_script(
                            self.demo_builder.data_population_results,
                            self._last_schema_name,
                            skip_modifications=is_template
                        )
                        
                        if success:
                            response = f"✅ **Population Successful!**\n\n{msg}\n\nDemo deployed to Snowflake! 🎉"
                            del self._last_population_error
                            del self._last_schema_name
                        else:
                            response = f"❌ Still failed: {msg[:200]}...\n\nTry 'truncate' (or '2') or 'fix' (or '3')?"
                        
                    except Exception as e:
                        import traceback
                        error_details = traceback.format_exc()
                        self.log_feedback(f"❌ Retry error: {error_details}")
                        response = f"❌ **Retry failed with error:**\n\n```\n{str(e)}\n```\n\nPlease try 'truncate' (or '2') to clear tables first."
                    
                    chat_history[-1] = (message, response)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return
                
                elif 'truncate' in message_lower or message_lower.strip() == '2':
                    # Truncate tables and retry
                    chat_history[-1] = (message, "Truncating tables and retrying...")
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    
                    try:
                        # Check required attributes exist
                        if not hasattr(self, '_last_schema_name') or not self._last_schema_name:
                            response = "❌ **Error:** Schema name not found. Please run deployment again first."
                            chat_history[-1] = (message, response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            return
                        
                        if not hasattr(self.demo_builder, 'data_population_results') or not self.demo_builder.data_population_results:
                            response = "❌ **Error:** Population code not found. Please run population again first."
                            chat_history[-1] = (message, response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            return
                        
                        from cdw_connector import SnowflakeDeployer
                        from demo_prep import execute_population_script
                        
                        deployer = SnowflakeDeployer()
                        deployer.connect()
                        
                        # Truncate all tables in schema
                        try:
                            cursor = deployer.connection.cursor()
                            cursor.execute(f"USE SCHEMA {self._last_schema_name}")
                            cursor.execute("SHOW TABLES")
                            tables = cursor.fetchall()
                            
                            for table in tables:
                                table_name = table[1]
                                self.log_feedback(f"Truncating {table_name}...")
                                cursor.execute(f"TRUNCATE TABLE {table_name}")
                            
                            cursor.close()
                            deployer.disconnect()
                            self.log_feedback("✅ Tables truncated")
                        except Exception as e:
                            self.log_feedback(f"⚠️ Truncate warning: {e}")
                            if deployer.connection:
                                deployer.disconnect()
                        
                        # Retry population
                        is_template = getattr(self.demo_builder, 'population_code_source', 'llm') == 'template'
                        success, msg = execute_population_script(
                            self.demo_builder.data_population_results,
                            self._last_schema_name,
                            skip_modifications=is_template
                        )
                        
                        if success:
                            response = f"✅ **Population Successful!**\n\n{msg}\n\nDemo deployed to Snowflake! 🎉"
                            del self._last_population_error
                            del self._last_schema_name
                        else:
                            response = f"❌ Still failed: {msg[:200]}...\n\nTry 'fix' (or '3') to let AI correct the code?"
                    
                    except Exception as e:
                        import traceback
                        error_details = traceback.format_exc()
                        self.log_feedback(f"❌ Truncate/retry error: {error_details}")
                        response = f"❌ **Truncate/retry failed with error:**\n\n```\n{str(e)}\n```\n\nPlease check the error details above."
                    
                    chat_history[-1] = (message, response)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return
                
                elif 'fix' in message_lower or message_lower.strip() == '3':
                    # Regenerate the code using the fixed template
                    chat_history[-1] = (message, "🔧 Regenerating population code with fixed template...")
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    
                    try:
                        # Check required attributes exist
                        if not hasattr(self.demo_builder, 'schema_generation_results') or not self.demo_builder.schema_generation_results:
                            response = "❌ **Error:** DDL schema not found. Please run DDL creation again first."
                            chat_history[-1] = (message, response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            return
                        
                        if not hasattr(self, '_last_schema_name') or not self._last_schema_name:
                            response = "❌ **Error:** Schema name not found. Please run deployment again first."
                            chat_history[-1] = (message, response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            return
                        
                        self.log_feedback("🔧 Regenerating population code from scratch...")
                        
                        # Regenerate using the reliable template
                        from schema_utils import parse_ddl_schema
                        
                        schema_info = parse_ddl_schema(self.demo_builder.schema_generation_results)
                        if not schema_info:
                            response = "❌ Failed to parse DDL schema. Cannot regenerate."
                            chat_history[-1] = (message, response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            return
                        
                        # Generate new code using the template (which includes all fixes)
                        fixed_code = self.get_fallback_population_code(schema_info)
                        
                        # Validate it compiles
                        try:
                            compile(fixed_code, '<regenerated>', 'exec')
                            self.log_feedback("✅ Regenerated code validated")
                        except SyntaxError as e:
                            response = f"❌ Template generation bug: {e}\n\nPlease contact support."
                            chat_history[-1] = (message, response)
                            yield chat_history, current_stage, current_model, company, use_case, ""
                            return
                        
                        # Update the code and mark as template-generated
                        self.demo_builder.data_population_results = fixed_code
                        self.population_code = fixed_code
                        self.demo_builder.population_code_source = "template"  # Mark as template
                        
                        self.log_feedback("🔧 Code regenerated, retrying deployment...")
                        
                        # Truncate and retry
                        from cdw_connector import SnowflakeDeployer
                        from demo_prep import execute_population_script
                        
                        deployer = SnowflakeDeployer()
                        deployer.connect()
                        
                        try:
                            cursor = deployer.connection.cursor()
                            cursor.execute(f"USE SCHEMA {self._last_schema_name}")
                            cursor.execute("SHOW TABLES")
                            tables = cursor.fetchall()
                            for table in tables:
                                cursor.execute(f"TRUNCATE TABLE {table[1]}")
                            cursor.close()
                            deployer.disconnect()
                        except Exception as e:
                            self.log_feedback(f"⚠️ Truncate warning: {e}")
                            if deployer.connection:
                                deployer.disconnect()
                        
                        success, msg = execute_population_script(
                            fixed_code,
                            self._last_schema_name,
                            skip_modifications=True  # Template code, don't modify
                        )
                        
                        if success:
                            response = f"✅ **Fixed and Successful!**\n\n{msg}\n\nDemo deployed to Snowflake! 🎉"
                            del self._last_population_error
                            del self._last_schema_name
                        else:
                            response = f"❌ AI fix didn't work: {msg[:200]}...\n\nTry 'fix' again or 'retry'?"
                    
                    except Exception as e:
                        import traceback
                        error_details = traceback.format_exc()
                        self.log_feedback(f"❌ Fix/regenerate error: {error_details}")
                        response = f"❌ **Fix/regenerate failed with error:**\n\n```\n{str(e)}\n```\n\nPlease check the error details above."
                    
                    chat_history[-1] = (message, response)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return
                return
        
        elif current_stage == 'outlier_adjustment':
            # Handle outlier adjustment stage
            if 'done' in message_lower or 'finish' in message_lower or 'complete' in message_lower:
                # Close adjuster connection
                if hasattr(self, '_adjuster'):
                    self._adjuster.close()
                    
                response = """✅ **Demo Creation Complete!**

Your demo is fully deployed with custom outliers!

**Summary:**
- ✅ Research completed
- ✅ DDL schema created
- ✅ Data populated
- ✅ Deployed to Snowflake
- ✅ ThoughtSpot objects created
- ✅ Custom outliers added

**Access your demo:**
- ThoughtSpot Liveboard: Check your ThoughtSpot instance
- Snowflake Data: Query your schema

🎉 **Ready to present!**"""
                chat_history[-1] = (message, response)
                yield chat_history, current_stage, current_model, company, use_case, ""
                return
            
            # Check if waiting for confirmation
            if hasattr(self, '_pending_adjustment'):
                if 'yes' in message_lower or 'execute' in message_lower or 'confirm' in message_lower:
                    # Execute the pending adjustment
                    try:
                        adjuster = self._adjuster
                        strategy = self._pending_adjustment['strategy']
                        
                        chat_history[-1] = (message, "**Executing SQL...**")
                        yield chat_history, current_stage, current_model, company, use_case, ""
                        
                        result = adjuster.execute_sql(strategy['sql'])
                        
                        if result['success']:
                            response = f"""✅ **SUCCESS!**

Updated {result['rows_affected']} rows.

**Next steps:**
- 🔄 Refresh your ThoughtSpot liveboard to see changes
- Or make another adjustment
- Type **'done'** when finished

**What else would you like to adjust?**"""
                        else:
                            response = f"""❌ **FAILED**

{result['error']}

**Common issues:**
- Number too large for column (try smaller value)
- Database connection issue

Try a different adjustment or type **'done'** to finish."""
                        
                        del self._pending_adjustment
                        chat_history[-1] = (message, response)
                        yield chat_history, current_stage, current_model, company, use_case, ""
                        return
                        
                    except Exception as e:
                        del self._pending_adjustment
                        response = f"""❌ **Execution Error**

{str(e)}

Try a different adjustment or type **'done'** to finish."""
                        chat_history[-1] = (message, response)
                        yield chat_history, current_stage, current_model, company, use_case, ""
                        return
                
                elif 'no' in message_lower or 'cancel' in message_lower:
                    del self._pending_adjustment
                    response = """❌ **Cancelled**

No changes made. Try another adjustment or type **'done'** to finish."""
                    chat_history[-1] = (message, response)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return
            
            # Check if adjuster is available
            if not hasattr(self, '_adjuster'):
                response = """❌ **Adjuster Not Available**
                
Smart data adjuster was not initialized properly.

Type **'done'** to finish the workflow."""
                chat_history[-1] = (message, response)
                yield chat_history, current_stage, current_model, company, use_case, ""
                return
            
            # Process adjustment request
            try:
                adjuster = self._adjuster
                
                # Show processing message
                chat_history[-1] = (message, "🤔 **Analyzing request...**")
                yield chat_history, current_stage, current_model, company, use_case, ""
                
                # Match request to visualization
                match = adjuster.match_request_to_viz(message)
                
                if not match:
                    response = """❌ **Could not understand request**

I couldn't match your request to a visualization.

**Examples:**
- "make 1080p webcam 40B"
- "increase smart watch by 20%"
- "viz 3, increase laptop to 50B"

Try again or type **'done'** to finish."""
                    chat_history[-1] = (message, response)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return
                
                # Pick metric column (use hint from match if available)
                metric_hint = match.get('metric_hint')
                metric_column = adjuster._pick_metric_column(metric_hint)
                if not metric_column:
                    response = "❌ Could not identify a metric column in your schema. Try specifying the column name."
                    chat_history[-1] = (message, response)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return

                # Get current value (new 4-tuple return: value, matched_name, dim_table, fact_table)
                entity_type = match.get('entity_type')
                current_value, matched_entity, dim_table, fact_table = adjuster.get_current_value(
                    match['entity_value'], metric_column, entity_type
                )

                if current_value == 0 or matched_entity is None:
                    response = (
                        f"❌ **No data found** for `{match['entity_value']}`.\n\n"
                        f"Check the spelling or try a different entity. Type **'done'** to finish."
                    )
                    chat_history[-1] = (message, response)
                    yield chat_history, current_stage, current_model, company, use_case, ""
                    return

                # Calculate target
                target_value = match.get('target_value')
                percentage = match.get('percentage') if match.get('is_percentage') else None
                if percentage is not None:
                    target_value = current_value * (1 + percentage / 100)
                    match['target_value'] = target_value

                # Generate strategy
                strategy = adjuster.generate_strategy(
                    match['entity_value'],
                    metric_column,
                    current_value,
                    target_value=target_value,
                    percentage=percentage,
                    entity_type=entity_type,
                )

                # Present smart confirmation
                confirmation = adjuster.present_smart_confirmation(match, current_value, strategy, metric_column)
                
                # Store for execution if user confirms
                self._pending_adjustment = {
                    'match': match,
                    'strategy': strategy,
                    'current_value': current_value
                }
                
                response = f"""{confirmation}

**Type 'yes' to execute, 'no' to cancel, or make another adjustment request.**"""
                
                chat_history[-1] = (message, response)
                yield chat_history, current_stage, current_model, company, use_case, ""
                return
                
            except Exception as e:
                import traceback
                error_details = traceback.format_exc()
                self.log_feedback(f"❌ Adjustment error: {error_details}")
                
                response = f"""❌ **Adjustment Error**

{str(e)}

Try a different request or type **'done'** to finish."""
                chat_history[-1] = (message, response)
                yield chat_history, current_stage, current_model, company, use_case, ""
                return
        
        # Fallback to regular message processing
        response = self.process_regular_message(message, current_stage, company, use_case)
        chat_history[-1] = (message, response)
        yield chat_history, current_stage, current_model, company, use_case, ""
    
    def extract_company_from_message(self, message):
        """Extract company name from message"""
        import re
        
        # Pre-clean the message: fix common typos
        # Replace multiple dots with single dot (e.g., "Comscore..com" -> "Comscore.com")
        cleaned_message = re.sub(r'\.{2,}', '.', message)
        # Remove spaces around dots in URLs (e.g., "Nike . com" -> "Nike.com")
        cleaned_message = re.sub(r'\s*\.\s*', '.', cleaned_message)
        
        # Simpler, more robust patterns - capture domain.tld format
        # Don't include trailing punctuation in the URL pattern
        patterns = [
            # With colon: "company: tinder.com"
            r'company:\s*([a-zA-Z0-9-]+\.[a-zA-Z]{2,})',
            # Without colon: "for the company tinder.com" or "company tinder.com"
            r'for\s+(?:the\s+)?company\s+([a-zA-Z0-9-]+\.[a-zA-Z]{2,})',
            r'company\s+([a-zA-Z0-9-]+\.[a-zA-Z]{2,})',
            # "demo for tinder.com"
            r'demo\s+for\s+([a-zA-Z0-9-]+\.[a-zA-Z]{2,})',
        ]
        
        for pattern in patterns:
            match = re.search(pattern, cleaned_message, re.IGNORECASE)
            if match:
                company = match.group(1).strip()
                # Clean up trailing dots just in case
                company = company.rstrip('.')
                return company

        # Fallback: any bare domain.tld in the message (e.g. "Nike.com, Retail Sales")
        bare_url = re.search(r'\b([a-zA-Z0-9-]+\.[a-zA-Z]{2,})\b', cleaned_message, re.IGNORECASE)
        if bare_url:
            return bare_url.group(1).rstrip('.')

        return None
    
    def extract_use_case_from_message(self, message):
        """Extract use case from message"""
        import re
        
        # Patterns for extracting use case - ordered by specificity
        patterns = [
            r'use\s+case\s+of\s+([^,\n]+?)(?:\s*$)',              # "use case of XYZ"
            r'use\s+case:\s*([^,\n]+?)(?:\s*$|\s+and)',           # "use case: XYZ"
            r'(?:the\s+)?use\s+case\s+is\s+([^,\n]+?)(?:\s*$)',   # "the use case is XYZ"
            r'for\s+(?:the\s+)?use\s+case:\s*([^,\n]+?)(?:\s*$|\s+and)',  # "for the use case: XYZ"
            r'(?:and|\.)\s+use\s+case\s+(?:of\s+)?([^,\n]+?)(?:\s*$)',    # ". use case of XYZ" or "and use case XYZ"
            r'with\s+(?:the\s+)?use\s+(?:case\s+)?([^,\n]+?)(?:\s*$)',    # "with the use XYZ" or "with the use case XYZ"
            r'(?:for|with)\s+(?:the\s+)?use:\s*([^,\n]+?)(?:\s*$)',       # "for the use: XYZ"
            r'focused on\s+([^,\n]+?)(?:\s*$|\s+and)',            # "focused on XYZ"
            # "with a X use case" format - capture everything between "with a" and "use case"
            r'with\s+(?:a|an)\s+([^,\n]+?)\s+use\s+case',         # "with a Subscription use case"
            # "with a X" at end of message (no trailing "use case")
            r'with\s+(?:a|an)\s+([^,\n]+?)(?:\s*$)',              # "with a Subscription Conversion & Pricing Impact"
        ]
        
        for pattern in patterns:
            match = re.search(pattern, message, re.IGNORECASE)
            if match:
                use_case = match.group(1).strip()
                # Clean up trailing punctuation
                use_case = use_case.rstrip('.')
                # Don't return if it looks like a company/url (contains .com, .org, etc)
                if re.search(r'\.(com|org|net|io|co|ai)\b', use_case, re.IGNORECASE):
                    continue
                return use_case

        # Fallback 1: text after a comma or dash following a domain.tld
        # Handles: "Nike.com, Retail Sales" / "Nike.com - Supply Chain"
        after_domain = re.search(
            r'[a-zA-Z0-9-]+\.[a-zA-Z]{2,}[\s]*[,\-–—]\s*(.+)',
            message, re.IGNORECASE
        )
        if after_domain:
            use_case = after_domain.group(1).strip().rstrip('.')
            if use_case and not re.search(r'\.(com|org|net|io|co|ai)\b', use_case, re.IGNORECASE):
                return use_case

        # Fallback 2: text after domain.tld separated only by a space
        # Handles: "Caterpillar.com Manufacturing Supply Chain"
        after_domain_space = re.search(
            r'[a-zA-Z0-9-]+\.[a-zA-Z]{2,}\s+(.+)',
            message, re.IGNORECASE
        )
        if after_domain_space:
            use_case = after_domain_space.group(1).strip().rstrip('.')
            if use_case and not re.search(r'\.(com|org|net|io|co|ai)\b', use_case, re.IGNORECASE):
                return use_case

        return None
    
    def handle_override(self, message):
        """Handle /over command to change company or use case"""
        parts = message.strip().split(maxsplit=1)
        if len(parts) < 2:
            return """🔧 **Override Command**

To change settings, use:
- `/over company: [new company]` - Change company
- `/over usecase: [new use case]` - Change use case  
- `/over company: [company] usecase: [use case]` - Change both

**Example:**
`/over company: Amazon.com usecase: supply chain`
"""
        
        override_text = parts[1]
        new_company = None
        new_usecase = None
        
        # Parse override text
        if 'company:' in override_text.lower():
            company_part = override_text.lower().split('company:')[1]
            if 'usecase:' in company_part:
                new_company = company_part.split('usecase:')[0].strip()
            else:
                new_company = company_part.strip()
        
        if 'usecase:' in override_text.lower():
            usecase_part = override_text.lower().split('usecase:')[1].strip()
            new_usecase = usecase_part
        
        if new_company or new_usecase:
            response = "✅ **Settings Updated!**\n\n"
            if new_company:
                response += f"📍 Company: **{new_company}**\n"
            if new_usecase:
                response += f"🎯 Use Case: **{new_usecase}**\n"
            response += "\nWhat would you like to do next?"
            return response
        
        return "❌ Could not parse override. Use format: `/over company: [name] usecase: [case]`"
    
    def log_feedback(self, message):
        """Add message to AI feedback log and pipeline status"""
        import datetime
        timestamp = datetime.datetime.now().strftime("%H:%M:%S")
        entry = f"[{timestamp}] {message}"
        self.ai_feedback_log.append(entry)
        # Limit log size to prevent memory issues (keep last 500 entries)
        if len(self.ai_feedback_log) > 500:
            self.ai_feedback_log = self.ai_feedback_log[-500:]
        # Also feed into phase_log so Pipeline Status panel stays in sync
        self.phase_log.append(entry)
        if len(self.phase_log) > 200:
            self.phase_log = self.phase_log[-200:]
        print(f"[AI Feedback] {message}")  # Also print to console
        return "\n".join(self.ai_feedback_log)
    
    def run_research_streaming(self, company, use_case, generic_context=""):
        """Run the research phase with streaming updates

        Args:
            company: Company URL/name
            use_case: Use case name
            generic_context: Additional context provided by user for generic use cases
        """
        _slog = self._session_logger
        if _slog:
            model_setting = self.settings.get('model', DEFAULT_LLM_MODEL)
            try:
                provider_name, model_name = map_llm_display_to_provider(model_setting)
                resolved_model = f"{provider_name}/{model_name}"
            except Exception:
                resolved_model = str(model_setting or DEFAULT_LLM_MODEL)
            _slog.log(
                "run", "run started",
                company=company or '',
                use_case=use_case or '',
                interface=getattr(self, '_run_source', 'chat'),
                vertical=getattr(self, 'vertical', '') or '',
                line=getattr(self, 'line', '') or '',
                function=getattr(self, 'function', '') or '',
                is_custom=getattr(self, 'is_generic_use_case', False),
                additional_context=(getattr(self, 'generic_use_case_context', '') or '')[:500],
                model=resolved_model,
                model_setting=model_setting,
                payload=self._snapshot_run_payload(),
            )
        password_block = self._temporary_password_block_message()
        if password_block:
            if _slog:
                _slog.log("auth", "temporary password blocked pipeline start")
            yield password_block
            return
        _t = _slog.log_start("research") if _slog else None

        print(f"\n\n[CACHE DEBUG] === run_research_streaming called ===")
        print(f"[CACHE DEBUG] company: {company}")
        print(f"[CACHE DEBUG] use_case: {use_case}\n\n")
        
        import time
        import os
        from main_research import ResultsManager
        
        # Validate that we have actual values
        if not company:
            yield "❌ **Error:** No company provided. Please specify a company URL."
            return
        
        if not use_case:
            yield f"❌ **Error:** No use case provided. Please specify what you want to analyze.\n\n**Company provided:** {company}\n**Use case needed:** Tell me what analytics you want!"
            return
        
        progress_message = "🔍 **Starting Research**\n\n"
        yield progress_message
        
        try:
            # Initialize demo builder if needed OR if company/use_case changed
            # CRITICAL: Always create fresh DemoBuilder when company/use_case changes
            # to avoid persisting prompts/data from previous runs
            needs_new_builder = (
                not self.demo_builder or
                self.demo_builder.company_url != company or
                self.demo_builder.use_case != use_case
            )
            
            if needs_new_builder:
                print(f"[Research] Initializing DemoBuilder for {company}", flush=True)
                progress_message += "✓ Initializing DemoBuilder...\n"
                yield progress_message
                
                self.demo_builder = DemoBuilder(
                    use_case=use_case,
                    company_url=company
                )
            
            # Prepare URL - clean up any extra text that might have been captured
            # Remove "use case:" and anything after it, and clean whitespace
            import re
            clean_company = re.sub(r'\s+use\s+case:.*$', '', company, flags=re.IGNORECASE).strip()
            clean_company = re.sub(r'\s+and\s+.*$', '', clean_company, flags=re.IGNORECASE).strip()
            url = clean_company if clean_company.startswith('http') else f"https://{clean_company}"
            
            # Check for cached research results
            domain = url.replace('https://', '').replace('http://', '').replace('www.', '').split('/')[0]
            safe_domain = domain.replace('.', '_').replace(':', '_')
            # Strip newlines and truncate — use_case can be multi-line; long filenames crash on Linux
            use_case_clean = use_case.replace('\n', ' ').replace('\r', ' ').strip()
            use_case_safe = use_case_clean.lower().replace(' ', '_').replace('/', '_')[:60]
            if generic_context and generic_context.strip():
                import hashlib
                context_hash = hashlib.sha256(generic_context.strip().encode("utf-8")).hexdigest()[:8]
                use_case_safe = f"{use_case_safe}_{context_hash}"
            
            # Try new format first (with use case)
            # Use absolute path to ensure we find cache regardless of CWD
            script_dir = os.path.dirname(os.path.abspath(__file__))
            results_dir = os.path.join(script_dir, "results")
            cache_filename = f"{safe_domain}_{use_case_safe}.json"
            cache_filepath = os.path.join(results_dir, cache_filename)
            
            # If exact match doesn't exist, try fuzzy matching for similar use cases
            if not os.path.exists(cache_filepath):
                import glob
                print(f"[CACHE DEBUG] Current working directory: {os.getcwd()}")
                print(f"[CACHE DEBUG] Script directory: {script_dir}")
                print(f"[CACHE DEBUG] Results directory: {results_dir}")
                similar_files = glob.glob(os.path.join(results_dir, f"{safe_domain}_*.json"))
                print(f"[CACHE DEBUG] Exact file {cache_filepath} not found")
                print(f"[CACHE DEBUG] Glob pattern: {results_dir}/{safe_domain}_*.json")
                print(f"[CACHE DEBUG] Similar files found: {similar_files}")
                if similar_files:
                    # Found similar cache files for this company
                    cache_filepath = similar_files[0]  # Use the first one found
                    cache_filename = os.path.basename(cache_filepath)
                    print(f"[CACHE DEBUG] Using similar file: {cache_filename}")
                    self.log_feedback(f"📋 Found similar cache file: {cache_filename}")
                elif not os.path.exists(cache_filepath):
                    # Try old format (without use case)
                    old_cache_filename = f"research_{safe_domain}.json"
                    old_cache_filepath = os.path.join(results_dir, old_cache_filename)
                    if os.path.exists(old_cache_filepath):
                        cache_filename = old_cache_filename
                        cache_filepath = old_cache_filepath
            
            cached_results = None
            cache_age_hours = None
            
            # Check for cached research and use automatically if valid
            print(f"[CACHE DEBUG] Final cache_filepath: {cache_filepath}, exists: {os.path.exists(cache_filepath)}")
            if os.path.exists(cache_filepath):
                try:
                    # Check cache age (5 day expiry)
                    cache_mtime = os.path.getmtime(cache_filepath)
                    cache_age = time.time() - cache_mtime
                    cache_age_hours = cache_age / 3600  # Convert to hours
                    
                    if cache_age_hours <= 120:  # Cache valid for 5 days (120 hours)
                        self.log_feedback(f"📋 Using cached research (age: {cache_age_hours:.1f} hours)")
                        progress_message += f"📋 **Using Cached Research** ({cache_age_hours:.1f} hours old)\n\n"
                        
                        # Load cached results automatically
                        with open(cache_filepath, 'r') as f:
                            cached_data = json.load(f)
                        
                        self.demo_builder.company_analysis_results = cached_data.get('company_summary', '')
                        self.demo_builder.industry_research_results = cached_data.get('research_paper', '')
                        self.demo_builder.combined_research_results = self.demo_builder.get_research_context()
                        self.demo_builder.company_url = cached_data.get('url', url)
                        self.demo_builder.advance_stage()
                        
                        progress_message += "✅ **Research loaded from cache!**"

                        self.log_feedback("✅ Research loaded from cache")
                        yield progress_message
                        if _slog and _t:
                            _slog.log_end("research", _t)
                        return
                    else:
                        self.log_feedback(f"📋 Cache too old ({cache_age_hours:.1f} hours), running fresh research")
                        progress_message += f"📋 Cache expired ({cache_age_hours:.1f} hours old), running fresh research...\n"
                        yield progress_message
                except Exception as e:
                    self.log_feedback(f"⚠️ Could not load cache: {str(e)}")
                    progress_message += f"⚠️ Could not load cache, running fresh research...\n"
                    yield progress_message
            else:
                progress_message += "📋 No cached research found, running fresh research...\n"
                yield progress_message
            
            # No valid cache, proceed with fresh research
            yield from self._run_fresh_research(
                company,
                use_case,
                url,
                progress_message,
                cache_filename,
                results_dir,
                generic_context,
            )
            if _slog and _t:
                _slog.log_end("research", _t)

        except Exception as e:
            import traceback
            trace = traceback.format_exc()
            error_msg = f"❌ Research failed: {str(e)}\n{trace}"
            self.log_feedback(error_msg)
            yield f"❌ Research failed: {str(e)}"
            if _slog and _t:
                _slog.log_end("research", _t, error=str(e), traceback=trace)
    
    def _run_fresh_research(
        self,
        company,
        use_case,
        url,
        progress_message,
        cache_filename,
        results_dir,
        generic_context="",
    ):
        """Run fresh research (no cache)
        
        Args:
            results_dir: Absolute path to the research cache directory.
            generic_context: Additional context for generic use cases
        """
        import os
        from datetime import datetime
        from main_research import ResultsManager

        _slog = self._session_logger
        
        # Extract website content (silent — failure is handled gracefully)
        website = Website(url)
        self.demo_builder.website_data = website
        if not website.text or len(website.text) == 0:
            print(f"[Research] Website unavailable for {url}, using internet research", flush=True)
            website.text = f"Website content unavailable. Company URL: {url}. Please research this company using general knowledge and the use case context."
            website.title = company
            website.css_links = []
            website.logo_candidates = []

        # Get LLM provider
        model = self.settings.get('model', DEFAULT_LLM_MODEL)
        provider_name, model_name = map_llm_display_to_provider(model)
        if _slog:
            _slog.log('pipeline', 'model resolved', model=f"{provider_name}/{model_name}", model_setting=model)
        progress_message += f"✓ Using {provider_name}/{model_name}\n\n"
        yield progress_message
        
        # Initialize researcher
        researcher = MultiLLMResearcher(provider=provider_name, model=model_name)
        
        # Company Analysis
        progress_message += "🔍 **Phase 1: Company Analysis**\n"
        yield progress_message
        
        self.log_feedback("Analyzing company...")
        system_prompt, user_prompt = build_company_analysis_prompt(
            use_case, 
            website.title, 
            url, 
            website.text, 
            len(website.css_links),
            website.logo_candidates
        )
        
        # Inject generic use case context if provided
        if generic_context:
            user_prompt += f"\n\n**Additional Context from User:**\n{generic_context}"

        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt}
        ]

        response = researcher.make_request(messages, temperature=0.3, max_tokens=4000, stream=True)
        company_analysis = ""
        chunk_count = 0
        for chunk in response:
            chunk_text = researcher.extract_chunk_content(chunk)
            if chunk_text:
                company_analysis += chunk_text
                chunk_count += 1
                if chunk_count % 5 == 0:  # Update every 5 chunks
                    progress_message_temp = progress_message + f"Analyzing... ({len(company_analysis)} chars)\n"
                    yield progress_message_temp
        
        # Log the prompt/response
        from prompt_logger import log_researcher_call
        log_researcher_call("research_company", researcher, messages, company_analysis, logger=self._prompt_logger)
        
        self.demo_builder.company_analysis_results = company_analysis
        progress_message += f"✅ Company analysis complete\n\n"
        yield progress_message
        
        # Industry Research
        progress_message += "🔍 **Phase 2: Industry Research**\n"
        yield progress_message
        
        system_prompt, user_prompt = build_industry_research_prompt(use_case, company_analysis)
        
        # Inject generic use case context if provided
        if generic_context:
            user_prompt += f"\n\n**Additional Context from User:**\n{generic_context}"
        
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt}
        ]
        
        # Stream the response
        response = researcher.make_request(messages, temperature=0.4, max_tokens=4000, stream=True)
        industry_research = ""
        chunk_count = 0
        for chunk in response:
            chunk_text = researcher.extract_chunk_content(chunk)
            if chunk_text:
                industry_research += chunk_text
                chunk_count += 1
                if chunk_count % 5 == 0:  # Update every 5 chunks
                    progress_message_temp = progress_message + f"Researching... ({len(industry_research)} chars)\n"
                    yield progress_message_temp
        
        # Log the prompt/response
        log_researcher_call("research_industry", researcher, messages, industry_research, logger=self._prompt_logger)
        
        self.demo_builder.industry_research_results = industry_research
        self.demo_builder.combined_research_results = self.demo_builder.get_research_context()
        self.log_feedback("✅ Industry research complete!")
        progress_message += f"✅ Industry research complete ({len(industry_research)} chars)\n\n"
        yield progress_message
        
        # Save to cache
        try:
            self.log_feedback("💾 Saving research to cache...")
            research_results = {
                'company_summary': company_analysis,
                'research_paper': industry_research,
                'url': url,
                'use_case': use_case,
                'generated_at': datetime.now().isoformat(),
            }
            os.makedirs(results_dir, exist_ok=True)
            ResultsManager.save_results(research_results, cache_filename, results_dir)
        except Exception as e:
            # Cache failure is non-fatal — log internally but don't surface to UI
            print(f"[CACHE] Could not save research cache: {e}", flush=True)
            if _slog:
                _slog.log_verbose("research", "cache write failed", error=str(e))
        
        # Update stage
        self.demo_builder.advance_stage()
        self.demo_builder.set_ready()
        
        self.research_results = {
            'company': company,
            'use_case': use_case,
            'completed': True
        }
        
        
        # Generate synopsis
        company_name = self.demo_builder.extract_company_name()
        synopsis = f"""✅ **Research Complete!**

**Company:** {company_name}
**Use Case:** {use_case}

**What I learned:**
- Analyzed company website and business model
- Researched industry best practices for {use_case}
- Generated context for building realistic demo data

"""

        yield synopsis

    def _load_cached_research(self, cache_filepath, company, use_case):
        """Load research from cache"""
        from main_research import ResultsManager
        
        try:
            cached_results = ResultsManager.load_results(cache_filepath)
            if not isinstance(cached_results, dict):
                return None
            
            # Store in demo_builder
            self.demo_builder.company_analysis_results = cached_results.get('company_summary', '')
            self.demo_builder.industry_research_results = cached_results.get('research_paper', '')
            
            # Also need to get website data for company name extraction
            url = company if company.startswith('http') else f"https://{company}"
            self.demo_builder.website_data = Website(url)
            
            # Update stage
            self.demo_builder.advance_stage()
            self.demo_builder.set_ready()
            
            self.research_results = {
                'company': company,
                'use_case': use_case,
                'completed': True
            }
            
            return True
        except Exception as e:
            self.log_feedback(f"❌ Error loading cache: {str(e)}")
            return None
    
    def _generate_spotter_questions(self, use_case: str, ddl_code: str) -> list:
        """Generate use-case specific Spotter questions based on the schema.
        
        Priority order:
        1. liveboard_questions.spotter_qs from the vertical×function config
        2. FUNCTIONS[fn].spotter_templates from the config
        3. Hardcoded fallbacks per use case
        4. Generic questions
        """
        import re

        # Priority 1: Use spotter_qs from liveboard_questions in use case config
        try:
            uc_config = self.use_case_config or get_use_case_config(
                self.vertical or "Generic", self.function or "Generic"
            )
            lq = uc_config.get("liveboard_questions", [])
            configured_questions = []
            for q in lq:
                for sq in q.get('spotter_qs', []):
                    configured_questions.append({
                        'question': sq,
                        'purpose': f'Reveals {q["title"]} pattern'
                    })
            if configured_questions:
                v = self.vertical or "Generic"
                f = self.function or "Generic"
                self.log_feedback(f"📋 Using {len(configured_questions)} Spotter questions from {v}×{f} config")
                return configured_questions[:8]
        except Exception as e:
            self.log_feedback(f"⚠️ Spotter questions not available: {e}")
        
        # Priority 2: Try FUNCTIONS config for spotter_templates
        try:
            uc_config = self.use_case_config or get_use_case_config(
                self.vertical or "Generic", self.function or "Generic"
            )
            spotter_templates = uc_config.get('spotter_templates', [])
            if spotter_templates:
                template_questions = [{'question': t, 'purpose': 'From use case config'} for t in spotter_templates]
                self.log_feedback(f"📋 Using {len(template_questions)} Spotter templates from config")
                return template_questions[:8]
        except:
            pass
        
        # Priority 3+4: Fallback to DDL-based and hardcoded questions
        # Extract column names from DDL
        columns = []
        if ddl_code:
            # Find column definitions (word followed by data type)
            col_pattern = r'^\s+(\w+)\s+(VARCHAR|NUMBER|DATE|BOOLEAN|INT|DECIMAL|FLOAT|TIMESTAMP)'
            for match in re.finditer(col_pattern, ddl_code, re.MULTILINE | re.IGNORECASE):
                col_name = match.group(1).lower().replace('_', ' ')
                columns.append(col_name)
        
        # Find likely metrics (columns with revenue, sales, cost, amount, count, total, etc.)
        metrics = [c for c in columns if any(m in c for m in ['revenue', 'sales', 'cost', 'amount', 'count', 'total', 'spend', 'clicks', 'impressions', 'conversions', 'rate', 'roi', 'ctr', 'cpm', 'cpc'])]
        # Find likely dimensions (columns with name, type, category, region, channel, segment, etc.)
        dimensions = [c for c in columns if any(d in c for d in ['name', 'type', 'category', 'region', 'channel', 'segment', 'campaign', 'audience', 'product', 'brand', 'status'])]
        
        # Use-case specific question templates (legacy fallback)
        use_case_questions = {
            'Marketing Analytics': [
                {'question': 'What is total spend by channel this quarter?', 'purpose': 'Shows channel performance analysis'},
                {'question': 'Which campaigns have the highest ROI?', 'purpose': 'Shows ranking and efficiency metrics'},
                {'question': 'How have conversions changed month over month?', 'purpose': 'Shows trend analysis and change detection'},
                {'question': 'What is the conversion rate by audience segment?', 'purpose': 'Shows segmentation analysis'},
            ],
            'Sales Analytics': [
                {'question': 'What is total revenue by region this quarter?', 'purpose': 'Shows geographic performance'},
                {'question': 'Who are the top 10 sales reps by revenue?', 'purpose': 'Shows ranking capabilities'},
                {'question': 'How has pipeline changed compared to last month?', 'purpose': 'Shows change detection'},
                {'question': 'What is win rate by product category?', 'purpose': 'Shows conversion analysis'},
            ],
            'Demand/Inventory Planning': [
                {'question': 'What is current inventory by product category?', 'purpose': 'Shows inventory overview'},
                {'question': 'Which products are at risk of stockout?', 'purpose': 'Shows proactive alerting'},
                {'question': 'How has demand changed compared to forecast?', 'purpose': 'Shows variance analysis'},
                {'question': 'What is days of supply by warehouse?', 'purpose': 'Shows operational metrics'},
            ],
            'Merchandising': [
                {'question': 'What is sales performance by product category?', 'purpose': 'Shows category analysis'},
                {'question': 'Which products have the highest margin?', 'purpose': 'Shows profitability ranking'},
                {'question': 'How has sell-through rate changed this month?', 'purpose': 'Shows trend analysis'},
                {'question': 'What is inventory turnover by store?', 'purpose': 'Shows store-level metrics'},
            ],
            'Loss Prevention Analytics': [
                {'question': 'What is total shrinkage by store?', 'purpose': 'Shows loss by location'},
                {'question': 'Which stores have the highest shrinkage rate?', 'purpose': 'Shows risk ranking'},
                {'question': 'How has shrinkage trended over the past 6 months?', 'purpose': 'Shows trend analysis'},
                {'question': 'What are the top causes of loss?', 'purpose': 'Shows root cause analysis'},
            ],
        }
        
        # Get questions for this use case, or fall back to generic
        questions = use_case_questions.get(use_case, [
            {'question': 'What are the key metrics this month?', 'purpose': 'Shows summary view'},
            {'question': 'What changed compared to last period?', 'purpose': 'Shows change detection'},
            {'question': 'What are the top performers?', 'purpose': 'Shows ranking capabilities'},
            {'question': 'How have trends changed over time?', 'purpose': 'Shows time-series analysis'},
        ])
        
        # If we found actual metrics/dimensions in DDL, try to make questions more specific
        if metrics and dimensions:
            metric = metrics[0].title()
            dimension = dimensions[0].title()
            questions[0] = {'question': f'What is total {metric} by {dimension}?', 'purpose': 'Shows core metric breakdown'}
        
        return questions
    
    def _get_demo_tips(self, use_case: str) -> str:
        """Get use-case specific demo tips"""
        tips = {
            'Marketing Analytics': """- **Lead with ROI**: Marketing leaders care about efficiency, show spend vs. results
- **Highlight attribution**: Show how ThoughtSpot can break down performance by channel/campaign
- **Show real-time**: Marketing moves fast - emphasize live data and quick answers
- **Monitor setup**: Demo alerting for campaign performance thresholds""",
            
            'Sales Analytics': """- **Focus on pipeline**: Sales leaders want to see deal flow and forecasting
- **Show rep performance**: Ranking and leaderboards resonate with sales teams
- **Highlight forecasting**: Show how AI can predict outcomes
- **Territory analysis**: Geographic breakdowns are always compelling""",
            
            'Demand/Inventory Planning': """- **Lead with stockouts**: Show how to prevent lost sales
- **Forecast vs. actual**: Variance analysis is key for planners
- **Supplier performance**: Show lead time and reliability metrics
- **Seasonal patterns**: Highlight time-series capabilities""",
            
            'Merchandising': """- **Category performance**: Start with what's selling
- **Margin analysis**: Profitability is always top of mind
- **Assortment optimization**: Show breadth vs. depth analysis
- **Store comparisons**: Regional and store-level drill-down""",
            
            'Loss Prevention Analytics': """- **Risk scoring**: Show high-risk locations or categories
- **Trend detection**: Highlight anomaly detection capabilities
- **Root cause**: Drill into why losses are occurring
- **ROI of prevention**: Connect to business impact""",
        }
        
        return tips.get(use_case, """- **Start broad**: Begin with executive summary metrics
- **Then drill down**: Show the ability to explore details
- **Ask questions**: Let the AI demonstrate natural language
- **End with action**: Show how insights lead to decisions""")
    
    def _generate_ai_spotter_story(self, company_name: str, use_case: str,
                                    model_name: str = None, model_url: str = None,
                                    liveboard_name: str = None,
                                    actual_columns: list = None) -> str:
        """Pure AI-generated Spotter Viz story — no matrix reference.
        AI decides what a compelling liveboard story looks like for this company + use case.
        """
        from prompts import build_prompt
        from demo_personas import parse_use_case

        v, f = parse_use_case(use_case or '')
        vertical = v or "Generic"
        function = f or "Generic"

        company_context = f"Company: {company_name}\nUse Case: {use_case}"
        if model_name:
            company_context += f"\nData Source/Model: {model_name}"
        if model_url:
            company_context += f"\nModel URL: {model_url}"
        if liveboard_name:
            company_context += f"\nLiveboard Name: {liveboard_name}"
        if hasattr(self, 'demo_builder') and self.demo_builder:
            research = getattr(self.demo_builder, 'company_summary', '') or ''
            if research:
                company_context += f"\n\nCompany Research:\n{research[:1500]}"
        if actual_columns:
            company_context += (
                f"\n\nACTUAL columns in the deployed data model (use these exact names "
                f"when referencing metrics or dimensions in Spotter prompts — do not invent column names "
                f"that don't appear in this list):\n{', '.join(actual_columns)}"
            )

        try:
            prompt = build_prompt(
                stage="spotter_viz_story",
                vertical=vertical,
                function=function,
                company_context=company_context,
            )
            llm_model = self.settings.get('model', DEFAULT_LLM_MODEL)
            self.log_feedback(f"🎬 Generating AI Spotter Viz story ({llm_model})...")
            provider_name, model_name_str = map_llm_display_to_provider(llm_model)
            researcher = MultiLLMResearcher(provider=provider_name, model=model_name_str)
            messages = [{"role": "user", "content": prompt}]
            result = researcher.make_request(messages, max_tokens=2000, temperature=0.7)
            from prompt_logger import log_researcher_call
            log_researcher_call("spotter_viz_story_ai", researcher, messages, result or "", logger=self._prompt_logger)
            return result
        except Exception as e:
            self.log_feedback(f"⚠️ AI Spotter Viz story generation failed: {e}")
            return f"*(Generation failed: {e})*"

    def _generate_matrix_spotter_story(self, company_name: str, use_case: str,
                                        model_name: str = None, model_url: str = None,
                                        liveboard_name: str = None,
                                        actual_columns: list = None) -> str:
        """Matrix/ThoughtSpot-recommended Spotter Viz story.
        Builds from the vertical×function matrix (KPIs, liveboard_questions, story controls, persona).
        AI writes it — adds narrative — but every step comes from the matrix.
        actual_columns: real column names from the deployed DDL — used to ground metric references.
        """
        from prompts import build_prompt
        from demo_personas import parse_use_case, get_use_case_config

        v, f = parse_use_case(use_case or '')
        uc_cfg = get_use_case_config(v or "Generic", f or "Generic")
        data_source = model_name or f"{company_name} model"

        # Build rich matrix context for the prompt
        kpis = uc_cfg.get("kpis", [])
        lq = uc_cfg.get("liveboard_questions", [])
        story_controls = uc_cfg.get("story_controls", {})
        persona = uc_cfg.get("persona", "")
        business_problem = uc_cfg.get("business_problem", "")
        use_case_name = uc_cfg.get("use_case_name", use_case)

        matrix_context = f"Company: {company_name}\nUse Case: {use_case_name}\nData Source/Model: {data_source}"
        if model_url:
            matrix_context += f"\nModel URL: {model_url}"
        if liveboard_name:
            matrix_context += f"\nLiveboard Name: {liveboard_name}"
        if actual_columns:
            matrix_context += (
                f"\n\nACTUAL columns in the deployed data model (map matrix KPIs to the closest "
                f"matching actual column name — do not reference columns that don't appear in this list):\n"
                f"{', '.join(actual_columns)}"
            )
        if persona:
            matrix_context += f"\nTarget Persona: {persona}"
        if business_problem:
            matrix_context += f"\nBusiness Problem: {business_problem}"

        if kpis:
            kpi_lines = "\n".join(
                f"  - {k['name']}: {k.get('definition', '')}" if isinstance(k, dict) else f"  - {k}"
                for k in kpis
            )
            matrix_context += f"\n\nKPIs (from ThoughtSpot matrix):\n{kpi_lines}"

        if lq:
            matrix_context += "\n\nLiveboard Questions (in order):"
            for q in lq:
                req = " [required]" if q.get("required") else ""
                matrix_context += f"\n  - {q['title']} ({q.get('viz_type','chart')}){req}: {q['viz_question']}"
                if q.get('insight'):
                    matrix_context += f"\n    Insight: {q['insight']}"

        if story_controls:
            dims = story_controls.get("dimensions", [])
            if dims:
                matrix_context += f"\n\nKey Dimensions: {', '.join(dims)}"
            seasonal = story_controls.get("seasonal_strength") or story_controls.get("seasonal")
            if seasonal:
                matrix_context += f"\nSeasonal pattern: {seasonal}"

        try:
            prompt = build_prompt(
                stage="spotter_viz_story_matrix",
                vertical=v or "Generic",
                function=f or "Generic",
                company_context=matrix_context,
            )
            llm_model = self.settings.get('model', DEFAULT_LLM_MODEL)
            self.log_feedback(f"🎬 Generating Matrix Spotter Viz story ({llm_model})...")
            provider_name, model_name_str = map_llm_display_to_provider(llm_model)
            researcher = MultiLLMResearcher(provider=provider_name, model=model_name_str)
            messages = [{"role": "user", "content": prompt}]
            result = researcher.make_request(messages, max_tokens=2000, temperature=0.6)
            from prompt_logger import log_researcher_call
            log_researcher_call("spotter_viz_story_matrix", researcher, messages, result or "", logger=self._prompt_logger)
            return result
        except Exception as e:
            self.log_feedback(f"⚠️ Matrix Spotter Viz story generation failed: {e}")
            return f"*(Generation failed: {e})*"
    
    def _build_fallback_spotter_story(self, company_name: str, use_case: str, 
                                       model_name: str = None) -> str:
        """Build a basic Spotter Viz story without LLM, using available context."""
        data_source = model_name or f"{company_name} model"
        
        # Get spotter questions from use case config
        spotter_qs = []
        try:
            from demo_personas import parse_use_case, get_use_case_config
            v, f = parse_use_case(use_case or '')
            uc_cfg = get_use_case_config(v or "Generic", f or "Generic")
            for q in uc_cfg.get("liveboard_questions", []):
                if q.get("required") and q.get("spotter_qs"):
                    spotter_qs.append(q["spotter_qs"][0])
        except:
            pass
        
        story = f"""# Spotter Viz Story: {company_name}
## {use_case}

*Copy these prompts into ThoughtSpot Spotter Viz to build this liveboard interactively.*

---

### Step 1: Set Context
> "Create a new liveboard for {company_name} {use_case} using the {data_source} data source."

**Expected result:** Empty liveboard created with the correct data source connected.

### Step 2: Add Key KPIs
> "Add KPI cards showing the main metrics with weekly sparklines."

**Expected result:** KPI tiles with sparkline trends at the top of the liveboard.

### Step 3: Add Trend Analysis
> "Add a line chart showing how the primary metric has trended over the last 12 months."

**Expected result:** Time-series visualization showing monthly trends.

### Step 4: Add Category Breakdown
> "Show a bar chart breaking down performance by the main dimension."

**Expected result:** Categorical breakdown chart.

### Step 5: Add Comparison
> "Add a comparison showing this period vs. last period."

**Expected result:** Period-over-period comparison visualization.
"""
        if spotter_qs:
            story += "\n### Step 6: Explore with Spotter Questions\n"
            for i, q in enumerate(spotter_qs[:3]):
                story += f'> "{q}"\n\n'
        
        story += """
---

*Refine the liveboard further by asking Spotter Viz to adjust colors, reorganize tiles, or add filters.*
"""
        return story

    def run_research(self, company, use_case):
        """Run the research phase"""
        import time
        
        self.log_feedback(f"🔍 Starting research for {company} - {use_case}")
        
        try:
            # Initialize demo builder if needed OR if company/use_case changed
            # CRITICAL: Always create fresh DemoBuilder when company/use_case changes
            # to avoid persisting prompts/data from previous runs
            needs_new_builder = (
                not self.demo_builder or
                self.demo_builder.company_url != company or
                self.demo_builder.use_case != use_case
            )
            
            if needs_new_builder:
                if self.demo_builder:
                    self.log_feedback(f"🔄 Company/use case changed - creating fresh DemoBuilder (was: {self.demo_builder.company_url}/{self.demo_builder.use_case})")
                else:
                    self.log_feedback("Initializing DemoBuilder...")
                self.demo_builder = DemoBuilder(
                    use_case=use_case,
                    company_url=company
                )
            
            # Extract website content
            url = company if company.startswith('http') else f"https://{company}"
            website = Website(url)
            self.demo_builder.website_data = website

            # Check if website extraction failed — silent, use fallback
            if not website.text or len(website.text) == 0:
                print(f"[Research] Website unavailable for {url}, using internet research", flush=True)
                website.text = f"Website content unavailable. Company URL: {url}. Please research this company using general knowledge and the use case context."
                website.title = company
                website.css_links = []
                website.logo_candidates = []

            # Get LLM provider
            model = self.settings.get('model', DEFAULT_LLM_MODEL)
            provider_name, model_name = map_llm_display_to_provider(model)

            # Initialize researcher
            researcher = MultiLLMResearcher(provider=provider_name, model=model_name)

            # Company Analysis
            system_prompt, user_prompt = build_company_analysis_prompt(
                use_case, 
                website.title, 
                url, 
                website.text,
                len(website.css_links),
                website.logo_candidates
            )
            
            messages = [
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": user_prompt}
            ]
            
            # Stream the response
            response = researcher.make_request(messages, temperature=0.3, max_tokens=4000, stream=True)
            company_analysis = ""
            for chunk in response:
                chunk_text = researcher.extract_chunk_content(chunk)
                if chunk_text:
                    company_analysis += chunk_text
            from prompt_logger import log_researcher_call
            log_researcher_call("research_company", researcher, messages, company_analysis, logger=self._prompt_logger)
            self.demo_builder.company_analysis_results = company_analysis
            self.log_feedback("✅ Company analysis complete!")
            
            # Industry Research
            self.log_feedback("Researching industry best practices...")
            system_prompt, user_prompt = build_industry_research_prompt(use_case, company_analysis)
            
            messages = [
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": user_prompt}
            ]
            
            # Stream the response
            response = researcher.make_request(messages, temperature=0.4, max_tokens=4000, stream=True)
            industry_research = ""
            for chunk in response:
                chunk_text = researcher.extract_chunk_content(chunk)
                if chunk_text:
                    industry_research += chunk_text
            log_researcher_call("research_industry", researcher, messages, industry_research, logger=self._prompt_logger)
            self.demo_builder.industry_research_results = industry_research
            self.log_feedback("✅ Industry research complete!")
            
            # Update stage
            self.demo_builder.advance_stage()
            self.demo_builder.set_ready()
            
            self.research_results = {
                'company': company,
                'use_case': use_case,
                'completed': True
            }
            
                
            # Generate synopsis
            company_name = self.demo_builder.extract_company_name()
            synopsis = f"""✅ **Research Complete!**

**Company:** {company_name}
**Use Case:** {use_case}

**What I learned:**
- Analyzed company website and business model
- Researched industry best practices for {use_case}
- Generated context for building realistic demo data

"""

            return synopsis
            
        except Exception as e:
            import traceback
            error_msg = f"❌ Research failed: {str(e)}\n{traceback.format_exc()}"
            self.log_feedback(error_msg)
            return f"❌ Research failed: {str(e)}"
    
    def run_ddl_creation(self):
        """Run DDL creation"""
        self.log_feedback("📝 Starting DDL creation...")
        _slog = self._session_logger
        _t_ddl = _slog.log_start("ddl") if _slog else None

        try:
            # Get timestamp for schema naming
            from datetime import datetime
            now = datetime.now()
            yymmdd = now.strftime("%y%m%d")
            hhmmss = now.strftime("%H%M%S")
            
            # Clean company and use case names (5 chars company, 3 chars use case)
            company_clean = self.demo_builder.extract_company_name().replace(" ", "").replace(".", "")[:5]
            usecase_clean = self.demo_builder.use_case.replace(" ", "").replace("-", "")[:3]
            schema_name = f"DM{yymmdd}_{hhmmss}_{company_clean}_{usecase_clean}"

            self._demo_bundle = None
            from demoprep_app.pipeline.build_demo import build_demo

            row_count_guidance = int(self.settings.get("fact_table_size", 5000) or 5000)
            company_name = self.demo_builder.extract_company_name()
            build = build_demo(
                company_name=company_name,
                company_url=self.demo_builder.company_url,
                use_case=self.demo_builder.use_case,
                vertical=self.vertical,
                function=self.function,
                row_count_guidance=row_count_guidance,
                research_context="\n\n".join(
                    part for part in [
                        getattr(self.demo_builder, "combined_research_results", "") or "",
                        getattr(self, "generic_use_case_context", "") or "",
                    ]
                    if part
                ),
                user_request=getattr(self, "generic_use_case_context", "") or self.demo_builder.use_case,
                llm_model=self.settings.get("model", DEFAULT_LLM_MODEL),
                progress_callback=self.log_feedback,
                prompt_logger=self._prompt_logger,
            )
            if not build.ddl or "CREATE TABLE" not in build.ddl.upper():
                raise RuntimeError(
                    "Blueprint pipeline produced empty DDL — the blueprint likely had no fact tables or dimensions. "
                    "Check the warnings above and retry."
                )
            self._demo_bundle = build.dataset
            self.demo_builder.schema_generation_results = build.ddl
            self.ddl_code = build.ddl
            for warning in build.warnings:
                self.log_feedback(f"⚠️ {warning}")
            self.demo_builder.advance_stage()
            if _slog:
                _slog.log_end(
                    "ddl",
                    _t_ddl,
                    table_count=build.ddl.upper().count("CREATE TABLE"),
                    generation_mode="blueprint",
                    scenario_type=build.scenario.scenario_type,
                    fact_grain=build.scenario.fact_grain,
                    contract_source=build.scenario.metadata.get("contract_source"),
                    business_domain=build.scenario.metadata.get("business_domain"),
                    table_names=[table.name for table in build.dataset.tables],
                )
            self.log_feedback("✅ Schema created successfully!")

            insight_lines = ""
            if build.blueprint and build.blueprint.insights:
                headlines = [f"  • {i.headline}" for i in build.blueprint.insights]
                insight_lines = "\n\n**Planted Demo Insights:**\n" + "\n".join(headlines)

            response = f"""✅ **Schema Creation Complete!**

**Schema:** {schema_name}

**Scenario:** {build.scenario.scenario_type}
**Fact grain:** {build.scenario.fact_grain}
**Generated rows:** {sum(len(table.rows) for table in build.dataset.tables):,}{insight_lines}"""
            return response, self.ddl_code
            
        except Exception as e:
            import traceback
            error_msg = f"❌ DDL creation failed: {str(e)}\n{traceback.format_exc()}"
            if _slog: _slog.log_end("ddl", _t_ddl, error=str(e))
            self.log_feedback(error_msg)
            self.demo_builder.schema_generation_results = ""
            self.ddl_code = ""
            return error_msg, ""
    
    def get_fallback_population_code(self, schema_info, fact_rows=10000, dim_rows=100):
        """Generate a simple, reliable fallback population code using plain strings
        
        Args:
            schema_info: Dict of table definitions
            fact_rows: Number of rows for fact tables (default: 10000)
            dim_rows: Number of rows for dimension tables (default: 100)
        """
        # schema_info is a dict: {'table_name': {'columns': [...], 'raw_definition': '...'}}
        
        def is_fact_table(table_name, columns):
            """Detect if table is likely a fact table (has measures/metrics)"""
            table_lower = table_name.lower()
            
            # FIRST: Explicitly classify dimension tables (these are NEVER facts)
            dimension_keywords = ['customer', 'product', 'seller', 'vendor', 'user', 'employee',
                                'center', 'warehouse', 'store', 'location', 'region', 'category',
                                'fulfillment', 'supplier', 'account', 'item_master', 'channel']
            if any(keyword in table_lower for keyword in dimension_keywords):
                return False
            
            # SECOND: Common fact table name patterns
            fact_keywords = ['transaction', 'order', 'sale', 'event', 'log', 'activity', 
                           'purchase', 'payment', 'shipment', 'invoice', 'fact', 'line']
            if any(keyword in table_lower for keyword in fact_keywords):
                return True
            
            # THIRD: Check for numeric/measure columns (amount, quantity, price, etc.)
            measure_count = 0
            for col in columns:
                col_name = col['name'].lower()
                col_type = col.get('type', '').upper()
                if any(word in col_name for word in ['amount', 'quantity', 'price', 'total', 'cost', 'revenue']):
                    measure_count += 1
                if 'DECIMAL' in col_type or 'FLOAT' in col_type or 'DOUBLE' in col_type:
                    measure_count += 1
            
            # If has 3+ measure-like columns, likely a fact table (raised threshold)
            return measure_count >= 3
        
        code_parts = []
        
        # Header
        code_parts.append("from dotenv import load_dotenv")
        code_parts.append("import os")
        code_parts.append("import snowflake.connector")
        code_parts.append("from faker import Faker")
        code_parts.append("import random")
        code_parts.append("from datetime import datetime, timedelta")
        code_parts.append("")
        code_parts.append("load_dotenv()")
        code_parts.append("")
        code_parts.append("from snowflake_auth import get_snowflake_connection_params")
        code_parts.append("")
        
        # Build populate functions
        table_names = []
        for table_name, table_data in schema_info.items():
            columns = table_data['columns']
            
            # Build column lists (skip auto-increment IDs and malformed names)
            col_names = []
            col_types = []  # Track types for each valid column
            
            for col in columns:
                col_name = col['name']
                col_type = col.get('type', 'VARCHAR').upper()
                
                # Skip IDs (let database auto-generate)
                if col_name.lower() in ['id', table_name.lower() + '_id']:
                    continue
                if 'IDENTITY' in col_type or 'AUTOINCREMENT' in col_type:
                    continue
                    
                # Skip malformed column names (numbers, special chars, etc)
                if not col_name.replace('_', '').replace(' ', '').isalnum():
                    continue
                if col_name.isdigit():
                    continue
                if any(char in col_name for char in ['(', ')', ',', ';']):
                    continue
                    
                col_names.append(col_name)
                col_types.append(col_type)
            
            if not col_names:
                continue  # Skip tables with no insertable columns
            
            # Only add to table_names if we're actually creating a function for it
            # Determine row count based on table type
            is_fact = is_fact_table(table_name, columns)
            row_count = fact_rows if is_fact else dim_rows
            table_names.append((table_name, row_count))
            
            placeholders = ', '.join(['%s'] * len(col_names))
            col_list = ', '.join(col_names)
            
            # Build fake data generation (ONLY for valid columns in col_names)
            fake_values = []
            for i, col_name in enumerate(col_names):
                col_type = col_types[i]  # Use the col_type we saved earlier
                
                # Check for special business columns FIRST (by name pattern)
                col_name_upper = col_name.upper()
                
                # FLAG columns
                if 'FLAG' in col_name_upper or col_name_upper.startswith('IS_'):
                    if 'INT' in col_type or 'NUMBER' in col_type:
                        fake_values.append("random.choice([0, 1])")
                    else:
                        fake_values.append("random.choice(['Y', 'N'])")
                
                # QUARTER columns - Q1, Q2, Q3, Q4
                elif 'QUARTER' in col_name_upper or col_name_upper == 'QTR':
                    if 'INT' in col_type or 'NUMBER' in col_type:
                        fake_values.append("random.randint(1, 4)")
                    else:
                        fake_values.append("random.choice(['Q1', 'Q2', 'Q3', 'Q4'])")
                
                # MONTH columns
                elif 'MONTH' in col_name_upper:
                    if 'INT' in col_type or 'NUMBER' in col_type:
                        fake_values.append("random.randint(1, 12)")
                    else:
                        fake_values.append("random.choice(['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])")
                
                # YEAR columns
                elif 'YEAR' in col_name_upper:
                    fake_values.append("random.randint(2020, 2024)")
                
                # STATUS columns (short codes)
                elif 'STATUS' in col_name_upper and 'CHAR' in col_type:
                    fake_values.append("random.choice(['ACTIVE', 'PENDING', 'CLOSED'])")
                
                # TYPE columns (short codes) - but NOT CATEGORY (handled below with full names)
                elif 'TYPE' in col_name_upper and 'CHAR' in col_type and 'CATEGORY' not in col_name_upper:
                    fake_values.append("random.choice(['A', 'B', 'C'])")
                
                elif 'VARCHAR' in col_type or 'TEXT' in col_type or 'STRING' in col_type or 'CHAR' in col_type:
                    # Extract VARCHAR length - always truncate generated values to fit
                    import re
                    length_match = re.search(r'\((\d+)\)', col_type)
                    varchar_length = int(length_match.group(1)) if length_match else 255
                    
                    # Generate domain-specific realistic data based on column name, then truncate to fit
                    base_value = None
                    if 'NAME' in col_name_upper and 'COMPANY' not in col_name_upper:
                        # Check domain-specific name columns BEFORE falling back to fake.name()
                        if 'DRUG' in col_name_upper or 'MEDICATION' in col_name_upper or 'THERAPEUTIC' in col_name_upper:
                            base_value = "random.choice(['Lipitor', 'Humira', 'Eliquis', 'Keytruda', 'Revlimid', 'Opdivo', 'Ozempic', 'Dupixent', 'Trulicity', 'Entresto', 'Metformin', 'Atorvastatin', 'Lisinopril', 'Amlodipine', 'Metoprolol', 'Omeprazole', 'Simvastatin', 'Losartan', 'Albuterol', 'Gabapentin'])"
                        elif 'PRODUCT' in col_name_upper:
                            base_value = "random.choice(['Laptop Pro 15', 'Wireless Mouse 2.4GHz', 'USB-C Cable 6ft', 'Monitor Stand Adjustable', 'Mechanical Keyboard RGB', 'Noise Canceling Headphones', '1080p Webcam', 'Portable SSD 1TB', 'Power Bank 20000mAh', 'Tablet 10 inch', 'Smart Watch', 'Bluetooth Speaker', 'Gaming Mouse Pad', 'Phone Case', 'Screen Protector', 'Charging Cable', 'Desk Lamp LED', 'Laptop Bag', 'Wireless Earbuds', 'USB Hub'])"
                        elif 'CUSTOMER' in col_name_upper or 'USER' in col_name_upper:
                            base_value = "fake.name()"
                        elif 'SELLER' in col_name_upper or 'VENDOR' in col_name_upper:
                            base_value = "random.choice(['Amazon', 'Best Buy', 'Walmart', 'Target', 'Costco', 'Home Depot', 'Lowes', 'Macys', 'Nordstrom', 'Kohls'])"
                        else:
                            base_value = "fake.name()"
                    elif 'CATEGORY' in col_name_upper:
                        base_value = "random.choice(['Electronics', 'Home & Kitchen', 'Books', 'Clothing', 'Sports', 'Toys', 'Beauty', 'Automotive'])"
                    elif 'BRAND' in col_name_upper:
                        base_value = "random.choice(['Samsung', 'Apple', 'Sony', 'LG', 'Dell', 'HP', 'Lenovo', 'Amazon Basics', 'Anker', 'Logitech'])"
                    elif 'CHANNEL' in col_name_upper or 'SOURCE' in col_name_upper:
                        # Marketing channels for lead generation / call tracking
                        base_value = "random.choice(['Google Ads Search', 'Bing Ads', 'Facebook Ads', 'LinkedIn Ads', 'Instagram Ads', 'Twitter Ads', 'Display Network', 'Programmatic Display', 'Retargeting', 'TV Commercial', 'Radio Ads', 'Billboard', 'Print Ads', 'Direct Mail', 'Email Newsletter', 'Organic Search', 'Social Media Organic', 'Google My Business', 'Referral', 'Affiliate Marketing', 'Content Marketing', 'Webinar', 'Podcast Sponsorship'])"
                    elif 'CAMPAIGN' in col_name_upper and ('NAME' in col_name_upper or col_name_upper == 'CAMPAIGN_NAME'):
                        # Marketing campaign names (usually reference the channel)
                        base_value = "random.choice(['Google Ads Q4 Lead Gen', 'Facebook Black Friday Promo', 'LinkedIn Spring Campaign', 'Instagram New Product Launch', 'Email Brand Awareness', 'Display Holiday Special', 'Google Ads Summer Sale', 'Facebook Back to School', 'LinkedIn Valentine Promo', 'Google Shopping Cyber Monday', 'Email Free Trial Offer', 'Webinar Registration Q3', 'Email Nurture Series', 'Display Retargeting Q3', 'Google Ads Demo Request', 'Referral Rewards Program', 'Google Ads Year End Sale', 'Facebook New Year Campaign', 'Instagram Flash Sale', 'Email Limited Time Offer', 'Google Ads Early Bird', 'LinkedIn VIP Member Drive', 'Facebook Product Teaser', 'Display Conference Promo', 'Email Partner Campaign', 'Google Ads Seasonal', 'Facebook Customer Appreciation', 'Email Win Back Campaign', 'LinkedIn Upsell Drive', 'Display Cross-Sell Q4'])"
                    elif ('CENTER' in col_name_upper and 'NAME' in col_name_upper) or ('CALL' in col_name_upper and 'CENTER' in col_name_upper):
                        # Call center names
                        base_value = "random.choice(['New York Contact Center', 'Los Angeles Support Hub', 'Chicago Call Center', 'Dallas Operations Center', 'Phoenix Customer Care', 'Philadelphia Service Center', 'San Diego Support Center', 'Miami Contact Hub', 'Atlanta Operations', 'Denver Call Center', 'Seattle Support Center', 'Boston Customer Service', 'Portland Contact Center', 'Austin Operations Hub', 'Las Vegas Call Center', 'Toronto Support Center', 'Offshore Manila Center', 'Offshore Bangalore Hub', 'Remote East Coast Team', 'Remote West Coast Team', 'Central Support Center', 'National Call Center', 'Regional North Hub', 'Regional South Hub', 'Enterprise Support Center'])"
                    elif 'DESCRIPTION' in col_name_upper or 'DESC' in col_name_upper:
                        base_value = "random.choice(['High quality product', 'Best seller', 'Customer favorite', 'New arrival', 'Limited edition', 'Premium quality'])"
                    elif 'EMAIL' in col_name_upper:
                        base_value = "fake.email()"
                    elif 'PHONE' in col_name_upper:
                        base_value = "f'{random.randint(200, 999)}-{random.randint(200, 999)}-{random.randint(1000, 9999)}'"
                    elif 'ADDRESS' in col_name_upper or 'STREET' in col_name_upper:
                        base_value = "f'{random.randint(1, 9999)} {random.choice([\"Main\", \"Oak\", \"Park\", \"Maple\", \"Cedar\", \"Elm\", \"Washington\", \"Lake\", \"Hill\", \"Broadway\"])} {random.choice([\"St\", \"Ave\", \"Blvd\", \"Dr\", \"Ln\"])}'"
                    elif 'CITY' in col_name_upper:
                        base_value = "random.choice(['New York', 'Los Angeles', 'Chicago', 'Houston', 'Phoenix', 'Philadelphia', 'San Antonio', 'San Diego', 'Dallas', 'San Jose', 'Austin', 'Seattle', 'Denver', 'Boston', 'Portland', 'Miami', 'Atlanta', 'Detroit', 'Las Vegas', 'Toronto'])"
                    elif 'STATE' in col_name_upper or 'PROVINCE' in col_name_upper:
                        base_value = "random.choice(['California', 'Texas', 'New York', 'Florida', 'Illinois', 'Ohio', 'Georgia', 'Washington', 'Virginia', 'Arizona', 'Colorado', 'Oregon', 'Nevada', 'Utah', 'Iowa'])"
                    elif 'COUNTRY' in col_name_upper:
                        base_value = "random.choice(['USA', 'Canada', 'UK', 'Germany', 'France', 'Japan', 'Australia', 'India', 'China', 'Brazil', 'Mexico', 'Spain', 'Italy', 'Netherlands', 'Sweden'])"
                    elif 'ZIP' in col_name_upper or 'POSTAL' in col_name_upper:
                        base_value = "random.choice(['10001', '90210', '60601', '77001', '85001', '19101', '78201', '92101', '75201', '95101', '78701', '98101', '80201', '02101', '97201'])"
                    elif 'COMPANY' in col_name_upper:
                        base_value = "random.choice(['Amazon', 'Microsoft', 'Apple Inc', 'Google LLC', 'Meta', 'Tesla Inc', 'Netflix', 'Adobe Inc', 'Oracle Corp', 'Salesforce', 'IBM Corp', 'Intel Corp', 'Cisco Systems', 'Dell Technologies', 'HP Inc'])"
                    # --- People names (columns without NAME in them) ---
                    elif any(kw in col_name_upper for kw in ['SALES_REP', 'SALESREP', 'ACCOUNT_REP', 'REP_ID']):
                        base_value = "fake.name()"
                    elif col_name_upper in ('REP', 'AGENT', 'REPRESENTATIVE'):
                        base_value = "fake.name()"
                    elif any(kw in col_name_upper for kw in ['MANAGER', 'SUPERVISOR', 'DIRECTOR', 'OWNER']):
                        base_value = "fake.name()"
                    elif any(kw in col_name_upper for kw in ['EMPLOYEE', 'EMP_', 'STAFF', 'ASSOCIATE']):
                        base_value = "fake.name()"
                    elif any(kw in col_name_upper for kw in ['CONTACT', 'ASSIGNED_TO', 'CREATED_BY', 'UPDATED_BY', 'APPROVED_BY']):
                        base_value = "fake.name()"
                    elif any(kw in col_name_upper for kw in ['PHYSICIAN', 'DOCTOR', 'PROVIDER', 'PRESCRIBER', 'HCP']):
                        base_value = "fake.name()"
                    elif any(kw in col_name_upper for kw in ['PATIENT', 'MEMBER', 'SUBSCRIBER']):
                        base_value = "fake.name()"
                    # --- Geographic / organizational ---
                    elif 'REGION' in col_name_upper:
                        base_value = "random.choice(['Northeast', 'Southeast', 'Midwest', 'Southwest', 'West', 'Pacific Northwest', 'Mid-Atlantic', 'New England', 'South Central', 'Mountain West', 'Great Lakes', 'Gulf Coast'])"
                    elif 'TERRITORY' in col_name_upper:
                        base_value = "random.choice(['Northeast Territory', 'Southeast Territory', 'Central Territory', 'Western Territory', 'Pacific Territory', 'Mountain Territory', 'Great Lakes Territory', 'Southern Territory', 'Mid-Atlantic Territory', 'Texas Territory'])"
                    elif 'DEPARTMENT' in col_name_upper or 'DEPT' in col_name_upper:
                        base_value = "random.choice(['Sales', 'Marketing', 'Finance', 'Operations', 'Engineering', 'Human Resources', 'Customer Success', 'Legal', 'Product', 'IT', 'Supply Chain', 'Research'])"
                    elif 'SEGMENT' in col_name_upper:
                        base_value = "random.choice(['Enterprise', 'Mid-Market', 'SMB', 'Consumer', 'Government', 'Education', 'Healthcare', 'Premium', 'Standard', 'Basic'])"
                    elif 'TIER' in col_name_upper:
                        base_value = "random.choice(['Platinum', 'Gold', 'Silver', 'Bronze', 'Premium', 'Standard', 'Basic'])"
                    elif 'PRIORITY' in col_name_upper:
                        base_value = "random.choice(['Critical', 'High', 'Medium', 'Low', 'Urgent'])"
                    # --- Business domain ---
                    elif 'PAYMENT' in col_name_upper and ('METHOD' in col_name_upper or 'TYPE' in col_name_upper):
                        base_value = "random.choice(['Credit Card', 'Debit Card', 'ACH Transfer', 'Wire Transfer', 'Check', 'PayPal', 'Apple Pay', 'Google Pay'])"
                    elif 'SHIPPING' in col_name_upper or 'SHIP_METHOD' in col_name_upper:
                        base_value = "random.choice(['Standard Ground', 'Express 2-Day', 'Next Day Air', 'Economy', 'Freight', 'Same Day', 'International Standard', 'Priority Mail'])"
                    elif 'WAREHOUSE' in col_name_upper or 'FULFILLMENT' in col_name_upper:
                        base_value = "random.choice(['East Coast DC', 'West Coast DC', 'Central Hub', 'Southeast Warehouse', 'Pacific Distribution', 'Northeast Fulfillment', 'Texas DC', 'Midwest Hub', 'Mountain West DC', 'Southern Distribution'])"
                    elif 'STORE' in col_name_upper or 'LOCATION' in col_name_upper:
                        base_value = "random.choice(['Downtown Flagship', 'Mall of America', 'Northgate Plaza', 'Southside Center', 'Airport Terminal', 'University District', 'Waterfront Promenade', 'Tech Park', 'Suburban Commons', 'Metro Center', 'Eastside Galleria', 'Westfield Mall'])"
                    elif 'TITLE' in col_name_upper or 'JOB' in col_name_upper or 'ROLE' in col_name_upper or 'POSITION' in col_name_upper:
                        base_value = "random.choice(['VP Sales', 'Account Executive', 'Sales Manager', 'Director of Marketing', 'Product Manager', 'Data Analyst', 'Regional Manager', 'CFO', 'Operations Lead', 'Supply Chain Manager', 'Customer Success Manager', 'Business Analyst'])"
                    elif 'SUBCATEGORY' in col_name_upper or 'SUB_CATEGORY' in col_name_upper:
                        base_value = "random.choice(['Laptops', 'Smartphones', 'Headphones', 'Monitors', 'Tablets', 'Accessories', 'Networking', 'Storage', 'Printers', 'Cameras', 'Wearables', 'Audio'])"
                    elif 'CURRENCY' in col_name_upper:
                        base_value = "random.choice(['USD', 'EUR', 'GBP', 'CAD', 'JPY', 'AUD', 'MXN'])"
                    elif 'REASON' in col_name_upper:
                        base_value = "random.choice(['Price', 'Quality', 'Delivery Delay', 'Wrong Item', 'Defective', 'Changed Mind', 'Better Alternative', 'Budget Cut', 'Not as Described'])"
                    elif 'OUTCOME' in col_name_upper or 'RESULT' in col_name_upper or 'DISPOSITION' in col_name_upper:
                        base_value = "random.choice(['Won', 'Lost', 'Pending', 'Qualified', 'Disqualified', 'No Decision', 'Deferred'])"
                    elif 'STAGE' in col_name_upper or 'PHASE' in col_name_upper:
                        base_value = "random.choice(['Prospecting', 'Qualification', 'Proposal', 'Negotiation', 'Closed Won', 'Closed Lost', 'Discovery', 'Demo', 'Contract Review'])"
                    elif 'RATING' in col_name_upper or 'GRADE' in col_name_upper or 'SCORE' in col_name_upper:
                        base_value = "random.choice(['A', 'B', 'C', 'D', 'Excellent', 'Good', 'Average', 'Below Average'])"
                    elif 'COLOR' in col_name_upper:
                        base_value = "random.choice(['Black', 'White', 'Navy', 'Red', 'Blue', 'Gray', 'Green', 'Brown', 'Beige', 'Olive'])"
                    elif 'SIZE' in col_name_upper:
                        base_value = "random.choice(['XS', 'S', 'M', 'L', 'XL', 'XXL', 'One Size'])"
                    elif 'GENDER' in col_name_upper or 'SEX' in col_name_upper:
                        base_value = "random.choice(['Male', 'Female', 'Non-Binary', 'Prefer Not to Say'])"
                    elif 'INDUSTRY' in col_name_upper or 'VERTICAL' in col_name_upper:
                        base_value = "random.choice(['Technology', 'Healthcare', 'Financial Services', 'Retail', 'Manufacturing', 'Education', 'Government', 'Media', 'Energy', 'Real Estate'])"
                    elif 'SPECIALTY' in col_name_upper or 'SPECIALIZATION' in col_name_upper:
                        base_value = "random.choice(['Cardiology', 'Oncology', 'Orthopedics', 'Neurology', 'Pediatrics', 'Dermatology', 'Internal Medicine', 'Family Medicine', 'Psychiatry', 'Radiology'])"
                    elif 'DRUG' in col_name_upper or 'MEDICATION' in col_name_upper or 'THERAPEUTIC' in col_name_upper:
                        base_value = "random.choice(['Lipitor', 'Humira', 'Eliquis', 'Keytruda', 'Revlimid', 'Opdivo', 'Ozempic', 'Dupixent', 'Trulicity', 'Entresto'])"
                    elif 'SKU' in col_name_upper or 'ITEM_CODE' in col_name_upper or 'PRODUCT_CODE' in col_name_upper:
                        base_value = "f'SKU-{random.randint(10000, 99999)}'"
                    elif 'ORDER_NUMBER' in col_name_upper or 'ORDER_NUM' in col_name_upper or 'INVOICE_NUM' in col_name_upper:
                        base_value = "f'ORD-{random.randint(100000, 999999)}'"
                    elif 'URL' in col_name_upper or 'WEBSITE' in col_name_upper:
                        base_value = "fake.url()"
                    elif 'NOTES' in col_name_upper or 'COMMENT' in col_name_upper:
                        base_value = "random.choice(['Follow up next week', 'Priority customer', 'Pending review', 'No issues', 'Escalated', 'Resolved', 'Awaiting approval', 'On track'])"
                    else:
                        # Default: use faker word as last resort
                        base_value = "fake.word()"
                    
                    # Always truncate to VARCHAR length - simple and works for all cases
                    fake_values.append(f"({base_value})[:{varchar_length}]")
                elif 'INT' in col_type or 'NUMBER' in col_type or 'BIGINT' in col_type:
                    fake_values.append("random.randint(1, 1000)")
                elif 'DECIMAL' in col_type or 'FLOAT' in col_type or 'DOUBLE' in col_type or 'NUMERIC' in col_type:
                    # Extract precision and scale from DECIMAL(p,s)
                    import re
                    decimal_match = re.search(r'\((\d+),\s*(\d+)\)', col_type)
                    if decimal_match:
                        precision = int(decimal_match.group(1))
                        scale = int(decimal_match.group(2))
                        # Max value is 10^(precision-scale) - 1, with 'scale' decimal places
                        # E.g., DECIMAL(3,2) -> max is 9.99, DECIMAL(5,2) -> max is 999.99
                        max_val = (10 ** (precision - scale)) - 1
                        fake_values.append(f"round(random.uniform(0, {max_val}), {scale})")
                    else:
                        # No precision specified, use safe defaults
                        fake_values.append("round(random.uniform(10, 1000), 2)")
                elif 'DATE' in col_type and 'TIME' not in col_type:  # DATE but not DATETIME/TIMESTAMP
                    fake_values.append("fake.date_between(start_date='-2y', end_date='today')")
                elif 'TIMESTAMP' in col_type or 'DATETIME' in col_type:
                    fake_values.append("fake.date_time_between(start_date='-2y', end_date='now')")
                elif 'BOOL' in col_type or 'BOOLEAN' in col_type:
                    # For Snowflake BOOLEAN, use True/False which will be converted to SQL TRUE/FALSE
                    fake_values.append("random.choice([True, False])")
                else:
                    # Unknown type - default to string
                    fake_values.append("str(fake.word())")
            
            fake_values_str = ', '.join(fake_values) if fake_values else "fake.word()"
            
            # Build function
            code_parts.append(f"def populate_{table_name.lower()}(cursor, fake):")
            code_parts.append(f'    """Populate {table_name} table ({"fact" if is_fact else "dimension"})"""')
            code_parts.append("    data = []")
            code_parts.append(f"    for _ in range({row_count}):")
            code_parts.append(f"        data.append(({fake_values_str},))")
            code_parts.append("")
            code_parts.append("    cursor.executemany(")
            code_parts.append(f'        "INSERT INTO {table_name} ({col_list}) VALUES ({placeholders})",')
            code_parts.append("        data")
            code_parts.append("    )")
            code_parts.append(f'    print(f"✅ Inserted {{len(data)}} rows into {table_name}")')
            code_parts.append("")
        
        # Safety check - ensure we have at least one table
        if not table_names:
            raise Exception("No valid tables found in schema. All tables were skipped due to having no insertable columns.")
        
        # Main function
        code_parts.append("def main():")
        code_parts.append("    conn_params = get_snowflake_connection_params()")
        code_parts.append("    conn_params.pop('schema', None)  # Remove to avoid duplicate")
        code_parts.append("    conn = snowflake.connector.connect(**conn_params, schema=os.getenv('SNOWFLAKE_SCHEMA'), autocommit=False)")
        code_parts.append("")
        code_parts.append("    try:")
        code_parts.append("        cursor = conn.cursor()")
        code_parts.append("        fake = Faker()")
        code_parts.append("")
        
        # Add function calls
        for table_name, row_count in table_names:
            code_parts.append(f"        populate_{table_name.lower()}(cursor, fake)")
        
        code_parts.append("")
        code_parts.append("        conn.commit()")
        code_parts.append('        print("✅ All data committed successfully")')
        code_parts.append("    except Exception as e:")
        code_parts.append("        conn.rollback()")
        code_parts.append('        print(f"❌ Error: {str(e)}")')
        code_parts.append("        raise")
        code_parts.append("    finally:")
        code_parts.append("        cursor.close()")
        code_parts.append("        conn.close()")
        code_parts.append("")
        code_parts.append('if __name__ == "__main__":')
        code_parts.append("    main()")
        
        code = '\n'.join(code_parts)
        
        # Validate the generated code compiles
        try:
            compile(code, '<template_generation>', 'exec')
            self.log_feedback("✅ Template validated successfully before return")
        except SyntaxError as e:
            self.log_feedback(f"❌ TEMPLATE GENERATION BUG: {e}")
            self.log_feedback(f"   Error at line {e.lineno}: {e.msg}")
            # Show the problematic lines
            lines = code.split('\n')
            if e.lineno:
                start = max(0, e.lineno - 3)
                end = min(len(lines), e.lineno + 2)
                self.log_feedback(f"\n   Context:")
                for i in range(start, end):
                    marker = ">>> " if i == e.lineno - 1 else "    "
                    self.log_feedback(f"{marker}{i+1:3}: {lines[i]}")
            raise Exception(f"Template generation has a bug: {e}")
        
        # Debug: Log first 1000 chars
        self.log_feedback(f"Generated template preview:\n{code[:1000]}")
        
        return code
    
    def run_population(self):
        """Run data population code generation"""
        self.log_feedback("🔢 Starting data population...")
        
        try:
            from schema_utils import parse_ddl_schema, generate_schema_constrained_prompt
            import re
            
            # Parse use case into vertical and function
            vertical, function = parse_use_case(self.demo_builder.use_case)
            config = get_use_case_config(vertical or "Generic", function or "Generic")
            
            # Build business context for population
            # Handle both new config structure and backward compatibility
            target_persona = config.get('target_persona', 'Business Leader')
            business_problem = config.get('business_problem', 'Need for faster, data-driven decisions')
            demo_objectives = config.get('demo_objectives', 'Show self-service analytics and business insights')
            
            # For generic cases, use the use_case_name
            use_case_display = config.get('use_case_name', self.demo_builder.use_case)
            
            business_context = f"""
BUSINESS CONTEXT:
- Use Case: {use_case_display}
- Target Persona: {target_persona}
- Business Problem: {business_problem}
- Demo Objectives: {demo_objectives}

MANDATORY CONNECTION CODE (MUST BE COMPLETE):
```python
from dotenv import load_dotenv
import os
import snowflake.connector
from faker import Faker
import random
from datetime import datetime, timedelta

load_dotenv()

from snowflake_auth import get_snowflake_connection_params

def main():
    conn_params = get_snowflake_connection_params()
    conn = snowflake.connector.connect(**conn_params, schema=os.getenv('SNOWFLAKE_SCHEMA'), autocommit=False)
    
    try:
        cursor = conn.cursor()
        fake = Faker()
        
        # [YOUR POPULATION CODE HERE - populate tables]
        
        conn.commit()
        print("✅ All data committed successfully")
    except Exception as e:
        conn.rollback()
        print(f"❌ Error: {{str(e)}}")
        raise
    finally:
        cursor.close()
        conn.close()

if __name__ == "__main__":
    main()
```

CRITICAL REQUIREMENTS:
1. **COMPLETE try/except/finally blocks** - NO incomplete blocks
2. Use cursor.executemany() for batch inserts with %s placeholders (NOT ?)
3. Create baseline normal data (1000+ rows per table)
4. Include strategic outliers with structured comments
5. NO explanatory text, just executable Python code
6. **DO NOT leave try blocks incomplete** - always include except and finally
7. Use Faker library for realistic data generation
8. **PROPER INDENTATION** - Use 4 spaces per indent level, NO TABS
9. **SYNTAX CHECK** - Ensure all code is valid Python with correct indentation

RETURN FORMAT:
- Return ONLY the complete Python code
- Start with imports, end with if __name__ == "__main__"
- NO markdown, NO explanations, NO comments outside the code
- All code must be properly indented and executable
"""
            
            # Parse schema and generate prompt
            self.log_feedback("Parsing DDL schema...")
            
            # Validate DDL exists
            if not self.demo_builder.schema_generation_results:
                raise Exception("❌ DDL is missing. Please create DDL first.")
            
            self.log_feedback(f"DDL length: {len(self.demo_builder.schema_generation_results)} characters")
            
            schema_info = parse_ddl_schema(self.demo_builder.schema_generation_results)
            
            if not schema_info:
                raise Exception("❌ Failed to parse DDL schema. DDL may be malformed.")
            
            self.log_feedback(f"Parsed {len(schema_info)} tables from DDL")
            
            self.log_feedback("Using LegitData for data generation...")
            
            self.demo_builder.data_population_results = "LEGITDATA"
            self.demo_builder.population_code_source = "legitdata"
            self.population_code = "# Data generated by LegitData"
            
            self.demo_builder.advance_stage()
            
            self.log_feedback("✅ Ready for deployment with LegitData!")
            
            response = f"""✅ **Data Population Ready!**

LegitData will generate realistic, AI-powered data.

**When you're ready, type 'deploy'** to:
- Create Snowflake schema & tables
- Populate with generated data  
- Create ThoughtSpot model & liveboard

⏱️ *This takes 2-5 minutes - watch the terminal for progress.*"""
            
            return response, self.population_code
            
        except Exception as e:
            import traceback
            error_msg = f"❌ Population failed: {str(e)}\n\n{traceback.format_exc()}\n\n**Would you like to retry?** (Type 'yes' to retry)"
            self.log_feedback(error_msg)
            return error_msg, self.population_code if hasattr(self, 'population_code') else ""
    
    def run_deployment(self):
        """Run deployment to Snowflake using LegitData (non-streaming version)"""
        # Consume the streaming version and return final result
        result = None
        for update in self.run_deployment_streaming():
            result = update
        return result
    
    def run_deployment_streaming(self):
        """Run deployment to Snowflake using LegitData - yields progress updates"""
        _slog = self._session_logger
        _t_deploy = _slog.log_start("deploy") if _slog else None
        _deploy_error = None
        _deploy_meta = {}

        progress = ""
        schema_name = None
        company_name = None

        # Clear and initialize live progress for Snowflake deployment
        self.live_progress_log = ["=" * 60, "SNOWFLAKE DEPLOYMENT STARTING", "=" * 60, ""]
        self._dq_warnings = []  # reset data-quality gate warnings for this run

        def log_progress(msg):
            """Log to live progress tab only — not pipeline status"""
            print(f"[Deploy] {msg}", flush=True)
            self.live_progress_log.append(msg)

        try:
            # Ensure deploy-time modules that still use os.getenv() see Supabase admin settings.
            inject_admin_settings_to_env()

            from cdw_connector import SnowflakeDeployer
            
            # Step 1: Connect
            progress = "**Step 1/3: Connecting to Snowflake...**"
            yield progress
            
            log_progress("Connecting to Snowflake...")
            deployer = SnowflakeDeployer()
            
            success, message = deployer.connect()
            if not success:
                if _slog: _slog.log("deploy", "snowflake connect failed", error=message)
                raise Exception(f"Snowflake connection failed: {message}")

            if _slog: _slog.log_verbose("deploy", "snowflake connected")
            progress += f"\n[OK] {message}"
            log_progress(f"[OK] {message}")
            yield progress
            
            # Step 2: Create schema and tables
            company_name = self.demo_builder.extract_company_name()
            self._slack_deployment_company = company_name
            from slack_notifier import notify_deployment_event
            notify_deployment_event(
                "DemoPrep deployment",
                "Started",
                [
                    ("Company", company_name),
                    ("Use case", self.demo_builder.use_case),
                ],
            )
            progress += f"\n\n**Step 2/3: Creating schema and tables...**"
            progress += f"\n  Company: {company_name}"
            progress += f"\n  Use Case: {self.demo_builder.use_case}"
            yield progress
            
            log_progress(f"Creating schema and deploying DDL for {company_name}...")

            # Generate base name for schema
            from demo_prep import generate_demo_base_name
            naming_prefix = self.settings.get('object_naming_prefix', '')
            base_name = generate_demo_base_name(naming_prefix, company_name)
            
            success, schema_name, deploy_message = deployer.create_demo_schema_and_deploy(
                base_name,
                self.demo_builder.schema_generation_results
            )
            
            
            if not success:
                if _slog: _slog.log("deploy", "ddl push failed", error=deploy_message)
                log_progress(f"[ERROR] DDL Deployment failed!")
                raise Exception(f"Schema deployment failed: {deploy_message}")

            if _slog: _slog.log_verbose("deploy", "ddl pushed", schema=schema_name, schema_name=schema_name)
            progress += f"\n[OK] Schema created: {schema_name}"
            progress += f"\n[OK] Tables created"
            log_progress(f"[OK] Schema created: {schema_name}")
            log_progress(f"[OK] Tables created successfully")
            yield progress
            
            # Step 3: Populate Snowflake from the generated dataset bundle
            import threading
            import time as time_module
            
            # Determine size from settings
            fact_rows = int(self.settings.get('fact_table_size', 5000))
            if fact_rows <= 100:
                size = "small"
            elif fact_rows <= 1000:
                size = "medium"
            elif fact_rows <= 5000:
                size = "standard"
            elif fact_rows <= 10000:
                size = "large"
            else:
                size = "xl"

            # Show size details
            size_details = {
                "small": "~500 rows total",
                "medium": "~1,500 rows total",
                "standard": "~5,500 rows total",
                "large": "~15,000 rows total",
                "xl": "~50,000+ rows total"
            }
            
            progress += f"\n\n**Step 3/3: Populating tables with data...**"
            progress += f"\n  Size: {size} ({size_details.get(size, '')})"
            progress += f"\n  Loading dataset into Snowflake... (typically under 1 minute)"
            progress += f"\n  *(detailed progress in Live Progress tab)*"
            yield progress

            log_progress("")
            log_progress("Loading dataset into Snowflake...")
            
            log_progress(f"  Size preset: {size}")
            log_progress(f"  Company URL: {self.demo_builder.company_url}")
            
            # Track population progress
            pop_messages = []
            def pop_callback(msg):
                log_progress(msg)
                pop_messages.append(msg)
                if _slog:
                    _slog.log("deploy", f"population progress: {str(msg)[:120]}")
            
            # Run populate_dataset_bundle in a background thread so we can yield progress
            pop_result = {"success": None, "message": None, "results": None, "done": False}
            
            def run_population():
                try:
                    from demoprep_app.integrations.snowflake import populate_dataset_bundle

                    pop_callback("Loading dataset into Snowflake...")
                    results = populate_dataset_bundle(
                        deployer.connection,
                        schema_name,
                        self._demo_bundle,
                        progress_callback=pop_callback,
                    )
                    total_rows = sum(results.values())
                    pop_callback(f"Load complete: {total_rows:,} rows inserted")
                    pop_result["success"] = True
                    pop_result["message"] = f"Load complete: {total_rows:,} rows inserted"
                    pop_result["results"] = results
                except Exception as e:
                    pop_result["success"] = False
                    pop_result["message"] = str(e)
                    pop_result["results"] = None
                finally:
                    pop_result["done"] = True
            
            pop_thread = threading.Thread(target=run_population)
            pop_thread.start()

            # Yield progress updates while LegitData runs
            POP_TIMEOUT = 900
            spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
            spinner_idx = 0
            start_time = time_module.time()
            last_msg_count = 0
            last_heartbeat = start_time

            while not pop_result["done"]:
                time_module.sleep(2)
                elapsed = time_module.time() - start_time

                # Hard timeout — break out so UI doesn't freeze forever
                if elapsed > POP_TIMEOUT:
                    timeout_err = f"Data generation timed out after {int(POP_TIMEOUT/60)} minutes"
                    pop_result["done"] = True
                    pop_result["success"] = False
                    pop_result["message"] = timeout_err
                    if _slog:
                        _slog.log("deploy", "LegitData timeout", error=timeout_err)
                    break

                # Heartbeat every 60 seconds so we can tell live vs. stuck
                if _slog and (time_module.time() - last_heartbeat) >= 60:
                    _slog.log("deploy", f"dataset loader heartbeat - {int(elapsed/60)}m {int(elapsed%60)}s elapsed")
                    last_heartbeat = time_module.time()

                mins = int(elapsed) // 60
                secs = int(elapsed) % 60

                spinner_char = spinner[spinner_idx % len(spinner)]
                spinner_idx += 1

                progress_update = progress + f"\n\n{spinner_char} **LegitData running...** ({mins}:{secs:02d} elapsed)"

                # Show recent progress messages from callback
                if len(pop_messages) > last_msg_count:
                    recent_msgs = pop_messages[last_msg_count:last_msg_count + 3]
                    for msg in recent_msgs:
                        if msg.strip():
                            progress_update += f"\n  • {msg[:60]}..."
                    last_msg_count = len(pop_messages)

                yield progress_update
            
            # Get results from thread
            pop_success = pop_result["success"]
            pop_message = pop_result["message"]
            results = pop_result["results"]
            
            if not pop_success:
                self._last_population_error = pop_message
                self._last_schema_name = schema_name
                log_progress(f"[ERROR] Population error: {pop_message}")
                raise Exception(f"Population failed: {pop_message[:200]}")
            
            progress += f"\n[OK] Data populated"
            log_progress(f"[OK] {pop_message}")

            # Data-quality gate: after every load, profile the MEASURE columns
            # that were actually written to Snowflake and fail loudly if any is
            # entirely zero/null (recurring zero-measure defect — derived
            # measures like TOTAL_*_USD loading as all zeros → blank tiles).
            from demoprep_app.integrations.snowflake import (
                ZeroMeasureError,
                run_measure_quality_gate,
            )
            progress += f"\n\n**Verifying data quality (measure columns)...**"
            yield progress
            log_progress("")
            log_progress("Running data-quality gate on measure columns...")
            try:
                dq_profile = run_measure_quality_gate(
                    deployer.connection, schema_name, progress_callback=log_progress
                )
            except ZeroMeasureError as e:
                self._last_population_error = str(e)
                self._last_schema_name = schema_name
                _deploy_meta = {
                    "schema_name": schema_name,
                    "data_quality": {
                        "failures": e.failures,
                        "warnings": e.profile.get("warnings", []),
                    },
                }
                log_progress(f"[ERROR] {e}")
                log_progress(f"  Schema {schema_name} left in place for inspection.")
                if _slog:
                    _slog.log("deploy", "data quality gate failed", error=str(e))
                raise Exception(str(e))

            dq_warnings = dq_profile.get("warnings", [])
            self._dq_warnings = dq_warnings
            progress += (
                f"\n[OK] Data quality gate passed "
                f"({dq_profile['measure_columns_checked']} measure columns "
                f"across {dq_profile['tables_checked']} tables)"
            )
            for _dq_w in dq_warnings:
                progress += f"\n[WARN] {_dq_w}"
            if _slog:
                _slog.log_verbose(
                    "deploy", "data quality gate passed",
                    measure_columns_checked=dq_profile["measure_columns_checked"],
                    tables_checked=dq_profile["tables_checked"],
                    dq_warnings=dq_warnings,
                )

            self._deployed_schema_name = schema_name
            _deploy_meta = {
                "schema_name": schema_name,
                "generation_mode": "blueprint",
                "table_names": [table.name for table in self._demo_bundle.tables] if self._demo_bundle else [],
                "row_counts": {table.name: len(table.rows) for table in self._demo_bundle.tables} if self._demo_bundle else {},
                "data_quality": {
                    "measure_columns_checked": dq_profile["measure_columns_checked"],
                    "tables_checked": dq_profile["tables_checked"],
                    "warnings": dq_warnings,
                },
            }
            
            log_progress("")
            log_progress("=" * 60)
            log_progress("SNOWFLAKE DEPLOYMENT COMPLETE")
            log_progress("=" * 60)
            
            # Check validation_mode setting to decide whether to auto-continue
            from supabase_client import load_gradio_settings, get_admin_setting
            settings = load_gradio_settings(self._get_effective_user_email())
            validation_mode = settings.get('validation_mode', 'Off')
            
            if validation_mode == 'On':
                # Validation mode - show message asking to type 'thoughtspot'
                final_response = f"""{progress}

**Deployment Complete**

Schema: **{schema_name}**
Tables: Created and populated
Status: Ready for ThoughtSpot

**Next:** Type **'thoughtspot'** to create ThoughtSpot objects"""
                notify_deployment_event(
                    "DemoPrep deployment",
                    "Ready for ThoughtSpot",
                    [
                        ("Company", company_name),
                        ("Use case", self.demo_builder.use_case),
                        ("Schema", schema_name),
                    ],
                )
                if _slog:
                    _slog.log(
                        "run",
                        "run waiting for user",
                        checkpoint="thoughtspot",
                        reason="validation mode requested manual ThoughtSpot start",
                        schema_name=schema_name,
                    )
                yield (final_response, "thoughtspot")
            else:
                # Auto-continue to ThoughtSpot deployment (default behavior)
                final_response = f"""{progress}

**Deployment Complete**

Schema: **{schema_name}**
Tables: Created and populated

**Auto-continuing to ThoughtSpot deployment...**"""
                log_progress("Auto-continuing to ThoughtSpot deployment...")
                with open('/tmp/demoprep_debug.log', 'a') as f:
                    f.write(f"ABOUT TO YIELD auto_ts tuple, schema={schema_name}\n")
                yield (final_response, "auto_ts", schema_name)
                with open('/tmp/demoprep_debug.log', 'a') as f:
                    f.write(f"YIELD COMPLETED for auto_ts\n")
            
        except Exception as e:
            import traceback
            error_msg = str(e)
            _deploy_error = error_msg
            log_progress(f"[ERROR] {error_msg}")
            try:
                from slack_notifier import notify_deployment_event
                notify_deployment_event(
                    "DemoPrep deployment",
                    "Failed",
                    [
                        ("Company", company_name),
                        ("Use case", getattr(self.demo_builder, "use_case", None)),
                        ("Schema", schema_name),
                        ("Error", error_msg[:500]),
                    ],
                )
            except Exception:
                pass
            raise Exception(f"Deployment failed: {error_msg}")
        finally:
            if _slog and _t_deploy:
                _slog.log_end("deploy", _t_deploy, error=_deploy_error, **_deploy_meta)

    def _run_thoughtspot_deployment(self, schema_name, company, use_case, on_progress=None):
        """
        Generator that runs ThoughtSpot deployment and yields progress updates.
        
        Yields:
            str: Progress messages during deployment
            dict: Final result with 'response' and 'stage' keys
        """
        # Debug log at entry
        with open('/tmp/demoprep_debug.log', 'a') as f:
            f.write(f"_run_thoughtspot_deployment: ENTERED, schema={schema_name}\n")
        
        import os
        from thoughtspot_deployer import ThoughtSpotDeployer
        from cdw_connector import SnowflakeDeployer
        from supabase_client import load_gradio_settings, get_admin_setting
        from demo_prep import generate_demo_base_name
        from slack_notifier import notify_deployment_event
        
        _slog = self._session_logger
        _t_ts = _slog.log_start("thoughtspot") if _slog else None
        _ts_error = None
        _ts_meta = {}
        slack_company = getattr(self, "_slack_deployment_company", None) or company

        with open('/tmp/demoprep_debug.log', 'a') as f:
            f.write(f"_run_thoughtspot_deployment: imports done, about to yield first message\n")

        yield "**Starting ThoughtSpot Deployment...**\n\n"
        
        with open('/tmp/demoprep_debug.log', 'a') as f:
            f.write(f"_run_thoughtspot_deployment: first yield complete\n")
        
        self.log_feedback("Deploying to ThoughtSpot...")
        
        # Check if using existing model mode
        use_existing_model = self.settings.get('use_existing_model', False)
        existing_model_guid = self.settings.get('existing_model_guid', '')
        
        if use_existing_model and existing_model_guid:
            # Skip all table creation - go directly to liveboard creation
            yield f"**Using Existing Model Mode**\n\nModel GUID: `{existing_model_guid}`\n\nSkipping table/model creation...\n\n"
            self.log_feedback(f"Using existing model: {existing_model_guid}")
            
            try:
                from liveboard_creator import create_liveboard_from_model_mcp
                from thoughtspot_deployer import ThoughtSpotDeployer

                # Get ThoughtSpot settings
                ts_url = (self.settings.get('thoughtspot_url') or '').strip()
                ts_secret = (self.settings.get('thoughtspot_trusted_auth_key') or '').strip()
                if not ts_url or not ts_secret:
                    raise ValueError("ThoughtSpot environment not set — select a TS environment from the dropdown")
                ts_user = self._get_effective_user_email()

                # Clean company name for display (strip .com, .org, etc)
                clean_company = company.split('.')[0].title() if '.' in company else company
                liveboard_name = self.settings.get('liveboard_name', '') or f"{clean_company} - {use_case}"

                # Get company data for liveboard
                company_data = {
                    'name': clean_company,
                    'url': getattr(self.demo_builder, 'company_url', company),
                    'logo_url': getattr(self.demo_builder, 'logo_url', None),
                    'primary_color': getattr(self.demo_builder, 'primary_color', '#3498db'),
                    'secondary_color': getattr(self.demo_builder, 'secondary_color', '#2c3e50'),
                    'additional_context': getattr(self, 'generic_use_case_context', '') or '',
                }

                yield f"**Creating Liveboard from Existing Model**\n\nModel: `{existing_model_guid}`\n\n"

                # Auth a deployer so we can pass ts_client to MCP
                ts_client = ThoughtSpotDeployer(ts_url, ts_user, ts_secret)
                if not ts_client.authenticate():
                    raise ValueError("ThoughtSpot authentication failed.")

                llm_model = self.settings.get('model', DEFAULT_LLM_MODEL)

                # Create liveboard through the supported MCP path.
                liveboard_result = create_liveboard_from_model_mcp(
                    ts_client=ts_client,
                    model_id=existing_model_guid,
                    model_name="Existing Model",
                    company_data=company_data,
                    use_case=use_case,
                    num_visualizations=8,
                    liveboard_name=liveboard_name,
                    llm_model=llm_model,
                    prompt_logger=self._prompt_logger,
                )
                
                if liveboard_result.get('success'):
                    liveboard_url = liveboard_result.get('liveboard_url', '')
                    yield {
                        'response': f"""✅ **Liveboard Created from Existing Model**

**Model GUID:** `{existing_model_guid}`
**Liveboard:** [{liveboard_name}]({liveboard_url})

The liveboard was created using the existing model. No new tables or models were created.""",
                        'stage': 'complete'
                    }
                    return
                else:
                    error = liveboard_result.get('error', 'Unknown error')
                    yield {
                        'response': f"""❌ **Liveboard Creation Failed**

**Model GUID:** `{existing_model_guid}`
**Error:** {error}

Please verify the model GUID is correct and you have access to it.""",
                        'stage': 'thoughtspot'
                    }
                    return
                    
            except Exception as e:
                import traceback
                yield {
                    'response': f"""❌ **Error Using Existing Model**

**Error:** {str(e)}

**Details:**
```
{traceback.format_exc()}
```""",
                    'stage': 'thoughtspot'
                }
                return
        
        try:
            # FIRST: Verify schema exists in Snowflake
            yield "**Starting ThoughtSpot Deployment...**\n\nVerifying Snowflake schema..."
            
            sf_deployer = SnowflakeDeployer()
            sf_deployer.connect()
            cursor = sf_deployer.connection.cursor()
            
            # Get DDL and database info
            ddl = self.demo_builder.schema_generation_results
            from snowflake_auth import get_demo_database
            database = get_demo_database()
            
            # Check if schema exists
            cursor.execute(f"USE DATABASE {database}")
            cursor.execute(f"SHOW SCHEMAS LIKE '{schema_name}'")
            schemas = cursor.fetchall()
            
            if not schemas:
                yield {
                    'response': f"""❌ **Schema Not Found in Snowflake**

The schema `{database}.{schema_name}` doesn't exist in Snowflake.

**Did the deployment complete successfully?** 

If not, try these options:
1. Type **'deploy'** - Deploy to Snowflake first
2. Check if tables were actually created in Snowflake

Cannot deploy to ThoughtSpot without valid Snowflake schema.""",
                    'stage': 'deploy'
                }
                return
            
            # Check if tables exist
            cursor.execute(f'USE SCHEMA "{schema_name}"')
            cursor.execute(f"SHOW TABLES")
            tables = cursor.fetchall()
            
            if not tables:
                yield {
                    'response': f"""❌ **No Tables Found**

The schema `{database}.{schema_name}` exists but has no tables!

**What happened:**
- Schema was created but table creation may have failed
- Or tables were dropped/truncated

**Next steps:**
1. Type **'deploy'** - Re-run the full Snowflake deployment
2. Or check Snowflake manually to see what's there

Cannot deploy to ThoughtSpot without tables.""",
                    'stage': 'deploy'
                }
                return
            
            sf_deployer.connection.close()
            
            yield f"**Starting ThoughtSpot Deployment...**\n\nSchema verified: {database}.{schema_name}\nFound {len(tables)} tables\n\n"
            
            # Create deployer — prefer session-selected env (from TS env dropdown),
            # TS env must be selected from dropdown — no admin fallback.
            ts_url = (self.settings.get('thoughtspot_url') or '').strip()
            ts_secret = (self.settings.get('thoughtspot_trusted_auth_key') or '').strip()
            if not ts_url or not ts_secret:
                raise ValueError("ThoughtSpot environment not set — select a TS environment from the dropdown")
            ts_user = self._get_effective_user_email()
            
            deployer = ThoughtSpotDeployer(
                base_url=ts_url,
                username=ts_user,
                secret_key=ts_secret
            )
            
            # Apply column naming style from settings
            deployer.column_naming_style = self.settings.get('column_naming_style', 'Regular Case')
            deployer.prompt_logger = self._prompt_logger
            
            # Clear and prepare progress capture
            self.live_progress_log = ["=" * 60, "THOUGHTSPOT DEPLOYMENT STARTING", "=" * 60, ""]
            progress_messages = []
            
            def progress_callback(msg):
                progress_messages.append(msg)
                self.live_progress_log.append(msg)
                self.log_feedback(msg)
                safe_print(msg, flush=True)
                # Optional external hook (e.g. the MCP status updater): fires for
                # every deploy line so a live status can track real progress instead
                # of freezing between the generator's sparse yields.
                if on_progress is not None:
                    try:
                        on_progress(msg)
                    except Exception:
                        pass
            
            # Show initial message
            yield {
                'stage': 'thoughtspot',
                'response': """**Starting ThoughtSpot Deployment...**

Authenticating with ThoughtSpot...

**This takes 2-5 minutes.**

**Switch to the "Live Progress" tab** to watch real-time progress.

Steps:
1. Schema creation
2. Data generation
3. Model creation
4. Liveboard creation

This chat will update when complete."""
            }
            
            safe_print("\n" + "="*60, flush=True)
            safe_print("THOUGHTSPOT DEPLOYMENT STARTING", flush=True)
            safe_print("="*60, flush=True)
            safe_print("Watch Live Progress tab for real-time updates...\n", flush=True)
            
            # Load settings
            settings = load_gradio_settings(self._get_effective_user_email())
            liveboard_name = self.settings.get('liveboard_name', '')
            
            # Default liveboard name to company name (without .com) if blank
            if not liveboard_name:
                liveboard_name = company.replace('.com', '').replace('.', ' ').strip().title()
            
            llm_model = settings.get('default_llm', self.settings.get('model', DEFAULT_LLM_MODEL))
            tag_name_value = self.settings.get('tag_name') or settings.get('tag_name')
            naming_prefix = self.settings.get('object_naming_prefix') or settings.get('object_naming_prefix', '')
            # Extract base_name from schema_name (e.g., BLA_01311648_EAX_sch -> BLA_01311648_EAX)
            # DO NOT regenerate - must match what Snowflake deployment used
            base_name = schema_name.replace('_sch', '') if schema_name.endswith('_sch') else schema_name
            
            print(f"🔍 DEBUG: tag_name='{tag_name_value}'")
            
            # Run deployment in a thread with progress spinner (like LegitData)
            import threading
            import time as time_module
            
            ts_result = {"done": False, "results": None, "error": None}
            
            def run_ts_deployment():
                try:
                    ts_result["results"] = deployer.deploy_all(
                        ddl=ddl,
                        database=database,
                        schema=schema_name,
                        base_name=base_name,
                        company_name=company,
                        use_case=use_case,
                        liveboard_name=liveboard_name,
                        llm_model=llm_model,
                        tag_name=tag_name_value,
                        share_with=self.settings.get('share_with', '').strip() or None,
                        company_research=self.demo_builder.get_research_context() if self.demo_builder else None,
                        additional_context=getattr(self, 'generic_use_case_context', '') or '',
                        vertical=getattr(self, 'vertical', None),
                        line=getattr(self, 'line', None),
                        function=getattr(self, 'function', None),
                        progress_callback=progress_callback,
                        session_logger=_slog,
                    )
                except Exception as e:
                    ts_result["error"] = str(e)
                finally:
                    ts_result["done"] = True
            
            ts_thread = threading.Thread(target=run_ts_deployment)
            ts_thread.start()

            # Yield progress updates with spinner while ThoughtSpot deployment runs.
            # ThoughtSpot metadata imports can outlive the 300s gateway timeout; do
            # not mark the run failed while the worker is still running.
            TS_SOFT_WARNING_SECONDS = 2700
            spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
            spinner_idx = 0
            start_time = time_module.time()
            last_msg_count = 0
            last_heartbeat = start_time
            soft_warning_logged = False

            base_progress = """**ThoughtSpot Deployment in Progress...**

Steps:
1. ✅ Authenticating
2. 🔄 Creating connection & tables
3. ⏳ Creating model
4. ⏳ Creating liveboard

**Watch "Live Progress" tab for real-time details.**"""

            while not ts_result["done"]:
                time_module.sleep(2)
                elapsed = time_module.time() - start_time

                if elapsed > TS_SOFT_WARNING_SECONDS and not soft_warning_logged:
                    warning = (
                        f"ThoughtSpot deployment still running after "
                        f"{int(TS_SOFT_WARNING_SECONDS/60)} minutes"
                    )
                    if _slog:
                        _slog.log("thoughtspot", "TS deployment still running", elapsed_s=round(elapsed, 1))
                    progress_messages.append(f"⚠️ {warning}; continuing to wait for the worker to finish")
                    soft_warning_logged = True

                # Heartbeat every 60 seconds
                if _slog and (time_module.time() - last_heartbeat) >= 60:
                    _slog.log_verbose("thoughtspot", f"TS heartbeat - {int(elapsed/60)}m {int(elapsed%60)}s elapsed")
                    last_heartbeat = time_module.time()

                mins = int(elapsed) // 60
                secs = int(elapsed) % 60

                spinner_char = spinner[spinner_idx % len(spinner)]
                spinner_idx += 1

                progress_update = base_progress + f"\n\n{spinner_char} **ThoughtSpot deploying...** ({mins}:{secs:02d} elapsed)"

                # Show recent progress messages from callback
                if len(progress_messages) > last_msg_count:
                    recent_msgs = progress_messages[-3:]
                    progress_update += "\n\n**Recent:**"
                    for msg in recent_msgs:
                        if msg.strip():
                            clean_msg = msg.replace('[ThoughtSpot]', '').replace('[AI Feedback]', '').strip()
                            if clean_msg:
                                progress_update += f"\n  • {clean_msg[:70]}"
                    last_msg_count = len(progress_messages)

                yield progress_update
            
            # Get results from thread
            if ts_result["error"]:
                raise Exception(ts_result["error"])
            
            results = ts_result["results"]
            self.record_deployment_completion(results, database, schema_name, use_case)
            _ts_meta = {
                "schema_name": schema_name,
                "connection": results.get("connection"),
                "model_guid": results.get("model_guid"),
                "liveboard_guid": results.get("liveboard_guid") or results.get("liveboard_id"),
                "liveboard_url": results.get("liveboard_url"),
                "table_names": results.get("tables", []),
                "success": results.get("success"),
                "warnings": results.get("warnings", []),
            }
            
            safe_print("\n" + "="*60, flush=True)
            if results.get('success'):
                safe_print("DEPLOYMENT COMPLETE", flush=True)
                self.live_progress_log.extend(["", "=" * 60, "DEPLOYMENT COMPLETE", "=" * 60])
                # Capture any non-fatal errors (e.g. enhance failure) so they reach session_logs
                if results.get('errors'):
                    _ts_error = '; '.join(results['errors'])
            else:
                safe_print("DEPLOYMENT FAILED", flush=True)
                safe_print(f"Errors: {results.get('errors', [])}", flush=True)
                self.live_progress_log.extend(["", "DEPLOYMENT FAILED", f"Errors: {results.get('errors', [])}"])
                if results.get('errors'):
                    _ts_error = '; '.join(results['errors'])
            safe_print("="*60 + "\n", flush=True)

            notify_deployment_event(
                "DemoPrep deployment",
                "Complete" if results.get("success") else "Failed",
                [
                    ("Company", slack_company),
                    ("Use case", use_case),
                    ("Schema", f"{database}.{schema_name}"),
                    ("Connection", results.get("connection")),
                    ("Model GUID", results.get("model_guid")),
                    ("Liveboard", results.get("liveboard_url") or results.get("liveboard_guid") or results.get("liveboard_id")),
                    ("Errors", "; ".join(results.get("errors", []))[:500] if results.get("errors") else None),
                ],
            )
            
            progress_log = '\n'.join(progress_messages) if progress_messages else 'No progress messages captured'
            
            # Generate Demo Pack on success
            if results.get('success'):
                try:
                    company_name = company.replace('.com', '').replace('.', ' ').title()
                    
                    # Generate use-case specific Spotter questions
                    spotter_questions = self._generate_spotter_questions(use_case, self.ddl_code)
                    spotter_section = "\n".join([f'{i+1}. **"{q["question"]}"** - {q["purpose"]}' 
                                                  for i, q in enumerate(spotter_questions)])
                    
                    # Use-case specific demo tips
                    demo_tips = self._get_demo_tips(use_case)
                    
                    # Build URLs for demo pack
                    ts_base = (self.settings.get('thoughtspot_url') or '').rstrip('/')
                    pack_model_url = f"{ts_base}/#/data/tables/{results.get('model_guid', '')}" if results.get('model_guid') and ts_base else results.get('model', 'N/A')
                    pack_lb_url = results.get('liveboard_url') or (f"{ts_base}/#/pinboard/{liveboard_guid}" if liveboard_guid and ts_base else results.get('liveboard', 'N/A'))
                    
                    self.demo_pack_content = f"""# {company_name} Demo Pack
## {use_case}

*Generated: {__import__('datetime').datetime.now().strftime('%Y-%m-%d %H:%M')}*

---

## Deployment Summary

- **Liveboard:** {pack_lb_url}
- **Model:** {pack_model_url}  
- **Tables:** {len(results.get('tables', []))} imported

---

## Suggested Spotter Questions

Ask these questions to showcase ThoughtSpot's AI capabilities:

{spotter_section}

---

## Demo Flow

1. Start with the overview liveboard to set context
2. Ask a Spotter question to show natural language search
3. Drill into an interesting metric to show interactivity
4. Show Monitor for proactive alerting

---

## Tips for {use_case}

{demo_tips}

---

*Pro tip: Use "what changed" questions to show ThoughtSpot's change detection!*
"""
                    safe_print("Demo Pack generated - check the Demo Pack tab.", flush=True)
                    self.live_progress_log.append("Demo Pack generated")
                except Exception as e:
                    safe_print(f"Could not generate demo pack: {e}", flush=True)
                    self.demo_pack_content = f"*Demo pack generation failed: {e}*"
                
                # Generate Spotter Viz Stories (both matrix and AI versions)
                try:
                    _model_name = results.get('model', None)
                    _model_guid = results.get('model_guid', None)
                    _ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/')
                    if _model_guid and _ts_url:
                        _model_url = f"{_ts_url}/#/data/tables/{_model_guid}"
                    else:
                        _model_url = None

                    # Extract actual column names from the deployed DDL so the story
                    # generator uses real column names instead of idealized matrix ones.
                    _actual_columns = []
                    if self.ddl_code:
                        import re as _re
                        _skip = {'CREATE', 'TABLE', 'IF', 'NOT', 'EXISTS', 'PRIMARY', 'FOREIGN',
                                 'KEY', 'REFERENCES', 'UNIQUE', 'INDEX', 'CONSTRAINT', 'DEFAULT',
                                 'NULL', 'AUTO_INCREMENT', 'IDENTITY'}
                        for _m in _re.finditer(r'^\s+"?([A-Za-z_]\w*)"?\s+\w', self.ddl_code, _re.MULTILINE):
                            _col = _m.group(1)
                            if _col.upper() not in _skip:
                                _actual_columns.append(_col)
                        # Deduplicate while preserving order
                        _seen = set()
                        _actual_columns = [c for c in _actual_columns if not (_seen.add(c) or c in _seen)]

                    _story_args = dict(
                        company_name=company_name,
                        use_case=use_case,
                        model_name=_model_name,
                        model_url=_model_url,
                        liveboard_name=results.get('liveboard', None),
                        actual_columns=_actual_columns[:80],  # cap to avoid prompt bloat
                    )
                    self.spotter_story_matrix = self._generate_matrix_spotter_story(**_story_args)
                    self.spotter_story_ai = self._generate_ai_spotter_story(**_story_args)
                    safe_print("Spotter Viz Stories generated - check the Spotter Viz Story tab.", flush=True)
                    self.live_progress_log.append("Spotter Viz Stories generated")
                except Exception as e:
                    safe_print(f"Could not generate Spotter Viz story: {e}", flush=True)
                    _err = f"*(Generation failed: {e})*"
                    self.spotter_story_matrix = _err
                    self.spotter_story_ai = _err
            
            # Build final response
            if results.get('success'):
                # Safely extract GUIDs - handle None, 'None', 'N/A', empty string
                liveboard_guid = results.get('liveboard_guid') or results.get('liveboard_id')
                if not liveboard_guid or liveboard_guid in ('None', 'N/A', ''):
                    liveboard_guid = None
                
                liveboard_name_result = results.get('liveboard', 'N/A')
                self._liveboard_guid = liveboard_guid
                self._liveboard_name = liveboard_name_result
                
                final_stage = 'deploy'
                
                # Try to load adjuster for outliers stage
                if liveboard_guid:
                    try:
                        from smart_data_adjuster import SmartDataAdjuster
                        
                        # Pass selected LLM model + session-selected TS env to adjuster
                        llm_model = self.settings.get('model', DEFAULT_LLM_MODEL)
                        adjuster = SmartDataAdjuster(
                            database, schema_name, liveboard_guid,
                            llm_model=llm_model,
                            ts_url=self.settings.get('thoughtspot_url') or None,
                            ts_secret=self.settings.get('thoughtspot_trusted_auth_key') or None,
                            username=self._get_effective_user_email(),
                            prompt_logger=self._prompt_logger,
                        )
                        adjuster.connect()
                        
                        if adjuster.load_liveboard_context():
                            self._adjuster = adjuster
                            
                            viz_list = "\n".join([
                                f"  [{i+1}] {v['name']}"
                                for i, v in enumerate(adjuster.visualizations)
                            ])
                            
                            # Build clickable links - validate GUIDs before creating URLs
                            ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/')
                            model_guid = results.get('model_guid') or ''
                            
                            # Only create liveboard URL if we have a valid GUID
                            lb_url = results.get('liveboard_url', '')
                            if not lb_url and liveboard_guid and ts_url:
                                lb_url = f"{ts_url}/#/pinboard/{liveboard_guid}"
                            
                            # Format table names
                            table_names = results.get('tables', [])
                            tables_list = ', '.join(table_names) if table_names else 'N/A'
                            
                            # Build URLs for easy Slack pasting
                            model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else None
                            
                            response = f"""**ThoughtSpot Deployment Complete**

**Created:**
- Connection: {results.get('connection', 'N/A')}
- Tables: {tables_list}
- Model: {model_url if model_url else results.get('model', 'N/A')}
- Liveboard: {lb_url if lb_url else liveboard_name_result}

**Your demo is ready!** 🎉

---

**🎯 Ready for Outlier Adjustments!**

I've loaded your liveboard context. Here are the visualizations:

{viz_list}

**What you can do:**
- Naturally request changes: "make 1080p webcam 40B"
- Adjust by percentage: "increase smart watch by 20%"
- Reference by viz number: "viz 3, increase laptop to 50B"

**Try an adjustment now, or type 'done' to finish!**"""
                            final_stage = 'outlier_adjustment'
                        else:
                            ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/')
                            model_guid = results.get('model_guid') or ''
                            lb_url = results.get('liveboard_url', '')
                            if not lb_url and liveboard_guid and ts_url:
                                lb_url = f"{ts_url}/#/pinboard/{liveboard_guid}"
                            table_names = results.get('tables', [])
                            tables_list = ', '.join(table_names) if table_names else 'N/A'
                            
                            # Build URLs for easy Slack pasting
                            model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else None
                            
                            response = f"""**ThoughtSpot Deployment Complete**

**Created:**
- Connection: {results.get('connection', 'N/A')}
- Tables: {tables_list}
- Model: {model_url if model_url else results.get('model', 'N/A')}
- Liveboard: {lb_url if lb_url else liveboard_name_result}

Your demo is ready!

Note: Could not load liveboard context for adjustments.
Type **'done'** to finish."""
                    except Exception as e:
                        self.log_feedback(f"Failed to load liveboard context: {e}")
                        ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/')
                        model_guid = results.get('model_guid') or ''
                        lb_url = results.get('liveboard_url', '')
                        if not lb_url and liveboard_guid and ts_url:
                            lb_url = f"{ts_url}/#/pinboard/{liveboard_guid}"
                        table_names = results.get('tables', [])
                        tables_list = ', '.join(table_names) if table_names else 'N/A'
                        
                        # Build URLs for easy Slack pasting
                        model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else None
                        
                        response = f"""**ThoughtSpot Deployment Complete**

**Created:**
- Connection: {results.get('connection', 'N/A')}
- Tables: {tables_list}
- Model: {model_url if model_url else results.get('model', 'N/A')}
- Liveboard: {lb_url if lb_url else liveboard_name_result}

Your demo is ready!

Note: Could not load liveboard context for adjustments: {str(e)}
Type **'done'** to finish."""
                else:
                    # Deployed OK but no liveboard GUID returned — treat as partial success
                    ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/')
                    model_guid = results.get('model_guid') or ''
                    table_names = results.get('tables', [])
                    tables_list = ', '.join(table_names) if table_names else 'N/A'
                    model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else results.get('model', 'N/A')

                    response = f"""⚠️ **Partial Success — Dataset & Model Created**

Your Snowflake data and ThoughtSpot model were deployed successfully.
The liveboard GUID couldn't be retrieved — it may still have been created in ThoughtSpot.

**Created:**
- Connection: {results.get('connection', 'N/A')}
- Tables: {tables_list}
- Model: {model_url}

**What you can do:**
- Check the **Spotter Viz Story** tab to recreate the liveboard manually
- Log into ThoughtSpot to verify whether the liveboard was created
- Type **'retry liveboard'** to try building it again

Type **'done'** to finish."""

                # Surface data-quality warnings from the load gate in the completion panel
                _dq_warnings = getattr(self, '_dq_warnings', None)
                if _dq_warnings:
                    response += "\n\n---\n\n**⚠️ Data quality warnings:**\n" + \
                        "\n".join(f"- {w}" for w in _dq_warnings)

                yield {'response': response, 'stage': final_stage}
            else:
                errors = results.get('errors', ['Unknown error'])
                error_details = '\n'.join(errors)

                # Check for partial success: Snowflake + model deployed OK but liveboard failed
                model_ok = bool(results.get('model_guid'))
                liveboard_errors = [e for e in errors if 'liveboard' in e.lower()]
                non_liveboard_errors = [e for e in errors if 'liveboard' not in e.lower()]

                if model_ok and liveboard_errors and not non_liveboard_errors:
                    # Partial success — dataset and model are live, only liveboard failed
                    ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/')
                    model_guid = results.get('model_guid') or ''
                    table_names = results.get('tables', [])
                    tables_list = ', '.join(table_names) if table_names else 'N/A'
                    model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else results.get('model', 'N/A')
                    lb_error = liveboard_errors[0]

                    yield {
                        'response': f"""⚠️ **Partial Success — Dataset & Model Created**

Your Snowflake data and ThoughtSpot model were deployed successfully.
The liveboard couldn't be built automatically.

**Created:**
- Connection: {results.get('connection', 'N/A')}
- Tables: {tables_list}
- Model: {model_url}

**Liveboard error:**
```
{lb_error}
```

**What you can do:**
- Check the **Spotter Viz Story** tab — use that sequence to recreate the liveboard manually in Spotter Viz
- Type **'retry liveboard'** to try building it again
- Or continue in ThoughtSpot using your model directly""",
                        'stage': 'deploy'
                    }
                else:
                    if 'schema validation' in error_details.lower() or 'schema' in error_details.lower():
                        guidance = "**Root Cause:** The model TML has validation errors."
                    elif 'connection' in error_details.lower():
                        guidance = "**Root Cause:** Connection issue with ThoughtSpot or Snowflake"
                    elif 'authenticate' in error_details.lower() or 'auth' in error_details.lower():
                        guidance = "**Root Cause:** Authentication failed"
                    else:
                        guidance = "**Check the progress log above for details.**"

                    yield {
                        'response': f"""❌ **ThoughtSpot Deployment Failed**

**Error Details:**
```
{error_details}
```

**Progress Log:**
```
{progress_log}
```
{guidance}

**Next Steps:**
- Type **'retry'** to try again
- Or fix the issues above first""",
                        'stage': 'deploy'
                    }
                
        except Exception as e:
            import traceback
            error_details = traceback.format_exc()
            _ts_error = str(e)
            self.log_feedback(f"❌ ThoughtSpot deployment error: {error_details}")
            try:
                from slack_notifier import notify_deployment_event
                notify_deployment_event(
                    "DemoPrep deployment",
                    "Failed",
                    [
                        ("Company", slack_company),
                        ("Use case", use_case),
                        ("Schema", schema_name),
                        ("Error", str(e)[:500]),
                    ],
                )
            except Exception:
                pass

            yield {
                'response': f"""❌ **ThoughtSpot Deployment Error**

**Error:** {str(e)}

**Full Details:**
```
{error_details}
```

**Common Causes:**
- Missing ThoughtSpot credentials in .env
- Snowflake connection issues
- Invalid schema or table names

**Next Steps:**
- Verify credentials are correct
- Type **'retry'** to try again""",
                'stage': 'deploy'
            }
        finally:
            if _slog and _t_ts:
                _slog.log_end("thoughtspot", _t_ts, error=_ts_error, **_ts_meta)

    def process_regular_message(self, message, current_stage, company, use_case):
        """Process regular chat messages"""
        message_lower = message.lower()
        
        # Check if message contains company and use case info
        if "creating a demo for" in message_lower or "create a demo for" in message_lower:
            # Extract company and use case from message
            extracted_company = self.extract_company_from_message(message)
            extracted_use_case = self.extract_use_case_from_message(message)
            
            if extracted_company:
                company = extracted_company
            if extracted_use_case:
                # Use vertical × function system to resolve use case
                v, f = parse_use_case(extracted_use_case)
                config = get_use_case_config(v or "Generic", f or "Generic")
                use_case = config.get('use_case_name', extracted_use_case)
            
            # Automatically trigger research
            return f"""✅ **Got it!**

**Company:** {company}
**Use Case:** {use_case}

🔍 **Starting Research...**

I'll analyze {company} and research {use_case} best practices.
This will take about 2-3 minutes.

(Type 'stop' if you want to change anything)"""
        
        # Simple intent detection (Phase 1)
        if any(word in message_lower for word in ['research', 'start', 'begin', 'analyze']):
            # Just confirm and tell them to use the proper format
            return f"""To start research, please use this format:

```
I'm creating a demo for company: {company} use case: {use_case}
```

This will automatically begin the research process!"""
        
        elif any(word in message_lower for word in ['configure', 'settings', 'change']):
            return """⚙️ **Configuration Options**

You can change:
- **Company**: `/over company: [new company]`
- **Use Case**: `/over usecase: [new use case]`
- **AI Model**: Use the dropdown on the right →

What would you like to adjust?"""
        
        elif 'help' in message_lower:
            return """💡 **How to Use This Interface**

**Commands:**
- `/over company: [name]` - Change company
- `/over usecase: [case]` - Change use case

**What You Can Say:**
- "Start research" - Begin demo creation
- "Configure settings" - Adjust parameters
- "What stage are we at?" - Check progress
- Ask any question naturally!

**Current Stage:** You can see it on the right side →
**AI Model:** Editable in the dropdown on the right →

What would you like to do?"""
        
        elif any(word in message_lower for word in ['stage', 'progress', 'status', 'where']):
            return f"""📊 **Current Status**

**Stage:** {current_stage.replace('_', ' ').title()}
**Company:** {company}
**Use Case:** {use_case}

**What's Next:**
Tell me what you'd like to do, and I'll guide you through the process!"""
        
        else:
            # Generic helpful response - shouldn't normally reach here
            return f"""I'm ready to work on **{use_case}** for **{company}**.

To get started, just say:
```
I'm creating a demo for company: {company} use case: {use_case}
```

Or if you want to change something, use `/over` to adjust."""


def create_chat_interface():
    """
    Create the new chat-based demo builder interface
    
    IMPORTANT: Each browser session gets its own ChatDemoInterface instance
    via gr.State() to enable multi-user concurrent access.
    """
    
    # Bootstrap defaults before authenticated user-specific settings are loaded
    default_settings = {
        "company": "",
        "use_case": "",
        "model": DEFAULT_LLM_MODEL,
        "stage": "initialization",
    }
    
    _remember_me_js = """
(function() {
    var KEY = 'demoprep_remembered_user';
    var _done = false;
    function injectIntoForm(loginRoot) {
        if (_done || !loginRoot) return;
        var formDiv = loginRoot.querySelector('div.form');
        if (!formDiv) return;
        var inputs = formDiv.querySelectorAll('input, textarea');
        var uInput = null;
        for (var i = 0; i < inputs.length; i++) {
            var tp = (inputs[i].type || inputs[i].tagName).toLowerCase();
            if (tp !== 'password' && tp !== 'submit' && tp !== 'checkbox' && tp !== 'hidden') {
                uInput = inputs[i]; break;
            }
        }
        if (!uInput || loginRoot.querySelector('#dp-rmb')) return;
        var saved = localStorage.getItem(KEY);
        if (saved && !uInput.value) {
            try {
                var desc = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value') ||
                           Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value');
                if (desc && desc.set) desc.set.call(uInput, saved);
                else uInput.value = saved;
                uInput.dispatchEvent(new Event('input', { bubbles: true }));
            } catch(e) { uInput.value = saved; }
            var pwInput = formDiv.querySelector('input[type="password"]');
            if (pwInput) setTimeout(function() { pwInput.focus(); }, 50);
        }
        var lbl = document.createElement('label');
        lbl.style.cssText = 'display:flex;align-items:center;gap:6px;font-size:13px;margin:8px 0 12px;cursor:pointer;color:#374151;';
        var cb = document.createElement('input');
        cb.type = 'checkbox'; cb.id = 'dp-rmb'; cb.checked = !!saved; cb.style.cursor = 'pointer';
        lbl.appendChild(cb);
        lbl.appendChild(document.createTextNode(' Remember me'));
        var btn = loginRoot.querySelector('button');
        if (btn && btn.parentNode) btn.parentNode.insertBefore(lbl, btn);
        else loginRoot.appendChild(lbl);
        function save() {
            if (cb.checked && uInput.value) localStorage.setItem(KEY, uInput.value);
            else localStorage.removeItem(KEY);
        }
        if (btn) btn.addEventListener('click', save);
        _done = true;
        console.log('[DemoPrep] Remember me injected');
    }
    function tryInject() {
        if (_done) return;
        if (!window.gradio_config || !window.gradio_config.auth_required) return;
        var wrap = document.querySelector('div.wrap');
        if (wrap) { injectIntoForm(wrap); if (_done) return; }
        document.querySelectorAll('div.form').forEach(function(f) {
            if (!_done) injectIntoForm(f.parentElement);
        });
    }
    var t = setInterval(function() { tryInject(); if (_done) clearInterval(t); }, 200);
    setTimeout(function() { clearInterval(t); }, 10000);
})();
"""

    with gr.Blocks(
        title=f"{'[TEST] ' if IS_TEST else ''}ThoughtSpot Demo Builder - Chat",
        theme=gr.themes.Soft(primary_hue="blue", secondary_hue="cyan"),
        css="""
.tabitem { padding-top: 6px !important; }
/* Remove chatbot box — let content sit directly on page */
#main-chatbot { border: none !important; background: transparent !important; box-shadow: none !important; padding: 0 !important; }
#main-chatbot > div { border: none !important; background: transparent !important; box-shadow: none !important; }
#main-chatbot .bubble-wrap { background: transparent !important; padding-top: 0 !important; padding-bottom: 0 !important; }
#main-chatbot .wrap { background: transparent !important; border: none !important; }
#env-banner { background: #92400e; color: #fef3c7; padding: 8px 16px; text-align: center; font-weight: 600; font-size: 14px; border-radius: 6px; margin-bottom: 8px; }
""",
    ) as interface:
        
        # SESSION STATE: Each user gets their own ChatDemoInterface instance
        # This is the key fix for multi-user support!
        chat_controller_state = gr.State(None)  # Initialized on first interaction
        
        # State variables
        current_stage = gr.State(default_settings['stage'])
        current_model = gr.State(default_settings['model'])
        current_company = gr.State(default_settings['company'])
        current_usecase = gr.State(default_settings['use_case'])
        current_liveboard_name = gr.State(default_settings.get('liveboard_name', ''))
        
        # Test environment banner
        if IS_TEST:
            gr.HTML('<div id="env-banner">⚠️ TEST ENVIRONMENT — thoughtspot-dp-test-demoprep.hf.space</div>')

        # Header with logout link
        with gr.Row(equal_height=True):
            gr.Markdown("""
            # 💬 ThoughtSpot Demo Builder
            ### AI-Powered Conversational Demo Creation
            """)
            gr.HTML("""
            <div style="display:flex; align-items:center; justify-content:flex-end; padding:8px 0;">
                <a href="/logout" style="
                    color: #6b7280;
                    text-decoration: none;
                    font-size: 14px;
                    padding: 6px 14px;
                    border: 1px solid #d1d5db;
                    border-radius: 6px;
                    white-space: nowrap;
                ">
                    Sign Out →
                </a>
            </div>
            """)
        
        # Additional state for new tabs
        ai_feedback_state = gr.State("")
        ddl_code_state = gr.State("")
        population_code_state = gr.State("")
        live_progress_state = gr.State("")
        demo_pack_state = gr.State("")

        with gr.Group(visible=False) as password_gate:
            gr.Markdown("""
            ## Change Password Required

            You are signed in with a temporary password. Set a new password before using DemoPrep.
            """)
            with gr.Row():
                with gr.Column(scale=1):
                    gate_current_password = gr.Textbox(
                        label="Temporary Password",
                        type="password",
                        placeholder="Enter the password you just used to sign in",
                    )
                    gate_new_password = gr.Textbox(
                        label="New Password",
                        type="password",
                        placeholder="At least 8 characters",
                    )
                    gate_confirm_password = gr.Textbox(
                        label="Confirm New Password",
                        type="password",
                        placeholder="Repeat new password",
                    )
                    gate_change_password_btn = gr.Button("Change Password", variant="primary")
                    gate_password_status = gr.Markdown("")
        
        with gr.Tabs(visible=True) as main_tabs:
            with gr.Tab("📱 App"):
                chat_components = create_chat_tab(
                    chat_controller_state, default_settings, current_stage, current_model, 
                    current_company, current_usecase,
                    ai_feedback_state, ddl_code_state, population_code_state
                )
            
            with gr.Tab("🤖 AI Feedback"):
                gr.Markdown("### AI Processing")
                gr.Markdown("*Research, LLM calls, and AI decisions*")
                ai_feedback_display = gr.TextArea(
                    label="AI Processing Log",
                    value="",
                    lines=15,
                    max_lines=15,
                    interactive=False,
                    show_label=False
                )
                gr.Markdown("### Deploy Log")
                gr.Markdown("*Snowflake + ThoughtSpot deployment steps*")
                live_progress_display = gr.TextArea(
                    label="Deploy Log",
                    value="Waiting for deployment to start...\n\nGo to Chat tab and type 'thoughtspot' or 'deploy' to begin.",
                    lines=15,
                    max_lines=15,
                    interactive=False,
                    show_label=False,
                    elem_classes=["live-progress-area"]
                )

                # Timer-based refresh for live progress (every 2 seconds)
                # INACTIVE by default - activated when deployment starts, deactivated when done
                # Otherwise idle tabs consume concurrency slots and block new sessions
                live_progress_timer = gr.Timer(value=2, active=False)

                def refresh_live_progress(controller):
                    """Poll and return current progress log"""
                    if controller is None:
                        return "Waiting for deployment to start...\n\nGo to Chat tab and type 'thoughtspot' or 'deploy' to begin."
                    live_progress = getattr(controller, 'live_progress_log', [])
                    if live_progress:
                        return "\n".join(live_progress)
                    return "Waiting for deployment to start...\n\nGo to Chat tab and type 'thoughtspot' or 'deploy' to begin."

                live_progress_timer.tick(
                    fn=refresh_live_progress,
                    inputs=[chat_controller_state],
                    outputs=[live_progress_display]
                )
            
            with gr.Tab("🎬 Demo Assets"):
                _spotter_default = "Story will be generated after liveboard creation.\n\n**What is Spotter Viz?** An AI agent in ThoughtSpot that builds Liveboards through natural language — type a request, it creates and refines the dashboard step by step."
                with gr.Tabs():
                    with gr.Tab("🗺️ SpotterViz TS"):
                        gr.Markdown("*Built from the ThoughtSpot-recommended KPIs and visualizations for this vertical × function.*")
                        spotter_matrix_display = gr.Markdown(
                            value=_spotter_default,
                            elem_classes=["spotter-viz-story-content"]
                        )
                    with gr.Tab("✨ SpotterViz AI"):
                        gr.Markdown("*Pure AI — no matrix constraints. What the AI thinks would make a compelling liveboard story.*")
                        spotter_ai_display = gr.Markdown(
                            value=_spotter_default,
                            elem_classes=["spotter-viz-story-content"]
                        )
                    with gr.Tab("📋 Demo Pack"):
                        gr.Markdown("### Demo Pack - Talking Points & Spotter Questions")
                        gr.Markdown("*Generated automatically after deployment completes*")
                        demo_pack_display = gr.Markdown(
                            value="Demo pack will be generated after deployment completes.\n\nThis will include:\n- Key insights/outliers\n- Spotter questions to ask\n- Talking points for the demo",
                            elem_classes=["demo-pack-content"]
                        )

            with gr.Tab("⚙️ Settings"):
                settings_components = create_settings_tab()

            with gr.Tab("📄 DDL Code"):
                ddl_display = gr.Code(
                    label="Generated DDL",
                    language="sql",
                    value="-- DDL will appear here after generation",
                    lines=30,
                    interactive=False
                )

            with gr.Tab("👑 Admin", visible=False) as admin_tab:
                with gr.Tabs():
                    with gr.Tab("👤 User Management") as user_mgmt_tab:
                        gr.Markdown("### User Management (Admin Only)")
                        gr.Markdown("*Add, deactivate, or reset passwords for DemoPrep users.*")

                        with gr.Row():
                            with gr.Column(scale=2):
                                gr.Markdown("#### Current Users")
                                user_list_display = gr.Dataframe(
                                    headers=["Email", "Display Name", "Admin", "Active", "Must Change PW", "Last Login EST"],
                                    datatype=["str", "str", "bool", "bool", "bool", "str"],
                                    interactive=False,
                                    label="Users"
                                )
                                refresh_users_btn = gr.Button("🔄 Refresh User List", size="sm")

                            with gr.Column(scale=1):
                                gr.Markdown("#### Add New User")
                                new_user_email = gr.Textbox(label="Email", placeholder="user@company.com")
                                new_user_password = gr.Textbox(
                                    label="Temporary Password (optional)",
                                    type="password",
                                    placeholder="Leave blank to generate one"
                                )
                                new_user_display = gr.Textbox(label="Display Name", placeholder="Jane Doe")
                                new_user_admin = gr.Checkbox(label="Admin?", value=False)
                                add_user_btn = gr.Button("➕ Add User", variant="primary")
                                invite_message = gr.Textbox(
                                    label="Slack invite message",
                                    lines=9,
                                    interactive=True,
                                    show_copy_button=True,
                                    placeholder="Add a user to generate the message to send in Slack."
                                )

                                gr.Markdown("---")
                                gr.Markdown("#### User Actions")
                                action_email = gr.Textbox(label="User Email (for actions below)")
                                with gr.Row():
                                    deactivate_btn = gr.Button("🚫 Deactivate", size="sm")
                                    activate_btn = gr.Button("✅ Activate", size="sm")
                                new_password = gr.Textbox(label="New Password", type="password")
                                reset_pw_btn = gr.Button("🔑 Reset Password", size="sm")

                                user_mgmt_status = gr.Textbox(label="Status", interactive=False)

                        def load_user_list():
                            """Load user list from Supabase."""
                            try:
                                from supabase_client import UserManager
                                from zoneinfo import ZoneInfo
                                um = UserManager()
                                users = um.list_users()
                                users = sorted(users, key=lambda u: u.get('last_login') or '', reverse=True)
                                eastern = ZoneInfo('America/New_York')
                                rows = []
                                for u in users:
                                    raw_login = u.get('last_login')
                                    if raw_login:
                                        try:
                                            from datetime import datetime
                                            dt = datetime.fromisoformat(str(raw_login).replace('Z', '+00:00'))
                                            last_login_str = dt.astimezone(eastern).strftime('%Y-%m-%d %H:%M')
                                        except Exception:
                                            last_login_str = str(raw_login)[:19]
                                    else:
                                        last_login_str = 'Never'
                                    rows.append([
                                        u.get('email', ''),
                                        u.get('display_name', ''),
                                        u.get('is_admin', False),
                                        u.get('is_active', True),
                                        u.get('must_change_password', False),
                                        last_login_str
                                    ])
                                return rows
                            except Exception as e:
                                return [[f"Error: {e}", "", False, False, False, ""]]
                
                        def add_user_handler(email, password, display_name, is_admin, request: gr.Request = None):
                            """Add a new user."""
                            if not email:
                                return load_user_list(), "Email is required.", ""
                            try:
                                from supabase_client import UserManager
                                um = UserManager()
                                temp_password = password or um.generate_temp_password()
                                success = um.add_user(
                                    email,
                                    temp_password,
                                    display_name,
                                    is_admin,
                                    must_change_password=True,
                                )
                                if success:
                                    clean_email = email.lower().strip()
                                    display = (display_name or clean_email.split("@")[0]).strip()
                                    app_url = resolve_app_url_for_invite(request).rstrip("/") + "/"
                                    invite = (
                                        f"Hi {display}, welcome to DemoPrep.\n\n"
                                        f"The app is located here: {app_url}\n"
                                        "The quick start guide and other documentation are here: "
                                        "https://thoughtspot-dp-demoprep-doc.static.hf.space/index.html\n\n"
                                        f"Username: {clean_email}\n"
                                        f"Temporary password: {temp_password}\n\n"
                                        "When you first sign in, DemoPrep will ask you to choose a new password. "
                                        "After that, you can start building demos from the main screen.\n\n"
                                        "If you have any problems, please Slack Mike Boone."
                                    )
                                    if not um._supports_must_change_password():
                                        invite += (
                                            "\n\nAdmin note: temporary-password enforcement is not active "
                                            "until the demoprep_users.must_change_password migration is applied."
                                        )
                                    return load_user_list(), f"User {clean_email} added. Slack invite generated below.", invite
                                else:
                                    return load_user_list(), f"Failed to add user {email}.", ""
                            except Exception as e:
                                return load_user_list(), f"Error: {e}", ""

                        def deactivate_handler(email):
                            if not email:
                                return load_user_list(), "Enter an email first."
                            try:
                                from supabase_client import UserManager
                                um = UserManager()
                                um.deactivate_user(email)
                                return load_user_list(), f"User {email} deactivated."
                            except Exception as e:
                                return load_user_list(), f"Error: {e}"

                        def activate_handler(email):
                            if not email:
                                return load_user_list(), "Enter an email first."
                            try:
                                from supabase_client import UserManager
                                um = UserManager()
                                um.activate_user(email)
                                return load_user_list(), f"User {email} activated."
                            except Exception as e:
                                return load_user_list(), f"Error: {e}"

                        def reset_password_handler(email, new_pw):
                            if not email or not new_pw:
                                return "Enter email and new password."
                            try:
                                from supabase_client import UserManager
                                um = UserManager()
                                um.reset_password(email, new_pw, must_change_password=True)
                                return f"Password reset for {email}."
                            except Exception as e:
                                return f"Error: {e}"

                        refresh_users_btn.click(fn=load_user_list, inputs=[], outputs=[user_list_display])
                        add_user_btn.click(fn=add_user_handler, inputs=[new_user_email, new_user_password, new_user_display, new_user_admin], outputs=[user_list_display, user_mgmt_status, invite_message])
                        deactivate_btn.click(fn=deactivate_handler, inputs=[action_email], outputs=[user_list_display, user_mgmt_status])
                        activate_btn.click(fn=activate_handler, inputs=[action_email], outputs=[user_list_display, user_mgmt_status])
                        reset_pw_btn.click(fn=reset_password_handler, inputs=[action_email, new_password], outputs=[user_mgmt_status])
                        interface.load(fn=load_user_list, inputs=[], outputs=[user_list_display])

                    with gr.Tab("⚙️ Admin Settings") as admin_settings_tab:
                        gr.Markdown("### System-Wide Settings")
                        gr.Markdown("These settings apply to all users. Only admins can view and edit.")

                        # Hidden fields — values still saved/loaded but not shown in UI
                        admin_ts_url = gr.Textbox(visible=False)
                        admin_openai_key = gr.Textbox(visible=False)
                        admin_google_key = gr.Textbox(visible=False)

                        with gr.Row():
                            with gr.Column():
                                gr.Markdown("#### ThoughtSpot Connection")
                                admin_share_with = gr.Textbox(
                                    label="Default Share With (User or Group)",
                                    placeholder="user@company.com or group-name",
                                    info="System-wide default: model + liveboard shared here after every build"
                                )
                                admin_log_level = gr.Dropdown(
                                    label="Log Level",
                                    choices=["off", "regular", "verbose"],
                                    value="regular",
                                    info="off = no Supabase logging · regular = stage start/end + TS milestones · verbose = full sub-step detail"
                                )

                            with gr.Column():
                                gr.Markdown("#### Snowflake Connection")
                                admin_sf_account = gr.Textbox(label="Snowflake Account")
                                admin_sf_kp_user = gr.Textbox(label="Key Pair User")
                                admin_sf_kp_pk = gr.Textbox(label="Private Key (PEM)", lines=3, type="password")
                                admin_sf_kp_pass = gr.Textbox(label="Private Key Passphrase", type="password")
                                admin_sf_role = gr.Textbox(label="Role")
                                admin_sf_warehouse = gr.Textbox(label="Warehouse")
                                admin_sf_database = gr.Textbox(label="Database")
                                admin_sf_sso_user = gr.Textbox(label="SSO User (for Snowflake browser auth)")
                
                        with gr.Row():
                            load_admin_btn = gr.Button("🔄 Load Current Settings", size="sm")
                            save_admin_btn = gr.Button("💾 Save Admin Settings", variant="primary", size="sm")
                        admin_settings_status = gr.Textbox(label="Status", interactive=False)

                        admin_fields = [
                            admin_ts_url,
                            admin_openai_key, admin_google_key,
                            admin_sf_account, admin_sf_kp_user, admin_sf_kp_pk,
                            admin_sf_kp_pass, admin_sf_role, admin_sf_warehouse,
                            admin_sf_database, admin_sf_sso_user,
                            admin_share_with,
                            admin_log_level,
                        ]

                        admin_keys_order = [
                            "THOUGHTSPOT_URL",
                            "OPENAI_API_KEY", "GOOGLE_API_KEY",
                            "SNOWFLAKE_ACCOUNT", "SNOWFLAKE_KP_USER", "SNOWFLAKE_KP_PK",
                            "SNOWFLAKE_KP_PASSPHRASE", "SNOWFLAKE_ROLE", "SNOWFLAKE_WAREHOUSE",
                            "SNOWFLAKE_DATABASE", "SNOWFLAKE_SSO_USER",
                            "SHARE_WITH",
                            "LOG_LEVEL",
                        ]

                        def load_admin_settings_handler():
                            try:
                                from supabase_client import load_admin_settings
                                settings = load_admin_settings(force_refresh=True)
                                values = [settings.get(k, "") for k in admin_keys_order]
                                return values + ["Settings loaded from Supabase."]
                            except Exception as e:
                                return [""] * len(admin_keys_order) + [f"Error loading: {e}"]

                        def save_admin_settings_handler(*field_values):
                            try:
                                from supabase_client import save_admin_settings
                                settings_dict = {key: (val or "") for key, val in zip(admin_keys_order, field_values)}
                                success = save_admin_settings(settings_dict)
                                return "✅ Admin settings saved to Supabase and applied." if success else "❌ Failed to save some admin settings."
                            except Exception as e:
                                return f"❌ Error saving: {e}"

                        load_admin_btn.click(fn=load_admin_settings_handler, inputs=[], outputs=admin_fields + [admin_settings_status])
                        save_admin_btn.click(fn=save_admin_settings_handler, inputs=admin_fields, outputs=[admin_settings_status])
                        interface.load(fn=load_admin_settings_handler, inputs=[], outputs=admin_fields + [admin_settings_status])

                        # --- Session Log Viewer ---
                        gr.Markdown("---")
                        gr.Markdown("### 📋 Session Logs")

                        with gr.Row():
                            log_user_filter = gr.Textbox(label="Filter by user (email, blank=all)", scale=2)
                            log_limit = gr.Dropdown(label="Show", choices=["25", "50", "100"], value="50", scale=1)
                            log_refresh_btn = gr.Button("🔄 Refresh", scale=1)

                        session_log_display = gr.Dataframe(
                            headers=["Time", "User", "Stage", "Event", "Duration (ms)", "Error"],
                            label="Recent Sessions",
                            interactive=False,
                            wrap=True,
                        )

                        def load_session_logs(user_filter, limit):
                            try:
                                from supabase_client import SupabaseSettings
                                ss = SupabaseSettings()
                                if not ss.is_enabled():
                                    return [["Supabase not configured", "", "", "", "", ""]]
                                query = ss.client.table("session_logs").select(
                                    "ts,user_email,stage,event,duration_ms,error"
                                ).order("ts", desc=True).limit(int(limit))
                                if user_filter and user_filter.strip():
                                    query = query.ilike("user_email", f"%{user_filter.strip()}%")
                                result = query.execute()
                                rows = []
                                for r in result.data:
                                    ts = r.get("ts", "")[:19].replace("T", " ")
                                    rows.append([ts, r.get("user_email", ""), r.get("stage", ""), r.get("event", ""), str(r.get("duration_ms", "") or ""), r.get("error", "") or ""])
                                return rows if rows else [["No logs found", "", "", "", "", ""]]
                            except Exception as e:
                                return [[f"Error: {e}", "", "", "", "", ""]]

                        log_refresh_btn.click(fn=load_session_logs, inputs=[log_user_filter, log_limit], outputs=[session_log_display])

                    with gr.Tab("📝 Prompt Log") as prompt_log_tab:
                        gr.Markdown("### Prompt Log — What We Send to LLMs")
                        gr.Markdown("*Every prompt and response is logged here for review. Updates every 5 seconds.*")
                        prompt_log_display = gr.Markdown(
                            value="No prompts logged yet. Start a demo build to see LLM calls here.",
                            elem_classes=["prompt-log-content"]
                        )
                        prompt_log_timer = gr.Timer(value=5, active=True)

                        def refresh_prompt_log(controller):
                            try:
                                if controller is not None and controller._prompt_logger is not None:
                                    return controller._prompt_logger.get_summary()
                            except Exception:
                                pass
                            return "No prompts logged yet."

                        prompt_log_timer.tick(fn=refresh_prompt_log, inputs=[chat_controller_state], outputs=[prompt_log_display])

                    with gr.Tab("🧩 Matrix"):
                        matrix_components = create_matrix_tab(interface)

                    with gr.Tab("📊 Run History"):
                        gr.Markdown("### Pipeline Run History")
                        gr.Markdown("*Every pipeline run — who ran it, whether it succeeded, and where it failed.*")
                        with gr.Row():
                            run_history_refresh_btn = gr.Button("🔄 Refresh", size="sm")
                            run_history_email_filter = gr.Textbox(label="Filter by email", placeholder="user@company.com", scale=2)
                            run_history_show_tests = gr.Checkbox(
                                label="Show test runs",
                                value=False,
                                scale=1,
                            )
                            run_history_limit = gr.Dropdown(
                                label="Show",
                                choices=["10", "50", "100", "All"],
                                value="10",
                                scale=1,
                            )
                        run_history_display = gr.Dataframe(
                            headers=["Time (UTC)", "User", "Company", "Use Case", "Interface", "Status", "Failed At", "Duration"],
                            datatype=["str", "str", "str", "str", "str", "str", "str", "str"],
                            column_widths=["130px", "200px", "130px", "150px", "100px", "90px", "200px", "80px"],
                            interactive=False,
                            label="Runs",
                            wrap=False,
                        )
                        full_use_cases_state = gr.State([])
                        use_case_detail = gr.Textbox(
                            label="Full Use Case",
                            lines=4,
                            interactive=False,
                            visible=False,
                        )

                        def load_run_history(email_filter="", show_tests=False, limit_choice="10"):
                            try:
                                from supabase_client import SupabaseSettings
                                from datetime import datetime as _dt, timezone as _timezone
                                ss = SupabaseSettings()
                                if not ss.is_enabled():
                                    return [["Supabase not configured", "", "", "", "", "", ""]]

                                display_limit = None if limit_choice == "All" else int(limit_choice)
                                # Fetch enough raw rows to aggregate into desired number of sessions
                                fetch_limit = 2000 if limit_choice == "All" else max(500, (display_limit or 10) * 20)

                                query = ss.client.table("session_logs") \
                                    .select("session_id,user_email,ts,stage,event,duration_ms,error,meta") \
                                    .order("ts", desc=True) \
                                    .limit(fetch_limit)
                                if email_filter and email_filter.strip():
                                    query = query.eq("user_email", email_filter.strip())
                                elif not show_tests:
                                    query = query.neq("user_email", "testrunner@thoughtspot.com")
                                result = query.execute()
                                rows = result.data or []

                                # Group by session_id
                                sessions = {}
                                for row in rows:
                                    sid = row.get('session_id', '')
                                    if not sid:
                                        continue
                                    if sid not in sessions:
                                        sessions[sid] = {
                                            'user': row.get('user_email', ''),
                                            'events': [],
                                            'start_ts': row.get('ts', ''),
                                            'end_ts': row.get('ts', ''),
                                            'errors': [],
                                            'stages': [],
                                            'meta': {},
                                            'last_stage': row.get('stage', ''),
                                            'last_event': row.get('event', ''),
                                            'row_count': 0,
                                        }
                                    s = sessions[sid]
                                    s['row_count'] += 1
                                    s['events'].append(row.get('event', ''))
                                    ts = row.get('ts', '')
                                    if ts and ts < s['start_ts']:
                                        s['start_ts'] = ts
                                    if ts and ts > s['end_ts']:
                                        s['end_ts'] = ts
                                        s['last_stage'] = row.get('stage', '')
                                        s['last_event'] = row.get('event', '')
                                    if row.get('error'):
                                        s['errors'].append(f"{row.get('stage','?')}: {row.get('error','')[:100]}")
                                    stage = row.get('stage')
                                    if stage and stage not in s['stages']:
                                        s['stages'].append(stage)
                                    if row.get('meta'):
                                        s['meta'].update(row.get('meta') or {})

                                sorted_sessions = sorted(sessions.items(), key=lambda x: x[1]['start_ts'], reverse=True)
                                if display_limit:
                                    sorted_sessions = sorted_sessions[:display_limit]

                                display_rows = []
                                full_use_cases = []
                                now_utc = _dt.now(_timezone.utc)
                                for sid, s in sorted_sessions:
                                    company = s['meta'].get('company', '') or s['meta'].get('company_name', '')
                                    use_case = s['meta'].get('use_case', '')
                                    interface = s['meta'].get('interface', '')

                                    terminal_events = set(e or "" for e in s['events'])
                                    has_explicit_failure = any(e.endswith('failed') for e in s['events'])
                                    has_errors = len(s['errors']) > 0
                                    has_final_complete = 'thoughtspot completed' in s['events']
                                    has_run_complete = 'run completed' in terminal_events
                                    has_waiting = 'run waiting for user' in terminal_events
                                    has_interrupted = 'run interrupted' in terminal_events
                                    has_no_input = (
                                        not company
                                        and not use_case
                                        and s['row_count'] <= 2
                                        and all((e or '') == 'run started' for e in s['events'])
                                    )

                                    try:
                                        end_for_age = _dt.fromisoformat(s['end_ts'].replace('Z', '+00:00'))
                                        if end_for_age.tzinfo is None:
                                            end_for_age = end_for_age.replace(tzinfo=_timezone.utc)
                                        age_s = int((now_utc - end_for_age).total_seconds())
                                    except Exception:
                                        age_s = 0

                                    if has_final_complete and has_errors:
                                        status = '⚠️ Partial Success'
                                        failed_at = s['errors'][0][:80]
                                    elif has_final_complete or has_run_complete:
                                        status = '✅ Success'
                                        failed_at = ''
                                    elif has_waiting:
                                        status = '⏸ Waiting for User'
                                        failed_at = s.get('last_event') or ''
                                    elif has_no_input:
                                        status = '⚪ No Run Started'
                                        failed_at = 'No company/use case submitted'
                                    elif has_interrupted:
                                        status = '⚠️ Interrupted'
                                        failed_at = s['errors'][0][:80] if s['errors'] else (s.get('last_event') or '')
                                    elif has_explicit_failure or has_errors:
                                        status = '❌ Failed'
                                        if s['errors']:
                                            failed_at = s['errors'][0][:80]
                                        else:
                                            failed_at = next((e for e in s['events'] if e.endswith('failed')), 'unknown')
                                    elif age_s > 30 * 60:
                                        status = '⚠️ Stale / Interrupted'
                                        failed_at = f"Last event: {s.get('last_stage')}: {s.get('last_event')}"
                                    else:
                                        status = '⏳ In Progress'
                                        failed_at = ''

                                    try:
                                        start = _dt.fromisoformat(s['start_ts'].replace('Z', '+00:00'))
                                        end = _dt.fromisoformat(s['end_ts'].replace('Z', '+00:00'))
                                        dur_s = int((end - start).total_seconds())
                                        dur_str = f"{dur_s//60}m {dur_s%60}s" if dur_s >= 60 else f"{dur_s}s"
                                    except Exception:
                                        dur_str = ''

                                    use_case_display = (use_case[:60] + '…') if len(use_case) > 60 else use_case
                                    full_use_cases.append(use_case)
                                    display_rows.append([
                                        s['start_ts'][:16].replace('T', ' '),
                                        s['user'],
                                        company,
                                        use_case_display,
                                        interface,
                                        status,
                                        failed_at,
                                        dur_str,
                                    ])

                                if not display_rows:
                                    return [["No runs found", "", "", "", "", "", ""]], []
                                return display_rows, full_use_cases
                            except Exception as e:
                                return [[f"Error: {e}", "", "", "", "", "", ""]], []

                        def show_full_use_case(evt: gr.SelectData, full_use_cases):
                            row_idx = evt.index[0]
                            if row_idx < len(full_use_cases) and len(full_use_cases[row_idx]) > 60:
                                return gr.Textbox(value=full_use_cases[row_idx], visible=True)
                            return gr.Textbox(visible=False)

                        run_history_refresh_btn.click(
                            fn=load_run_history,
                            inputs=[run_history_email_filter, run_history_show_tests, run_history_limit],
                            outputs=[run_history_display, full_use_cases_state]
                        )
                        interface.load(fn=load_run_history, inputs=[], outputs=[run_history_display, full_use_cases_state])
                        run_history_display.select(
                            fn=show_full_use_case,
                            inputs=[full_use_cases_state],
                            outputs=[use_case_detail]
                        )

                    with gr.Tab("🧾 TS Table Calls"):
                        gr.Markdown("### ThoughtSpot Table TML Calls")
                        gr.Markdown("*Every table TML import request and response persisted in session_logs.*")
                        with gr.Row():
                            table_calls_refresh_btn = gr.Button("🔄 Refresh", size="sm")
                            table_calls_session_filter = gr.Textbox(label="Session ID", placeholder="Leave blank for latest table-call run", scale=3)
                            table_calls_email_filter = gr.Textbox(label="Filter by email", placeholder="user@company.com", scale=2)
                            table_calls_limit = gr.Dropdown(label="Show", choices=["25", "50", "100", "250"], value="100", scale=1)
                        table_calls_display = gr.Dataframe(
                            headers=["#", "Phase", "Table(s)", "Request UTC", "Response UTC", "Status", "Seconds", "Payload", "Session", "User", "Environment", "Error"],
                            datatype=["number", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str"],
                            column_widths=["50px", "220px", "180px", "145px", "145px", "70px", "70px", "85px", "210px", "200px", "260px", "320px"],
                            interactive=False,
                            label="Table Calls",
                            wrap=False,
                        )

                        def load_table_tml_calls(session_filter="", email_filter="", limit_choice="100"):
                            try:
                                from supabase_client import SupabaseSettings
                                ss = SupabaseSettings()
                                if not ss.is_enabled():
                                    return [["", "Supabase not configured", "", "", "", "", "", "", "", "", "", ""]]

                                fetch_limit = max(500, int(limit_choice) * 6)
                                query = (
                                    ss.client.table("session_logs")
                                    .select("session_id,user_email,ts,event,error,meta")
                                    .order("ts", desc=True)
                                    .limit(fetch_limit)
                                )
                                if session_filter and session_filter.strip():
                                    query = query.eq("session_id", session_filter.strip())
                                elif email_filter and email_filter.strip():
                                    query = query.eq("user_email", email_filter.strip())

                                raw_rows = list(reversed(query.execute().data or []))
                                if not session_filter or not session_filter.strip():
                                    latest_sid = ""
                                    for row in reversed(raw_rows):
                                        event = row.get("event") or ""
                                        meta = row.get("meta") or {}
                                        if event in {
                                            "table tml import request started",
                                            "table tml import response received",
                                            "table tml import exception",
                                            "tml import HTTP error",
                                        } or meta.get("table_names"):
                                            latest_sid = row.get("session_id") or ""
                                            break
                                    if latest_sid:
                                        raw_rows = [r for r in raw_rows if r.get("session_id") == latest_sid]

                                calls = []
                                for row in raw_rows:
                                    meta = row.get("meta") or {}
                                    event = row.get("event") or ""
                                    phase = meta.get("phase") or ""
                                    table_names = ", ".join(meta.get("table_names") or [])
                                    if event == "table tml import request started":
                                        calls.append({
                                            "session": row.get("session_id") or "",
                                            "user": row.get("user_email") or "",
                                            "environment": meta.get("ts_environment") or meta.get("ts_env") or meta.get("ts_url") or "",
                                            "phase": phase,
                                            "tables": table_names,
                                            "request": (row.get("ts") or "")[:19].replace("T", " "),
                                            "response": "",
                                            "status": "",
                                            "seconds": "",
                                            "payload": str(meta.get("payload_bytes") or ""),
                                            "error": "",
                                        })
                                    elif event in {"table tml import response received", "table tml import exception", "tml import HTTP error"}:
                                        match = None
                                        for call in reversed(calls):
                                            if call["phase"] == phase and call["tables"] == table_names and not call["response"]:
                                                match = call
                                                break
                                        if match is None:
                                            match = {
                                                "session": row.get("session_id") or "",
                                                "user": row.get("user_email") or "",
                                                "environment": meta.get("ts_environment") or meta.get("ts_env") or meta.get("ts_url") or "",
                                                "phase": phase,
                                                "tables": table_names,
                                                "request": "",
                                                "response": "",
                                                "status": "",
                                                "seconds": "",
                                                "payload": str(meta.get("payload_bytes") or ""),
                                                "error": "",
                                            }
                                            calls.append(match)
                                        if not match.get("environment"):
                                            match["environment"] = meta.get("ts_environment") or meta.get("ts_env") or meta.get("ts_url") or ""
                                        match["response"] = (row.get("ts") or "")[:19].replace("T", " ")
                                        match["status"] = "EXCEPTION" if event == "table tml import exception" else str(meta.get("status_code") or "")
                                        match["seconds"] = str(meta.get("elapsed_s") or "")
                                        match["payload"] = str(meta.get("payload_bytes") or match["payload"])
                                        match["error"] = (row.get("error") or meta.get("response_text") or "")[:500]

                                calls = calls[-int(limit_choice):]
                                rows = [
                                    [
                                        idx,
                                        call["phase"],
                                        call["tables"],
                                        call["request"],
                                        call["response"] or "open",
                                        call["status"] or "-",
                                        call["seconds"] or "-",
                                        call["payload"],
                                        call["session"],
                                        call["user"],
                                        call.get("environment") or "",
                                        call["error"],
                                    ]
                                    for idx, call in enumerate(calls, start=1)
                                ]
                                return rows if rows else [["", "No table calls found", "", "", "", "", "", "", "", "", "", ""]]
                            except Exception as e:
                                return [["", f"Error: {e}", "", "", "", "", "", "", "", "", "", ""]]

                        table_calls_refresh_btn.click(
                            fn=load_table_tml_calls,
                            inputs=[table_calls_session_filter, table_calls_email_filter, table_calls_limit],
                            outputs=[table_calls_display],
                        )
                        interface.load(fn=load_table_tml_calls, inputs=[], outputs=[table_calls_display])

                # Check admin status and toggle admin-only settings visibility
        def check_admin_visibility(request: gr.Request):
            """Check if logged-in user is admin and toggle settings visibility."""
            username = getattr(request, 'username', None) or ''
            is_admin = False
            try:
                from supabase_client import UserManager
                um = UserManager()
                if um.enabled and username:
                    is_admin = um.is_admin(username)
                else:
                    is_admin = True  # Local dev mode - show everything
            except Exception:
                is_admin = True  # If check fails, show everything
            
            return (
                gr.update(visible=is_admin),  # admin_ai_accordion
                gr.update(visible=is_admin),  # admin_db_accordion
                gr.update(visible=is_admin),  # admin_tab
            )

        admin_outputs = [
            settings_components['_admin_ai_accordion'],
            settings_components['_admin_db_accordion'],
            admin_tab,
        ]
        
        interface.load(
            fn=check_admin_visibility,
            inputs=[],
            outputs=admin_outputs
        )

        def check_password_gate(request: gr.Request = None):
            """Show only the forced password-change panel for temp-password users."""
            try:
                user_email = require_authenticated_email(request)
                from supabase_client import UserManager
                um = UserManager()
                must_change = um.enabled and um.must_change_password(user_email)
                return gr.update(visible=must_change), gr.update(visible=not must_change)
            except Exception as e:
                print(f"[LOAD] password gate check skipped: {e}")
                return gr.update(visible=False), gr.update(visible=True)

        def complete_required_password_change(current, new_pw, confirm, request: gr.Request = None):
            if not current or not new_pw or not confirm:
                return "❌ All fields are required.", gr.update(), gr.update()
            if new_pw != confirm:
                return "❌ New passwords don't match.", gr.update(), gr.update()
            try:
                user_email = require_authenticated_email(request)
                from supabase_client import UserManager
                um = UserManager()
                if not um.enabled:
                    return "⚠️ Supabase not configured — password change unavailable.", gr.update(), gr.update()
                if not um.authenticate(user_email, current):
                    return "❌ Temporary password is incorrect.", gr.update(), gr.update()
                if not um.reset_password(user_email, new_pw, must_change_password=False):
                    return "❌ Password change failed. Please try again or ask an admin to reset it.", gr.update(), gr.update()
                um.clear_must_change_password(user_email)
                return (
                    "✅ Password changed. DemoPrep is ready.",
                    gr.update(visible=False),
                    gr.update(visible=True),
                )
            except Exception as e:
                return f"❌ Error: {e}", gr.update(), gr.update()

        interface.load(
            fn=check_password_gate,
            inputs=[],
            outputs=[password_gate, main_tabs],
        )

        gate_change_password_btn.click(
            fn=complete_required_password_change,
            inputs=[gate_current_password, gate_new_password, gate_confirm_password],
            outputs=[gate_password_status, password_gate, main_tabs],
        )
        
        # Create update function for tabs
        _spotter_waiting = "Story will be generated after liveboard creation."

        def update_all_tabs(controller):
            if controller is None:
                return (
                    "",
                    "-- DDL will appear here after generation",
                    "Progress will appear here during deployment...",
                    "Demo pack will be generated after deployment completes.\n\nThis will include:\n- Key insights/outliers\n- Spotter questions to ask\n- Talking points for the demo",
                    _spotter_waiting,
                    _spotter_waiting,
                )

            live_progress = getattr(controller, 'live_progress_log', [])
            live_progress_text = "\n".join(live_progress) if live_progress else "Progress will appear here during deployment..."

            demo_pack = getattr(controller, 'demo_pack_content', '')
            demo_pack_text = demo_pack if demo_pack else "Demo pack will be generated after deployment completes.\n\nThis will include:\n- Key insights/outliers\n- Spotter questions to ask\n- Talking points for the demo"

            matrix_story = getattr(controller, 'spotter_story_matrix', '') or _spotter_waiting
            ai_story = getattr(controller, 'spotter_story_ai', '') or _spotter_waiting

            return (
                "\n".join(controller.ai_feedback_log),
                controller.ddl_code if controller.ddl_code else "-- DDL will appear here after generation",
                live_progress_text,
                demo_pack_text,
                matrix_story,
                ai_story,
            )

        # Wire up tab updates on chat interactions
        chat_components['chatbot'].change(
            fn=update_all_tabs,
            inputs=[chat_controller_state],
            outputs=[ai_feedback_display, ddl_display, live_progress_display, demo_pack_display,
                     spotter_matrix_display, spotter_ai_display]
        )
        
        # Load settings from Supabase on startup (uses SETTINGS_SCHEMA)
        def load_settings_on_startup(request: gr.Request = None):
            """Load saved settings from Supabase - uses schema-driven helper"""
            try:
                user_email = require_authenticated_email(request)
                print(f"[LOAD] load_settings_on_startup for {user_email}")
                settings = load_gradio_settings(user_email)
                result = load_settings_values(settings, user_email)
                print(f"[LOAD] load_settings_on_startup OK — {len(result)} values")
                return result
            except Exception as e:
                print(f"[LOAD] No authenticated user; using local default settings: {e}")
                # Return schema-length list of defaults so Gradio doesn't crash
                return [default for _, _, default, _ in SETTINGS_SCHEMA]

        def load_session_state_on_startup(request: gr.Request = None):
            """Initialize chat/session state from the same authenticated user settings."""
            try:
                user_email = require_authenticated_email(request)
                print(f"[LOAD] load_session_state_on_startup for {user_email}")
            except Exception as e:
                print(f"[LOAD] No authenticated user; using local default session state: {e}")
                _ts_choices = get_ts_environments()
                return (
                    "initialization", DEFAULT_LLM_MODEL, "", "", "",
                    gr.update(value=DEFAULT_LLM_MODEL), gr.update(value=""),
                    gr.update(value="Small"), gr.update(value="USA Only"),
                    gr.update(value=""), gr.update(value="Regular Case"),
                    gr.update(value=""), gr.update(value=""), "",
                    gr.update(value=_ts_choices[0]) if _ts_choices else gr.update(),
                    gr.update(), gr.update(), gr.update(), gr.update(),
                )
            try:
                settings = load_gradio_settings(user_email)

                model = (str(settings.get("default_llm", "")).strip() or DEFAULT_LLM_MODEL)
                if model not in UI_MODEL_CHOICES:
                    model = DEFAULT_LLM_MODEL
                liveboard_name = str(settings.get("liveboard_name", "")).strip()

                # Data Size — prefer new field, fall back to legacy fact_table_size
                saved_size = str(settings.get("default_data_size", "")).strip()
                if not saved_size:
                    _ft_to_size = {"1000": "Small", "10000": "Medium"}
                    saved_size = _ft_to_size.get(str(settings.get("fact_table_size", "1000")), "Small")

                saved_geo = str(settings.get("geo_scope", "USA Only")).strip() or "USA Only"
                saved_tag = str(settings.get("tag_name", "")).strip()
                saved_col_naming = str(settings.get("column_naming_style", "Regular Case")).strip() or "Regular Case"
                saved_obj_prefix = str(settings.get("object_naming_prefix", "")).strip()
                saved_share_with = str(settings.get("share_with", "")).strip()

                # TS Environment
                _ts_choices = get_ts_environments()
                saved_ts_env = str(settings.get("default_ts_env", "")).strip()
                if saved_ts_env not in _ts_choices:
                    saved_ts_env = _ts_choices[0] if _ts_choices else ""

                # Optional run input defaults
                use_defaults = settings.get("use_default_inputs", False)
                if isinstance(use_defaults, str):
                    use_defaults = use_defaults.lower() == "true"
                def_vertical = str(settings.get("default_vertical", "")).strip()
                def_line = str(settings.get("default_line", "")).strip()
                def_function = str(settings.get("default_function", "")).strip()
                def_company_url = str(settings.get("default_company_url", "")).strip()

                # Use legacy default_company_url / default_use_case for state
                company = def_company_url if use_defaults else ""
                use_case = str(settings.get("default_use_case", "")).strip()

                print(f"[LOAD] load_session_state_on_startup OK — model={model}, ts_env={saved_ts_env}, use_defaults={use_defaults}")
                return (
                    "initialization",
                    model,
                    company,
                    use_case,
                    liveboard_name,
                    gr.update(value=model),
                    gr.update(value=liveboard_name),
                    gr.update(value=saved_size),
                    gr.update(value=saved_geo),
                    gr.update(value=saved_tag),
                    gr.update(value=saved_col_naming),
                    gr.update(value=saved_obj_prefix),
                    gr.update(value=saved_share_with),
                    "",
                    gr.update(value=saved_ts_env) if saved_ts_env else gr.update(),
                    gr.update(value=def_vertical) if (use_defaults and def_vertical) else gr.update(),
                    gr.update(value=def_line) if (use_defaults and def_line) else gr.update(),
                    gr.update(value=def_function) if (use_defaults and def_function) else gr.update(),
                    gr.update(value=def_company_url) if (use_defaults and def_company_url) else gr.update(),
                )
            except Exception as e:
                import traceback
                print(f"[LOAD ERROR] load_session_state_on_startup failed: {e}\n{traceback.format_exc()}")
                return (
                    "initialization", DEFAULT_LLM_MODEL, "", "", "",
                    gr.update(value=DEFAULT_LLM_MODEL), gr.update(value=""),
                    gr.update(value="Small"), gr.update(value="USA Only"),
                    gr.update(value=""), gr.update(value="Regular Case"),
                    gr.update(value=""), gr.update(value=""), "",
                    gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
                )

        # Wire up load handler - outputs follow SETTINGS_SCHEMA order
        interface.load(
            fn=load_settings_on_startup,
            inputs=[],
            outputs=[settings_components[key] for key, _, _, _ in SETTINGS_SCHEMA]
        )

        interface.load(
            fn=load_session_state_on_startup,
            inputs=[],
            outputs=[
                current_stage,
                current_model,
                current_company,
                current_usecase,
                current_liveboard_name,
                chat_components["model_dropdown"],
                chat_components["liveboard_name_input"],
                chat_components["data_size_dropdown"],
                chat_components["geo_scope_dropdown"],
                chat_components["tag_name_input"],
                chat_components["column_naming_dropdown"],
                chat_components["object_prefix_input"],
                chat_components["share_with_input"],
                chat_components["msg"],
                chat_components["ts_env_dropdown"],
                chat_components["vertical_dd"],
                chat_components["line_dd"],
                chat_components["function_dd"],
                chat_components["url_input"],
            ]
        )
    
    return interface


def create_chat_tab(chat_controller_state, settings, current_stage, current_model, current_company, current_usecase, 
                    ai_feedback_state=None, ddl_code_state=None, population_code_state=None):
    """Create the main chat interface tab
    
    IMPORTANT: chat_controller_state is a gr.State that holds each session's
    ChatDemoInterface instance. This enables multi-user concurrent access.
    """
    
    # Create initial welcome message (before any session exists)
    initial_controller = ChatDemoInterface()
    initial_welcome = initial_controller.format_welcome_message(
        settings['company'], 
        settings['use_case']
    )
    
    with gr.Row():
        # Left column - App form + Chat
        with gr.Column(scale=5):
            with gr.Tabs():
                # ── App tab: clean form, no chatbot ──────────────────────
                with gr.Tab("App"):
                    _vertical_choices = list(VERTICAL_LINES.keys())
                    _first_vertical = _vertical_choices[0]
                    _first_lines = VERTICAL_LINES[_first_vertical]
                    vertical_dd = gr.Dropdown(
                        label="Vertical",
                        choices=_vertical_choices,
                        value=_first_vertical,
                        interactive=True,
                    )
                    line_dd = gr.Dropdown(
                        label="Line",
                        choices=_first_lines,
                        value=_first_lines[0] if _first_lines else None,
                        interactive=True,
                    )
                    function_dd = gr.Dropdown(
                        label="Function",
                        choices=DEMO_FUNCTIONS,
                        value=DEMO_FUNCTIONS[0],
                        interactive=True,
                    )
                    with gr.Row():
                        url_input = gr.Textbox(
                            label="Company URL",
                            placeholder="e.g. Amazon.com",
                            lines=1,
                            scale=4,
                            interactive=True,
                        )
                        use_url_cb = gr.Checkbox(
                            label="Use URL",
                            value=True,
                            scale=1,
                            interactive=True,
                        )
                    additional_info_input = gr.Textbox(
                        label="Context",
                        placeholder="Any extra context for the demo...",
                        lines=2,
                        interactive=True,
                    )
                    go_btn = gr.Button("→ GO", variant="primary")

                # ── Chat tab: full conversational view ───────────────────
                with gr.Tab("Chat"):
                    welcome_md = gr.Markdown(value=initial_welcome, visible=True)
                    chatbot = gr.Chatbot(
                        value=[],
                        height=400,
                        label="Demo Builder Assistant",
                        show_label=False,
                        avatar_images=None,
                        type='tuples',
                        elem_id="main-chatbot",
                        visible=False,
                    )
                    with gr.Row():
                        msg = gr.Textbox(
                            label="Your message",
                            value="",
                            placeholder="e.g. Amazon.com Retail Sales — or continue a conversation here",
                            lines=1,
                            max_lines=1,
                            scale=5,
                            show_label=False,
                            interactive=True
                        )
                        send_btn = gr.Button("Send", variant="primary", scale=1)
        
        # Right column - Status & Settings
        with gr.Column(scale=2):
            # TS Environment selector (always visible)
            ts_env_choices = get_ts_environments()
            _saved_ts_env = str(settings.get('default_ts_env', '')).strip()
            _init_ts_env = _saved_ts_env if _saved_ts_env in ts_env_choices else (ts_env_choices[0] if ts_env_choices else None)
            ts_env_dropdown = gr.Dropdown(
                label="TS Environment",
                choices=ts_env_choices,
                value=_init_ts_env,
                interactive=True,
            )

            # AI Model selector (always visible)
            model_dropdown = gr.Dropdown(
                label="AI Model",
                choices=list(UI_MODEL_CHOICES),
                value=settings['model'],
                interactive=True,
                allow_custom_value=False,
                info="Temporary model failover active: using claude-sonnet-4-6."
            )

            # Run-time settings — collapsible, all pipeline knobs here
            settings_accordion = gr.Accordion("⚙️ Settings", open=False)
            with settings_accordion:
                liveboard_name_input = gr.Textbox(
                    label="Liveboard Name",
                    placeholder="Auto from company URL if blank",
                    value=settings.get('liveboard_name', ''),
                    lines=1,
                    interactive=True,
                )

                # Seed Data Size from default_data_size; fall back to fact_table_size for old records
                _init_data_size = str(settings.get('default_data_size', '')).strip()
                if not _init_data_size:
                    _ft_to_size = {"1000": "Small", "10000": "Medium"}
                    _init_data_size = _ft_to_size.get(str(settings.get('fact_table_size', '1000')), "Small")

                data_size_dropdown = gr.Dropdown(
                    label="Data Size",
                    choices=["Small", "Medium"],
                    value=_init_data_size,
                    interactive=True,
                    info="Small 1k rows · Medium 10k rows",
                )
                geo_scope_dropdown = gr.Dropdown(
                    label="Geographic Scope",
                    choices=["USA Only", "International"],
                    value=settings.get('geo_scope', 'USA Only'),
                    interactive=True,
                )
                tag_name_input = gr.Textbox(
                    label="Tag Name",
                    placeholder="e.g. Sales_Demo (blank = no tag)",
                    value=settings.get('tag_name', ''),
                    lines=1,
                    interactive=True,
                )
                column_naming_dropdown = gr.Dropdown(
                    label="Column Naming Style",
                    choices=["Regular Case", "snake_case", "camelCase", "PascalCase", "UPPER_CASE", "original"],
                    value=settings.get('column_naming_style', 'Regular Case'),
                    interactive=True,
                )
                object_prefix_input = gr.Textbox(
                    label="Object Naming Prefix",
                    placeholder="e.g. ACME_ (blank = none)",
                    value=settings.get('object_naming_prefix', ''),
                    lines=1,
                    interactive=True,
                )
                share_with_input = gr.Textbox(
                    label="Share With",
                    placeholder="user@company.com or group-name (blank = no share)",
                    value=settings.get('share_with', ''),
                    lines=1,
                    interactive=True,
                )

            gr.Markdown("### 📈 Progress")

            # Stage order used to determine done/current/upcoming
            _STAGE_ORDER = [
                'initialization', 'awaiting_context', 'research',
                'create_ddl', 'deploy', 'populate',
                'thoughtspot', 'outlier_adjustment', 'complete',
            ]
            # Each display step: (label, [stage keys that map to it])
            _PROGRESS_STEPS = [
                ('Init',         ['initialization']),
                ('Research',     ['awaiting_context', 'research']),
                ('DDL',          ['create_ddl']),
                ('Data',         ['deploy', 'populate']),
                ('ThoughtSpot',  ['thoughtspot']),
                ('Data Adjuster',['outlier_adjustment']),
                ('Complete',     ['complete']),
            ]

            def get_progress_html(stage):
                """Generate progress HTML showing done/current/upcoming states."""
                try:
                    current_idx = _STAGE_ORDER.index(stage)
                except ValueError:
                    current_idx = 0

                # Find which display step is current
                current_step = None
                for step_label, step_keys in _PROGRESS_STEPS:
                    for k in step_keys:
                        if stage == k:
                            current_step = step_label
                            break

                # Build ordered list of display steps that are reached
                reached = set()
                for step_label, step_keys in _PROGRESS_STEPS:
                    for k in step_keys:
                        try:
                            if _STAGE_ORDER.index(k) <= current_idx:
                                reached.add(step_label)
                        except ValueError:
                            pass

                html = "<div style='padding:8px 4px; font-size:13px; line-height:1.8;'>"
                for step_label, _ in _PROGRESS_STEPS:
                    # Skip Data Adjuster unless it's active
                    if step_label == 'Data Adjuster' and step_label not in reached:
                        continue
                    if step_label == current_step:
                        html += (f"<div style='margin:3px 0; color:#3b82f6; font-weight:bold;'>"
                                 f"▶ {step_label}</div>")
                    elif step_label in reached:
                        html += (f"<div style='margin:3px 0; color:#22c55e;'>"
                                 f"✓ {step_label}</div>")
                    else:
                        html += (f"<div style='margin:3px 0; color:#9ca3af;'>"
                                 f"○ {step_label}</div>")
                html += "</div>"
                return html

            progress_html = gr.HTML(get_progress_html('initialization'))

            # Phase log stream — scrolling status updates from controller.phase_log
            phase_log_display = gr.Textbox(
                label="Pipeline Status",
                value="",
                lines=6,
                max_lines=6,
                interactive=False,
                placeholder="Pipeline status will appear here when the GO button is pressed...",
                elem_classes=["phase-log-stream"],
            )

            deployment_links_panel = gr.HTML(value="", visible=False)

            # Timer polls controller.phase_log every 2 seconds (activated when GO is pressed)
            phase_log_timer = gr.Timer(value=2, active=True)

        # Event handlers - each creates/uses session-specific controller
        def send_message(controller, message, history, stage, model, company, usecase, env_label=None, liveboard_name_ui=None, request: gr.Request = None):
            """Handle sending a message - creates controller if needed"""
            import traceback
            username = getattr(request, 'username', None) if request else None
            if controller is None:
                controller = ChatDemoInterface(user_email=username)
                print(f"[SESSION] Created new ChatDemoInterface for {username or 'anonymous'}")
            elif username:
                controller.user_email = username
            # Apply the selected TS environment on every message, not just on the
            # first — so a changed env dropdown re-targets the deploy on a reused session.
            if env_label:
                _url = get_ts_env_url(env_label)
                _key_value = get_ts_env_auth_key(env_label)
                if _url:
                    controller.settings['thoughtspot_url'] = _url
                if _key_value:
                    controller.settings['thoughtspot_trusted_auth_key'] = _key_value
            # Always use the current UI values — take priority over DB-loaded defaults
            if liveboard_name_ui is not None:
                controller.settings['liveboard_name'] = liveboard_name_ui
            if model:
                controller.settings['model'] = model
            hide_welcome = gr.update(visible=False)
            try:
                for result in controller.process_chat_message(
                    message, history, stage, model, company, usecase
                ):
                    new_stage = result[1] if len(result) > 1 else stage
                    progress = get_progress_html(new_stage)
                    chatbot_update = gr.update(value=result[0], visible=True)
                    yield (controller, chatbot_update) + result[1:] + (
                        progress,
                        hide_welcome,
                        controller.render_deployment_completion_html(),
                    )
            except Exception as e:
                err_tb = traceback.format_exc()
                print(f"[ERROR] send_message unhandled exception:\n{err_tb}")
                err_msg = (
                    f"❌ **An unexpected error occurred**\n\n"
                    f"`{type(e).__name__}: {e}`\n\n"
                    f"The pipeline has been interrupted. You can try again or start a new session."
                )
                history = history or []
                history.append((message, err_msg))
                completion_update = controller.render_deployment_completion_html() if controller else gr.update(value="", visible=False)
                yield (
                    controller,
                    gr.update(value=history, visible=True),
                    stage,
                    model,
                    company,
                    usecase,
                    "",
                    get_progress_html(stage),
                    hide_welcome,
                    completion_update,
                )

        # Wire up send button and enter key
        _send_inputs = [chat_controller_state, msg, chatbot, current_stage, current_model, current_company, current_usecase, ts_env_dropdown, liveboard_name_input]
        _send_outputs = [
            chat_controller_state, chatbot, current_stage, current_model,
            current_company, current_usecase, msg, progress_html, welcome_md,
            deployment_links_panel,
        ]

        msg.submit(fn=send_message, inputs=_send_inputs, outputs=_send_outputs)
        send_btn.click(fn=send_message, inputs=_send_inputs, outputs=_send_outputs)
        
        # App tab: vertical → line + function cascade
        def update_line_on_vertical(vertical):
            lines = VERTICAL_LINES.get(vertical, [])
            if vertical == "* CUSTOM *" or not lines:
                return (
                    gr.Dropdown(choices=["— not used —"], value="— not used —", interactive=False, label="Line (n/a for custom)"),
                    gr.Dropdown(choices=["— not used —"], value="— not used —", interactive=False, label="Function (n/a for custom)"),
                    gr.Textbox(label="Context *", placeholder="Describe your use case, industry, and key metrics...", interactive=True),
                )
            return (
                gr.Dropdown(choices=lines, value=lines[0], interactive=True, label="Line"),
                gr.Dropdown(choices=DEMO_FUNCTIONS, value=DEMO_FUNCTIONS[0], interactive=True, label="Function"),
                gr.Textbox(label="Context", placeholder="Any extra context for the demo...", interactive=True),
            )

        vertical_dd.change(
            fn=update_line_on_vertical,
            inputs=[vertical_dd],
            outputs=[line_dd, function_dd, additional_info_input]
        )

        # Defined tab: GO button handler
        def defined_go(controller, vertical, line, function, url, use_url, additional_info,
                       history, stage, model, company, usecase, env_label, lb_name,
                       data_size, geo_scope, tag_name, col_naming, obj_prefix, share_with,
                       request: gr.Request = None):
            import traceback
            function_clean = (function or "").strip()
            username = getattr(request, 'username', None) if request else None
            if controller is None:
                controller = ChatDemoInterface(user_email=username)
                print(f"[SESSION] Created new ChatDemoInterface for {username or 'anonymous'}")
            elif username:
                controller.user_email = username
            # Apply the selected TS environment on EVERY GO, not only when a new
            # controller is created. A reused session (e.g. the e2e runs all 8 demos
            # on one login) otherwise keeps the first/default env and silently ignores
            # a changed dropdown — deploying to the wrong ThoughtSpot instance.
            if env_label:
                _url = get_ts_env_url(env_label)
                _key_value = get_ts_env_auth_key(env_label)
                if _url:
                    controller.settings['thoughtspot_url'] = _url
                if _key_value:
                    controller.settings['thoughtspot_trusted_auth_key'] = _key_value
            if lb_name is not None:
                controller.settings['liveboard_name'] = lb_name
            if model:
                controller.settings['model'] = model
            if data_size:
                _size_map = {
                    "Small":  ("1000",  "50"),
                    "Medium": ("10000", "500"),
                }
                _ft, _dt = _size_map.get(data_size, ("1000", "100"))
                controller.settings['fact_table_size'] = _ft
                controller.settings['dim_table_size'] = _dt
            if geo_scope:
                controller.settings['geo_scope'] = geo_scope
            if tag_name is not None:
                controller.settings['tag_name'] = tag_name
            if col_naming:
                controller.settings['column_naming_style'] = col_naming
            if obj_prefix is not None:
                controller.settings['object_naming_prefix'] = obj_prefix
            if share_with is not None:
                controller.settings['share_with'] = share_with

            # Capture the raw GO form payload so the run logger records exactly
            # what was submitted (logged secret-redacted via _snapshot_run_payload)
            controller._run_payload = {
                'vertical': vertical,
                'line': line,
                'function': function_clean,
                'url': url,
                'use_url': use_url,
                'additional_info': additional_info,
                'model': model,
                'ts_environment': env_label,
                'liveboard_name': lb_name,
                'data_size': data_size,
                'geo_scope': geo_scope,
                'tag_name': tag_name,
                'column_naming': col_naming,
                'object_prefix': obj_prefix,
                'share_with': share_with,
            }

            # Derive company name from URL or line+function label
            if use_url and url.strip():
                raw_company = url.strip()
            else:
                raw_company = f"{line or vertical} Demo"

            is_custom = (vertical == "* CUSTOM *")
            hide_welcome = gr.update(visible=False)
            password_block = controller._temporary_password_block_message()
            if password_block:
                history = history or []
                history.append(("GO", password_block))
                controller.phase_log = ["Password change required before starting pipeline."]
                yield (
                    controller,
                    gr.update(value=history, visible=True),
                    stage,
                    model,
                    company,
                    usecase,
                    get_progress_html(stage),
                    hide_welcome,
                    gr.update(open=False),
                    gr.update(value="", visible=False),
                )
                return

            if is_custom:
                # Custom: context field drives the use case
                use_case_str = (additional_info or "").strip() or "Custom Demo"
                controller.vertical = None
                controller.line = None
                controller.function = None
                controller.use_case_config = get_use_case_config("Generic", "Generic")
                controller.is_generic_use_case = True
                controller.generic_use_case_context = use_case_str
            else:
                # Use line + function as the use case (vertical is context only)
                use_case_str = f"{line} {function_clean}" if line else f"{vertical} {function_clean}"
                controller.vertical = vertical
                controller.line = line
                controller.function = function_clean
                controller.use_case_config = get_use_case_config(
                    line or vertical or "Generic",
                    function_clean or "Generic",
                    vertical_fallback=vertical if line else None,
                )
                is_known = (line and function_clean
                            and not controller.use_case_config.get('is_generic'))
                controller.is_generic_use_case = not is_known
                controller.generic_use_case_context = additional_info.strip() if additional_info else ""

            controller.pending_generic_company = raw_company
            # For Custom, the display name IS the context text; for standard, use config name
            if is_custom:
                use_case_display = use_case_str
            else:
                use_case_display = controller.use_case_config.get('use_case_name', use_case_str)
            controller.pending_generic_use_case = use_case_display

            # Clear phase log for a fresh run
            controller.phase_log = [f"→ GO received — preparing {raw_company} · {use_case_display}"]
            controller.clear_deployment_completion()

            # Tag the run source so session logging can record which interface was used
            controller._run_source = 'app_custom' if is_custom else 'app_defined'

            # Drive straight to awaiting_context → 'proceed' to skip confirmation dialog
            # For Custom, pass the context as the message so process_chat_message stores it;
            # for standard, use 'proceed' (context already set above).
            proceed_msg = use_case_str if is_custom else "proceed"
            history = history or []
            try:
                for result in controller.process_chat_message(
                    proceed_msg, history, 'awaiting_context', model,
                    raw_company, use_case_display
                ):
                    new_stage = result[1] if len(result) > 1 else stage
                    progress = get_progress_html(new_stage)
                    chatbot_update = gr.update(value=result[0], visible=True)
                    yield (controller, chatbot_update) + result[1:5] + (
                        progress,
                        hide_welcome,
                        gr.update(open=False),
                        controller.render_deployment_completion_html(),
                    )
            except Exception as e:
                err_tb = traceback.format_exc()
                print(f"[ERROR] defined_go unhandled exception:\n{err_tb}")
                controller.phase_log.append(f"❌ GO failed before pipeline start: {type(e).__name__}: {e}")
                err_msg = (
                    f"❌ **An unexpected error occurred**\n\n"
                    f"`{type(e).__name__}: {e}`\n\n"
                    f"The pipeline has been interrupted. You can try again or start a new session."
                )
                history.append((f"GO: {use_case_str}", err_msg))
                yield (
                    controller,
                    gr.update(value=history, visible=True),
                    stage,
                    model,
                    company,
                    usecase,
                    get_progress_html(stage),
                    hide_welcome,
                    gr.update(open=False),
                    controller.render_deployment_completion_html(),
                )

        _go_inputs = [
            chat_controller_state, vertical_dd, line_dd, function_dd, url_input, use_url_cb,
            additional_info_input, chatbot, current_stage, current_model,
            current_company, current_usecase, ts_env_dropdown, liveboard_name_input,
            data_size_dropdown, geo_scope_dropdown, tag_name_input,
            column_naming_dropdown, object_prefix_input, share_with_input,
        ]
        _go_outputs = [
            chat_controller_state, chatbot, current_stage, current_model,
            current_company, current_usecase, progress_html, welcome_md,
            settings_accordion, deployment_links_panel,
        ]
        go_btn.click(fn=defined_go, inputs=_go_inputs, outputs=_go_outputs)

        # Phase log timer — poll controller.phase_log and render as newline-joined text
        def refresh_phase_log(controller):
            if controller is None:
                return gr.update()
            log = getattr(controller, 'phase_log', [])
            return "\n".join(log) if log else ""

        phase_log_timer.tick(
            fn=refresh_phase_log,
            inputs=[chat_controller_state],
            outputs=[phase_log_display],
            trigger_mode="always_last",
        )

        # Model dropdown change
        def update_model(new_model, controller, history):
            if controller is not None:
                controller.settings['model'] = new_model
            return new_model, history

        model_dropdown.change(
            fn=update_model,
            inputs=[model_dropdown, chat_controller_state, chatbot],
            outputs=[current_model, chatbot]
        )

        # Liveboard name blur — update controller settings when user leaves the field.
        # Using blur (not change) avoids per-keystroke queue events that cause the
        # Gradio progress spinner/timer to appear on this component while a deploy runs.
        # No output needed — the value is already in the textbox; we only update the dict.
        def update_liveboard_name(name, controller):
            if controller is not None:
                controller.settings['liveboard_name'] = name

        liveboard_name_input.blur(
            fn=update_liveboard_name,
            inputs=[liveboard_name_input, chat_controller_state],
            outputs=[]
        )

        # TS environment change — update controller settings in real-time
        def update_ts_env(label, controller):
            url = get_ts_env_url(label)
            auth_key_value = get_ts_env_auth_key(label)
            if controller is not None:
                if url:
                    controller.settings['thoughtspot_url'] = url
                if auth_key_value:
                    controller.settings['thoughtspot_trusted_auth_key'] = auth_key_value
            return label

        ts_env_dropdown.change(
            fn=update_ts_env,
            inputs=[ts_env_dropdown, chat_controller_state],
            outputs=[]
        )
    
    # Return components for external access
    return {
        'chatbot': chatbot,
        'msg': msg,
        'model_dropdown': model_dropdown,
        'send_btn': send_btn,
        'ts_env_dropdown': ts_env_dropdown,
        'liveboard_name_input': liveboard_name_input,
        'data_size_dropdown': data_size_dropdown,
        'geo_scope_dropdown': geo_scope_dropdown,
        'tag_name_input': tag_name_input,
        'column_naming_dropdown': column_naming_dropdown,
        'object_prefix_input': object_prefix_input,
        'share_with_input': share_with_input,
        'progress_html': progress_html,
        'phase_log_display': phase_log_display,
        'phase_log_timer': phase_log_timer,
        # App tab form inputs — seeded from default run inputs if enabled
        'vertical_dd': vertical_dd,
        'line_dd': line_dd,
        'function_dd': function_dd,
        'url_input': url_input,
    }


def create_matrix_tab(interface):
    """Matrix editor tab — view and edit Vertical × Function cell configs."""

    def _cell_data(vertical, function):
        """Compute all display values for a given cell."""
        config = get_use_case_config(vertical, function)
        has_override = (vertical, function) in MATRIX_OVERRIDES

        # Coverage line
        if has_override:
            coverage = "✅ **Override** — enriched persona, custom questions, tuned story controls"
        else:
            coverage = "🔵 **Base merge** — vertical + function defaults combined"

        # Persona / problem (overrides only)
        persona = config.get('target_persona', '')
        problem = config.get('business_problem', '')
        persona_md = ""
        if persona:
            persona_md += f"**Target persona:** {persona}  \n"
        if problem:
            persona_md += f"**Business problem:** {problem}"

        # KPIs
        kpis = config.get('kpis', [])
        kpi_defs = config.get('kpi_definitions', {})
        lines = []
        for k in kpis:
            d = kpi_defs.get(k, '')
            lines.append(f"**{k}** — {d}" if d else f"**{k}**")
        kpi_md = "  \n".join(lines) if lines else "_None defined_"

        # Liveboard questions → dataframe rows
        questions = config.get('liveboard_questions', [])
        rows = []
        for q in questions:
            spotter = ", ".join(q.get('spotter_qs', []))
            rows.append([
                q.get('title', ''),
                q.get('viz_type', ''),
                q.get('required', False),
                q.get('viz_question', ''),
                q.get('insight', ''),
                spotter,
            ])

        # Story controls as formatted text
        sc = config.get('story_controls', {})
        sc_lines = []
        for k, v in sc.items():
            if isinstance(v, dict):
                sc_lines.append(f"**{k}:** {v}")
            elif isinstance(v, list):
                sc_lines.append(f"**{k}:** {', '.join(str(x) for x in v)}")
            elif isinstance(v, float):
                sc_lines.append(f"**{k}:** {v:.4g}")
            else:
                sc_lines.append(f"**{k}:** {v}")
        sc_md = "  \n".join(sc_lines) if sc_lines else "_None_"

        return coverage, persona_md, kpi_md, rows, sc_md

    # --- Initial values ---
    init_v = list(VERTICALS.keys())[0]
    init_f = list(FUNCTIONS.keys())[0]
    init_cov, init_persona, init_kpi, init_rows, init_sc = _cell_data(init_v, init_f)

    gr.Markdown("## 🧩 Matrix Editor")
    gr.Markdown(
        "The matrix defines what gets built for every Vertical × Function combination — "
        "KPIs, visualizations, story controls, and persona.  \n"
        "Changes here are **in-session only** for now; persistence via Supabase is coming next."
    )

    # --- Selectors ---
    with gr.Row():
        m_vertical = gr.Dropdown(
            label="Vertical",
            choices=list(VERTICALS.keys()),
            value=init_v,
            interactive=True,
            scale=1,
        )
        m_function = gr.Dropdown(
            label="Function",
            choices=list(FUNCTIONS.keys()),
            value=init_f,
            interactive=True,
            scale=1,
        )

    coverage_md = gr.Markdown(value=init_cov)
    persona_md = gr.Markdown(value=init_persona)

    gr.Markdown("---")

    # --- KPIs ---
    with gr.Accordion("📊 KPIs", open=True):
        kpi_md = gr.Markdown(value=init_kpi)

    # --- Liveboard Questions ---
    with gr.Accordion("📋 Liveboard Questions", open=True):
        gr.Markdown(
            "*Edit cells directly. Add rows with the + button. "
            "Changes affect this session until persistence is wired up.*"
        )
        questions_df = gr.Dataframe(
            value=init_rows,
            headers=["Title", "Viz Type", "Required", "Question", "Insight", "Spotter Questions"],
            datatype=["str", "str", "bool", "str", "str", "str"],
            interactive=True,
            row_count=(len(init_rows), "dynamic"),
            col_count=(6, "fixed"),
            wrap=True,
        )

    # --- Story Controls ---
    with gr.Accordion("⚙️ Story Controls", open=False):
        sc_md = gr.Markdown(value=init_sc)

    # --- Wire dropdowns ---
    def on_cell_change(vertical, function):
        cov, persona, kpi, rows, sc = _cell_data(vertical, function)
        return cov, persona, kpi, rows, sc

    _outputs = [coverage_md, persona_md, kpi_md, questions_df, sc_md]
    m_vertical.change(fn=on_cell_change, inputs=[m_vertical, m_function], outputs=_outputs)
    m_function.change(fn=on_cell_change, inputs=[m_vertical, m_function], outputs=_outputs)

    # Populate on tab load
    interface.load(
        fn=on_cell_change,
        inputs=[m_vertical, m_function],
        outputs=_outputs,
    )

    return {'vertical': m_vertical, 'function': m_function, 'questions_df': questions_df}


def create_settings_tab():
    """Create the settings configuration tab - returns components for loading
    
    Uses module-level SETTINGS_SCHEMA for consistency with load/save functions.
    """
    gr.Markdown("## ⚙️ Configuration Settings")
    gr.Markdown("Configure your demo builder preferences")

    # ── Default Settings — mirrors App tab right panel ────────────────────────
    gr.Markdown("### ⭐ Default Settings")
    gr.Markdown("*These seed the App tab right panel on every page load.*")

    default_ai_model = gr.Dropdown(
        label="Default AI Model",
        choices=list(UI_MODEL_CHOICES),
        value=DEFAULT_LLM_MODEL,
        info="Temporary model failover active: using claude-sonnet-4-6.",
        allow_custom_value=False,
    )
    _ts_env_choices = get_ts_environments()
    default_ts_env = gr.Dropdown(
        label="Default TS Environment",
        choices=_ts_env_choices,
        value=_ts_env_choices[0] if _ts_env_choices else None,
        info="Which ThoughtSpot environment to select on load",
        allow_custom_value=True,
    )
    liveboard_name = gr.Textbox(
        label="Default Liveboard Name",
        placeholder="Auto from company URL if blank",
        value="",
        info="Leave blank to auto-derive from company URL",
    )
    default_data_size = gr.Dropdown(
        label="Default Data Size",
        choices=["Small", "Medium"],
        value="Small",
        info="Small = 1k rows · Medium = 10k rows",
    )
    geo_scope = gr.Dropdown(
        label="Default Geographic Scope",
        choices=["USA Only", "International"],
        value="USA Only",
        info="USA Only = US states/cities/USD · International = global",
    )
    tag_name = gr.Textbox(
        label="Default Tag Name",
        placeholder="e.g. Sales_Demo (blank = no tag)",
        value="",
        info="Tag applied to all TS objects after each build",
    )
    column_naming_style = gr.Dropdown(
        label="Default Column Naming Style",
        choices=["Regular Case", "snake_case", "camelCase", "PascalCase", "UPPER_CASE", "original"],
        value="Regular Case",
        info="Regular Case = State Id, Total Revenue",
    )
    object_naming_prefix = gr.Textbox(
        label="Default Object Naming Prefix",
        placeholder="e.g. ACME_ (blank = none)",
        value="",
    )
    share_with = gr.Textbox(
        label="Default Share With",
        placeholder="user@company.com or group-name (blank = no share)",
        value="",
        info="Model + liveboard shared after every build",
    )

    # ── Optional default run inputs ────────────────────────────────────────────
    gr.Markdown("---")
    gr.Markdown("### 🏁 Default Run Inputs *(optional)*")
    gr.Markdown("*When enabled, these pre-fill Vertical, Line, Function, and Company URL on the App tab at load time.*")

    use_default_inputs = gr.Checkbox(
        label="Pre-fill App tab on load",
        value=False,
    )
    _vert_choices = list(VERTICAL_LINES.keys())
    default_vertical = gr.Dropdown(
        label="Default Vertical",
        choices=_vert_choices,
        value=_vert_choices[0] if _vert_choices else None,
        allow_custom_value=True,
    )
    _first_lines = VERTICAL_LINES.get(_vert_choices[0], []) if _vert_choices else []
    default_line = gr.Dropdown(
        label="Default Line",
        choices=_first_lines,
        value=_first_lines[0] if _first_lines else None,
        allow_custom_value=True,
        info="Updates when Default Vertical changes",
    )
    default_function = gr.Dropdown(
        label="Default Function",
        choices=DEMO_FUNCTIONS,
        value=DEMO_FUNCTIONS[0] if DEMO_FUNCTIONS else None,
        allow_custom_value=True,
    )
    default_company_url = gr.Textbox(
        label="Default Company URL",
        placeholder="e.g. Nike.com",
        value="",
    )

    # Wire vertical → line cascade
    def _update_default_line(vertical):
        lines = VERTICAL_LINES.get(vertical, [])
        return gr.update(choices=lines, value=lines[0] if lines else None)
    default_vertical.change(fn=_update_default_line, inputs=[default_vertical], outputs=[default_line])

    # ── Other settings ─────────────────────────────────────────────────────────
    gr.Markdown("---")
    gr.Markdown("### 🌍 Other Settings")

    validation_mode = gr.Radio(
        label="Validation Mode",
        choices=["On", "Off"],
        value="Off",
        info="On: Pause at DDL & TS checkpoints. Off: Auto-run entire pipeline after context question.",
    )

    # Hidden legacy fields — kept in schema for backward compat
    default_use_case = gr.Textbox(visible=False, value="Sales Analytics")
    fact_table_size = gr.Textbox(visible=False, value="1000")
    dim_table_size = gr.Textbox(visible=False, value="100")
    use_existing_model = gr.Checkbox(visible=False, value=False)
    existing_model_guid = gr.Textbox(visible=False, value="")
    
    # Advanced AI Settings (Admin Only)
    admin_ai_accordion = gr.Accordion("🤖 Advanced AI Settings (Admin)", open=False, visible=True)
    with admin_ai_accordion:
        with gr.Row():
            temperature_slider = gr.Slider(
                minimum=0.0,
                maximum=1.0,
                value=0.3,
                step=0.1,
                label="Temperature",
                info="Controls randomness (lower = more focused)"
            )
            
            max_tokens = gr.Number(
                value=4000,
                label="Max Tokens",
                info="Maximum tokens for AI responses"
            )
        
        with gr.Row():
            batch_size = gr.Slider(
                minimum=1000,
                maximum=50000,
                value=5000,
                step=1000,
                label="Batch Size",
                info="Rows per batch for bulk operations"
            )
            
            thread_count = gr.Slider(
                minimum=1,
                maximum=16,
                value=4,
                step=1,
                label="Thread Count",
                info="Parallel threads for data generation"
            )
    
    # Database Connections (Admin Only)
    admin_db_accordion = gr.Accordion("💾 Database Connections (Admin)", open=False, visible=True)
    with admin_db_accordion:
        gr.Markdown("""
        **⚠️ Note:** These fields are **legacy placeholders** and are **not used by deploy runtime**.  
        Active deployment credentials come from **Admin Settings** (system-wide `__admin__` values in Supabase).  
        This section is for reference/future use only.
        """)
        
        with gr.Row():
            with gr.Column():
                gr.Markdown("### ❄️ Snowflake Connection")
                
                sf_account = gr.Textbox(
                    label="Snowflake Account",
                    placeholder="xy12345.us-east-1",
                    info="Your Snowflake account identifier"
                )
                
                sf_user = gr.Textbox(
                    label="Snowflake User",
                    placeholder="your_username",
                    info="Snowflake username (password in .env)"
                )
                
                sf_role = gr.Textbox(
                    label="Snowflake Role",
                    value="ACCOUNTADMIN",
                    info="Default role for connections"
                )
                
                default_warehouse = gr.Textbox(
                    label="Default Warehouse",
                    value="COMPUTE_WH",
                    info="Snowflake warehouse for demos"
                )
                
                default_database = gr.Textbox(
                    label="Default Database",
                    value="DEMO_DB",
                    info="Snowflake database for demos"
                )
                
                default_schema = gr.Textbox(
                    label="Default Schema",
                    value="PUBLIC",
                    info="Default schema for demos"
                )
                
            with gr.Column():
                gr.Markdown("### 📊 ThoughtSpot Settings")
                
                # ts_instance_url removed — replaced by TS Environment dropdown on front page
                ts_instance_url = gr.Textbox(visible=False)
                
                ts_username = gr.Textbox(
                    label="ThoughtSpot Username",
                    placeholder="your.email@company.com",
                    info="Your ThoughtSpot login"
                )
                
                # ts_password removed — auth is handled by TS Environment dropdown (API key),
                # not username/password. This field was never wired to SETTINGS_SCHEMA.
    
    gr.Markdown("---")
    gr.Markdown("### 🔧 Data Adjuster")
    gr.Markdown("*Jump straight to data adjustment on an existing liveboard — skips the build pipeline entirely.*")

    with gr.Row():
        with gr.Column():
            data_adjuster_url = gr.Textbox(
                label="Liveboard URL",
                placeholder="https://your-instance.thoughtspot.cloud/#/pinboard/guid",
                value="",
                info="Paste a ThoughtSpot liveboard URL to open it directly in Data Adjuster"
            )

    gr.Markdown("---")
    with gr.Row():
        save_settings_btn = gr.Button("💾 Save Settings", variant="primary", size="lg")
        reset_settings_btn = gr.Button("🔄 Reset to Defaults", size="lg")
    
    settings_status = gr.Markdown("")
    
    def save_settings_handler(request: gr.Request, *args):
        """Save all settings to Supabase - uses schema-driven helper"""
        try:
            from supabase_client import SupabaseSettings
            
            user_email = require_authenticated_email(request)
            settings_client = SupabaseSettings()
            
            if not settings_client.is_enabled():
                return "⚠️ Supabase not configured. Settings saved locally only."
            
            # Build save dict from args using schema
            settings_to_save = build_settings_save_dict(list(args))
            
            success = settings_client.save_all_settings(user_email, settings_to_save)
            
            if success:
                return f"✅ **Settings saved successfully!**\n\nSaved for user: `{user_email}`"
            else:
                return "❌ Error saving some settings. Check console for details."
                
        except Exception as e:
            return f"❌ Error saving settings: {str(e)}"
    
    def reset_settings_handler():
        return "🔄 Settings reset to defaults! (Refresh page to see defaults)"
    
    # Build components dict matching SETTINGS_SCHEMA order
    # This is the single mapping from schema keys to component variables
    all_components = {
        # Panel defaults (mirrors App tab right panel)
        'default_ai_model': default_ai_model,
        'default_ts_env': default_ts_env,
        'liveboard_name': liveboard_name,
        'default_data_size': default_data_size,
        'geo_scope': geo_scope,
        'tag_name': tag_name,
        'column_naming_style': column_naming_style,
        'object_naming_prefix': object_naming_prefix,
        'share_with': share_with,
        # Optional run input defaults
        'use_default_inputs': use_default_inputs,
        'default_vertical': default_vertical,
        'default_line': default_line,
        'default_function': default_function,
        'default_company_url': default_company_url,
        # Other settings
        'validation_mode': validation_mode,
        # Legacy hidden fields
        'default_use_case': default_use_case,
        'fact_table_size': fact_table_size,
        'dim_table_size': dim_table_size,
        'use_existing_model': use_existing_model,
        'existing_model_guid': existing_model_guid,
        # Advanced AI Settings
        'temperature_slider': temperature_slider,
        'max_tokens': max_tokens,
        'batch_size': batch_size,
        'thread_count': thread_count,
        # Database Connection Settings
        'sf_account': sf_account,
        'sf_user': sf_user,
        'sf_role': sf_role,
        'default_warehouse': default_warehouse,
        'default_database': default_database,
        'default_schema': default_schema,
        'ts_instance_url': ts_instance_url,
        'ts_username': ts_username,
        'data_adjuster_url': data_adjuster_url,
        # Status
        'settings_status': settings_status,
        # Admin-only visibility toggles
        '_admin_ai_accordion': admin_ai_accordion,
        '_admin_db_accordion': admin_db_accordion,
    }
    
    # Get inputs in schema order (exclude settings_status which is output only)
    save_inputs = [all_components[key] for key, storage_key, _, _ in SETTINGS_SCHEMA if storage_key is not None]
    
    save_settings_btn.click(
        fn=save_settings_handler,
        inputs=save_inputs,
        outputs=[settings_status]
    )
    
    reset_settings_btn.click(
        fn=reset_settings_handler,
        inputs=[],
        outputs=[settings_status]
    )

    # -----------------------------------------------------------------------
    # Change Password
    # -----------------------------------------------------------------------
    gr.Markdown("---")
    with gr.Accordion("🔒 Change Password", open=False):
        with gr.Row():
            with gr.Column(scale=1):
                cp_current = gr.Textbox(label="Current Password", type="password", placeholder="Your current password")
                cp_new = gr.Textbox(label="New Password", type="password", placeholder="New password")
                cp_confirm = gr.Textbox(label="Confirm New Password", type="password", placeholder="Repeat new password")
                cp_btn = gr.Button("Change Password", variant="primary")
                cp_status = gr.Markdown("")
            with gr.Column(scale=1):
                gr.Markdown("""
                **Password requirements:**
                - Current password required to confirm identity

                *Forgot your password? Ask an admin to reset it via the Admin tab, then change it here after signing in.*
                """)

    def change_password_handler(current, new_pw, confirm, request: gr.Request = None):
        if not current or not new_pw or not confirm:
            return "❌ All fields are required."
        if new_pw != confirm:
            return "❌ New passwords don't match."
        try:
            user_email = require_authenticated_email(request)
            from supabase_client import UserManager
            um = UserManager()
            if not um.enabled:
                return "⚠️ Supabase not configured — password change unavailable."
            if not um.authenticate(user_email, current):
                return "❌ Current password is incorrect."
            um.reset_password(user_email, new_pw)
            um.clear_must_change_password(user_email)
            return "✅ Password changed successfully. Use your new password next time you sign in."
        except Exception as e:
            return f"❌ Error: {e}"

    cp_btn.click(
        fn=change_password_handler,
        inputs=[cp_current, cp_new, cp_confirm],
        outputs=[cp_status]
    )

    # Return components dict for loading (follows schema order)
    return all_components


def authenticate_user(username: str, password: str) -> bool:
    """
    Gradio auth callback — validates against Supabase demoprep_users table.
    Falls back to no-auth if Supabase is not configured (local dev).
    """
    try:
        from supabase_client import UserManager
        um = UserManager()
        if not um.enabled:
            # Supabase not configured — allow anyone (local dev mode)
            print(f"[Auth] Supabase not configured, allowing login for: {username}")
            return True
        
        user = um.authenticate(username, password)
        if user:
            # Keep env-backed modules aligned with Supabase admin settings after login.
            inject_admin_settings_to_env()
            print(f"[Auth] Login successful: {username} (admin={user.get('is_admin', False)})")
            return True
        else:
            print(f"[Auth] Login failed: {username}")
            return False
    except Exception as e:
        print(f"[Auth] Error during authentication: {e}")
        # If auth system is broken, don't lock everyone out
        return False


_REMEMBER_ME_SCRIPT = """
\t\t\t<script>
\t\t\t/* DemoPrep: Remember Me — Gradio 4.44.1 uses div.form, not <form>, no shadow DOM */
\t\t\t(function() {
\t\t\t\tvar KEY = 'demoprep_remembered_user';
\t\t\t\tvar _done = false;

\t\t\t\tfunction injectIntoForm(loginRoot) {
\t\t\t\t\tif (_done || !loginRoot) return;
\t\t\t\t\tvar formDiv = loginRoot.querySelector('div.form');
\t\t\t\t\tif (!formDiv) return;
\t\t\t\t\tvar inputs = formDiv.querySelectorAll('input, textarea');
\t\t\t\t\tvar uInput = null;
\t\t\t\t\tfor (var i = 0; i < inputs.length; i++) {
\t\t\t\t\t\tvar tp = (inputs[i].type || inputs[i].tagName).toLowerCase();
\t\t\t\t\t\tif (tp !== 'password' && tp !== 'submit' && tp !== 'checkbox' && tp !== 'hidden') {
\t\t\t\t\t\t\tuInput = inputs[i]; break;
\t\t\t\t\t\t}
\t\t\t\t\t}
\t\t\t\t\tif (!uInput || loginRoot.querySelector('#dp-rmb')) return;

\t\t\t\t\tvar saved = localStorage.getItem(KEY);
\t\t\t\t\tif (saved && !uInput.value) {
\t\t\t\t\t\ttry {
\t\t\t\t\t\t\tvar desc = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value') ||
\t\t\t\t\t\t\t           Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value');
\t\t\t\t\t\t\tif (desc && desc.set) desc.set.call(uInput, saved);
\t\t\t\t\t\t\telse uInput.value = saved;
\t\t\t\t\t\t\tuInput.dispatchEvent(new Event('input', { bubbles: true }));
\t\t\t\t\t\t} catch(e) { uInput.value = saved; }
\t\t\t\t\t\tvar pwInput = formDiv.querySelector('input[type="password"]');
\t\t\t\t\t\tif (pwInput) setTimeout(function() { pwInput.focus(); }, 50);
\t\t\t\t\t}

\t\t\t\t\tvar lbl = document.createElement('label');
\t\t\t\t\tlbl.style.cssText = 'display:flex;align-items:center;gap:6px;font-size:13px;margin:8px 0 12px;cursor:pointer;color:#374151;';
\t\t\t\t\tvar cb = document.createElement('input');
\t\t\t\t\tcb.type = 'checkbox'; cb.id = 'dp-rmb'; cb.checked = !!saved; cb.style.cursor = 'pointer';
\t\t\t\t\tlbl.appendChild(cb);
\t\t\t\t\tlbl.appendChild(document.createTextNode(' Remember me'));
\t\t\t\t\tvar btn = loginRoot.querySelector('button');
\t\t\t\t\tif (btn && btn.parentNode) btn.parentNode.insertBefore(lbl, btn);
\t\t\t\t\telse loginRoot.appendChild(lbl);

\t\t\t\t\tfunction save() {
\t\t\t\t\t\tif (cb.checked && uInput.value) localStorage.setItem(KEY, uInput.value);
\t\t\t\t\t\telse localStorage.removeItem(KEY);
\t\t\t\t\t}
\t\t\t\t\tif (btn) btn.addEventListener('click', save);

\t\t\t\t\t_done = true;
\t\t\t\t\tconsole.log('[DemoPrep] Remember me injected');
\t\t\t\t}

\t\t\t\tfunction tryInject() {
\t\t\t\t\tif (_done) return;
\t\t\t\t\tif (!window.gradio_config || !window.gradio_config.auth_required) return;
\t\t\t\t\tvar wrap = document.querySelector('div.wrap');
\t\t\t\t\tif (wrap) { injectIntoForm(wrap); if (_done) return; }
\t\t\t\t\tdocument.querySelectorAll('div.form').forEach(function(f) {
\t\t\t\t\t\tif (!_done) injectIntoForm(f.parentElement);
\t\t\t\t\t});
\t\t\t\t}

\t\t\t\tvar t = setInterval(function() { tryInject(); if (_done) clearInterval(t); }, 200);
\t\t\t\tsetTimeout(function() { clearInterval(t); }, 10000);
\t\t\t})();
\t\t\t</script>"""

_REMEMBER_ME_MARKER = "/* DemoPrep: Remember Me"
_REMEMBER_ME_ANCHOR = '</head>'


def _patch_gradio_template():
    """Inject the Remember Me script into Gradio's index.html template.
    Runs at startup — idempotent, works on any machine including HuggingFace.
    """
    import gradio
    template_path = os.path.join(
        os.path.dirname(gradio.__file__),
        "templates", "frontend", "index.html"
    )
    try:
        content = open(template_path, encoding="utf-8").read()
        if _REMEMBER_ME_MARKER in content:
            print("[DemoPrep] Gradio template already patched — skipping")
            return
        if _REMEMBER_ME_ANCHOR not in content:
            print("[DemoPrep] WARNING: Could not find anchor in Gradio template — Remember Me not injected")
            return
        patched = content.replace(_REMEMBER_ME_ANCHOR, _REMEMBER_ME_SCRIPT + _REMEMBER_ME_ANCHOR)
        open(template_path, "w", encoding="utf-8").write(patched)
        print(f"[DemoPrep] Gradio template patched with Remember Me script")
    except Exception as e:
        print(f"[DemoPrep] WARNING: Could not patch Gradio template: {e}")


if __name__ == "__main__":
    """Launch the chat interface standalone"""
    print("Starting Chat-Based Demo Builder...")

    _patch_gradio_template()

    app = create_chat_interface()
    
    # Enable queue with concurrency to handle multiple requests
    app.queue(
        default_concurrency_limit=10,  # Allow up to 10 concurrent requests
        api_open=False
    )
    
    
    # Bypass Gradio localhost accessibility check (httpx 0.28 compatibility)
    import gradio.networking as _gn
    _gn.url_ok = lambda url: True

    # Determine auth mode
    # If DEMOPREP_NO_AUTH=true, skip login (local dev override)
    no_auth = os.getenv('DEMOPREP_NO_AUTH', 'false').lower() in ('true', '1', 'yes')
    auth_fn = None if no_auth else authenticate_user

    app.launch(
        server_name="0.0.0.0",
        server_port=7863,  # Different port from main app (7860) and old chat (7861)
        share=False,
        inbrowser=False,
        debug=True,
        auth=auth_fn,

        max_threads=20  # Allow multiple threads for concurrent requests
    )