feat: App/Chat tab split, Spotter Viz dual stories, Custom vertical, bug fixes

UI:
- Rename Chat tab → App; split input into App (form) and Chat (freeform) sub-tabs
- Remove quick action buttons (Start Research, Configure, Help)
- Add Custom vertical option — disables Function dd, makes Context required
- Context field renamed from "Additional context (optional)" → "Context"
- Phase log timer now activates on Send + Enter, not just GO
- GO button bypasses init confirmation dialog, goes straight to pipeline

Spotter Viz Story:
- Split into two sub-tabs: ThoughtSpot Story (matrix-driven) and AI Story (pure LLM)
- New _generate_matrix_spotter_story() — builds from vertical×function KPIs/liveboard_questions
- New _generate_ai_spotter_story() — pure LLM, no matrix constraints
- New spotter_viz_story_matrix prompt in prompts.py

Bug fixes:
- NaN/Inf floats in LegitData population now replaced with NULL (fixes Snowflake 22018 errors)
- TS auth key: KEY_VAR value is the key directly, not a pointer to another env var
- Async pipeline plan added to sprint doc

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

chat_interface.py

@@ -19,10 +19,11 @@ from demo_builder_class import DemoBuilder
 from supabase_client import load_gradio_settings, get_admin_setting, inject_admin_settings_to_env
 from main_research import MultiLLMResearcher, Website
 from demo_personas import (
-    build_company_analysis_prompt, 
+    build_company_analysis_prompt,
     build_industry_research_prompt,
     VERTICALS,
     FUNCTIONS,
+    MATRIX_OVERRIDES,
     get_use_case_config,
     parse_use_case
 )
@@ -69,7 +70,7 @@ def get_ts_env_url(label: str) -> str:
 
 def get_ts_env_auth_key(label: str) -> str:
     """Return the actual auth key value for a given environment label.
-    TS_ENV_N_KEY_VAR holds the name of the env var containing the token.
+    TS_ENV_N_KEY_VAR holds the trusted auth key directly.
     """
     i = 1
     while True:
@@ -77,8 +78,7 @@ def get_ts_env_auth_key(label: str) -> str:
         if not env_label:
             break
         if env_label == label:
-            key_var = os.getenv(f'TS_ENV_{i}_KEY_VAR', '').strip()
-            return os.getenv(key_var, '').strip() if key_var else ''
+            return os.getenv(f'TS_ENV_{i}_KEY_VAR', '').strip()
         i += 1
     return ''
 
@@ -232,8 +232,10 @@ class ChatDemoInterface:
         self.pending_generic_use_case = None
         # New tab content
         self.live_progress_log = []  # Real-time deployment progress
-        self.demo_pack_content = ""   # Generated demo pack markdown
-        self.spotter_viz_story = ""   # Spotter Viz story (NL prompts for Spotter Viz agent)
+        self.phase_log = []          # High-level pipeline status (shown in right panel)
+        self.demo_pack_content = ""    # Generated demo pack markdown
+        self.spotter_story_ai = ""     # Pure AI-generated Spotter Viz story
+        self.spotter_story_matrix = "" # Matrix/ThoughtSpot-recommended Spotter Viz story
         # Per-session loggers (NOT module-level singletons — avoids cross-session contamination)
         self._session_logger = None
         self._prompt_logger = None
@@ -247,8 +249,8 @@ class ChatDemoInterface:
         """Load settings from Supabase or defaults"""
         # Fallback defaults (ONLY used if settings not found)
         defaults = {
-            'company': 'Amazon.com',
-            'use_case': 'Sales Analytics',
+            'company': '',
+            'use_case': '',
             'model': DEFAULT_LLM_MODEL,
             'fact_table_size': '1000',
             'dim_table_size': '100',
@@ -291,50 +293,28 @@ class ChatDemoInterface:
     
     def format_welcome_message(self, company, use_case):
         """Create the initial welcome message"""
-        # Always show the setup message - user must explicitly provide company and use case
-        needs_setup = True
-        
-        if needs_setup:
-            # Build use case examples from VERTICALS × FUNCTIONS
-            uc_examples = []
-            for v_name in list(VERTICALS.keys())[:3]:  # Show first 3 verticals
-                for f_name in list(FUNCTIONS.keys())[:2]:  # Show first 2 functions each
-                    uc_examples.append(f"   - {v_name} {f_name}")
-            uc_list = "\n".join(uc_examples)
-            
-            return f"""👋 **Welcome to ThoughtSpot Demo Builder!**
-
-Tell me who you're demoing to and what they care about — I'll build the whole thing.
-
-**Just say something like:**
-- *"Nike.com, Retail Sales"*
-- *"Salesforce.com, Software Sales"*
-- *"Target.com — supply chain analytics for their VP of Operations"*
-- *"Pfizer.com — analyzing clinical trial pipeline for a CMO persona"*
-
-**Pre-configured use cases** (KPIs, outliers, and Spotter questions ready to go):
-{uc_list}
-   - ...and more!
+        return """## Welcome to ThoughtSpot Demo Builder
 
-**Or describe any custom use case** — AI will research and build it from scratch.
+I'll research a company, build a Snowflake schema, generate realistic data, and deploy a ThoughtSpot model and liveboard — all from a single prompt.
 
-> 💡 **Tip:** Select your ThoughtSpot environment and AI model in the panel to the right before starting.
+**How to start:**
+- **Defined** — pick a vertical and function from the dropdowns below, add a company URL if you have one, and hit **→ GO**.
+- **Custom** — describe what you want in your own words in the chat box.
 
-*What company and use case are you working on?*"""
-        else:
-            return f"""👋 **Welcome to ThoughtSpot Demo Builder!**
-
-I'm ready to create a perfect ThoughtSpot demo!
+<details>
+<summary>📋 Example use cases</summary>
 
-**Company:** {company}  
-**Use Case:** {use_case}
+| Company | Vertical | Function | Story |
+|---------|----------|----------|-------|
+| Target.com | Retail | Sales | ASP decline, regional variance, holiday surge |
+| Walmart.com | Retail | Supply Chain | Stockout risk, OTIF, days on hand |
+| Chase.com | Banking | Marketing | Funnel drop-off, channel CTR, cost per acquisition |
+| Salesforce.com | Software | Sales | ARR by segment, pipeline coverage, win rate |
+| Caterpillar.com | Manufacturing | Supply Chain | Inventory levels, supplier performance |
 
-**Ready to start?**
-- Type "start" to begin research
-- Type `/over` to change company or use case
-- Or just tell me what you'd like to do!
+</details>
 
-What's your first step?"""
+> 💡 Select your ThoughtSpot environment in the right panel before starting."""
     
     def validate_required_settings(self) -> list:
         """
@@ -346,8 +326,6 @@ What's your first step?"""
         # Check admin settings (from environment, injected from Supabase)
         admin_checks = {
             'THOUGHTSPOT_URL': get_admin_setting('THOUGHTSPOT_URL', required=False),
-            'THOUGHTSPOT_TRUSTED_AUTH_KEY': get_admin_setting('THOUGHTSPOT_TRUSTED_AUTH_KEY', required=False),
-            'THOUGHTSPOT_ADMIN_USER': get_admin_setting('THOUGHTSPOT_ADMIN_USER', required=False),
             'SNOWFLAKE_ACCOUNT': get_admin_setting('SNOWFLAKE_ACCOUNT', required=False),
             'SNOWFLAKE_KP_USER': get_admin_setting('SNOWFLAKE_KP_USER', required=False),
             'SNOWFLAKE_KP_PK': get_admin_setting('SNOWFLAKE_KP_PK', required=False),
@@ -440,8 +418,10 @@ What's your first step?"""
                     from supabase_client import get_admin_setting
 
                     ts_url = (self.settings.get('thoughtspot_url') or '').strip() or get_admin_setting('THOUGHTSPOT_URL')
-                    ts_user = get_admin_setting('THOUGHTSPOT_ADMIN_USER')
-                    ts_secret = (self.settings.get('thoughtspot_trusted_auth_key') or '').strip() or get_admin_setting('THOUGHTSPOT_TRUSTED_AUTH_KEY')
+                    ts_user = self._get_effective_user_email()
+                    ts_secret = (self.settings.get('thoughtspot_trusted_auth_key') or '').strip()
+                    if not ts_secret:
+                        raise ValueError("ThoughtSpot auth key not set — select a TS environment from the dropdown")
 
                     ts_client = ThoughtSpotDeployer(base_url=ts_url, username=ts_user, secret_key=ts_secret)
                     ts_client.authenticate()
@@ -495,7 +475,7 @@ Type **done** when finished."""
                 # User provided just a URL - ask them to include the use case too
                 detected_company = standalone_url.group(1)
                 # Build dynamic use case list from VERTICALS × FUNCTIONS
-                uc_opts = [f"- {v} {f}" for v in list(VERTICALS.keys())[:3] for f in FUNCTIONS.keys()]
+                uc_opts = [f"- {v} {f}" for v in VERTICALS.keys() for f in FUNCTIONS.keys()]
                 uc_opts_str = "\n".join(uc_opts)
                 response = f"""I see you want to use **{detected_company}** - great choice!
 
@@ -511,7 +491,7 @@ I'm creating a demo for company: {detected_company} use case: Retail Sales
 
 What use case would you like for {detected_company}?"""
                 chat_history[-1] = (message, response)
-                yield chat_history, current_stage, current_model, company, use_case, ""
+                yield chat_history, current_stage, current_model, detected_company, use_case, ""
                 return
             
             # Check if user is providing company and use case
@@ -703,14 +683,14 @@ This may take 1-2 minutes. Watch the AI Feedback tab for progress!"""
                 return
 
             elif extracted_company and not extracted_use_case:
-                uc_opts = "\n".join([f"- {v} {f}" for v in list(VERTICALS.keys())[:3] for f in FUNCTIONS.keys()])
+                uc_opts = "\n".join([f"- {v} {f}" for v in VERTICALS.keys() for f in FUNCTIONS.keys()])
                 response = f"""Got it — **{extracted_company}**!
 
 What use case are we building? A few options:
 {uc_opts}
 - Or describe any custom scenario!"""
                 chat_history[-1] = (message, response)
-                yield chat_history, current_stage, current_model, company, use_case, ""
+                yield chat_history, current_stage, current_model, extracted_company, use_case, ""
                 return
 
             else:
@@ -748,8 +728,10 @@ Try something like:
             
             if validation_mode == 'Off':
                 # AUTO-RUN MODE: Run entire pipeline without any more prompts
-                current_stage = 'deploy'
-                
+                current_stage = 'research'
+                self.phase_log.append(f"🚀 Starting pipeline — {company} · {use_case}")
+                self.phase_log.append("🔍 Phase 1: Research")
+
                 auto_run_msg = f"""✅ **Starting Auto-Run Mode**
 
 **Company:** {company}
@@ -776,23 +758,29 @@ Watch the AI Feedback tab for real-time progress!"""
                 
                 self.log_feedback(f"DEBUG AUTO-RUN: Research loop EXITED after {research_yield_count} yields")
                 
-                # Show research complete
+                # Show research complete, move to DDL stage
+                current_stage = 'create_ddl'
+                self.phase_log.append("✅ Research complete")
+                self.phase_log.append("📐 Phase 2: DDL creation")
                 chat_history[-1] = (message, "✅ **Research Complete!**\n\n📝 **Creating DDL...**")
                 yield chat_history, current_stage, current_model, company, use_case, ""
                 self.log_feedback("DEBUG AUTO-RUN: Moving to DDL creation...")
-                
+
                 # Auto-create DDL
                 ddl_response, ddl_code = self.run_ddl_creation()
-                
+
                 # Check if DDL creation failed
                 if not ddl_code or ddl_code.strip() == "":
                     chat_history[-1] = (message, f"{ddl_response}\n\n❌ **Cannot proceed without valid DDL.** Please fix the error and try again.")
                     yield chat_history, current_stage, current_model, company, use_case, ""
                     return
-                
+
+                current_stage = 'deploy'
+                self.phase_log.append("✅ DDL created")
+                self.phase_log.append("🏗️ Phase 3: Snowflake deploy")
                 chat_history[-1] = (message, f"✅ DDL Created\n\n🚀 **Deploying to Snowflake...**")
                 yield chat_history, current_stage, current_model, company, use_case, ""
-                
+
                 # Auto-deploy
                 with open('/tmp/demoprep_debug.log', 'a') as f:
                     f.write(f"AUTO-RUN: Starting deployment streaming\n")
@@ -825,9 +813,11 @@ Watch the AI Feedback tab for real-time progress!"""
                             deploy_response, _, auto_schema = final_result
                             with open('/tmp/demoprep_debug.log', 'a') as f:
                                 f.write(f"AUTO-RUN: auto_ts detected, schema={auto_schema}\n")
+                            self.phase_log.append("✅ Snowflake deploy complete")
+                            self.phase_log.append("🔷 Phase 4: ThoughtSpot")
                             chat_history[-1] = (message, deploy_response)
                             yield chat_history, current_stage, current_model, company, use_case, ""
-                            
+
                             # Run ThoughtSpot deployment with detailed logging
                             with open('/tmp/demoprep_debug.log', 'a') as f:
                                 f.write(f"AUTO-RUN: Creating TS generator...\n")
@@ -848,6 +838,8 @@ Watch the AI Feedback tab for real-time progress!"""
                                 yield chat_history, current_stage, current_model, company, use_case, ""
                             with open('/tmp/demoprep_debug.log', 'a') as f:
                                 f.write(f"AUTO-RUN: TS deployment loop complete, {ts_update_count} updates\n")
+                            self.phase_log.append("✅ ThoughtSpot complete")
+                            self.phase_log.append("🎉 Pipeline done!")
                         else:
                             # Not auto_ts, just show the result
                             if len(final_result) >= 2:
@@ -1658,7 +1650,7 @@ Try a different request or type **'done'** to finish."""
                     continue
                 return use_case
 
-        # Fallback: text after a comma or dash following a domain.tld
+        # Fallback 1: text after a comma or dash following a domain.tld
         # Handles: "Nike.com, Retail Sales" / "Nike.com - Supply Chain"
         after_domain = re.search(
             r'[a-zA-Z0-9-]+\.[a-zA-Z]{2,}[\s]*[,\-–—]\s*(.+)',
@@ -1669,6 +1661,17 @@ Try a different request or type **'done'** to finish."""
             if use_case and not re.search(r'\.(com|org|net|io|co|ai)\b', use_case, re.IGNORECASE):
                 return use_case
 
+        # Fallback 2: text after domain.tld separated only by a space
+        # Handles: "Caterpillar.com Manufacturing Supply Chain"
+        after_domain_space = re.search(
+            r'[a-zA-Z0-9-]+\.[a-zA-Z]{2,}\s+(.+)',
+            message, re.IGNORECASE
+        )
+        if after_domain_space:
+            use_case = after_domain_space.group(1).strip().rstrip('.')
+            if use_case and not re.search(r'\.(com|org|net|io|co|ai)\b', use_case, re.IGNORECASE):
+                return use_case
+
         return None
     
     def handle_override(self, message):
@@ -2267,96 +2270,108 @@ To change settings, use:
 - **Ask questions**: Let the AI demonstrate natural language
 - **End with action**: Show how insights lead to decisions""")
     
-    def _generate_spotter_viz_story(self, company_name: str, use_case: str,
-                                     model_name: str = None, liveboard_name: str = None) -> str:
-        """Generate a Spotter Viz story — a conversational sequence of NL prompts
-        that can be entered into ThoughtSpot's Spotter Viz agent to build a liveboard.
-
-        Returns two sections:
-        1. Persona-driven story built from liveboard_questions config (always present)
-        2. AI-generated story from LLM (appended after a divider)
+    def _generate_ai_spotter_story(self, company_name: str, use_case: str,
+                                    model_name: str = None, liveboard_name: str = None) -> str:
+        """Pure AI-generated Spotter Viz story — no matrix reference.
+        AI decides what a compelling liveboard story looks like for this company + use case.
         """
-        from demo_personas import parse_use_case, get_use_case_config
+        from prompts import build_prompt
+        from demo_personas import parse_use_case
 
         v, f = parse_use_case(use_case or '')
-        uc_cfg = get_use_case_config(v or "Generic", f or "Generic")
-        lq = uc_cfg.get("liveboard_questions", [])
-        data_source = model_name or f"{company_name} model"
-
-        # --- Section 1: Persona-driven story from liveboard_questions ---
-        header = f"""# Spotter Viz Story: {company_name}
-## {use_case}
-
-*Copy these prompts into ThoughtSpot Spotter Viz to build this liveboard interactively.*
-
----
-
-## Part 1: Structured Demo Flow
-
-"""
-        steps = []
-        step_num = 1
-        for q in lq:
-            title = q['title']
-            viz_q = q['viz_question']
-            insight = q.get('insight', '')
-            spotter_qs = q.get('spotter_qs', [])
-
-            step = f"### Step {step_num}: {title}\n"
-            step += f'> "{viz_q}"\n\n'
-            if insight:
-                step += f"**What to look for:** {insight}\n\n"
-            if spotter_qs:
-                step += "**Follow-up Spotter questions:**\n"
-                for sq in spotter_qs[:2]:
-                    step += f'> "{sq}"\n\n'
-            steps.append(step)
-            step_num += 1
-
-        if steps:
-            persona_section = header + "\n".join(steps)
-        else:
-            persona_section = header + f'> "Build a {use_case} dashboard for {company_name} using {data_source}"\n\n'
-
-        persona_section += "\n---\n\n"
+        vertical = v or "Generic"
+        function = f or "Generic"
+
+        company_context = f"Company: {company_name}\nUse Case: {use_case}"
+        if model_name:
+            company_context += f"\nData Source/Model: {model_name}"
+        if liveboard_name:
+            company_context += f"\nLiveboard Name: {liveboard_name}"
+        if hasattr(self, 'demo_builder') and self.demo_builder:
+            research = getattr(self.demo_builder, 'company_summary', '') or ''
+            if research:
+                company_context += f"\n\nCompany Research:\n{research[:1500]}"
 
-        # --- Section 2: AI-generated story ---
         try:
-            from prompts import build_prompt
-
-            vertical = v or "Generic"
-            function = f or "Generic"
-
-            company_context = f"Company: {company_name}\nUse Case: {use_case}"
-            if model_name:
-                company_context += f"\nData Source/Model: {model_name}"
-            if liveboard_name:
-                company_context += f"\nLiveboard Name: {liveboard_name}"
-            if hasattr(self, 'demo_builder') and self.demo_builder:
-                research = getattr(self.demo_builder, 'company_summary', '') or ''
-                if research:
-                    company_context += f"\n\nCompany Research:\n{research[:1500]}"
-
             prompt = build_prompt(
                 stage="spotter_viz_story",
                 vertical=vertical,
                 function=function,
                 company_context=company_context,
             )
-
             llm_model = self.settings.get('model', DEFAULT_LLM_MODEL)
             self.log_feedback(f"🎬 Generating AI Spotter Viz story ({llm_model})...")
-
             provider_name, model_name_str = map_llm_display_to_provider(llm_model)
             researcher = MultiLLMResearcher(provider=provider_name, model=model_name_str)
-            ai_story = researcher.make_request(prompt, max_tokens=2000, temperature=0.7)
-            ai_section = "## Part 2: AI-Generated Story\n\n" + ai_story
-
+            return researcher.make_request(prompt, max_tokens=2000, temperature=0.7)
         except Exception as e:
             self.log_feedback(f"⚠️ AI Spotter Viz story generation failed: {e}")
-            ai_section = "## Part 2: AI-Generated Story\n\n*(Generation failed — use the structured flow above.)*"
+            return f"*(Generation failed: {e})*"
+
+    def _generate_matrix_spotter_story(self, company_name: str, use_case: str,
+                                        model_name: str = None, liveboard_name: str = None) -> str:
+        """Matrix/ThoughtSpot-recommended Spotter Viz story.
+        Builds from the vertical×function matrix (KPIs, liveboard_questions, story controls, persona).
+        AI writes it — adds narrative — but every step comes from the matrix.
+        """
+        from prompts import build_prompt
+        from demo_personas import parse_use_case, get_use_case_config
+
+        v, f = parse_use_case(use_case or '')
+        uc_cfg = get_use_case_config(v or "Generic", f or "Generic")
+        data_source = model_name or f"{company_name} model"
+
+        # Build rich matrix context for the prompt
+        kpis = uc_cfg.get("kpis", [])
+        lq = uc_cfg.get("liveboard_questions", [])
+        story_controls = uc_cfg.get("story_controls", {})
+        persona = uc_cfg.get("persona", "")
+        business_problem = uc_cfg.get("business_problem", "")
+        use_case_name = uc_cfg.get("use_case_name", use_case)
+
+        matrix_context = f"Company: {company_name}\nUse Case: {use_case_name}\nData Source/Model: {data_source}"
+        if liveboard_name:
+            matrix_context += f"\nLiveboard Name: {liveboard_name}"
+        if persona:
+            matrix_context += f"\nTarget Persona: {persona}"
+        if business_problem:
+            matrix_context += f"\nBusiness Problem: {business_problem}"
+
+        if kpis:
+            kpi_lines = "\n".join(f"  - {k['name']}: {k.get('definition', '')}" for k in kpis)
+            matrix_context += f"\n\nKPIs (from ThoughtSpot matrix):\n{kpi_lines}"
+
+        if lq:
+            matrix_context += "\n\nLiveboard Questions (in order):"
+            for q in lq:
+                req = " [required]" if q.get("required") else ""
+                matrix_context += f"\n  - {q['title']} ({q.get('viz_type','chart')}){req}: {q['viz_question']}"
+                if q.get('insight'):
+                    matrix_context += f"\n    Insight: {q['insight']}"
+
+        if story_controls:
+            dims = story_controls.get("dimensions", [])
+            if dims:
+                matrix_context += f"\n\nKey Dimensions: {', '.join(dims)}"
+            seasonal = story_controls.get("seasonal_strength") or story_controls.get("seasonal")
+            if seasonal:
+                matrix_context += f"\nSeasonal pattern: {seasonal}"
 
-        return persona_section + ai_section
+        try:
+            prompt = build_prompt(
+                stage="spotter_viz_story_matrix",
+                vertical=v or "Generic",
+                function=f or "Generic",
+                company_context=matrix_context,
+            )
+            llm_model = self.settings.get('model', DEFAULT_LLM_MODEL)
+            self.log_feedback(f"🎬 Generating Matrix Spotter Viz story ({llm_model})...")
+            provider_name, model_name_str = map_llm_display_to_provider(llm_model)
+            researcher = MultiLLMResearcher(provider=provider_name, model=model_name_str)
+            return researcher.make_request(prompt, max_tokens=2000, temperature=0.6)
+        except Exception as e:
+            self.log_feedback(f"⚠️ Matrix Spotter Viz story generation failed: {e}")
+            return f"*(Generation failed: {e})*"
     
     def _build_fallback_spotter_story(self, company_name: str, use_case: str, 
                                        model_name: str = None) -> str:
@@ -3538,9 +3553,11 @@ Tables: Created and populated
                 
                 # Get ThoughtSpot settings
                 ts_url = get_admin_setting('THOUGHTSPOT_URL')
-                ts_user = get_admin_setting('THOUGHTSPOT_ADMIN_USER')
-                ts_secret = get_admin_setting('THOUGHTSPOT_TRUSTED_AUTH_KEY')
-                
+                ts_user = self._get_effective_user_email()
+                ts_secret = self.settings.get('thoughtspot_trusted_auth_key')
+                if not ts_secret:
+                    raise ValueError("ThoughtSpot trusted auth key not set. Select a TS environment from the dropdown.")
+
                 liveboard_method = 'HYBRID'  # Only HYBRID method is supported
                 
                 # Clean company name for display (strip .com, .org, etc)
@@ -3672,10 +3689,12 @@ Cannot deploy to ThoughtSpot without tables.""",
             yield f"**Starting ThoughtSpot Deployment...**\n\nSchema verified: {database}.{schema_name}\nFound {len(tables)} tables\n\n"
             
             # Create deployer — prefer session-selected env (from TS env dropdown),
-            # fall back to admin settings
+            # fall back to admin settings. Use logged-in user so TS objects are owned by them.
             ts_url = (self.settings.get('thoughtspot_url') or '').strip() or get_admin_setting('THOUGHTSPOT_URL')
-            ts_user = get_admin_setting('THOUGHTSPOT_ADMIN_USER')
-            ts_secret = (self.settings.get('thoughtspot_trusted_auth_key') or '').strip() or get_admin_setting('THOUGHTSPOT_TRUSTED_AUTH_KEY')
+            ts_user = self._get_effective_user_email()
+            ts_secret = self.settings.get('thoughtspot_trusted_auth_key')
+            if not ts_secret:
+                raise ValueError("ThoughtSpot trusted auth key not set. Select a TS environment from the dropdown.")
             
             deployer = ThoughtSpotDeployer(
                 base_url=ts_url,
@@ -3760,6 +3779,7 @@ This chat will update when complete."""
                         tag_name=tag_name_value,
                         liveboard_method=liveboard_method,
                         share_with=self.settings.get('share_with', '').strip() or None,
+                        company_research=self.demo_builder.get_research_context() if self.demo_builder else None,
                         progress_callback=progress_callback
                     )
                 except Exception as e:
@@ -3822,10 +3842,15 @@ Steps:
             if results.get('success'):
                 safe_print("DEPLOYMENT COMPLETE", flush=True)
                 self.live_progress_log.extend(["", "=" * 60, "DEPLOYMENT COMPLETE", "=" * 60])
+                # Capture any non-fatal errors (e.g. enhance failure) so they reach session_logs
+                if results.get('errors'):
+                    _ts_error = '; '.join(results['errors'])
             else:
                 safe_print("DEPLOYMENT FAILED", flush=True)
                 safe_print(f"Errors: {results.get('errors', [])}", flush=True)
                 self.live_progress_log.extend(["", "DEPLOYMENT FAILED", f"Errors: {results.get('errors', [])}"])
+                if results.get('errors'):
+                    _ts_error = '; '.join(results['errors'])
             safe_print("="*60 + "\n", flush=True)
             
             progress_log = '\n'.join(progress_messages) if progress_messages else 'No progress messages captured'
@@ -3894,21 +3919,23 @@ Ask these questions to showcase ThoughtSpot's AI capabilities:
                     safe_print(f"Could not generate demo pack: {e}", flush=True)
                     self.demo_pack_content = f"*Demo pack generation failed: {e}*"
                 
-                # Generate Spotter Viz Story
+                # Generate Spotter Viz Stories (both matrix and AI versions)
                 try:
-                    model_name_for_story = results.get('model', None)
-                    liveboard_name_for_story = results.get('liveboard', None)
-                    self.spotter_viz_story = self._generate_spotter_viz_story(
+                    _story_args = dict(
                         company_name=company_name,
                         use_case=use_case,
-                        model_name=model_name_for_story,
-                        liveboard_name=liveboard_name_for_story
+                        model_name=results.get('model', None),
+                        liveboard_name=results.get('liveboard', None),
                     )
-                    safe_print("Spotter Viz Story generated - check the Spotter Viz Story tab.", flush=True)
-                    self.live_progress_log.append("Spotter Viz Story generated")
+                    self.spotter_story_matrix = self._generate_matrix_spotter_story(**_story_args)
+                    self.spotter_story_ai = self._generate_ai_spotter_story(**_story_args)
+                    safe_print("Spotter Viz Stories generated - check the Spotter Viz Story tab.", flush=True)
+                    self.live_progress_log.append("Spotter Viz Stories generated")
                 except Exception as e:
                     safe_print(f"Could not generate Spotter Viz story: {e}", flush=True)
-                    self.spotter_viz_story = f"*Spotter Viz story generation failed: {e}*"
+                    _err = f"*(Generation failed: {e})*"
+                    self.spotter_story_matrix = _err
+                    self.spotter_story_ai = _err
             
             # Build final response
             if results.get('success'):
@@ -4265,15 +4292,23 @@ def create_chat_interface():
     
     # Bootstrap defaults before authenticated user-specific settings are loaded
     default_settings = {
-        "company": "Amazon.com",
-        "use_case": "Sales Analytics",
+        "company": "",
+        "use_case": "",
         "model": DEFAULT_LLM_MODEL,
         "stage": "initialization",
     }
     
     with gr.Blocks(
         title="ThoughtSpot Demo Builder - Chat",
-        theme=gr.themes.Soft(primary_hue="blue", secondary_hue="cyan")
+        theme=gr.themes.Soft(primary_hue="blue", secondary_hue="cyan"),
+        css="""
+.tabitem { padding-top: 6px !important; }
+/* Remove chatbot box — let content sit directly on page */
+#main-chatbot { border: none !important; background: transparent !important; box-shadow: none !important; padding: 0 !important; }
+#main-chatbot > div { border: none !important; background: transparent !important; box-shadow: none !important; }
+#main-chatbot .bubble-wrap { background: transparent !important; padding-top: 0 !important; padding-bottom: 0 !important; }
+#main-chatbot .wrap { background: transparent !important; border: none !important; }
+""",
     ) as interface:
         
         # SESSION STATE: Each user gets their own ChatDemoInterface instance
@@ -4317,7 +4352,7 @@ def create_chat_interface():
         demo_pack_state = gr.State("")
         
         with gr.Tabs():
-            with gr.Tab("💬 Chat"):
+            with gr.Tab("📱 App"):
                 chat_components = create_chat_tab(
                     chat_controller_state, default_settings, current_stage, current_model, 
                     current_company, current_usecase,
@@ -4366,16 +4401,28 @@ def create_chat_interface():
                 )
             
             with gr.Tab("🎬 Spotter Viz Story"):
-                gr.Markdown("### Spotter Viz Story — Natural Language Liveboard Builder")
-                gr.Markdown("*Conversational prompts you can enter into ThoughtSpot Spotter Viz to recreate this liveboard.*")
-                spotter_viz_story_display = gr.Markdown(
-                    value="Spotter Viz story will be generated after liveboard creation.\n\n**What is Spotter Viz?**\nSpotter Viz is an AI agent in ThoughtSpot that creates, structures, and styles Liveboards through natural language prompts. The agent reviews the data, proposes layouts, generates KPIs and visualizations, and allows conversational refinement.",
-                    elem_classes=["spotter-viz-story-content"]
-                )
+                gr.Markdown("*Conversational prompts to enter into ThoughtSpot Spotter Viz. Generated after liveboard creation.*")
+                _spotter_default = "Story will be generated after liveboard creation.\n\n**What is Spotter Viz?** An AI agent in ThoughtSpot that builds Liveboards through natural language — type a request, it creates and refines the dashboard step by step."
+                with gr.Tabs():
+                    with gr.Tab("🗺️ ThoughtSpot Story"):
+                        gr.Markdown("*Built from the ThoughtSpot-recommended KPIs and visualizations for this vertical × function.*")
+                        spotter_matrix_display = gr.Markdown(
+                            value=_spotter_default,
+                            elem_classes=["spotter-viz-story-content"]
+                        )
+                    with gr.Tab("✨ AI Story"):
+                        gr.Markdown("*Pure AI — no matrix constraints. What the AI thinks would make a compelling liveboard story.*")
+                        spotter_ai_display = gr.Markdown(
+                            value=_spotter_default,
+                            elem_classes=["spotter-viz-story-content"]
+                        )
             
             with gr.Tab("⚙️ Settings"):
                 settings_components = create_settings_tab()
-            
+
+            with gr.Tab("🧩 Matrix"):
+                matrix_components = create_matrix_tab(interface)
+
             with gr.Tab("🤖 AI Feedback"):
                 ai_feedback_display = gr.TextArea(
                     label="AI Processing Log",
@@ -4567,14 +4614,12 @@ def create_chat_interface():
                 
                 # Hidden fields — values still saved/loaded but not shown in UI
                 admin_ts_url = gr.Textbox(visible=False)
-                admin_ts_auth_key = gr.Textbox(visible=False)
                 admin_openai_key = gr.Textbox(visible=False)
                 admin_google_key = gr.Textbox(visible=False)
 
                 with gr.Row():
                     with gr.Column():
                         gr.Markdown("#### ThoughtSpot Connection")
-                        admin_ts_user = gr.Textbox(label="ThoughtSpot Admin Username", placeholder="admin@company.com")
                         admin_share_with = gr.Textbox(
                             label="Default Share With (User or Group)",
                             placeholder="user@company.com or group-name",
@@ -4599,7 +4644,7 @@ def create_chat_interface():
                 
                 # Admin settings field list (order matches ADMIN_SETTINGS_KEYS)
                 admin_fields = [
-                    admin_ts_url, admin_ts_auth_key, admin_ts_user,
+                    admin_ts_url,
                     admin_openai_key, admin_google_key,
                     admin_sf_account, admin_sf_kp_user, admin_sf_kp_pk,
                     admin_sf_kp_pass, admin_sf_role, admin_sf_warehouse,
@@ -4608,7 +4653,7 @@ def create_chat_interface():
                 ]
 
                 admin_keys_order = [
-                    "THOUGHTSPOT_URL", "THOUGHTSPOT_TRUSTED_AUTH_KEY", "THOUGHTSPOT_ADMIN_USER",
+                    "THOUGHTSPOT_URL",
                     "OPENAI_API_KEY", "GOOGLE_API_KEY",
                     "SNOWFLAKE_ACCOUNT", "SNOWFLAKE_KP_USER", "SNOWFLAKE_KP_PK",
                     "SNOWFLAKE_KP_PASSPHRASE", "SNOWFLAKE_ROLE", "SNOWFLAKE_WAREHOUSE",
@@ -4751,8 +4796,8 @@ def create_chat_interface():
         )
         
         # Create update function for tabs
-        spotter_viz_default = "Spotter Viz story will be generated after liveboard creation.\n\n**What is Spotter Viz?**\nSpotter Viz is an AI agent in ThoughtSpot that creates, structures, and styles Liveboards through natural language prompts. The agent reviews the data, proposes layouts, generates KPIs and visualizations, and allows conversational refinement."
-        
+        _spotter_waiting = "Story will be generated after liveboard creation."
+
         def update_all_tabs(controller):
             if controller is None:
                 return (
@@ -4760,35 +4805,34 @@ def create_chat_interface():
                     "-- DDL will appear here after generation",
                     "Progress will appear here during deployment...",
                     "Demo pack will be generated after deployment completes.\n\nThis will include:\n- Key insights/outliers\n- Spotter questions to ask\n- Talking points for the demo",
-                    spotter_viz_default
+                    _spotter_waiting,
+                    _spotter_waiting,
                 )
-            
-            # Get live progress from controller (captures deployment output)
+
             live_progress = getattr(controller, 'live_progress_log', [])
             live_progress_text = "\n".join(live_progress) if live_progress else "Progress will appear here during deployment..."
-            
-            # Get demo pack from controller
+
             demo_pack = getattr(controller, 'demo_pack_content', '')
             demo_pack_text = demo_pack if demo_pack else "Demo pack will be generated after deployment completes.\n\nThis will include:\n- Key insights/outliers\n- Spotter questions to ask\n- Talking points for the demo"
-            
-            # Get Spotter Viz story from controller
-            spotter_story = getattr(controller, 'spotter_viz_story', '')
-            spotter_story_text = spotter_story if spotter_story else spotter_viz_default
-            
+
+            matrix_story = getattr(controller, 'spotter_story_matrix', '') or _spotter_waiting
+            ai_story = getattr(controller, 'spotter_story_ai', '') or _spotter_waiting
+
             return (
                 "\n".join(controller.ai_feedback_log),
                 controller.ddl_code if controller.ddl_code else "-- DDL will appear here after generation",
                 live_progress_text,
                 demo_pack_text,
-                spotter_story_text
+                matrix_story,
+                ai_story,
             )
-        
+
         # Wire up tab updates on chat interactions
-        # These will update after each message
         chat_components['chatbot'].change(
             fn=update_all_tabs,
             inputs=[chat_controller_state],
-            outputs=[ai_feedback_display, ddl_display, live_progress_display, demo_pack_display, spotter_viz_story_display]
+            outputs=[ai_feedback_display, ddl_display, live_progress_display, demo_pack_display,
+                     spotter_matrix_display, spotter_ai_display]
         )
         
         # Load settings from Supabase on startup (uses SETTINGS_SCHEMA)
@@ -4818,11 +4862,10 @@ def create_chat_interface():
             try:
                 settings = load_gradio_settings(user_email)
 
-                company = (str(settings.get("default_company_url", "")).strip() or "Amazon.com")
-                use_case = (str(settings.get("default_use_case", "")).strip() or "Sales Analytics")
+                company = str(settings.get("default_company_url", "")).strip()
+                use_case = str(settings.get("default_use_case", "")).strip()
                 model = (str(settings.get("default_llm", "")).strip() or DEFAULT_LLM_MODEL)
                 liveboard_name = str(settings.get("liveboard_name", "")).strip()
-                initial_message = build_initial_chat_message(company, use_case)
                 print(f"[LOAD] load_session_state_on_startup OK — model={model}, company={company}")
                 return (
                     "initialization",
@@ -4832,13 +4875,13 @@ def create_chat_interface():
                     liveboard_name,
                     gr.update(value=model),
                     gr.update(value=liveboard_name),
-                    initial_message,
+                    "",
                 )
             except Exception as e:
                 import traceback
                 print(f"[LOAD ERROR] load_session_state_on_startup failed: {e}\n{traceback.format_exc()}")
                 return (
-                    "initialization", DEFAULT_LLM_MODEL, "Amazon.com", "Sales Analytics", "",
+                    "initialization", DEFAULT_LLM_MODEL, "", "", "",
                     gr.update(value=DEFAULT_LLM_MODEL), gr.update(value=""), "",
                 )
 
@@ -4885,33 +4928,71 @@ def create_chat_tab(chat_controller_state, settings, current_stage, current_mode
     with gr.Row():
         # Left column - Chat
         with gr.Column(scale=3):
+            # Welcome shown as plain Markdown (no box); swapped out on first message
+            welcome_md = gr.Markdown(value=initial_welcome, visible=True)
             chatbot = gr.Chatbot(
-                value=[(None, initial_welcome)],
-                height=650,
+                value=[],
+                height=400,
                 label="Demo Builder Assistant",
                 show_label=False,
-                avatar_images=None,  # No avatars - keeps it clean
-                type='tuples'
+                avatar_images=None,
+                type='tuples',
+                elem_id="main-chatbot",
+                visible=False,
             )
-            
-            with gr.Row():
-                msg = gr.Textbox(
-                    label="Your message",
-                    value="",
-                    placeholder="Type your message here or use /over to change settings...",
-                    lines=1,
-                    max_lines=1,
-                    scale=5,
-                    show_label=False,
-                    interactive=True
-                )
-                send_btn = gr.Button("Send", variant="primary", scale=1)
-            
-            # Quick action buttons
-            with gr.Row():
-                start_btn = gr.Button("🔍 Start Research", size="sm")
-                configure_btn = gr.Button("⚙️ Configure", size="sm")
-                help_btn = gr.Button("💡 Help", size="sm")
+
+            # Input mode: App (dropdown form) or Chat (freeform)
+            with gr.Tabs():
+                with gr.Tab("App"):
+                    _override_keys = {(v, f) for v, f in MATRIX_OVERRIDES.keys()}
+                    _vertical_choices = list(VERTICALS.keys()) + ["Custom"]
+                    vertical_dd = gr.Dropdown(
+                        label="Vertical",
+                        choices=_vertical_choices,
+                        value=_vertical_choices[0],
+                        interactive=True,
+                    )
+                    function_dd = gr.Dropdown(
+                        label="Function",
+                        choices=list(FUNCTIONS.keys()),
+                        value=list(FUNCTIONS.keys())[0],
+                        interactive=True,
+                    )
+                    with gr.Row():
+                        url_input = gr.Textbox(
+                            label="Company URL",
+                            placeholder="e.g. Amazon.com",
+                            lines=1,
+                            scale=4,
+                            interactive=True,
+                        )
+                        use_url_cb = gr.Checkbox(
+                            label="Use URL",
+                            value=True,
+                            scale=1,
+                            interactive=True,
+                        )
+                    additional_info_input = gr.Textbox(
+                        label="Context",
+                        placeholder="Any extra context for the demo...",
+                        lines=2,
+                        interactive=True,
+                    )
+                    go_btn = gr.Button("→ GO", variant="primary")
+
+                with gr.Tab("Chat"):
+                    with gr.Row():
+                        msg = gr.Textbox(
+                            label="Your message",
+                            value="",
+                            placeholder="e.g. Amazon.com Retail Sales — or continue a conversation here",
+                            lines=1,
+                            max_lines=1,
+                            scale=5,
+                            show_label=False,
+                            interactive=True
+                        )
+                        send_btn = gr.Button("Send", variant="primary", scale=1)
         
         # Right column - Status & Settings
         with gr.Column(scale=1):
@@ -5004,7 +5085,21 @@ def create_chat_tab(chat_controller_state, settings, current_stage, current_mode
                 return html
 
             progress_html = gr.HTML(get_progress_html('initialization'))
-        
+
+            # Phase log stream — scrolling status updates from controller.phase_log
+            phase_log_display = gr.Textbox(
+                label="Pipeline Status",
+                value="",
+                lines=6,
+                max_lines=6,
+                interactive=False,
+                placeholder="Pipeline status will appear here when the GO button is pressed...",
+                elem_classes=["phase-log-stream"],
+            )
+
+            # Timer polls controller.phase_log every 2 seconds (activated when GO is pressed)
+            phase_log_timer = gr.Timer(value=2, active=False)
+
         # Event handlers - each creates/uses session-specific controller
         def send_message(controller, message, history, stage, model, company, usecase, env_label=None, liveboard_name_ui=None, request: gr.Request = None):
             """Handle sending a message - creates controller if needed"""
@@ -5024,13 +5119,15 @@ def create_chat_tab(chat_controller_state, settings, current_stage, current_mode
             # Always use the current UI value — takes priority over DB-loaded default
             if liveboard_name_ui is not None:
                 controller.settings['liveboard_name'] = liveboard_name_ui
+            hide_welcome = gr.update(visible=False)
             try:
                 for result in controller.process_chat_message(
                     message, history, stage, model, company, usecase
                 ):
                     new_stage = result[1] if len(result) > 1 else stage
                     progress = get_progress_html(new_stage)
-                    yield (controller,) + result + (progress,)
+                    chatbot_update = gr.update(value=result[0], visible=True)
+                    yield (controller, chatbot_update) + result[1:] + (progress, hide_welcome)
             except Exception as e:
                 err_tb = traceback.format_exc()
                 print(f"[ERROR] send_message unhandled exception:\n{err_tb}")
@@ -5041,15 +5138,52 @@ def create_chat_tab(chat_controller_state, settings, current_stage, current_mode
                 )
                 history = history or []
                 history.append((message, err_msg))
-                yield (controller, history, stage, model, company, usecase, "", get_progress_html(stage))
+                yield (controller, gr.update(value=history, visible=True), stage, model, company, usecase, "", get_progress_html(stage), hide_welcome)
+
+        # Wire up send button and enter key
+        _send_inputs = [chat_controller_state, msg, chatbot, current_stage, current_model, current_company, current_usecase, ts_env_dropdown, liveboard_name_input]
+        _send_outputs = [chat_controller_state, chatbot, current_stage, current_model, current_company, current_usecase, msg, progress_html, welcome_md]
 
-        def quick_action(controller, action_text, history, stage, model, company, usecase, env_label=None, liveboard_name_ui=None, request: gr.Request = None):
-            """Handle quick action button clicks"""
+        msg.submit(fn=send_message, inputs=_send_inputs, outputs=_send_outputs)
+        send_btn.click(fn=send_message, inputs=_send_inputs, outputs=_send_outputs)
+        
+        # App tab: update function dropdown and context field when vertical changes
+        def update_function_on_vertical(vertical):
+            """Reset function dropdown when vertical changes; mark overrides.
+            If Custom vertical: disable function dd, make context required."""
+            if vertical == "Custom":
+                return (
+                    gr.Dropdown(choices=list(FUNCTIONS.keys()), value=list(FUNCTIONS.keys())[0], interactive=False),
+                    gr.Textbox(label="Context *", placeholder="Describe your use case...", interactive=True),
+                )
+            choices = []
+            for fname in FUNCTIONS.keys():
+                if (vertical, fname) in MATRIX_OVERRIDES:
+                    choices.append(f"{fname} ✓")
+                else:
+                    choices.append(fname)
+            return (
+                gr.Dropdown(choices=choices, value=choices[0], interactive=True),
+                gr.Textbox(label="Context", placeholder="Any extra context for the demo...", interactive=True),
+            )
+
+        vertical_dd.change(
+            fn=update_function_on_vertical,
+            inputs=[vertical_dd],
+            outputs=[function_dd, additional_info_input]
+        )
+
+        # Defined tab: GO button handler
+        def defined_go(controller, vertical, function, url, use_url, additional_info,
+                       history, stage, model, company, usecase, env_label, lb_name,
+                       request: gr.Request = None):
+            import traceback
+            # Strip the "✓" suffix if present (added by update_function_on_vertical)
+            function_clean = function.replace(" ✓", "").strip() if function else function
             username = getattr(request, 'username', None) if request else None
             if controller is None:
                 controller = ChatDemoInterface(user_email=username)
                 print(f"[SESSION] Created new ChatDemoInterface for {username or 'anonymous'}")
-                # Apply selected TS environment settings to the new controller
                 if env_label:
                     _url = get_ts_env_url(env_label)
                     _key_value = get_ts_env_auth_key(env_label)
@@ -5057,42 +5191,103 @@ def create_chat_tab(chat_controller_state, settings, current_stage, current_mode
                         controller.settings['thoughtspot_url'] = _url
                     if _key_value:
                         controller.settings['thoughtspot_trusted_auth_key'] = _key_value
-            # Always use the current UI value — takes priority over DB-loaded default
-            if liveboard_name_ui is not None:
-                controller.settings['liveboard_name'] = liveboard_name_ui
+            if lb_name is not None:
+                controller.settings['liveboard_name'] = lb_name
 
-            for result in controller.process_chat_message(
-                action_text, history, stage, model, company, usecase
-            ):
-                new_stage = result[1] if len(result) > 1 else stage
-                progress = get_progress_html(new_stage)
-                yield (controller,) + result + (progress,)
-        
-        # Wire up send button and enter key
-        _send_inputs = [chat_controller_state, msg, chatbot, current_stage, current_model, current_company, current_usecase, ts_env_dropdown, liveboard_name_input]
-        _send_outputs = [chat_controller_state, chatbot, current_stage, current_model, current_company, current_usecase, msg, progress_html]
+            # Derive company name from URL or vertical+function label
+            if use_url and url.strip():
+                raw_company = url.strip()
+            else:
+                raw_company = f"{vertical} Demo"
 
-        msg.submit(fn=send_message, inputs=_send_inputs, outputs=_send_outputs)
-        send_btn.click(fn=send_message, inputs=_send_inputs, outputs=_send_outputs)
-        
-        # Quick action wrapper functions
-        def start_action(controller, history, stage, model, company, usecase, env_label, liveboard_name_ui):
-            yield from quick_action(controller, "Start research", history, stage, model, company, usecase, env_label, liveboard_name_ui)
+            is_custom = (vertical == "Custom")
 
-        def configure_action(controller, history, stage, model, company, usecase, env_label, liveboard_name_ui):
-            yield from quick_action(controller, "Configure settings", history, stage, model, company, usecase, env_label, liveboard_name_ui)
+            if is_custom:
+                # Custom vertical: context IS the use case; function dropdown is ignored
+                use_case_str = (additional_info or "").strip()
+                if not use_case_str:
+                    history = history or []
+                    history.append(("GO", "❌ Please describe your use case in the Context field when using Custom vertical."))
+                    yield (controller, gr.update(value=history, visible=True), stage, model, company, usecase, get_progress_html(stage), hide_welcome)
+                    return
+                controller.vertical = None
+                controller.function = None
+                controller.use_case_config = get_use_case_config("Generic", "Generic")
+                controller.is_generic_use_case = True
+                controller.generic_use_case_context = ""
+            else:
+                # Standard vertical: build use case from vertical + function
+                use_case_str = f"{vertical} {function_clean}"
+                controller.vertical = vertical
+                controller.function = function_clean
+                controller.use_case_config = get_use_case_config(
+                    vertical or "Generic", function_clean or "Generic"
+                )
+                is_known = (vertical and function_clean
+                            and not controller.use_case_config.get('is_generic'))
+                controller.is_generic_use_case = not is_known
+                controller.generic_use_case_context = additional_info.strip() if additional_info else ""
 
-        def help_action(controller, history, stage, model, company, usecase, env_label, liveboard_name_ui):
-            yield from quick_action(controller, "Help", history, stage, model, company, usecase, env_label, liveboard_name_ui)
+            controller.pending_generic_company = raw_company
+            use_case_display = controller.use_case_config.get('use_case_name', use_case_str)
+            controller.pending_generic_use_case = use_case_display
 
-        _action_inputs = [chat_controller_state, chatbot, current_stage, current_model, current_company, current_usecase, ts_env_dropdown, liveboard_name_input]
-        _action_outputs = [chat_controller_state, chatbot, current_stage, current_model, current_company, current_usecase, msg, progress_html]
+            # Clear phase log for a fresh run
+            controller.phase_log = []
+
+            # Drive straight to awaiting_context → 'proceed' to skip confirmation dialog
+            history = history or []
+            hide_welcome = gr.update(visible=False)
+            try:
+                for result in controller.process_chat_message(
+                    "proceed", history, 'awaiting_context', model,
+                    raw_company, use_case_display
+                ):
+                    new_stage = result[1] if len(result) > 1 else stage
+                    progress = get_progress_html(new_stage)
+                    chatbot_update = gr.update(value=result[0], visible=True)
+                    yield (controller, chatbot_update) + result[1:5] + (progress, hide_welcome)
+            except Exception as e:
+                err_tb = traceback.format_exc()
+                print(f"[ERROR] defined_go unhandled exception:\n{err_tb}")
+                err_msg = (
+                    f"❌ **An unexpected error occurred**\n\n"
+                    f"`{type(e).__name__}: {e}`\n\n"
+                    f"The pipeline has been interrupted. You can try again or start a new session."
+                )
+                history.append((f"GO: {use_case_str}", err_msg))
+                yield (controller, gr.update(value=history, visible=True), stage, model, company, usecase, get_progress_html(stage), hide_welcome)
+
+        _go_inputs = [
+            chat_controller_state, vertical_dd, function_dd, url_input, use_url_cb,
+            additional_info_input, chatbot, current_stage, current_model,
+            current_company, current_usecase, ts_env_dropdown, liveboard_name_input
+        ]
+        _go_outputs = [
+            chat_controller_state, chatbot, current_stage, current_model,
+            current_company, current_usecase, progress_html, welcome_md
+        ]
+        go_btn.click(fn=defined_go, inputs=_go_inputs, outputs=_go_outputs)
+
+        # Phase log timer — poll controller.phase_log and render as newline-joined text
+        def refresh_phase_log(controller):
+            if controller is None:
+                return gr.update()
+            log = getattr(controller, 'phase_log', [])
+            return "\n".join(log) if log else ""
+
+        phase_log_timer.tick(
+            fn=refresh_phase_log,
+            inputs=[chat_controller_state],
+            outputs=[phase_log_display],
+        )
+
+        # Activate phase log timer when GO or Send is pressed
+        _activate_timer = dict(fn=lambda: gr.update(active=True), inputs=[], outputs=[phase_log_timer], queue=False)
+        go_btn.click(**_activate_timer)
+        send_btn.click(**_activate_timer)
+        msg.submit(**_activate_timer)
 
-        # Quick action buttons
-        start_btn.click(fn=start_action, inputs=_action_inputs, outputs=_action_outputs)
-        configure_btn.click(fn=configure_action, inputs=_action_inputs, outputs=_action_outputs)
-        help_btn.click(fn=help_action, inputs=_action_inputs, outputs=_action_outputs)
-        
         # Model dropdown change
         def update_model(new_model, controller, history):
             if controller is not None:
@@ -5142,13 +5337,153 @@ def create_chat_tab(chat_controller_state, settings, current_stage, current_mode
         'msg': msg,
         'model_dropdown': model_dropdown,
         'send_btn': send_btn,
-        'send_btn_ref': send_btn,
         'ts_env_dropdown': ts_env_dropdown,
         'liveboard_name_input': liveboard_name_input,
-        'progress_html': progress_html
+        'progress_html': progress_html,
+        'phase_log_display': phase_log_display,
+        'phase_log_timer': phase_log_timer,
     }
 
 
+def create_matrix_tab(interface):
+    """Matrix editor tab — view and edit Vertical × Function cell configs."""
+
+    def _cell_data(vertical, function):
+        """Compute all display values for a given cell."""
+        config = get_use_case_config(vertical, function)
+        has_override = (vertical, function) in MATRIX_OVERRIDES
+
+        # Coverage line
+        if has_override:
+            coverage = "✅ **Override** — enriched persona, custom questions, tuned story controls"
+        else:
+            coverage = "🔵 **Base merge** — vertical + function defaults combined"
+
+        # Persona / problem (overrides only)
+        persona = config.get('target_persona', '')
+        problem = config.get('business_problem', '')
+        persona_md = ""
+        if persona:
+            persona_md += f"**Target persona:** {persona}  \n"
+        if problem:
+            persona_md += f"**Business problem:** {problem}"
+
+        # KPIs
+        kpis = config.get('kpis', [])
+        kpi_defs = config.get('kpi_definitions', {})
+        lines = []
+        for k in kpis:
+            d = kpi_defs.get(k, '')
+            lines.append(f"**{k}** — {d}" if d else f"**{k}**")
+        kpi_md = "  \n".join(lines) if lines else "_None defined_"
+
+        # Liveboard questions → dataframe rows
+        questions = config.get('liveboard_questions', [])
+        rows = []
+        for q in questions:
+            spotter = ", ".join(q.get('spotter_qs', []))
+            rows.append([
+                q.get('title', ''),
+                q.get('viz_type', ''),
+                q.get('required', False),
+                q.get('viz_question', ''),
+                q.get('insight', ''),
+                spotter,
+            ])
+
+        # Story controls as formatted text
+        sc = config.get('story_controls', {})
+        sc_lines = []
+        for k, v in sc.items():
+            if isinstance(v, dict):
+                sc_lines.append(f"**{k}:** {v}")
+            elif isinstance(v, list):
+                sc_lines.append(f"**{k}:** {', '.join(str(x) for x in v)}")
+            elif isinstance(v, float):
+                sc_lines.append(f"**{k}:** {v:.4g}")
+            else:
+                sc_lines.append(f"**{k}:** {v}")
+        sc_md = "  \n".join(sc_lines) if sc_lines else "_None_"
+
+        return coverage, persona_md, kpi_md, rows, sc_md
+
+    # --- Initial values ---
+    init_v = list(VERTICALS.keys())[0]
+    init_f = list(FUNCTIONS.keys())[0]
+    init_cov, init_persona, init_kpi, init_rows, init_sc = _cell_data(init_v, init_f)
+
+    gr.Markdown("## 🧩 Matrix Editor")
+    gr.Markdown(
+        "The matrix defines what gets built for every Vertical × Function combination — "
+        "KPIs, visualizations, story controls, and persona.  \n"
+        "Changes here are **in-session only** for now; persistence via Supabase is coming next."
+    )
+
+    # --- Selectors ---
+    with gr.Row():
+        m_vertical = gr.Dropdown(
+            label="Vertical",
+            choices=list(VERTICALS.keys()),
+            value=init_v,
+            interactive=True,
+            scale=1,
+        )
+        m_function = gr.Dropdown(
+            label="Function",
+            choices=list(FUNCTIONS.keys()),
+            value=init_f,
+            interactive=True,
+            scale=1,
+        )
+
+    coverage_md = gr.Markdown(value=init_cov)
+    persona_md = gr.Markdown(value=init_persona)
+
+    gr.Markdown("---")
+
+    # --- KPIs ---
+    with gr.Accordion("📊 KPIs", open=True):
+        kpi_md = gr.Markdown(value=init_kpi)
+
+    # --- Liveboard Questions ---
+    with gr.Accordion("📋 Liveboard Questions", open=True):
+        gr.Markdown(
+            "*Edit cells directly. Add rows with the + button. "
+            "Changes affect this session until persistence is wired up.*"
+        )
+        questions_df = gr.Dataframe(
+            value=init_rows,
+            headers=["Title", "Viz Type", "Required", "Question", "Insight", "Spotter Questions"],
+            datatype=["str", "str", "bool", "str", "str", "str"],
+            interactive=True,
+            row_count=(len(init_rows), "dynamic"),
+            col_count=(6, "fixed"),
+            wrap=True,
+        )
+
+    # --- Story Controls ---
+    with gr.Accordion("⚙️ Story Controls", open=False):
+        sc_md = gr.Markdown(value=init_sc)
+
+    # --- Wire dropdowns ---
+    def on_cell_change(vertical, function):
+        cov, persona, kpi, rows, sc = _cell_data(vertical, function)
+        return cov, persona, kpi, rows, sc
+
+    _outputs = [coverage_md, persona_md, kpi_md, questions_df, sc_md]
+    m_vertical.change(fn=on_cell_change, inputs=[m_vertical, m_function], outputs=_outputs)
+    m_function.change(fn=on_cell_change, inputs=[m_vertical, m_function], outputs=_outputs)
+
+    # Populate on tab load
+    interface.load(
+        fn=on_cell_change,
+        inputs=[m_vertical, m_function],
+        outputs=_outputs,
+    )
+
+    return {'vertical': m_vertical, 'function': m_function, 'questions_df': questions_df}
+
+
 def create_settings_tab():
     """Create the settings configuration tab - returns components for loading
     

legitdata_bridge.py

@@ -205,10 +205,16 @@ class KeyPairSnowflakeWriter:
         sql = f"INSERT INTO {table_name} ({col_list}) VALUES ({placeholders})"
         
         def convert_value(value, col_name):
+            import math
             if value is None:
                 return None
+            if isinstance(value, bool):
+                return int(value)
             if isinstance(value, Decimal):
                 value = float(value)
+            # NaN and Inf are not valid SQL values — replace with NULL
+            if isinstance(value, float) and (math.isnan(value) or math.isinf(value)):
+                return None
             if isinstance(value, datetime):
                 return value.strftime('%Y-%m-%d %H:%M:%S')
             if isinstance(value, date):
@@ -265,7 +271,9 @@ class KeyPairSnowflakeWriter:
                         self.cursor.execute(sql, row)
                         total_inserted += 1
                     except Exception as row_error:
-                        print(f"  Failed row: {row[:3]}... - {row_error}")
+                        col_val_pairs = list(zip(filtered_columns, row))
+                        print(f"  Failed row ({type(row_error).__name__}: {row_error})")
+                        print(f"  Values: {col_val_pairs[:6]}")
 
         self.connection.commit()
         print(f"Inserted {total_inserted} rows into {table_name}")

liveboard_creator.py

@@ -84,24 +84,18 @@ def _clean_viz_title(title: str) -> str:
     return ' '.join(result)
 
 
-def _get_direct_api_session():
+def _get_direct_api_session(username: str, secret_key: str):
     """
-    Get or create an authenticated session for direct ThoughtSpot API calls.
-    Uses trusted auth from environment variables.
+    Create an authenticated session for direct ThoughtSpot API calls.
+    Uses trusted auth — authenticates as the given username.
+    Returns (session, token) tuple, or (None, None) on failure.
     """
-    global _direct_api_token, _direct_api_session
-    
-    if _direct_api_session is not None:
-        return _direct_api_session
-    
     base_url = get_admin_setting('THOUGHTSPOT_URL').rstrip('/')
-    username = get_admin_setting('THOUGHTSPOT_ADMIN_USER')
-    secret_key = get_admin_setting('THOUGHTSPOT_TRUSTED_AUTH_KEY')
-    
+
     if not all([base_url, username, secret_key]):
-        print(f"⚠️  Direct API: Missing env vars (URL={bool(base_url)}, USER={bool(username)}, KEY={bool(secret_key)})")
-        return None
-    
+        print(f"⚠️  Direct API: Missing settings (URL={bool(base_url)}, USER={bool(username)}, KEY={bool(secret_key)})")
+        return None, None
+
     # Get auth token
     auth_url = f"{base_url}/api/rest/2.0/auth/token/full"
     resp = requests.post(auth_url, json={
@@ -109,42 +103,42 @@ def _get_direct_api_session():
         "secret_key": secret_key,
         "validity_time_in_sec": 3600
     })
-    
+
     if resp.status_code != 200:
         print(f"⚠️  Direct API auth failed: {resp.status_code}")
-        return None
-    
+        return None, None
+
     token = resp.json().get('token')
     if not token:
         print(f"⚠️  Direct API: No token in response")
-        return None
-    
-    _direct_api_token = token
-    _direct_api_session = requests.Session()
-    _direct_api_session.headers['Authorization'] = f'Bearer {token}'
-    _direct_api_session.headers['Content-Type'] = 'application/json'
-    _direct_api_session.headers['Accept'] = 'application/json'
-    
-    print(f"✅ Direct API: Authenticated to {base_url}")
-    return _direct_api_session
+        return None, None
+
+    session = requests.Session()
+    session.headers['Authorization'] = f'Bearer {token}'
+    session.headers['Content-Type'] = 'application/json'
+    session.headers['Accept'] = 'application/json'
 
+    print(f"✅ Direct API: Authenticated as {username} to {base_url}")
+    return session, token
 
-def _get_answer_direct(question: str, model_id: str) -> dict:
+
+def _get_answer_direct(question: str, model_id: str, username: str, secret_key: str) -> dict:
     """
     Get an answer from ThoughtSpot using direct API call instead of MCP.
-    
+
     This bypasses MCP's agent.thoughtspot.app proxy which may have issues
     with certain clusters (e.g., Spotter Agent v3 clusters).
-    
+
     Args:
         question: Natural language question
         model_id: ThoughtSpot model GUID
-        
+        username: ThoughtSpot username to authenticate as
+
     Returns:
         dict with answer data including session_identifier, tokens, question, etc.
         Returns None on failure.
     """
-    session = _get_direct_api_session()
+    session, _ = _get_direct_api_session(username, secret_key)
     if not session:
         return None
     
@@ -3024,20 +3018,20 @@ def create_liveboard_from_model_mcp(
             print(f"[MCP] MCP modules imported successfully (streamable HTTP transport)")
             
             # Get trusted auth token for bearer auth
-            print(f"[MCP] Getting trusted auth token...")
-            _get_direct_api_session()  # Ensures _direct_api_token is populated
-            if not _direct_api_token:
+            print(f"[MCP] Getting trusted auth token for {ts_client.username}...")
+            _, mcp_token = _get_direct_api_session(ts_client.username, ts_client.secret_key)
+            if not mcp_token:
                 return {
                     'success': False,
                     'error': 'Failed to get trusted auth token. Check THOUGHTSPOT_TRUSTED_AUTH_KEY in .env'
                 }
-            
+
             # Build bearer auth header: {token}@{host}
             ts_url = get_admin_setting('THOUGHTSPOT_URL').rstrip('/')
             ts_host = ts_url.replace('https://', '').replace('http://', '')
             mcp_endpoint = "https://agent.thoughtspot.app/bearer/mcp"
             headers = {
-                "Authorization": f"Bearer {_direct_api_token}@{ts_host}"
+                "Authorization": f"Bearer {mcp_token}@{ts_host}"
             }
             print(f"[MCP] Connecting to {mcp_endpoint} (bearer auth, host={ts_host})")
             
@@ -3140,7 +3134,7 @@ def create_liveboard_from_model_mcp(
                             
                             if use_direct_api:
                                 # Use direct ThoughtSpot API (bypasses MCP proxy issues)
-                                answer_data = _get_answer_direct(question_text, model_id)
+                                answer_data = _get_answer_direct(question_text, model_id, ts_client.username, ts_client.secret_key)
                                 if answer_data:
                                     # Clean up the viz title
                                     if 'question' in answer_data:

model_semantic_updater.py

@@ -1,8 +1,11 @@
 """
 Model Semantic Layer Updater
 
-Adds descriptions and synonyms to ThoughtSpot model columns to improve search relevance.
-Uses AI to generate business-friendly semantic metadata from company research.
+Enriches ThoughtSpot model TML with:
+  - A model-level description (business context for the whole dataset)
+  - Per-column descriptions (human-readable)
+  - Per-column synonyms (search terms business users would type)
+  - Per-column AI context (≤400 char coaching hints for Spotter)
 """
 
 import json
@@ -18,349 +21,306 @@ from llm_client_factory import create_openai_client
 
 
 class ModelSemanticUpdater:
-    """Add descriptions and synonyms to existing model columns"""
+    """Enrich a ThoughtSpot model with descriptions, synonyms, and AI context."""
 
     def __init__(self, ts_client, llm_model: str = DEFAULT_LLM_MODEL):
-        """
-        Initialize ModelSemanticUpdater
-
-        Args:
-            ts_client: ThoughtSpotDeployer instance (already authenticated)
-        """
         self.ts_client = ts_client
-        self.llm_model = resolve_model_name(llm_model)
-        if not is_openai_model_name(self.llm_model):
-            raise ValueError(
-                f"ModelSemanticUpdater only supports OpenAI GPT/Codex models. Received '{self.llm_model}'."
-            )
+        # Fall back to default if a non-OpenAI model is passed — this module uses
+        # json_object response_format which requires OpenAI.
+        resolved = resolve_model_name(llm_model)
+        self.llm_model = resolved if is_openai_model_name(resolved) else resolve_model_name(DEFAULT_LLM_MODEL)
         self.openai_client = create_openai_client()
 
-    def get_model_list(self) -> List[Dict]:
-        """
-        Fetch list of available models for user selection
+    # ------------------------------------------------------------------
+    # TML export / import helpers
+    # ------------------------------------------------------------------
 
-        Returns:
-            List of model metadata dictionaries with 'id' and 'name'
-        """
+    def get_model_tml(self, model_id: str) -> Optional[Dict]:
+        """Export existing model TML from ThoughtSpot, return parsed dict."""
         try:
             response = self.ts_client.session.post(
-                f"{self.ts_client.base_url}/api/rest/2.0/metadata/search",
-                headers=self.ts_client.headers,
+                f"{self.ts_client.base_url}/api/rest/2.0/metadata/tml/export",
                 json={
-                    "metadata": [{"type": "LOGICAL_TABLE"}],
-                    "record_size": 10000  # Get all models - there can be thousands
+                    "metadata": [{"identifier": model_id}],
+                    "export_associated": False,
+                    "export_fqn": True,
                 }
             )
-
             if response.status_code == 200:
                 result = response.json()
-                models = []
-
-                # API returns a list of metadata objects
-                if isinstance(result, list):
-                    for item in result:
-                        # Extract from metadata_header if present
-                        header = item.get('metadata_header', {})
-                        models.append({
-                            'id': item.get('metadata_id') or header.get('id'),
-                            'name': item.get('metadata_name') or header.get('name'),
-                            'description': header.get('description', ''),
-                            'author': header.get('authorName', ''),
-                            'created': header.get('created'),
-                            'modified': header.get('modified')
-                        })
-                else:
-                    # Fallback for dict response
-                    headers = result.get('headers', [])
-                    for header in headers:
-                        models.append({
-                            'id': header.get('id'),
-                            'name': header.get('name'),
-                            'description': header.get('description', ''),
-                            'author': header.get('author', {}).get('name', ''),
-                            'created': header.get('created'),
-                            'modified': header.get('modified')
-                        })
-
-                return models
+                if result.get('object') and len(result['object']) > 0:
+                    edoc = result['object'][0].get('edoc')
+                    if edoc:
+                        return yaml.safe_load(edoc)
             else:
-                print(f"Error fetching models: {response.status_code} - {response.text}")
-                return []
+                print(f"Error exporting model TML: {response.status_code} - {response.text}")
         except Exception as e:
-            print(f"Exception fetching models: {e}")
-            return []
-
-    def get_model_tml(self, model_id: str) -> Optional[Dict]:
-        """
-        Export existing model TML
-
-        Args:
-            model_id: GUID of the model to export
+            print(f"Exception exporting model TML: {e}")
+        return None
 
-        Returns:
-            Parsed model TML dictionary, or None if failed
-        """
+    def deploy_updated_model(self, model_tml_yaml: str) -> Dict:
+        """Re-import updated model TML (update in place, not create new)."""
         try:
             response = self.ts_client.session.post(
-                f"{self.ts_client.base_url}/api/rest/2.0/metadata/tml/export",
-                headers=self.ts_client.headers,
+                f"{self.ts_client.base_url}/api/rest/2.0/metadata/tml/import",
                 json={
-                    "metadata": [{"identifier": model_id}],
-                    "export_associated": False,
-                    "export_fqn": True
+                    "metadata_tmls": [model_tml_yaml],
+                    "import_policy": "PARTIAL",
+                    "force_create": False,
                 }
             )
-
             if response.status_code == 200:
                 result = response.json()
+                model_id = None
                 if result.get('object') and len(result['object']) > 0:
-                    edoc = result['object'][0].get('edoc')
-                    if edoc:
-                        return yaml.safe_load(edoc)
-                return None
+                    model_id = result['object'][0].get('response', {}).get('header', {}).get('id')
+                return {'success': True, 'model_id': model_id, 'response': result}
             else:
-                print(f"Error exporting model TML: {response.status_code} - {response.text}")
-                return None
+                return {'success': False, 'error': f"HTTP {response.status_code}: {response.text}"}
         except Exception as e:
-            print(f"Exception exporting model TML: {e}")
-            return None
+            return {'success': False, 'error': str(e)}
+
+    # ------------------------------------------------------------------
+    # LLM generation
+    # ------------------------------------------------------------------
+
+    def generate_model_description(
+        self,
+        company_research: str,
+        use_case: str,
+        company_name: str,
+        model_name: str,
+    ) -> str:
+        """
+        Generate a 2-3 sentence model-level description for ThoughtSpot.
+
+        Returns plain text (not JSON).
+        """
+        research_snippet = company_research[:2000] if len(company_research) > 2000 else company_research
+
+        prompt = f"""Write a 2-3 sentence business description for a ThoughtSpot analytics model.
+
+Company: {company_name}
+Use Case: {use_case}
+Model Name: {model_name}
+
+Company/Industry Context:
+{research_snippet}
+
+Requirements:
+- 2-3 sentences, plain text
+- Business-facing, no technical jargon
+- State what data the model covers and what business questions it answers
+- Mention the company name and use case
+- Do NOT use markdown formatting
+
+Example style:
+"Analyze Target's retail sales performance across stores, product categories, and regions. \
+Track key metrics including revenue, units sold, and basket size to identify trends and \
+opportunities. Designed for sales and merchandising teams to monitor performance and drive decisions."
+
+Write only the description, nothing else."""
+
+        try:
+            token_kwargs = build_openai_chat_token_kwargs(self.llm_model, 300)
+            response = self.openai_client.chat.completions.create(
+                model=self.llm_model,
+                messages=[{"role": "user", "content": prompt}],
+                temperature=0.4,
+                **token_kwargs,
+            )
+            return response.choices[0].message.content.strip()
+        except Exception as e:
+            print(f"Error generating model description: {e}")
+            return f"{company_name} {use_case} analytics model."
 
     def generate_column_semantics(
         self,
         company_research: str,
         model_tml: Dict,
-        use_case: str = ""
+        use_case: str = "",
+        company_name: str = "",
     ) -> Dict[str, Dict]:
         """
-        Use AI to generate descriptions and synonyms for columns
-        based on company research and business context
-
-        Args:
-            company_research: Text containing company research/analysis
-            model_tml: Parsed model TML dictionary
-            use_case: Optional use case context for more targeted semantics
-
-        Returns:
-            Dictionary mapping column names to semantic metadata:
-            {
-                "column_name": {
-                    "description": "Business-friendly description",
-                    "synonyms": ["synonym1", "synonym2", ...]
-                }
+        Generate per-column semantic metadata using the LLM.
+
+        Returns a dict keyed by column name:
+        {
+            "column_name": {
+                "description": "Human-readable 1-2 sentence description",
+                "synonyms": ["term1", "term2", ...],
+                "ai_context": "Spotter coaching hint ≤400 chars"
             }
+        }
         """
-        # Extract column information from model TML
         columns = model_tml.get('model', {}).get('columns', [])
-
         column_info = []
         for col in columns:
             column_info.append({
                 'name': col.get('name'),
-                'column_id': col.get('column_id', ''),
                 'type': col.get('properties', {}).get('column_type', 'UNKNOWN'),
-                'current_description': col.get('properties', {}).get('description', '')
+                'current_description': col.get('description', ''),
             })
 
-        # Truncate research if too long (to fit in prompt)
-        max_research_length = 3000
-        if len(company_research) > max_research_length:
-            company_research = company_research[:max_research_length] + "..."
+        research_snippet = company_research[:3000] if len(company_research) > 3000 else company_research
+        use_case_line = f"\nUse case: {use_case}" if use_case else ""
+        company_line = f"\nCompany: {company_name}" if company_name else ""
 
-        use_case_context = f"\nUse case context: {use_case}\n" if use_case else ""
+        prompt = f"""You are a data analyst enriching a ThoughtSpot analytics model with semantic metadata.
+{company_line}{use_case_line}
 
-        prompt = f"""You are a data analyst creating semantic metadata for a data model.
+Company/Industry Context:
+{research_snippet}
 
-Company Research:
-{company_research}
-{use_case_context}
-Data Model Columns:
+Columns in this model:
 {json.dumps(column_info, indent=2)}
 
-For each column, generate:
-1. **description**: A clear, business-friendly description (1-2 sentences) that helps users understand what the column represents and how to use it. Consider the company context and use case.
+For EVERY column listed, generate three fields:
 
-2. **synonyms**: 3-5 alternative terms or phrases that users might search for when looking for this data. Include:
-   - Common business terms
-   - Abbreviations and full forms
-   - Related concepts
-   - Industry-specific terminology
+1. description (1-2 sentences, business-friendly, helps users understand the column)
+2. synonyms (3-5 alternative terms a business user would type to find this data)
+3. ai_context (a Spotter coaching hint, ≤350 characters, written as clear instructions to an AI)
+   - Tell Spotter when/how to use the column, default aggregation, or key business rules
+   - Examples: "Use this for revenue analysis. Default to SUM. Combine with Date for trends."
+   - "Filter dimension for product category. Never aggregate. Use for grouping and comparison."
 
 Guidelines:
-- Make descriptions actionable and relevant to business users
-- Use natural language, not technical jargon
-- For measure columns, mention what aggregation makes sense (sum, average, count, etc.)
-- For dimension columns, mention what they're used for grouping/filtering
-- Synonyms should be realistic search terms a business user would type
-- Consider the company's industry and use case when choosing terminology
-
-Return a JSON object with this structure:
+- Make descriptions actionable, no technical jargon
+- For MEASURE columns: mention aggregation (sum, avg, count)
+- For ATTRIBUTE columns: mention how it's used for grouping/filtering
+- Synonyms should be realistic search terms
+- ai_context must be ≤350 characters — keep it concise and direct
+
+Return a JSON object keyed by the exact column name:
 {{
   "column_name": {{
-    "description": "Clear business description",
-    "synonyms": ["synonym1", "synonym2", "synonym3"]
-  }},
-  ...
+    "description": "...",
+    "synonyms": ["term1", "term2", "term3"],
+    "ai_context": "..."
+  }}
 }}"""
 
         try:
-            token_kwargs = build_openai_chat_token_kwargs(self.llm_model, 3000)
+            token_kwargs = build_openai_chat_token_kwargs(self.llm_model, 4000)
             response = self.openai_client.chat.completions.create(
                 model=self.llm_model,
                 messages=[{"role": "user", "content": prompt}],
                 response_format={"type": "json_object"},
-                temperature=0.3,  # Lower temperature for more consistent semantic metadata
-                **token_kwargs
+                temperature=0.3,
+                **token_kwargs,
             )
-
-            result = json.loads(response.choices[0].message.content)
-            return result
+            return json.loads(response.choices[0].message.content)
         except Exception as e:
             print(f"Error generating column semantics: {e}")
             return {}
 
-    def update_model_tml_semantics(self, model_tml: Dict, semantics: Dict[str, Dict]) -> str:
-        """
-        Update model TML with descriptions and synonyms
+    # ------------------------------------------------------------------
+    # TML mutation
+    # ------------------------------------------------------------------
 
-        Args:
-            model_tml: Parsed model TML dictionary
-            semantics: Dictionary mapping column names to semantic metadata
+    def apply_to_model_tml(
+        self,
+        model_tml: Dict,
+        column_semantics: Dict[str, Dict],
+        model_description: str = None,
+    ) -> str:
+        """
+        Apply model description + column semantics to the model TML dict.
 
-        Returns:
-            Updated model TML as YAML string
+        Returns updated TML as a YAML string.
         """
-        columns = model_tml.get('model', {}).get('columns', [])
+        model_section = model_tml.setdefault('model', {})
 
-        for column in columns:
+        # Model-level description
+        if model_description:
+            model_section['description'] = model_description
+
+        # Column-level enrichment
+        for column in model_section.get('columns', []):
             col_name = column.get('name')
-            if col_name in semantics:
-                semantic_data = semantics[col_name]
+            if col_name not in column_semantics:
+                continue
+            data = column_semantics[col_name]
 
-                # Add or update description
-                if 'description' in semantic_data and semantic_data['description']:
-                    column['properties']['description'] = semantic_data['description']
+            if data.get('description'):
+                column['description'] = data['description']
 
-                # Add or update synonyms
-                if 'synonyms' in semantic_data and semantic_data['synonyms']:
-                    column['properties']['synonyms'] = semantic_data['synonyms']
+            props = column.setdefault('properties', {})
 
-        return yaml.dump(model_tml, default_flow_style=False, sort_keys=False)
+            if data.get('synonyms'):
+                props['synonyms'] = data['synonyms']
+                props['synonym_type'] = 'USER_DEFINED'
 
-    def deploy_updated_model(self, model_tml_yaml: str) -> Dict:
-        """
-        Deploy updated model TML back to ThoughtSpot
+            if data.get('ai_context'):
+                # Enforce 400-char limit
+                column['ai_context'] = data['ai_context'][:400]
 
-        Args:
-            model_tml_yaml: Updated model TML as YAML string
+        return yaml.dump(model_tml, allow_unicode=True, sort_keys=False)
 
-        Returns:
-            Dictionary with deployment result:
-                - success: Boolean
-                - model_id: GUID of updated model
-                - error: Error message if failed
-        """
-        try:
-            response = self.ts_client.session.post(
-                f"{self.ts_client.base_url}/api/rest/2.0/metadata/tml/import",
-                headers=self.ts_client.headers,
-                json={
-                    "metadata_tmls": [model_tml_yaml],
-                    "import_policy": "PARTIAL",
-                    "force_create": False  # Update existing, don't create new
-                }
-            )
-
-            if response.status_code == 200:
-                result = response.json()
-
-                # Extract model ID from response
-                model_id = None
-                if result.get('object') and len(result['object']) > 0:
-                    model_id = result['object'][0].get('response', {}).get('header', {}).get('id')
-
-                return {
-                    'success': True,
-                    'model_id': model_id,
-                    'response': result
-                }
-            else:
-                return {
-                    'success': False,
-                    'error': f"API returned status {response.status_code}: {response.text}"
-                }
-        except Exception as e:
-            return {
-                'success': False,
-                'error': str(e)
-            }
+    # ------------------------------------------------------------------
+    # High-level entry point
+    # ------------------------------------------------------------------
 
-    def update_model_semantics(
+    def enrich_model(
         self,
         model_id: str,
         company_research: str,
-        use_case: str = ""
+        use_case: str = "",
+        company_name: str = "",
+        model_name: str = "",
     ) -> Dict:
         """
-        Complete workflow to update model semantics
-
-        Args:
-            model_id: GUID of model to update
-            company_research: Company research text
-            use_case: Optional use case context
-
-        Returns:
-            Dictionary with result:
-                - success: Boolean
-                - model_id: GUID of updated model
-                - columns_updated: Number of columns updated
-                - error: Error message if failed
+        Full workflow: export model TML → generate semantics → reimport.
+
+        Returns dict with success, columns_updated, error.
         """
-        # Export current model TML
         model_tml = self.get_model_tml(model_id)
         if not model_tml:
-            return {
-                'success': False,
-                'error': 'Could not export model TML'
-            }
-
-        # Generate semantic metadata
-        semantics = self.generate_column_semantics(company_research, model_tml, use_case)
+            return {'success': False, 'error': 'Could not export model TML'}
+
+        # Generate model description
+        model_description = self.generate_model_description(
+            company_research, use_case, company_name,
+            model_name or model_tml.get('model', {}).get('name', ''),
+        )
+
+        # Generate column semantics
+        semantics = self.generate_column_semantics(
+            company_research, model_tml, use_case, company_name
+        )
         if not semantics:
-            return {
-                'success': False,
-                'error': 'Could not generate semantic metadata'
-            }
-
-        # Update TML with semantics
-        updated_tml_yaml = self.update_model_tml_semantics(model_tml, semantics)
+            return {'success': False, 'error': 'Could not generate column semantics'}
 
-        # Deploy updated model
-        result = self.deploy_updated_model(updated_tml_yaml)
+        # Apply to TML
+        updated_yaml = self.apply_to_model_tml(model_tml, semantics, model_description)
 
+        # Deploy
+        result = self.deploy_updated_model(updated_yaml)
         if result['success']:
             result['columns_updated'] = len(semantics)
-
+            result['model_description'] = model_description
         return result
 
 
-def update_model_semantic_layer(
+# Convenience function for direct use
+def enrich_model_semantics(
     ts_client,
     model_id: str,
     company_research: str,
     use_case: str = "",
+    company_name: str = "",
+    model_name: str = "",
     llm_model: str = DEFAULT_LLM_MODEL,
 ) -> Dict:
-    """
-    Convenience function to update model semantic layer
-
-    Args:
-        ts_client: Authenticated ThoughtSpotDeployer instance
-        model_id: GUID of model to update
-        company_research: Company research text
-        use_case: Optional use case context
-
-    Returns:
-        Dictionary with update result
-    """
     updater = ModelSemanticUpdater(ts_client, llm_model=llm_model)
-    return updater.update_model_semantics(model_id, company_research, use_case)
+    return updater.enrich_model(
+        model_id=model_id,
+        company_research=company_research,
+        use_case=use_case,
+        company_name=company_name,
+        model_name=model_name,
+    )
+
+
+# Backwards-compatibility alias used by demo_prep.py
+update_model_semantic_layer = enrich_model_semantics

prompts.py

@@ -556,6 +556,32 @@ Create a bullet outline demo script with:
 - Spotter questions to ask live
 - Closing value proposition""",
 
+    "spotter_viz_story_matrix": """You are creating a ThoughtSpot-recommended Spotter Viz story.
+
+Spotter Viz is an AI agent in ThoughtSpot that builds Liveboards through natural language prompts.
+
+You have been given the ThoughtSpot-recommended KPIs, visualizations, and liveboard questions for this exact vertical and function. Your job is to turn those into a natural, conversational demo script — a sequence of prompts a user would type into Spotter Viz to build the liveboard step by step.
+
+{context}
+
+---
+
+Write the story as a numbered sequence of prompts. Every KPI and visualization listed above must appear as a step. Format each step as:
+
+### Step N: [Brief label]
+> "[The exact prompt to type into Spotter Viz]"
+
+**What to look for:** [1 sentence on what the viz reveals — use the insight from the matrix if provided]
+
+Rules:
+- Step 1: set context — company name, data source, overall goal
+- KPI steps: include time dimension and granularity (e.g. "revenue for the last 12 months as a monthly KPI sparkline")
+- Chart steps: keep the metric name and dimension from the matrix (e.g. "revenue by region as a bar chart")
+- Add 1-2 narrative sentences between steps to tell a story — what are we looking for, what does it mean?
+- Final step: the "aha moment" — name the key insight or outlier from the story controls
+- Use the company name and real metric names from the matrix above
+- Do not invent KPIs or visualizations that are not in the matrix""",
+
     "spotter_viz_story": """You are creating a Spotter Viz story for ThoughtSpot's AI-powered liveboard builder.
 
 Spotter Viz is an AI agent in ThoughtSpot that creates, structures, and styles Liveboards through natural language prompts. Users type conversational requests and the agent builds the dashboard step by step, allowing iterative refinement.

sprint_2026_03.md

@@ -110,8 +110,24 @@ This sprint covers hardening, settings, and new capabilities before that happens
 - [ ] **Existing Model Selection + Self-Join Skip** — may be done; needs confirmation test + verify self-join skip is working correctly
 - [ ] **Universal Context Prompt** — double-test this feature end-to-end
 - [ ] **Chat Adjustment Using Outlier System** — never got to this
+- [x] **Matrix Editor Tab** ✅ — 🧩 Matrix tab added; first draft view+edit
+  - Vertical + Function dropdowns; coverage badge (Override / Base merge)
+  - Target persona + business problem shown for overrides
+  - KPIs accordion with definitions
+  - Liveboard Questions editable dataframe (add/delete rows)
+  - Story Controls accordion (read-only for now)
+  - Persistence via Supabase deferred to next sprint
+  - Reference doc: `dev_notes/matrix_reference.md`
+- [x] **Remember Me on Login** ✅ — injected into Gradio venv `index.html` template
+  - `div.form` (not `<form>`) found via polling; no shadow DOM
+  - Saves username to localStorage on login; pre-fills on return visit
 - [ ] **Interface Mode Refactor** (`DemoWorkflowEngine` shared class concept)
-- [ ] **Wizard Tab UI** — not started
+- [x] **Wizard Tab UI** ✅ — Defined/Custom sub-tabs in Chat; Defined tab first; GO button wires into pipeline
+  - Vertical dropdown → Function dropdown (stacked, marks overrides with ✓)
+  - Company URL + "Use URL" checkbox + Additional context + → GO
+  - Custom tab = original chat flow unchanged
+  - Welcome message redesigned: title + How to start + collapsible example table
+  - Chatbot box removed (transparent) to reduce visual clutter
 - [ ] **Tag Assignment to Models** — returns 404 (works for tables, not models); needs investigation
 - [ ] **Spotter Viz Story Verification** — run end-to-end and verify story generation + blank viz (ASP, Total Sales Weekly) and brand colors rendering
 - [ ] **Fix Research Cache Not Loading** — relative path issue; fix was ready, needs test
@@ -157,6 +173,50 @@ This sprint covers hardening, settings, and new capabilities before that happens
 
 ---
 
+## Ideas to Think About (Not Implementing Yet)
+
+### Spotter Viz Story — Matrix-Grounded Single Story
+Current state: `_generate_spotter_viz_story()` generates two sections (persona-driven from matrix + AI-generated LLM story) combined into one Markdown block.
+
+**Proposed approach:**
+- Single cohesive story (no split tabs) — the matrix IS the foundation, AI adds glue
+- Take `liveboard_questions` from the config → generate high-level NL prompts (not granular)
+  - e.g. `"Show revenue for the last 3 months as a KPI"` / `"Revenue by region as a bar chart"`
+- AI's role: light narrative that connects the dots, names the story, adds 1-2 sentences per step
+- Keep it simple — the goal is a ready-to-use demo script, not documentation
+- Both sections currently generated sequentially after ThoughtSpot deploy; could stay that way
+
+### Async Pipeline — Parallel ThoughtSpot + Data Population
+Current pipeline is fully sequential: Research → DDL → Create Tables → Populate → TS Model → Liveboard
+
+**Dependency analysis:**
+```
+Research ──► DDL ──► Create Tables ─┬──► Populate Data (LegitData)  ──┐
+                                    │                                   ├──► MCP Liveboard
+                                    └──► TS Model + Connections ────────┘
+                                         (semantic layer, column naming,
+                                          sharing, tagging, Sage index)
+```
+- TS model creation needs **schema** (tables exist) but NOT data rows
+- Data population needs **tables** but not the TS model
+- MCP liveboard (Spotter Viz) needs **both** — reads actual data via the model
+- Everything from "Create Tables" onward can be parallelized except the final liveboard step
+
+**What runs in parallel after tables are created:**
+- Thread A: LegitData population
+- Thread B: TS Model creation → column naming → semantic update → sharing → tagging → Sage indexing
+
+**Estimated savings:** Model creation + semantic update is ~30-60s; LegitData is ~60-120s.
+Running in parallel saves roughly the model-creation time off the total wall clock.
+
+**Risks/considerations:**
+- Gradio streaming yields from a generator — parallel work needs to run in threads and feed a shared queue that the generator drains
+- Error handling: if one branch fails, need to cancel/report both and decide whether to proceed to liveboard
+- Progress reporting needs to interleave updates from both branches
+- Could implement with `concurrent.futures.ThreadPoolExecutor` + a `queue.Queue` for progress messages
+
+---
+
 ## Phase 3 (Future)
 
 - [ ] **OAuth/SSO Login** — swap Gradio auth for proper OAuth flow
@@ -175,6 +235,38 @@ This sprint covers hardening, settings, and new capabilities before that happens
 
 ## Done
 
+### Session: March 27, 2026 — ts_user Fix + Semantics Pipeline + UX Fixes
+
+- [x] **ts_user fix** ✅ — ThoughtSpot objects now created under logged-in user, not admin
+  - All 3 `ts_user` locations in `chat_interface.py` → `self._get_effective_user_email()`
+  - `ThoughtSpotDeployer` raises `ValueError` if username or secret_key not passed
+  - `_get_direct_api_session` / `_get_answer_direct` in `liveboard_creator.py`: accept `secret_key` param, no global singleton cache
+- [x] **THOUGHTSPOT_ADMIN_USER removed** ✅ — from all non-test Python files
+  - `supabase_client.py`: removed from `ADMIN_SETTINGS_KEYS` and `inject_admin_settings_to_env`
+  - `chat_interface.py`: removed startup check + admin hidden field
+- [x] **THOUGHTSPOT_TRUSTED_AUTH_KEY removed from Supabase settings** ✅
+  - All reads changed to `self.settings.get('thoughtspot_trusted_auth_key')` — set per-env via dropdown
+  - Removed `admin_ts_auth_key` hidden textbox from Settings UI
+- [x] **Model semantic enrichment wired into pipeline** ✅ — `thoughtspot_deployer.py` `deploy_all`
+  - Model description, per-column description + synonyms + ai_context generated via LLM in one call
+  - Applied to model TML and reimported in same Spotter-enable cycle
+  - `model_semantic_updater.py` rewritten: `generate_model_description`, `generate_column_semantics`, `apply_to_model_tml`, `enrich_model` + backwards-compat alias `update_model_semantic_layer`
+- [x] **Welcome page examples updated** ✅ — 5 hardcoded examples (Retail/Banking/Software/Manufacturing verticals)
+  - Persona column removed; examples guaranteed to work
+- [x] **Boolean type mismatch fixed** ✅ — `legitdata_bridge.py` `convert_value()`
+  - `isinstance(value, bool): return int(value)` added before int check (bool is subclass of int)
+  - Fixes SUPPLY_CHAIN_EVENTS population failure (`NUMBER(38,0)` vs `BOOLEAN`)
+- [x] **Company parsing fix** ✅ — "Caterpillar.com Manufacturing Supply Chain" now parses in one shot
+  - Added space-only separator pattern `r'[a-zA-Z0-9-]+\.[a-zA-Z]{2,}\s+(.+)'` to `extract_use_case_from_message`
+  - Company state preserved when only company detected (not overwritten with old value)
+- [x] **Manufacturing in use case list** ✅ — removed `[:3]` slice so all 4 verticals show
+- [x] **Progress bar stage fix** ✅ — auto-run now shows correct stage at each step
+  - `current_stage = 'research'` at start of research loop (was `'deploy'`)
+  - `current_stage = 'create_ddl'` after research, before DDL creation
+  - `current_stage = 'deploy'` after DDL, before Snowflake deployment
+- [x] **Remove default company from input bar** ✅ — input now starts blank
+  - All `"Amazon.com"` fallbacks replaced with `""` in `load_session_state_on_startup`, `get_session_defaults`, and `default_settings`
+
 ### Session: March 27, 2026 — Concurrency Fix + Logging Fix
 
 - [x] **Concurrent session UI interference fixed** ✅ — `chat_interface.py`

supabase_client.py

@@ -530,8 +530,6 @@ ADMIN_USER_KEY = "__admin__"
 ADMIN_SETTINGS_KEYS = {
     # ThoughtSpot
     "THOUGHTSPOT_URL": "ThoughtSpot Instance URL",
-    "THOUGHTSPOT_TRUSTED_AUTH_KEY": "ThoughtSpot Trusted Auth Key",
-    "THOUGHTSPOT_ADMIN_USER": "ThoughtSpot Admin Username",
     # LLM API Keys
     "OPENAI_API_KEY": "OpenAI API Key",
     "GOOGLE_API_KEY": "Google API Key",
@@ -642,8 +640,6 @@ def inject_admin_settings_to_env() -> bool:
     
     Called once after login, before any pipeline runs.
     
-    Also maps THOUGHTSPOT_ADMIN_USER -> THOUGHTSPOT_USERNAME for backwards compatibility.
-    
     Returns:
         True if settings were loaded and injected successfully.
     """
@@ -662,11 +658,6 @@ def inject_admin_settings_to_env() -> bool:
             os.environ[key] = str(value)
             injected += 1
     
-    # Map THOUGHTSPOT_ADMIN_USER -> THOUGHTSPOT_USERNAME for backwards compat
-    admin_user = settings.get("THOUGHTSPOT_ADMIN_USER", "")
-    if admin_user:
-        os.environ["THOUGHTSPOT_USERNAME"] = admin_user
-    
     print(
         f"✅ Injected {injected} admin settings from Supabase into environment "
         "(LLM keys remain environment-only)."

tests/conftest.py

@@ -0,0 +1,102 @@
+"""
+Pytest configuration and shared fixtures for DemoPrep E2E tests.
+"""
+import os
+import pytest
+from dotenv import load_dotenv
+from playwright.sync_api import Page, BrowserContext, Browser, expect
+
+load_dotenv(os.path.join(os.path.dirname(__file__), '..', '.env'))
+
+BASE_URL = "https://thoughtspot-dp-demoprep.hf.space"
+TEST_USER = os.getenv("TEST_USER")
+TEST_PASSWORD = os.getenv("TEST_PASSWORD")
+
+if not TEST_USER or not TEST_PASSWORD:
+    raise EnvironmentError("TEST_USER and TEST_PASSWORD must be set in .env")
+
+
+def _do_login(page: Page) -> None:
+    """Perform the login flow on a page."""
+    page.goto(BASE_URL, timeout=90000)
+    page.wait_for_selector('input[placeholder="Type here..."]', timeout=90000)
+    page.fill('input[type=text]', TEST_USER)
+    page.fill('input[type=password]', TEST_PASSWORD)
+    page.click('button:has-text("Login")')
+    page.wait_for_selector('.gradio-container', timeout=90000)
+    page.wait_for_timeout(3000)  # allow Gradio JS to settle
+
+
+def login(page: Page) -> None:
+    """Log in to the app with test credentials (for tests that need a fresh login)."""
+    _do_login(page)
+
+
+@pytest.fixture(scope="session")
+def auth_context(browser: Browser, tmp_path_factory):
+    """
+    Session-scoped authenticated browser context.
+
+    Logs in once, saves storage state, and yields a persistent context.
+    All logged_in_page fixtures share this context, avoiding repeated expensive
+    logins and connection setup against HF Spaces.
+    """
+    state_file = str(tmp_path_factory.mktemp("auth") / "state.json")
+
+    # Log in on a temp page to capture auth state
+    setup_ctx = browser.new_context(viewport={"width": 1280, "height": 900})
+    setup_page = setup_ctx.new_page()
+    _do_login(setup_page)
+    setup_ctx.storage_state(path=state_file)
+    setup_page.close()
+    setup_ctx.close()
+
+    # Create the long-lived context with saved auth state
+    ctx = browser.new_context(
+        storage_state=state_file,
+        viewport={"width": 1280, "height": 900},
+    )
+    yield ctx
+    ctx.close()
+
+
+@pytest.fixture()
+def logged_in_page(auth_context: BrowserContext):
+    """
+    Function-scoped: opens a fresh page in the shared authenticated context.
+    Each test gets an isolated page (own URL, own Gradio session) without
+    the overhead of a full login or new browser context.
+    """
+    page = auth_context.new_page()
+    for attempt in range(2):
+        try:
+            page.goto(BASE_URL, timeout=90000)
+            # Wait for the Gradio app to fully initialize — specifically the
+            # chat input and tab buttons, which appear after full JS execution.
+            page.wait_for_selector('button[role=tab]', timeout=90000)
+            page.wait_for_selector('input[placeholder*="Type your message"]', timeout=90000)
+            page.wait_for_timeout(1000)
+            break
+        except Exception:
+            if attempt == 1:
+                raise
+            page.wait_for_timeout(5000)
+    yield page
+    page.close()
+
+
+def get_chat_input(page: Page):
+    """Return the main chat message input field."""
+    return page.locator('input[placeholder*="Type your message"]')
+
+
+# Alias for backwards compatibility
+def get_chat_textarea(page: Page):
+    return get_chat_input(page)
+
+
+def wait_for_response(page: Page, timeout_ms: int = 90000) -> None:
+    """Wait until the assistant has finished responding."""
+    send_btn = page.locator('button:has-text("Send")')
+    expect(send_btn).to_be_enabled(timeout=timeout_ms)
+    page.wait_for_timeout(500)

tests/e2e_chat.py

@@ -0,0 +1,96 @@
+"""
+Chat interaction tests — verify the chat responds correctly to user input.
+
+Note: These tests send real messages to the app and wait for LLM responses.
+They are intentionally scoped to short/fast interactions. Pipeline tests
+(research → DDL → deploy) are in e2e_pipeline.py.
+
+Run with: pytest tests/e2e_chat.py -v
+"""
+import pytest
+from playwright.sync_api import Page, expect
+from tests.conftest import logged_in_page, get_chat_textarea, wait_for_response  # noqa: F401
+
+
+def send_message(page: Page, text: str):
+    """Type and send a chat message."""
+    chat_input = get_chat_textarea(page)
+    chat_input.click()
+    chat_input.fill(text)
+    page.click('button:has-text("Send")')
+
+
+class TestChatBasics:
+    def test_help_command_responds(self, logged_in_page: Page):
+        """Sending 'help' returns a response from the assistant."""
+        send_message(logged_in_page, "help")
+        wait_for_response(logged_in_page, timeout_ms=30000)
+        body_text = logged_in_page.inner_text('body')
+        # Should contain some meaningful help content
+        assert any(word in body_text.lower() for word in ["use case", "company", "demo", "help"])
+
+    def test_message_clears_after_send(self, logged_in_page: Page):
+        """Chat input is cleared after sending a message."""
+        chat_input = get_chat_textarea(logged_in_page)
+        chat_input.click()
+        chat_input.fill("hello")
+        logged_in_page.click('button:has-text("Send")')
+        # Input should be empty immediately after send
+        expect(chat_input).to_have_value("")
+
+    def test_invalid_input_gets_response(self, logged_in_page: Page):
+        """Nonsense input still gets a graceful response."""
+        send_message(logged_in_page, "asdfghjkl zxcvbnm")
+        wait_for_response(logged_in_page, timeout_ms=30000)
+        body_text = logged_in_page.inner_text('body')
+        # Should have more content than just the welcome message
+        assert "asdfghjkl" in body_text or len(body_text) > 500
+
+
+class TestQuickActions:
+    def test_start_research_button_sends_message(self, logged_in_page: Page):
+        """Clicking Start Research triggers a response."""
+        logged_in_page.click('button:has-text("🔍 Start Research")')
+        wait_for_response(logged_in_page, timeout_ms=30000)
+        body_text = logged_in_page.inner_text('body')
+        # Should prompt for company info or explain research
+        assert len(body_text) > 200
+
+    def test_configure_button_responds(self, logged_in_page: Page):
+        """Configure button triggers a response."""
+        logged_in_page.click('button:has-text("⚙️ Configure")')
+        wait_for_response(logged_in_page, timeout_ms=30000)
+        body_text = logged_in_page.inner_text('body')
+        assert len(body_text) > 200
+
+    def test_help_button_responds(self, logged_in_page: Page):
+        """Help button triggers a response."""
+        logged_in_page.click('button:has-text("💡 Help")')
+        wait_for_response(logged_in_page, timeout_ms=30000)
+        body_text = logged_in_page.inner_text('body')
+        assert len(body_text) > 200
+
+
+class TestTabNavigation:
+    def test_can_switch_to_settings_tab(self, logged_in_page: Page):
+        """Settings tab is clickable and shows settings content."""
+        logged_in_page.click('button[role=tab]:has-text("⚙️ Settings")')
+        logged_in_page.wait_for_timeout(1000)
+        body_text = logged_in_page.inner_text('body')
+        assert any(word in body_text.lower() for word in ["model", "settings", "liveboard", "save"])
+
+    def test_can_switch_back_to_chat(self, logged_in_page: Page):
+        """Can navigate away from and back to the Chat tab."""
+        logged_in_page.click('button[role=tab]:has-text("⚙️ Settings")')
+        logged_in_page.wait_for_timeout(500)
+        logged_in_page.click('button[role=tab]:has-text("💬 Chat")')
+        logged_in_page.wait_for_timeout(500)
+        expect(logged_in_page.locator('button:has-text("Send")')).to_be_visible()
+
+    def test_live_progress_tab_loads(self, logged_in_page: Page):
+        """Live Progress tab is accessible and renders."""
+        logged_in_page.click('button[role=tab]:has-text("Live Progress")')
+        logged_in_page.wait_for_timeout(1000)
+        # Tab should now be selected
+        tab = logged_in_page.locator('button[role=tab]:has-text("Live Progress")')
+        expect(tab).to_have_attribute('aria-selected', 'true')

tests/e2e_settings.py

@@ -0,0 +1,52 @@
+"""
+Settings tab tests — verify settings load, display, and save correctly.
+
+Run with: pytest tests/e2e_settings.py -v
+"""
+import pytest
+from playwright.sync_api import Page, expect
+from tests.conftest import logged_in_page  # noqa: F401
+
+
+def go_to_settings(page: Page):
+    """Navigate to the Settings tab."""
+    page.click('button[role=tab]:has-text("⚙️ Settings")')
+    page.wait_for_timeout(1500)
+
+
+class TestSettingsTab:
+    def test_settings_tab_accessible(self, logged_in_page: Page):
+        """Settings tab loads without error."""
+        go_to_settings(logged_in_page)
+        body_text = logged_in_page.inner_text('body')
+        assert any(word in body_text.lower() for word in ["settings", "model", "save"])
+
+    def test_save_settings_button_present(self, logged_in_page: Page):
+        """Save Settings button is visible in the Settings tab."""
+        go_to_settings(logged_in_page)
+        expect(logged_in_page.locator('button:has-text("💾 Save Settings")')).to_be_visible()
+
+    def test_reset_to_defaults_button_present(self, logged_in_page: Page):
+        """Reset to Defaults button is visible."""
+        go_to_settings(logged_in_page)
+        expect(logged_in_page.locator('button:has-text("🔄 Reset to Defaults")')).to_be_visible()
+
+    def test_liveboard_name_field_present(self, logged_in_page: Page):
+        """Liveboard name textarea is accessible in settings."""
+        go_to_settings(logged_in_page)
+        logged_in_page.wait_for_timeout(1000)
+        field = logged_in_page.locator('textarea[placeholder="My Demo Liveboard"]')
+        expect(field).to_be_visible(timeout=10000)
+
+    def test_change_password_section_present(self, logged_in_page: Page):
+        """Change Password section is accessible for the test user."""
+        go_to_settings(logged_in_page)
+        expect(logged_in_page.locator('button:has-text("🔒 Change Password")')).to_be_visible()
+
+    def test_admin_sections_not_visible_for_test_user(self, logged_in_page: Page):
+        """Admin-only sections (User Management) should not appear for testrunner."""
+        go_to_settings(logged_in_page)
+        body_text = logged_in_page.inner_text('body')
+        # testrunner is not an admin — user management table should not be visible
+        # (or at least not show other users' emails)
+        assert "mike.boone@thoughtspot.com" not in body_text

tests/e2e_smoke.py

@@ -0,0 +1,92 @@
+"""
+Smoke tests — verify the app is up and core UI elements are present.
+Run with: pytest tests/e2e_smoke.py -v
+"""
+import pytest
+from playwright.sync_api import Page, Browser, expect
+from tests.conftest import login, get_chat_textarea, BASE_URL, TEST_USER
+
+
+class TestAppAvailability:
+    def test_app_loads(self, page: Page):
+        """App responds and shows the login page."""
+        page.goto(BASE_URL, timeout=60000)
+        expect(page.locator('input[type=password]')).to_be_visible(timeout=45000)
+
+    def test_login_page_has_expected_fields(self, page: Page):
+        """Login form has username, password, and login button."""
+        page.goto(BASE_URL, timeout=60000)
+        page.wait_for_selector('input[type=text]', timeout=45000)
+        expect(page.locator('input[type=text]')).to_be_visible()
+        expect(page.locator('input[type=password]')).to_be_visible()
+        expect(page.locator('button:has-text("Login")')).to_be_visible()
+
+    def test_bad_credentials_rejected(self, page: Page):
+        """Wrong password does not log in."""
+        page.goto(BASE_URL, timeout=60000)
+        page.wait_for_selector('input[type=text]', timeout=45000)
+        page.fill('input[type=text]', TEST_USER)
+        page.fill('input[type=password]', 'definitely-wrong-password')
+        page.click('button:has-text("Login")')
+        page.wait_for_timeout(3000)
+        # Should still show the login form
+        expect(page.locator('input[type=password]')).to_be_visible()
+
+    def test_login_succeeds(self, page: Page):
+        """Valid credentials get past the login page."""
+        login(page)
+        # The login-specific password field (placeholder ••••••••) should be gone
+        expect(page.locator('input[placeholder="••••••••"]')).not_to_be_visible()
+        # App body should be present
+        expect(page.locator('.gradio-container')).to_be_visible()
+
+
+class TestMainUIStructure:
+    def test_all_tabs_present(self, logged_in_page: Page):
+        """All expected tabs are visible after login."""
+        expected_tabs = [
+            "💬 Chat",
+            "Live Progress",
+            "📋 Demo Pack",
+            "🎬 Spotter Viz Story",
+            "⚙️ Settings",
+            "🤖 AI Feedback",
+            "📄 DDL Code",
+        ]
+        for tab_text in expected_tabs:
+            expect(logged_in_page.locator(f'button[role=tab]:has-text("{tab_text}")')).to_be_visible()
+
+    def test_chat_tab_active_by_default(self, logged_in_page: Page):
+        """Chat tab is selected on first load."""
+        tab = logged_in_page.locator('button[role=tab]:has-text("💬 Chat")')
+        expect(tab).to_have_attribute('aria-selected', 'true')
+
+    def test_chat_input_visible(self, logged_in_page: Page):
+        """Chat textarea and Send button are visible."""
+        expect(logged_in_page.locator('button:has-text("Send")')).to_be_visible()
+        expect(logged_in_page.locator('textarea').first).to_be_visible()
+
+    def test_quick_action_buttons_present(self, logged_in_page: Page):
+        """Quick action buttons are visible in the chat tab."""
+        expect(logged_in_page.locator('button:has-text("🔍 Start Research")')).to_be_visible()
+        expect(logged_in_page.locator('button:has-text("⚙️ Configure")')).to_be_visible()
+        expect(logged_in_page.locator('button:has-text("💡 Help")')).to_be_visible()
+
+    def test_welcome_message_displayed(self, logged_in_page: Page):
+        """The welcome message appears in the chatbot on first load."""
+        body_text = logged_in_page.inner_text('body')
+        assert "Welcome to ThoughtSpot Demo Builder" in body_text
+
+    def test_sign_out_link_present(self, logged_in_page: Page):
+        """Sign Out link is visible after login."""
+        expect(logged_in_page.locator('text=Sign Out')).to_be_visible()
+
+    def test_sign_out_link_navigates_to_logout(self, logged_in_page: Page):
+        """The Sign Out link points to /logout (verified by href, not by clicking).
+
+        NOTE: We do NOT navigate to /logout here because doing so with the shared
+        test session would invalidate auth_context and break subsequent tests.
+        The logout flow is covered separately in e2e_auth.py which runs last.
+        """
+        link = logged_in_page.locator('a[href="/logout"]')
+        expect(link).to_be_visible()

tests/e2e_z_auth.py

@@ -0,0 +1,24 @@
+"""
+Auth teardown tests — verify logout works.
+
+This file is named with 'z_' prefix so it runs LAST alphabetically.
+These tests navigate to /logout which invalidates the shared test session,
+so they must not run before any test that depends on auth_context.
+
+Run with: pytest tests/e2e_z_auth.py -v
+"""
+import pytest
+from playwright.sync_api import Page, Browser, expect
+from tests.conftest import login, BASE_URL
+
+
+class TestLogout:
+    def test_sign_out_logs_out(self, browser: Browser):
+        """Navigating to /logout clears the session and shows the login page."""
+        # Use a dedicated fresh context so the shared auth_context is not affected.
+        ctx = browser.new_context(viewport={"width": 1280, "height": 900})
+        page = ctx.new_page()
+        login(page)
+        page.goto(f"{BASE_URL}/logout", timeout=30000)
+        expect(page.locator('button:has-text("Login")')).to_be_visible(timeout=15000)
+        ctx.close()

thoughtspot_deployer.py

@@ -121,8 +121,12 @@ class ThoughtSpotDeployer:
         Raises ValueError if any required setting is missing.
         """
         self.base_url = base_url if base_url else get_admin_setting('THOUGHTSPOT_URL')
-        self.username = username if username else get_admin_setting('THOUGHTSPOT_ADMIN_USER')
-        self.secret_key = secret_key if secret_key else get_admin_setting('THOUGHTSPOT_TRUSTED_AUTH_KEY')
+        if not username:
+            raise ValueError("ThoughtSpotDeployer requires username — pass the logged-in user's email")
+        self.username = username
+        if not secret_key:
+            raise ValueError("ThoughtSpotDeployer requires secret_key — pass the trusted auth key for the selected environment")
+        self.secret_key = secret_key
         
         # Snowflake connection details from environment (key pair auth)
         self.sf_account = get_admin_setting('SNOWFLAKE_ACCOUNT')
@@ -1823,6 +1827,7 @@ class ThoughtSpotDeployer:
                   use_case: str = None, liveboard_name: str = None,
                   llm_model: str = None, tag_name: str = None,
                   liveboard_method: str = None, share_with: str = None,
+                  company_research: str = None,
                   progress_callback=None) -> Dict:
         """
         Deploy complete data model to ThoughtSpot
@@ -2230,8 +2235,8 @@ class ThoughtSpotDeployer:
                             log_progress(f"Sharing model with '{_effective_share}'...")
                             self.share_objects([model_guid], 'LOGICAL_TABLE', _effective_share)
 
-                        # Step 3.5: Enable Spotter via TML update
-                        # create_new=True import ignores spotter_config — must export then re-import to set it
+                        # Step 3.5: Enable Spotter + enrich model semantics in a single export→update→reimport
+                        # (create_new=True import ignores spotter_config, so we always re-import here)
                         try:
                             export_resp = self.session.post(
                                 f"{self.base_url}/api/rest/2.0/metadata/tml/export",
@@ -2245,8 +2250,41 @@ class ThoughtSpotDeployer:
                                 export_data = export_resp.json()
                                 if export_data and 'edoc' in export_data[0]:
                                     model_tml_dict = json.loads(export_data[0]['edoc'])
-                                    # Set spotter_config inside properties (correct location per golden demo TML)
+
+                                    # Enable Spotter
                                     model_tml_dict.setdefault('model', {}).setdefault('properties', {})['spotter_config'] = {'is_spotter_enabled': True}
+
+                                    # Enrich with description, synonyms, and AI context
+                                    if company_research:
+                                        try:
+                                            from model_semantic_updater import ModelSemanticUpdater
+                                            log_progress("Generating model description, synonyms, and AI context...")
+                                            sem_start = time.time()
+                                            updater = ModelSemanticUpdater(self, llm_model=llm_model)
+
+                                            model_description = updater.generate_model_description(
+                                                company_research=company_research,
+                                                use_case=use_case or '',
+                                                company_name=company_name or '',
+                                                model_name=model_name,
+                                            )
+                                            column_semantics = updater.generate_column_semantics(
+                                                company_research=company_research,
+                                                model_tml=model_tml_dict,
+                                                use_case=use_case or '',
+                                                company_name=company_name or '',
+                                            )
+                                            # Apply to TML dict in-place (returns YAML string)
+                                            enriched_yaml = updater.apply_to_model_tml(
+                                                model_tml_dict, column_semantics, model_description
+                                            )
+                                            # Parse back so we can still dump consistently below
+                                            model_tml_dict = yaml.safe_load(enriched_yaml)
+                                            sem_time = time.time() - sem_start
+                                            log_progress(f"[OK] Semantics generated: {len(column_semantics)} columns enriched ({sem_time:.1f}s)")
+                                        except Exception as sem_err:
+                                            log_progress(f"[WARN] Semantic enrichment failed (non-fatal): {sem_err}")
+
                                     updated_tml = yaml.dump(model_tml_dict, allow_unicode=True, sort_keys=False)
                                     update_resp = self.session.post(
                                         f"{self.base_url}/api/rest/2.0/metadata/tml/import",
@@ -2257,15 +2295,15 @@ class ThoughtSpotDeployer:
                                         }
                                     )
                                     if update_resp.status_code == 200:
-                                        log_progress(f"🤖 Spotter enabled on model")
+                                        log_progress("🤖 Spotter enabled + model semantics applied")
                                     else:
-                                        log_progress(f"🤖 Spotter enable update failed: HTTP {update_resp.status_code} — {update_resp.text[:200]}")
+                                        log_progress(f"🤖 Model update failed: HTTP {update_resp.status_code} — {update_resp.text[:200]}")
                                 else:
-                                    log_progress(f"🤖 Spotter enable: export returned no edoc")
+                                    log_progress("🤖 Spotter enable: export returned no edoc")
                             else:
                                 log_progress(f"🤖 Spotter enable: export failed HTTP {export_resp.status_code}")
                         except Exception as spotter_error:
-                            log_progress(f"🤖 Spotter enable exception: {spotter_error}")
+                            log_progress(f"🤖 Spotter/semantics exception: {spotter_error}")
 
                         # Step 4: Auto-create Liveboard from model
                         lb_start = time.time()
@@ -2408,7 +2446,9 @@ class ThoughtSpotDeployer:
                                     if enhance_result.get('success'):
                                         log_progress(f"   [OK] Enhancement applied")
                                     else:
-                                        log_progress(f"   [WARN] Enhancement partial: {enhance_result.get('message', '')[:50]}")
+                                        enhance_err = f"TML enhancement failed: {enhance_result.get('message', 'unknown')[:100]}"
+                                        log_progress(f"   [ERROR] {enhance_err}")
+                                        results['errors'].append(enhance_err)
 
                             # Check result
                             print(f"🔍 DEBUG: Liveboard result received: {liveboard_result}")
@@ -2441,6 +2481,7 @@ class ThoughtSpotDeployer:
                         except Exception as lb_error:
                             error = f"Liveboard creation exception: {str(lb_error)}"
                             results['errors'].append(error)
+                            log_progress(f"[ERROR] {error}")
                     else:
                         # Extract detailed error information
                         obj_response = objects[0].get('response', {})