Spaces:
Running
Running
| """ | |
| Chat-based Demo Builder Interface | |
| Clean, conversational interface for creating ThoughtSpot demos | |
| """ | |
| import warnings | |
| warnings.filterwarnings('ignore', message='.*tuples.*format.*chatbot.*deprecated.*') | |
| import gradio as gr | |
| import os | |
| import sys | |
| import json | |
| import time | |
| import glob | |
| from datetime import datetime | |
| from dotenv import load_dotenv | |
| from demo_builder_class import DemoBuilder | |
| APP_ENV = os.getenv("APP_ENV", "production").lower() | |
| IS_TEST = APP_ENV == "test" | |
| from supabase_client import load_gradio_settings, get_admin_setting, inject_admin_settings_to_env | |
| from main_research import MultiLLMResearcher, Website | |
| from demo_personas import ( | |
| build_company_analysis_prompt, | |
| build_industry_research_prompt, | |
| VERTICALS, | |
| FUNCTIONS, | |
| MATRIX_OVERRIDES, | |
| VERTICAL_LINES, | |
| DEMO_FUNCTIONS, | |
| get_use_case_config, | |
| parse_use_case | |
| ) | |
| from llm_config import ( | |
| DEFAULT_LLM_MODEL, | |
| UI_MODEL_CHOICES, | |
| get_openai_api_key, | |
| map_llm_display_to_provider, | |
| ) | |
| load_dotenv(override=True) | |
| # ========================================================================== | |
| # TS ENVIRONMENT HELPERS | |
| # ========================================================================== | |
| def get_ts_environments() -> list: | |
| """Return list of environment labels from TS_ENV_N_LABEL/URL/.env entries.""" | |
| envs = [] | |
| i = 1 | |
| while True: | |
| label = os.getenv(f'TS_ENV_{i}_LABEL', '').strip() | |
| url = os.getenv(f'TS_ENV_{i}_URL', '').strip() | |
| if not label or not url: | |
| break | |
| envs.append(label) | |
| i += 1 | |
| return envs or ['(no environments configured)'] | |
| def get_ts_env_url(label: str) -> str: | |
| """Return the URL for a given environment label.""" | |
| i = 1 | |
| while True: | |
| env_label = os.getenv(f'TS_ENV_{i}_LABEL', '').strip() | |
| if not env_label: | |
| break | |
| if env_label == label: | |
| return os.getenv(f'TS_ENV_{i}_URL', '').strip().rstrip('/') | |
| i += 1 | |
| return '' | |
| def get_ts_env_auth_key(label: str) -> str: | |
| """Return the actual auth key value for a given environment label. | |
| TS_ENV_N_KEY_VAR holds the trusted auth key directly. | |
| """ | |
| i = 1 | |
| while True: | |
| env_label = os.getenv(f'TS_ENV_{i}_LABEL', '').strip() | |
| if not env_label: | |
| break | |
| if env_label == label: | |
| return os.getenv(f'TS_ENV_{i}_KEY_VAR', '').strip() | |
| i += 1 | |
| return '' | |
| # ========================================================================== | |
| # SETTINGS SCHEMA - Single source of truth for all settings | |
| # To add a new setting: add ONE entry here, then create the UI component | |
| # Format: (component_key, storage_key, default_value, converter_fn) | |
| # ========================================================================== | |
| SETTINGS_SCHEMA = [ | |
| # ── Panel defaults (mirrors App tab right panel) ────────────────────────── | |
| ('default_ai_model', 'default_llm', DEFAULT_LLM_MODEL, str), | |
| ('default_ts_env', 'default_ts_env', 'secloud - primary', str), | |
| ('liveboard_name', 'liveboard_name', '', str), | |
| ('default_data_size', 'default_data_size', 'Medium', str), | |
| ('geo_scope', 'geo_scope', 'USA Only', str), | |
| ('tag_name', 'tag_name', '', str), | |
| ('column_naming_style', 'column_naming_style', 'Regular Case', str), | |
| ('object_naming_prefix', 'object_naming_prefix', '', str), | |
| ('share_with', 'share_with', '', str), | |
| # ── Optional run input defaults ─────────────────────────────────────────── | |
| ('use_default_inputs', 'use_default_inputs', False, bool), | |
| ('default_vertical', 'default_vertical', '', str), | |
| ('default_line', 'default_line', '', str), | |
| ('default_function', 'default_function', '', str), | |
| ('default_company_url', 'default_company_url', '', str), | |
| # ── Other settings ──────────────────────────────────────────────────────── | |
| ('validation_mode', 'validation_mode', 'Off', str), | |
| # Legacy hidden fields — kept for backward compat, not shown in UI | |
| ('default_use_case', 'default_use_case', 'Sales Analytics', str), | |
| ('fact_table_size', 'fact_table_size', '1000', str), | |
| ('dim_table_size', 'dim_table_size', '100', str), | |
| ('use_existing_model', 'use_existing_model', False, bool), | |
| ('existing_model_guid', 'existing_model_guid', '', str), | |
| # Advanced AI Settings (collapsed accordion) | |
| ('temperature_slider', 'temperature', 0.3, float), | |
| ('max_tokens', 'max_tokens', 4000, int), | |
| ('batch_size', 'batch_size', 5000, int), | |
| ('thread_count', 'thread_count', 4, int), | |
| # Database Connection Settings (collapsed accordion) | |
| ('sf_account', 'snowflake_account', '', str), | |
| ('sf_user', 'snowflake_user', '', str), | |
| ('sf_role', 'snowflake_role', 'ACCOUNTADMIN', str), | |
| ('default_warehouse', 'default_warehouse', 'COMPUTE_WH', str), | |
| ('default_database', 'default_database', 'DEMO_DB', str), | |
| ('default_schema', 'default_schema', 'PUBLIC', str), | |
| # ts_instance_url removed — replaced by TS Environment dropdown on front page | |
| ('ts_username', 'thoughtspot_username', '', str), | |
| ('data_adjuster_url', 'data_adjuster_url', '', str), | |
| # Status (special - not saved, just displayed) | |
| ('settings_status', None, '', str), | |
| ] | |
| def get_settings_defaults(): | |
| """Return list of default values in schema order""" | |
| return [default for _, _, default, _ in SETTINGS_SCHEMA] | |
| def load_settings_values(settings_dict: dict, user_email: str = "") -> list: | |
| """Load settings from dict, returning values in schema order""" | |
| values = [] | |
| for component_key, storage_key, default, converter in SETTINGS_SCHEMA: | |
| if storage_key is None: # Special case: settings_status | |
| values.append(f"✅ Settings loaded for {user_email}" if user_email else "") | |
| else: | |
| raw_value = settings_dict.get(storage_key, default) | |
| try: | |
| if storage_key == 'default_llm' and raw_value not in UI_MODEL_CHOICES: | |
| raw_value = default | |
| if converter == bool: | |
| # Special handling: bool("False") returns True (non-empty string) | |
| # We need to check the actual string value | |
| if isinstance(raw_value, bool): | |
| values.append(raw_value) | |
| elif isinstance(raw_value, str): | |
| values.append(raw_value.lower() in ('true', '1', 'yes')) | |
| else: | |
| values.append(bool(raw_value) if raw_value else default) | |
| else: | |
| values.append(converter(raw_value) if raw_value else default) | |
| except (ValueError, TypeError): | |
| values.append(default) | |
| return values | |
| def build_settings_save_dict(values: list) -> dict: | |
| """Build dict for saving from values in schema order""" | |
| save_dict = {} | |
| for i, (component_key, storage_key, default, converter) in enumerate(SETTINGS_SCHEMA): | |
| if storage_key is None: # Skip settings_status | |
| continue | |
| if i < len(values): | |
| value = values[i] | |
| # Convert to string for storage (Supabase stores as text) | |
| save_dict[storage_key] = str(value) if value is not None else str(default) | |
| return save_dict | |
| def require_authenticated_email(request: gr.Request = None, user_email: str = None) -> str: | |
| """Require an authenticated user identity for settings operations.""" | |
| request_user = getattr(request, 'username', None) if request else None | |
| resolved_user = request_user or user_email | |
| if resolved_user and str(resolved_user).strip(): | |
| return str(resolved_user).strip().lower() | |
| # Local-dev no-auth mode is explicit and still requires a concrete user key. | |
| no_auth = os.getenv('DEMOPREP_NO_AUTH', 'false').lower() in ('true', '1', 'yes') | |
| if no_auth: | |
| dev_user_email = os.getenv('DEMOPREP_DEV_USER_EMAIL', '').strip().lower() | |
| if dev_user_email: | |
| return dev_user_email | |
| raise ValueError( | |
| "DEMOPREP_NO_AUTH=true requires DEMOPREP_DEV_USER_EMAIL to be set. " | |
| "Set both values and retry." | |
| ) | |
| raise ValueError( | |
| "Authenticated username is required for settings operations. " | |
| "Please sign in and retry." | |
| ) | |
| def resolve_app_url_for_invite(request: gr.Request = None) -> str: | |
| """Resolve the public app URL for copy/paste onboarding invites.""" | |
| configured_url = ( | |
| os.getenv("DEMOPREP_APP_URL", "").strip() | |
| or os.getenv("PUBLIC_APP_URL", "").strip() | |
| or os.getenv("SPACE_HOST", "").strip() | |
| ) | |
| if configured_url: | |
| if configured_url.startswith("http"): | |
| return configured_url.rstrip("/") | |
| return f"https://{configured_url.strip('/')}" | |
| try: | |
| headers = getattr(request, "headers", {}) if request else {} | |
| referer = headers.get("referer") or headers.get("origin") or "" | |
| if referer: | |
| return referer.split("?")[0].rstrip("/") | |
| except Exception: | |
| pass | |
| return "https://thoughtspot-dp-test-demoprep.hf.space" | |
| def build_initial_chat_message(company: str, use_case: str) -> str: | |
| """Build the pre-filled chat message from current settings.""" | |
| if company and use_case: | |
| return f"{company}, {use_case}" | |
| return "" | |
| def safe_print(*args, **kwargs): | |
| """Print that silently handles broken pipes (background processes)""" | |
| try: | |
| print(*args, **kwargs) | |
| except BrokenPipeError: | |
| pass # Ignore - happens when running in background | |
| class ChatDemoInterface: | |
| """ | |
| New chat-based interface for demo creation | |
| """ | |
| def __init__(self, user_email: str = None): | |
| self.user_email = user_email # Must be set by login system | |
| self.demo_builder = None | |
| self.conversation_history = [] | |
| self.settings = self.load_default_settings() | |
| self.ai_feedback_log = [] | |
| self.ddl_code = "" | |
| self.population_code = "" | |
| self.research_results = None | |
| # Vertical × Function use case system | |
| self.vertical = None | |
| self.function = None | |
| self.use_case_config = None | |
| # Generic use case handling | |
| self.is_generic_use_case = False | |
| self.generic_use_case_context = "" | |
| self.pending_generic_company = None | |
| self.pending_generic_use_case = None | |
| # New tab content | |
| self.live_progress_log = [] # Real-time deployment progress | |
| self.phase_log = [] # High-level pipeline status (shown in right panel) | |
| self.demo_pack_content = "" # Generated demo pack markdown | |
| self.spotter_story_ai = "" # Pure AI-generated Spotter Viz story | |
| self.spotter_story_matrix = "" # Matrix/ThoughtSpot-recommended Spotter Viz story | |
| self.deployment_completion = None # Final model/liveboard links shown in the right panel | |
| # Per-session loggers (NOT module-level singletons — avoids cross-session contamination) | |
| self._session_logger = None | |
| self._prompt_logger = None | |
| self._demo_bundle = None | |
| def clear_deployment_completion(self) -> None: | |
| self.deployment_completion = None | |
| def record_deployment_completion(self, results: dict, database: str, schema_name: str, use_case: str) -> None: | |
| """Store final ThoughtSpot artifact links for the UI completion panel.""" | |
| if not isinstance(results, dict): | |
| self.deployment_completion = None | |
| return | |
| ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/') | |
| model_guid = results.get('model_guid') or '' | |
| liveboard_guid = results.get('liveboard_guid') or results.get('liveboard_id') or '' | |
| liveboard_url = results.get('liveboard_url') or ( | |
| f"{ts_url}/#/pinboard/{liveboard_guid}" if ts_url and liveboard_guid else '' | |
| ) | |
| model_url = f"{ts_url}/#/data/tables/{model_guid}" if ts_url and model_guid else '' | |
| backup = bool(results.get('backup_liveboard')) or results.get('liveboard_creation_path') == 'spotter_tml_backup' | |
| warnings = results.get('warnings') or [] | |
| errors = results.get('errors') or [] | |
| if backup: | |
| status = "Backup liveboard created" | |
| note = ( | |
| "MCP/Spotter answer generation was unavailable, so DemoPrep created a clearly marked " | |
| "Spotter/TML backup liveboard. The model and data are still available." | |
| ) | |
| elif results.get('success') and liveboard_url: | |
| status = "MCP liveboard created" | |
| note = "The liveboard was created through the MCP path and enhanced after creation." | |
| elif model_url: | |
| status = "Model created" | |
| note = "The model was created, but a liveboard link was not returned. Use the model link to continue in ThoughtSpot." | |
| else: | |
| status = "Deployment finished" | |
| note = "Review the pipeline status for details." | |
| self.deployment_completion = { | |
| "status": status, | |
| "note": note, | |
| "code_version": results.get("code_version") or "", | |
| "ts_environment": results.get("ts_environment") or ts_url, | |
| "ts_username": results.get("ts_username") or "", | |
| "liveboard_creation_path": results.get("liveboard_creation_path") or ("spotter_tml_backup" if backup else "mcp" if liveboard_url else "none"), | |
| "fallback_reason": results.get("fallback_reason") or "", | |
| "model_url": model_url, | |
| "liveboard_url": liveboard_url, | |
| "model_guid": model_guid, | |
| "liveboard_guid": liveboard_guid, | |
| "connection": results.get('connection') or '', | |
| "schema": f"{database}.{schema_name}" if database and schema_name else schema_name, | |
| "use_case": use_case, | |
| "warnings": warnings, | |
| "errors": errors, | |
| "success": bool(results.get('success')), | |
| } | |
| def render_deployment_completion_html(self): | |
| """Render final artifact links for the completion panel.""" | |
| if not self.deployment_completion: | |
| return gr.update(value="", visible=False) | |
| import html as _html | |
| item = self.deployment_completion | |
| status = _html.escape(str(item.get("status") or "Deployment complete")) | |
| note = _html.escape(str(item.get("note") or "")) | |
| model_url = str(item.get("model_url") or "") | |
| liveboard_url = str(item.get("liveboard_url") or "") | |
| schema = _html.escape(str(item.get("schema") or "")) | |
| connection = _html.escape(str(item.get("connection") or "")) | |
| ts_environment = _html.escape(str(item.get("ts_environment") or "")) | |
| ts_username = _html.escape(str(item.get("ts_username") or "")) | |
| code_version = _html.escape(str(item.get("code_version") or "")) | |
| liveboard_path = _html.escape(str(item.get("liveboard_creation_path") or "unknown")) | |
| fallback_reason = _html.escape(str(item.get("fallback_reason") or "")) | |
| warnings = item.get("warnings") or [] | |
| errors = item.get("errors") or [] | |
| is_mcp = item.get("status") == "MCP liveboard created" | |
| border = "#22c55e" if is_mcp and item.get("success") and not item.get("errors") else "#f59e0b" | |
| badge_bg = "#dcfce7" if is_mcp else "#fef3c7" | |
| badge_color = "#166534" if is_mcp else "#92400e" | |
| def _link_button(label, url): | |
| if not url: | |
| return f"<span style='color:#6b7280;font-size:13px;'>{_html.escape(label)} unavailable</span>" | |
| safe_url = _html.escape(url, quote=True) | |
| return ( | |
| f"<a href='{safe_url}' target='_blank' rel='noopener noreferrer' " | |
| "style='display:inline-block;padding:9px 12px;margin:4px 6px 4px 0;" | |
| "border-radius:6px;background:#2563eb;color:white;text-decoration:none;" | |
| "font-weight:600;font-size:13px;'>" | |
| f"{_html.escape(label)}</a>" | |
| ) | |
| warning_html = "" | |
| if warnings: | |
| warning_items = "".join(f"<li>{_html.escape(str(w))}</li>" for w in warnings[:3]) | |
| warning_html = f"<ul style='margin:8px 0 0 18px;color:#92400e;font-size:12px;'>{warning_items}</ul>" | |
| error_html = "" | |
| if errors: | |
| error_items = "".join(f"<li>{_html.escape(str(e))}</li>" for e in errors[:2]) | |
| error_html = f"<ul style='margin:8px 0 0 18px;color:#991b1b;font-size:12px;'>{error_items}</ul>" | |
| html_value = f""" | |
| <div style="border:1px solid {border};border-left:5px solid {border};border-radius:8px;padding:12px;margin:10px 0;background:#fff;"> | |
| <div style="display:flex;align-items:center;justify-content:space-between;gap:8px;"> | |
| <div style="font-weight:700;color:#111827;">Deployment Links</div> | |
| <div style="padding:3px 8px;border-radius:999px;background:{badge_bg};color:{badge_color};font-size:12px;font-weight:700;">{status}</div> | |
| </div> | |
| <div style="margin-top:8px;color:#374151;font-size:13px;line-height:1.35;">{note}</div> | |
| <div style="margin-top:10px;"> | |
| {_link_button("Open Liveboard", liveboard_url)} | |
| {_link_button("Open Model", model_url)} | |
| </div> | |
| <div style="margin-top:8px;color:#6b7280;font-size:12px;line-height:1.4;"> | |
| <div><strong>Code:</strong> {code_version or "unknown"}</div> | |
| <div><strong>Environment:</strong> {ts_environment or "unknown"}</div> | |
| <div><strong>ThoughtSpot user:</strong> {ts_username or "unknown"}</div> | |
| <div><strong>Liveboard path:</strong> {liveboard_path}{f" · <strong>Fallback reason:</strong> {fallback_reason}" if fallback_reason else ""}</div> | |
| <div><strong>Schema:</strong> {schema or "n/a"}</div> | |
| <div><strong>Connection:</strong> {connection or "n/a"}</div> | |
| </div> | |
| {warning_html} | |
| {error_html} | |
| </div> | |
| """ | |
| return gr.update(value=html_value, visible=True) | |
| def _get_effective_user_email(self) -> str: | |
| """Resolve and cache effective user identity for settings access.""" | |
| self.user_email = require_authenticated_email(user_email=self.user_email) | |
| return self.user_email | |
| def _temporary_password_block_message(self) -> str: | |
| """Return a blocking message when a temp-password user tries to run the app.""" | |
| try: | |
| from supabase_client import UserManager | |
| user_email = self._get_effective_user_email() | |
| um = UserManager() | |
| if um.enabled and um.must_change_password(user_email): | |
| return ( | |
| "🔒 **Password change required**\n\n" | |
| "You are signed in with a temporary password. " | |
| "Open **Settings → Change Password**, set your own password, " | |
| "then come back and run DemoPrep." | |
| ) | |
| except Exception as e: | |
| print(f"[Auth] Unable to check temporary-password status: {e}") | |
| return "" | |
| def load_default_settings(self): | |
| """Load settings from Supabase or defaults""" | |
| # Fallback defaults (ONLY used if settings not found) | |
| defaults = { | |
| 'company': '', | |
| 'use_case': '', | |
| 'model': DEFAULT_LLM_MODEL, | |
| 'fact_table_size': '5000', | |
| 'dim_table_size': '100', | |
| 'stage': 'initialization', | |
| 'tag_name': None, | |
| 'validation_mode': 'Off', # Off = auto-run, On = pause at checkpoints | |
| 'geo_scope': 'USA Only', # USA Only or International | |
| } | |
| try: | |
| # Try to load from Supabase | |
| user_email = self._get_effective_user_email() | |
| if user_email: | |
| settings = load_gradio_settings(user_email) | |
| # Only override if values are meaningful (not generic placeholders) | |
| company = settings.get('default_company_url', '').strip() | |
| use_case = settings.get('default_use_case', '').strip() | |
| if company and company not in ['your company', 'yourcompany', '']: | |
| defaults['company'] = company | |
| if use_case and use_case not in ['analytics', '']: | |
| defaults['use_case'] = use_case | |
| if settings.get('default_llm'): | |
| saved_model = settings.get('default_llm') | |
| defaults['model'] = saved_model if saved_model in UI_MODEL_CHOICES else DEFAULT_LLM_MODEL | |
| if settings.get('fact_table_size'): | |
| defaults['fact_table_size'] = settings.get('fact_table_size') | |
| if settings.get('dim_table_size'): | |
| defaults['dim_table_size'] = settings.get('dim_table_size') | |
| if settings.get('tag_name'): | |
| defaults['tag_name'] = settings.get('tag_name') | |
| if settings.get('geo_scope'): | |
| defaults['geo_scope'] = settings.get('geo_scope') | |
| if settings.get('validation_mode'): | |
| defaults['validation_mode'] = settings.get('validation_mode') | |
| except Exception as e: | |
| print(f"Could not load settings from Supabase: {e}") | |
| print(f"DEBUG: Final settings - company: {defaults['company']}, use_case: {defaults['use_case']}") | |
| return defaults | |
| def format_welcome_message(self, company, use_case): | |
| """Create the initial welcome message""" | |
| return """## Welcome to ThoughtSpot Demo Builder | |
| I'll research a company, build a Snowflake schema, generate realistic data, and deploy a ThoughtSpot model and liveboard — all from a single prompt. | |
| **How to start:** | |
| - **Defined** — pick a vertical and function from the dropdowns below, add a company URL if you have one, and hit **→ GO**. | |
| - **Custom** — describe what you want in your own words in the chat box. | |
| <details> | |
| <summary>📋 Example use cases</summary> | |
| | Company | Vertical | Function | Story | | |
| |---------|----------|----------|-------| | |
| | Target.com | Retail | Sales | ASP decline, regional variance, holiday surge | | |
| | Walmart.com | Retail | Supply Chain | Stockout risk, OTIF, days on hand | | |
| | Chase.com | Banking | Marketing | Funnel drop-off, channel CTR, cost per acquisition | | |
| | Salesforce.com | Software | Sales | ARR by segment, pipeline coverage, win rate | | |
| | Caterpillar.com | Manufacturing | Supply Chain | Inventory levels, supplier performance | | |
| </details> | |
| > 💡 Select your ThoughtSpot environment in the right panel before starting.""" | |
| def validate_required_settings(self) -> list: | |
| """ | |
| Check that required admin and user settings are configured. | |
| Returns list of missing settings. Empty list = all good. | |
| """ | |
| missing = [] | |
| # Check admin settings (from environment, injected from Supabase) | |
| admin_checks = { | |
| 'SNOWFLAKE_ACCOUNT': get_admin_setting('SNOWFLAKE_ACCOUNT', required=False), | |
| 'SNOWFLAKE_KP_USER': get_admin_setting('SNOWFLAKE_KP_USER', required=False), | |
| 'SNOWFLAKE_KP_PK': get_admin_setting('SNOWFLAKE_KP_PK', required=False), | |
| 'SNOWFLAKE_ROLE': get_admin_setting('SNOWFLAKE_ROLE', required=False), | |
| 'SNOWFLAKE_WAREHOUSE': get_admin_setting('SNOWFLAKE_WAREHOUSE', required=False), | |
| 'SNOWFLAKE_DATABASE': get_admin_setting('SNOWFLAKE_DATABASE', required=False), | |
| } | |
| # LLM key now comes from environment only. | |
| has_llm = bool(get_openai_api_key(required=False)) | |
| for key, val in admin_checks.items(): | |
| if not val: | |
| missing.append(key) | |
| if not has_llm: | |
| missing.append('OPENAI_API_KEY') | |
| return missing | |
| def _create_run_loggers(self, force: bool = False): | |
| """Create a run-scoped session logger and prompt logger.""" | |
| if self._session_logger is not None and not force: | |
| return self._session_logger | |
| from session_logger import build_session_id, create_session_logger | |
| from prompt_logger import PromptLogger | |
| tag = self.settings.get('tag_name', '') | |
| session_id = build_session_id(tag) | |
| self._session_logger = create_session_logger(session_id, user_email=getattr(self, 'user_email', None)) | |
| self._prompt_logger = PromptLogger(session_id=session_id) | |
| model_setting = self.settings.get('model', DEFAULT_LLM_MODEL) | |
| try: | |
| provider_name, model_name = map_llm_display_to_provider(model_setting) | |
| resolved_model = f"{provider_name}/{model_name}" | |
| except Exception: | |
| resolved_model = str(model_setting or DEFAULT_LLM_MODEL) | |
| self._session_logger.log( | |
| 'pipeline', | |
| 'run started', | |
| test_tag=tag, | |
| model=resolved_model, | |
| model_setting=model_setting, | |
| payload=self._snapshot_run_payload(), | |
| ) | |
| return self._session_logger | |
| def _snapshot_run_payload(self): | |
| """ | |
| One JSON-safe record of everything that defines this run. | |
| Logged into session_logs meta on every run so Run History and ad-hoc | |
| queries can always answer "what was run": the raw GO form inputs | |
| (App tab), the resolved use case, and a secret-redacted snapshot of | |
| the controller settings. Secrets never leave the process — see | |
| sanitize_payload() in session_logger.py. | |
| """ | |
| from session_logger import sanitize_payload | |
| return sanitize_payload({ | |
| 'interface': getattr(self, '_run_source', 'chat'), | |
| 'company': getattr(self, 'pending_generic_company', '') or self.settings.get('company', ''), | |
| 'use_case': getattr(self, 'pending_generic_use_case', '') or self.settings.get('use_case', ''), | |
| 'vertical': getattr(self, 'vertical', None), | |
| 'line': getattr(self, 'line', None), | |
| 'function': getattr(self, 'function', None), | |
| 'is_custom': getattr(self, 'is_generic_use_case', False), | |
| 'additional_context': getattr(self, 'generic_use_case_context', '') or '', | |
| 'form': getattr(self, '_run_payload', None), | |
| 'settings': dict(self.settings), | |
| }) | |
| def process_chat_message(self, message, chat_history, current_stage, current_model, company, use_case): | |
| """ | |
| Process user message and return updated chat history and state (with streaming) | |
| Returns: (chat_history, current_stage, current_model, company, use_case, next_textbox_value) | |
| """ | |
| # Chat-tab runs enter at 'initialization' (App tab jumps straight to | |
| # 'awaiting_context' after stashing its form payload) — reset any | |
| # leftover App-tab run metadata so the logged payload matches this run. | |
| if current_stage == 'initialization': | |
| self._run_source = 'chat' | |
| self._run_payload = None | |
| # Pipeline starts from awaiting_context; always give it a fresh run log. | |
| self._create_run_loggers(force=(current_stage == 'awaiting_context')) | |
| _slog = self._session_logger | |
| _slog.log(current_stage or 'init', f"user message received: {message[:120]}") | |
| # Add user message to history | |
| chat_history.append((message, None)) | |
| password_block = self._temporary_password_block_message() | |
| if password_block and current_stage in {'initialization', 'awaiting_context'}: | |
| chat_history[-1] = (message, password_block) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # If data_adjuster_url is saved in settings and we're at init, inject it as the message | |
| # so the user lands directly in Data Adjuster without having to paste the URL manually | |
| da_url = self.settings.get('data_adjuster_url', '').strip() | |
| if da_url and current_stage == 'initialization' and 'pinboard/' in da_url: | |
| message = da_url | |
| chat_history[-1] = (da_url, None) | |
| # Validate required settings before proceeding | |
| missing = self.validate_required_settings() | |
| if missing and current_stage == 'initialization': | |
| missing_str = ", ".join(missing) | |
| error_msg = ( | |
| f"**⚠️ Missing required settings:**\n\n" | |
| f"`{missing_str}`\n\n" | |
| f"Please configure these values. " | |
| f"LLM key must be set in `.env` as `OPENAI_API_KEY`." | |
| ) | |
| chat_history[-1] = (message, error_msg) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Check for special commands | |
| if message.strip().lower().startswith('/over'): | |
| # Override command - extract new values | |
| response = self.handle_override(message) | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| message_lower = message.lower() | |
| # Pre-clean any URLs in the message (fix typos like double dots) | |
| import re | |
| cleaned_message = re.sub(r'\.{2,}', '.', message) | |
| # Stage-based processing | |
| if current_stage == 'initialization': | |
| # Check if user pasted a ThoughtSpot liveboard URL → jump straight to data adjuster | |
| lb_guid_match = re.search( | |
| r'pinboard/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', | |
| message, re.I | |
| ) | |
| if lb_guid_match: | |
| liveboard_guid = lb_guid_match.group(1) | |
| chat_history[-1] = (message, "🔍 **Resolving liveboard context...**") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| try: | |
| from smart_data_adjuster import SmartDataAdjuster, load_context_from_liveboard | |
| from thoughtspot_deployer import ThoughtSpotDeployer | |
| from supabase_client import get_admin_setting | |
| ts_url = (self.settings.get('thoughtspot_url') or '').strip() | |
| ts_secret = (self.settings.get('thoughtspot_trusted_auth_key') or '').strip() | |
| if not ts_url or not ts_secret: | |
| raise ValueError("ThoughtSpot environment not set — select a TS environment from the dropdown") | |
| ts_user = self._get_effective_user_email() | |
| ts_client = ThoughtSpotDeployer(base_url=ts_url, username=ts_user, secret_key=ts_secret) | |
| ts_client.authenticate() | |
| ctx = load_context_from_liveboard(liveboard_guid, ts_client) | |
| llm_model = self.settings.get('model', DEFAULT_LLM_MODEL) | |
| adjuster = SmartDataAdjuster( | |
| database=ctx['database'], | |
| schema=ctx['schema'], | |
| liveboard_guid=liveboard_guid, | |
| llm_model=llm_model, | |
| ts_url=ts_url, | |
| ts_secret=ts_secret, | |
| username=ts_user, | |
| prompt_logger=self._prompt_logger, | |
| ) | |
| adjuster.connect() | |
| if not adjuster.load_liveboard_context(): | |
| raise ValueError("Liveboard has no answer-based visualizations to adjust.") | |
| self._adjuster = adjuster | |
| current_stage = 'outlier_adjustment' | |
| viz_list = "\n".join( | |
| f" [{i+1}] {v['name']}" | |
| for i, v in enumerate(adjuster.visualizations) | |
| ) | |
| response = f"""✅ **Liveboard context loaded — ready for data adjustments** | |
| **Liveboard:** {ctx['liveboard_name']} | |
| **Model:** {ctx['model_name']} | |
| **Snowflake:** `{ctx['database']}`.`{ctx['schema']}` | |
| **Visualizations:** | |
| {viz_list} | |
| Tell me what to change — e.g. *"increase webcam revenue by 20%"* or *"make Acme Corp 50B"*. | |
| Type **done** when finished.""" | |
| except Exception as e: | |
| response = f"❌ **Could not load liveboard context**\n\n`{e}`" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Check if user just provided a standalone URL (e.g., "Comscore.com") | |
| standalone_url = re.search(r'^([a-zA-Z0-9-]+\.[a-zA-Z]{2,})$', cleaned_message.strip()) | |
| if standalone_url: | |
| # User provided just a URL - ask them to include the use case too | |
| detected_company = standalone_url.group(1) | |
| # Build dynamic use case list from VERTICALS × FUNCTIONS | |
| uc_opts = [f"- {v} {f}" for v in VERTICALS.keys() for f in FUNCTIONS.keys()] | |
| uc_opts_str = "\n".join(uc_opts) | |
| response = f"""I see you want to use **{detected_company}** - great choice! | |
| Now I need to know what kind of demo you want. Please tell me both together: | |
| ``` | |
| I'm creating a demo for company: {detected_company} use case: Retail Sales | |
| ``` | |
| **Configured use cases** (with KPIs, outliers, Spotter questions): | |
| {uc_opts_str} | |
| - Or any custom use case you want! | |
| What use case would you like for {detected_company}?""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, detected_company, use_case, "" | |
| return | |
| # Check if user is providing company and use case | |
| if "creating a demo for" in message_lower or "create a demo for" in message_lower: | |
| # Extract company and use case (use cleaned message for better matching) | |
| extracted_company = self.extract_company_from_message(cleaned_message) | |
| extracted_use_case = self.extract_use_case_from_message(cleaned_message) | |
| if extracted_company: | |
| company = extracted_company | |
| if extracted_use_case: | |
| # Parse use case into vertical × function using new system | |
| self.vertical, self.function = parse_use_case(extracted_use_case) | |
| self.use_case_config = get_use_case_config( | |
| self.vertical or "Generic", | |
| self.function or "Generic" | |
| ) | |
| # Determine if this is a known or generic use case | |
| is_known = self.vertical and self.function and not self.use_case_config.get('is_generic') | |
| use_case_display = self.use_case_config.get('use_case_name', extracted_use_case) | |
| # Store company/use case for context prompt | |
| if is_known: | |
| # Matched a configured vertical × function combination | |
| self.is_generic_use_case = False | |
| self.pending_generic_company = company | |
| self.pending_generic_use_case = use_case_display | |
| use_case_type_note = f"\n\n*Matched: **{self.vertical}** × **{self.function}** — using configured KPIs, outlier patterns, and Spotter questions.*" | |
| elif self.vertical or self.function: | |
| # Partial match - have vertical OR function but not both | |
| self.is_generic_use_case = True | |
| self.pending_generic_company = company | |
| self.pending_generic_use_case = extracted_use_case | |
| matched = self.vertical or self.function | |
| use_case_type_note = f"\n\n*Partial match: **{matched}** recognized — AI will fill in the gaps based on research.*" | |
| else: | |
| # Fully generic/custom use case | |
| self.is_generic_use_case = True | |
| self.pending_generic_company = company | |
| self.pending_generic_use_case = extracted_use_case | |
| use_case_type_note = "\n\n*Custom use case — AI will research the industry to build a relevant schema and KPIs.*" | |
| # ALWAYS ask for additional context (for both generic and standard use cases) | |
| context_prompt = f"""✅ **Demo Configuration** | |
| I am creating a demo for **{company}** with use case: **{self.pending_generic_use_case}**{use_case_type_note} | |
| **Default Schema:** | |
| - 1 fact table (transactions/events) | |
| - 3-4 dimension tables (customers, products, dates, etc.) | |
| **Want to customize?** You can add requirements like: | |
| - "I need 2 fact tables: SALES and INVENTORY" | |
| - "Include a RETURNS table" | |
| - "Focus on employee retention metrics" | |
| **Type your requirements, or say "proceed" to use defaults.**""" | |
| chat_history[-1] = (message, context_prompt) | |
| current_stage = 'awaiting_context' | |
| yield chat_history, current_stage, current_model, company, use_case, "proceed" | |
| return | |
| # Validate both are provided before proceeding | |
| if not extracted_company or not extracted_use_case: | |
| # Friendly message based on what's missing | |
| if extracted_company and not extracted_use_case: | |
| # Got company, need use case | |
| error_msg = f"""Great! I see you want to create a demo for **{extracted_company}**. | |
| Do you have a use case in mind? For example: | |
| - Sales Analytics | |
| - Supply Chain | |
| - Customer Analytics | |
| - Or any custom use case you'd like! | |
| **Just tell me like this:** | |
| ``` | |
| I'm creating a demo for company: {extracted_company} use case: Supply Chain | |
| ``` | |
| What would you like to analyze?""" | |
| elif extracted_use_case and not extracted_company: | |
| # Got use case, need company | |
| error_msg = f"""Perfect! I see you want to analyze **{extracted_use_case}**. | |
| Which company should I research? (Must be a real website) | |
| **Examples:** Nike.com, Target.com, Walmart.com | |
| **Just tell me like this:** | |
| ``` | |
| I'm creating a demo for company: Nike.com use case: {extracted_use_case} | |
| ``` | |
| What company URL should I use?""" | |
| else: | |
| # Got neither - shouldn't happen with our pattern matching, but just in case | |
| error_msg = """I need both a company and a use case to get started! | |
| **Example:** | |
| ``` | |
| I'm creating a demo for company: Nike.com use case: Supply Chain | |
| ``` | |
| What company and use case would you like?""" | |
| chat_history[-1] = (message, error_msg) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Update stage to research | |
| current_stage = 'research' | |
| # Show confirmation and starting message | |
| confirmation_msg = f"""✅ **Got it!** | |
| **Company:** {company} | |
| **Use Case:** {use_case} | |
| 🔍 **Starting Research...** | |
| I'm now researching {company}'s business model and {use_case} requirements. | |
| This may take 1-2 minutes. Watch the AI Feedback tab for progress!""" | |
| chat_history[-1] = (message, confirmation_msg) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Start research with streaming | |
| last_response = "" | |
| for response in self.run_research_streaming(company, use_case): | |
| chat_history[-1] = (message, response) | |
| last_response = response | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Stay in research stage - will move to create_ddl when user approves | |
| # After research completes, pre-fill "yes" if it ended with the approval question | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"RESEARCH COMPLETE, last_response contains 'Ready to create'? {'Ready to create the DDL now?' in last_response}\n") | |
| if "Ready to create the DDL now?" in last_response or "Would you like to use the cached results?" in last_response: | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"PRE-FILLING yes for DDL\n") | |
| yield chat_history, current_stage, current_model, company, use_case, "yes" | |
| else: | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # --- Catch-all: try to extract company + use case from any free-form message --- | |
| # Handles: "Nike.com, Retail Sales" / "Salesforce - Software Sales" / etc. | |
| extracted_company = self.extract_company_from_message(cleaned_message) | |
| extracted_use_case = self.extract_use_case_from_message(cleaned_message) | |
| if extracted_company and extracted_use_case: | |
| company = extracted_company | |
| self.vertical, self.function = parse_use_case(extracted_use_case) | |
| self.use_case_config = get_use_case_config( | |
| self.vertical or "Generic", self.function or "Generic" | |
| ) | |
| is_known = self.vertical and self.function and not self.use_case_config.get('is_generic') | |
| use_case_display = self.use_case_config.get('use_case_name', extracted_use_case) | |
| self.is_generic_use_case = not is_known | |
| self.pending_generic_company = company | |
| self.pending_generic_use_case = use_case_display | |
| if is_known: | |
| note = f"\n\n*Matched: **{self.vertical}** × **{self.function}** — KPIs, outliers, and Spotter questions ready.*" | |
| elif self.vertical or self.function: | |
| note = f"\n\n*Partial match: **{self.vertical or self.function}** recognized — AI will fill in the gaps.*" | |
| else: | |
| note = "\n\n*Custom use case — AI will research and build from scratch.*" | |
| context_prompt = f"""✅ **Demo Configuration** | |
| **Company:** {company} | |
| **Use Case:** {use_case_display}{note} | |
| **Want to add any requirements?** (or just say "proceed") | |
| - "Include a RETURNS table" | |
| - "Focus on enterprise accounts only" | |
| - "I need 2 fact tables: Sales and Inventory" | |
| *Type your requirements, or say **"proceed"** to use defaults.*""" | |
| chat_history[-1] = (message, context_prompt) | |
| current_stage = 'awaiting_context' | |
| yield chat_history, current_stage, current_model, company, use_case_display, "proceed" | |
| return | |
| elif extracted_company and not extracted_use_case: | |
| uc_opts = "\n".join([f"- {v} {f}" for v in VERTICALS.keys() for f in FUNCTIONS.keys()]) | |
| response = f"""Got it — **{extracted_company}**! | |
| What use case are we building? A few options: | |
| {uc_opts} | |
| - Or describe any custom scenario!""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, 'awaiting_use_case', current_model, extracted_company, use_case, "" | |
| return | |
| else: | |
| # Nothing useful extracted — show a brief prompt | |
| response = """I need a **company** and **use case** to get started. | |
| Try something like: | |
| - *"Nike.com, Retail Sales"* | |
| - *"Salesforce.com — Software pipeline analytics"* | |
| - *"Walmart.com, Supply Chain"*""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| elif current_stage == 'awaiting_use_case': | |
| # User is responding to "what use case?" — treat the entire message as the use case, | |
| # do NOT re-run company extraction (that's what caused "NJ.Products" bugs) | |
| extracted_use_case = self.extract_use_case_from_message(message) or message.strip() | |
| self.vertical, self.function = parse_use_case(extracted_use_case) | |
| self.use_case_config = get_use_case_config( | |
| self.vertical or "Generic", self.function or "Generic" | |
| ) | |
| is_known = self.vertical and self.function and not self.use_case_config.get('is_generic') | |
| use_case_display = self.use_case_config.get('use_case_name', extracted_use_case) | |
| self.is_generic_use_case = not is_known | |
| self.pending_generic_company = company | |
| self.pending_generic_use_case = use_case_display | |
| if is_known: | |
| note = f"\n\n*Matched: **{self.vertical}** × **{self.function}** — KPIs, outliers, and Spotter questions ready.*" | |
| elif self.vertical or self.function: | |
| note = f"\n\n*Partial match: **{self.vertical or self.function}** recognized — AI will fill in the gaps.*" | |
| else: | |
| note = "\n\n*Custom use case — AI will research and build from scratch.*" | |
| context_prompt = f"""✅ **Demo Configuration** | |
| **Company:** {company} | |
| **Use Case:** {use_case_display}{note} | |
| **Want to add any requirements?** (or just say "proceed") | |
| - "Include a RETURNS table" | |
| - "Focus on enterprise accounts only" | |
| - "I need 2 fact tables: Sales and Inventory" | |
| *Type your requirements, or say **"proceed"** to use defaults.*""" | |
| chat_history[-1] = (message, context_prompt) | |
| current_stage = 'awaiting_context' | |
| yield chat_history, current_stage, current_model, company, use_case_display, "proceed" | |
| return | |
| elif current_stage == 'awaiting_context': | |
| # User is providing context for ANY use case (both generic and established) | |
| if message_lower.strip() in ['proceed', 'continue', 'no', 'skip']: | |
| # User wants to proceed without additional context | |
| self.generic_use_case_context = "" | |
| context_note = "Proceeding with standard configuration." | |
| else: | |
| # User provided additional context - store it | |
| self.generic_use_case_context = message.strip() | |
| context_note = f"Using additional context:\n> {message.strip()}" | |
| # Get stored company and use case | |
| company = self.pending_generic_company | |
| use_case = self.pending_generic_use_case | |
| use_case_type = "generic" if self.is_generic_use_case else "established" | |
| # Check validation_mode setting. Local launcher runs without Gradio | |
| # auth, so fall back to the controller's in-memory settings when | |
| # user-scoped Supabase settings are unavailable. | |
| try: | |
| from supabase_client import load_gradio_settings | |
| settings = load_gradio_settings(self._get_effective_user_email()) | |
| except Exception as settings_error: | |
| print(f"[Settings] Using in-memory settings for validation_mode: {settings_error}", flush=True) | |
| settings = self.settings | |
| validation_mode = settings.get('validation_mode', 'Off') | |
| if validation_mode == 'Off': | |
| # AUTO-RUN MODE: Run entire pipeline without any more prompts | |
| _slog = self._session_logger | |
| current_stage = 'research' | |
| self.phase_log.append(f"🚀 Starting pipeline — {company} · {use_case}") | |
| self.phase_log.append("🔍 Phase 1: Research") | |
| auto_run_msg = f"""✅ **Starting Auto-Run Mode** | |
| **Company:** {company} | |
| **Use Case:** {use_case} ({use_case_type}) | |
| {context_note} | |
| 🚀 **Running complete pipeline...** | |
| - Research → Blueprint → Schema/data → Snowflake → ThoughtSpot | |
| Watch the AI Feedback tab for real-time progress!""" | |
| chat_history[-1] = (message, auto_run_msg) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Run research - yield every response to keep Gradio alive | |
| last_research_response = "" | |
| for response in self.run_research_streaming(company, use_case, self.generic_use_case_context): | |
| last_research_response = response | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| if str(last_research_response).lstrip().startswith("❌ Research failed"): | |
| self.phase_log.append("❌ Research failed") | |
| if _slog: | |
| _slog.log("run", "run failed", error="research failed", failed_stage="research") | |
| current_stage = 'research' | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Show research complete, move to schema stage | |
| current_stage = 'create_ddl' | |
| self.phase_log.append("✅ Research complete") | |
| self.phase_log.append("📐 Phase 2: Schema creation") | |
| chat_history[-1] = (message, "✅ **Research Complete!**\n\n📝 **Creating schema...**") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Auto-create DDL | |
| ddl_response, ddl_code = self.run_ddl_creation() | |
| # Check if DDL creation failed | |
| if not ddl_code or ddl_code.strip() == "": | |
| if _slog: | |
| _slog.log("run", "run failed", error="DDL creation returned no code", failed_stage="ddl") | |
| chat_history[-1] = (message, f"{ddl_response}\n\n❌ **Schema creation failed — check Pipeline Status for details.**") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| current_stage = 'deploy' | |
| self.phase_log.append("✅ Dataset schema created") | |
| self.phase_log.append("🏗️ Phase 3: Snowflake deploy") | |
| chat_history[-1] = (message, f"✅ Dataset schema created\n\n🚀 **Deploying to Snowflake...**") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Auto-deploy | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"AUTO-RUN: Starting deployment streaming\n") | |
| try: | |
| final_result = None | |
| deploy_update_count = 0 | |
| for progress_update in self.run_deployment_streaming(): | |
| if isinstance(progress_update, tuple): | |
| final_result = progress_update | |
| elif isinstance(progress_update, dict): | |
| current_stage = progress_update.get('stage', current_stage) | |
| chat_history[-1] = (message, progress_update['response']) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| else: | |
| deploy_update_count += 1 | |
| # Yield every update — run_deployment_streaming already sleeps 2s between yields | |
| chat_history[-1] = (message, progress_update) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"AUTO-RUN: Deployment loop EXITED, final_result={type(final_result)}, len={len(final_result) if final_result else 0}\n") | |
| if final_result: | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"AUTO-RUN: final_result[1]={final_result[1] if len(final_result) > 1 else 'N/A'}\n") | |
| # Check for auto_ts to continue to ThoughtSpot deployment | |
| if len(final_result) == 3 and final_result[1] == "auto_ts": | |
| deploy_response, _, auto_schema = final_result | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"AUTO-RUN: auto_ts detected, schema={auto_schema}\n") | |
| self.phase_log.append("✅ Snowflake deploy complete") | |
| self.phase_log.append("🔷 Phase 4: ThoughtSpot") | |
| chat_history[-1] = (message, deploy_response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Run ThoughtSpot deployment with detailed logging | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"AUTO-RUN: Creating TS generator...\n") | |
| ts_generator = self._run_thoughtspot_deployment(auto_schema, company, use_case) | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"AUTO-RUN: Generator created, starting iteration...\n") | |
| ts_update_count = 0 | |
| ts_final_stage = current_stage | |
| ts_final_response = "" | |
| for ts_update in ts_generator: | |
| ts_update_count += 1 | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| update_preview = str(ts_update)[:100] if ts_update else "None" | |
| f.write(f"AUTO-RUN: TS update #{ts_update_count}: {update_preview}\n") | |
| if isinstance(ts_update, dict): | |
| current_stage = ts_update.get('stage', current_stage) | |
| ts_final_stage = current_stage | |
| ts_final_response = str(ts_update.get('response', '')) | |
| chat_history[-1] = (message, ts_update['response']) | |
| else: | |
| ts_final_response = str(ts_update) | |
| chat_history[-1] = (message, ts_update) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"AUTO-RUN: TS deployment loop complete, {ts_update_count} updates\n") | |
| ts_failed = ( | |
| ts_final_stage != 'outlier_adjustment' | |
| and ( | |
| "ThoughtSpot Deployment Failed" in ts_final_response | |
| or "ThoughtSpot Deployment Error" in ts_final_response | |
| or "Partial Success" in ts_final_response | |
| ) | |
| ) | |
| if ts_failed: | |
| self.phase_log.append("❌ ThoughtSpot failed") | |
| if _slog: | |
| _slog.log("run", "run failed", error="ThoughtSpot deployment failed", failed_stage="thoughtspot") | |
| current_stage = 'deploy' | |
| else: | |
| self.phase_log.append("✅ ThoughtSpot complete") | |
| self.phase_log.append("🎉 Pipeline done!") | |
| if _slog: | |
| _slog.log("run", "run completed") | |
| current_stage = 'complete' | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| else: | |
| # Not auto_ts, just show the result | |
| if len(final_result) >= 2: | |
| final_response = final_result[0] | |
| next_msg = final_result[1] if len(final_result) > 1 else "" | |
| else: | |
| final_response = str(final_result) | |
| next_msg = "" | |
| if _slog: | |
| if next_msg == "thoughtspot": | |
| _slog.log( | |
| "run", | |
| "run waiting for user", | |
| checkpoint="thoughtspot", | |
| reason="validation mode requested manual ThoughtSpot start", | |
| ) | |
| else: | |
| _slog.log( | |
| "run", | |
| "run interrupted", | |
| error="Snowflake deploy completed but auto ThoughtSpot handoff was not returned", | |
| failed_stage="thoughtspot_handoff", | |
| next_message=next_msg, | |
| ) | |
| chat_history[-1] = (message, final_response) | |
| yield chat_history, current_stage, current_model, company, use_case, next_msg | |
| else: | |
| if _slog: | |
| _slog.log( | |
| "run", | |
| "run interrupted", | |
| error="Snowflake deploy stream ended without a final handoff result", | |
| failed_stage="thoughtspot_handoff", | |
| ) | |
| self.phase_log.append("⚠️ Pipeline interrupted after Snowflake deploy") | |
| chat_history[-1] = ( | |
| message, | |
| "⚠️ **Pipeline interrupted after Snowflake deploy.**\n\n" | |
| "Snowflake may have been created, but the app did not receive the ThoughtSpot handoff signal. " | |
| "Please retry ThoughtSpot deployment or check the run logs.", | |
| ) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| except Exception as e: | |
| import traceback | |
| full_trace = traceback.format_exc() | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"AUTO-RUN: EXCEPTION: {str(e)}\n{full_trace}\n") | |
| # Log real error to Supabase for investigation | |
| if _slog: | |
| _slog.log( | |
| current_stage or 'pipeline', | |
| "unhandled pipeline exception", | |
| error=str(e), | |
| traceback=full_trace, | |
| ) | |
| # Show clean message to user — real error goes to Live Progress + logs | |
| self.log_feedback(f"[ERROR] Pipeline exception: {str(e)}") | |
| self.log_feedback(full_trace) | |
| error_msg = "**❌ Something went wrong during the pipeline.** Our team has been notified. Check the Live Progress tab for details, or try again." | |
| chat_history[-1] = (message, error_msg) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| else: | |
| # VALIDATION MODE ON: Follow normal flow with pauses | |
| current_stage = 'research' | |
| confirmation_msg = f"""✅ **Got it!** | |
| **Company:** {company} | |
| **Use Case:** {use_case} ({use_case_type}) | |
| {context_note} | |
| 🔍 **Starting Research...** | |
| I'm now researching {company}'s business model and {use_case} requirements. | |
| This may take 1-2 minutes. Watch the AI Feedback tab for progress!""" | |
| chat_history[-1] = (message, confirmation_msg) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Start research with streaming | |
| last_response = "" | |
| for response in self.run_research_streaming(company, use_case, self.generic_use_case_context): | |
| chat_history[-1] = (message, response) | |
| last_response = response | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Pre-fill "yes" if ready for next step | |
| if "Ready to create the DDL now?" in last_response or "Would you like to use the cached results?" in last_response: | |
| yield chat_history, current_stage, current_model, company, use_case, "yes" | |
| else: | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| elif current_stage == 'research': | |
| # Check if we're waiting for cache response | |
| if hasattr(self, '_cache_available') and self._cache_available: | |
| if 'yes' in message_lower: | |
| # User wants to use cache | |
| chat_history[-1] = (message, "Loading cached research...") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| success = self._load_cached_research(self._cached_research_path, company, use_case) | |
| if success: | |
| company_name = self.demo_builder.extract_company_name() | |
| response = f"""✅ **Research Loaded from Cache!** | |
| **Company:** {company_name} | |
| **Use Case:** {use_case} | |
| **Cached results successfully loaded!** | |
| - Company analysis retrieved | |
| - Industry research retrieved | |
| - Ready to proceed | |
| """ | |
| chat_history[-1] = (message, response) | |
| current_stage = 'research' | |
| self._cache_available = False | |
| yield chat_history, current_stage, current_model, company, use_case, "yes" # Pre-fill "yes" | |
| return | |
| else: | |
| response = "❌ Failed to load cache, running fresh research..." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| self._cache_available = False | |
| # Fall through to run fresh research | |
| elif 'no' in message_lower: | |
| # User wants fresh research | |
| chat_history[-1] = (message, "Running fresh research...") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| self._cache_available = False | |
| # Fall through to run fresh research | |
| else: | |
| # Invalid response, ask again | |
| response = "Please type 'yes' to use cached results or 'no' to run fresh research." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # If we get here and cache was declined, run fresh research | |
| if not hasattr(self, '_cache_available') or not self._cache_available: | |
| last_response = "" | |
| for response in self.run_research_streaming(company, use_case): | |
| chat_history[-1] = (message, response) | |
| last_response = response | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Stay in research stage - will move to create_ddl when user approves | |
| # Pre-fill "yes" after research | |
| if "Ready to create the DDL now?" in last_response: | |
| yield chat_history, current_stage, current_model, company, use_case, "yes" | |
| else: | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Normal research stage handling (no cache prompt active) | |
| # This is only reached when validation_mode = On (otherwise auto-run handles it) | |
| if 'yes' in message_lower or 'proceed' in message_lower or 'continue' in message_lower: | |
| current_stage = 'create_ddl' | |
| response, ddl_code = self.run_ddl_creation() | |
| chat_history[-1] = (message, response) | |
| # With validation_mode On, we show DDL and wait for approval | |
| yield chat_history, current_stage, current_model, company, use_case, "yes" | |
| return | |
| elif 'no' in message_lower or 'redo' in message_lower: | |
| chat_history[-1] = (message, "Restarting research...") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| self._cache_available = False # Clear cache flag | |
| last_response = "" | |
| for response in self.run_research_streaming(company, use_case): | |
| chat_history[-1] = (message, response) | |
| last_response = response | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Stay in research stage, pre-fill "yes" | |
| if "Ready to create the DDL now?" in last_response: | |
| yield chat_history, current_stage, current_model, company, use_case, "yes" | |
| else: | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| elif current_stage == 'create_ddl': | |
| # Waiting for DDL approval | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"ENTERED create_ddl stage, message={message_lower[:50]}\n") | |
| if 'yes' in message_lower or 'approve' in message_lower or 'proceed' in message_lower: | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"DDL APPROVED, moving to deploy\n") | |
| current_stage = 'deploy' | |
| # Stream deployment progress directly to chat | |
| try: | |
| final_result = None | |
| for progress_update in self.run_deployment_streaming(): | |
| if isinstance(progress_update, tuple): | |
| final_result = progress_update | |
| elif isinstance(progress_update, dict): | |
| current_stage = progress_update.get('stage', current_stage) | |
| chat_history[-1] = (message, f"**DDL Approved - Deploying...**\n\n{progress_update['response']}") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| else: | |
| chat_history[-1] = (message, f"**DDL Approved - Deploying...**\n\n{progress_update}") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Handle final result | |
| def debug_log(msg): | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| import datetime | |
| f.write(f"[{datetime.datetime.now()}] {msg}\n") | |
| f.flush() | |
| self.log_feedback(msg) | |
| if final_result: | |
| debug_log(f"DEBUG create_ddl: final_result len={len(final_result)}, [1]='{final_result[1] if len(final_result) > 1 else 'N/A'}'") | |
| if len(final_result) == 3 and final_result[1] == "auto_ts": | |
| # Auto-continue to ThoughtSpot deployment | |
| deploy_response, _, auto_schema = final_result | |
| debug_log(f"DEBUG create_ddl: auto_ts, schema={auto_schema}") | |
| chat_history[-1] = (message, deploy_response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| debug_log("DEBUG create_ddl: About to call _run_thoughtspot_deployment") | |
| # Run ThoughtSpot deployment (mirrors 'thoughtspot' handler) | |
| ts_update_count = 0 | |
| for ts_update in self._run_thoughtspot_deployment(auto_schema, company, use_case): | |
| ts_update_count += 1 | |
| debug_log(f"DEBUG create_ddl: ts_update #{ts_update_count} type={type(ts_update)}") | |
| if isinstance(ts_update, dict): | |
| # Final result | |
| current_stage = ts_update.get('stage', current_stage) | |
| chat_history[-1] = (message, ts_update['response']) | |
| else: | |
| # Progress update | |
| chat_history[-1] = (message, ts_update) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| debug_log(f"DEBUG create_ddl: TS deployment loop complete, {ts_update_count} updates") | |
| current_stage = 'complete' | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| else: | |
| deploy_response, next_msg = final_result | |
| chat_history[-1] = (message, deploy_response) | |
| yield chat_history, current_stage, current_model, company, use_case, next_msg | |
| except Exception as e: | |
| import traceback | |
| error_msg = f"**Deployment Error:** {str(e)}\n\n```\n{traceback.format_exc()}\n```" | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"DEBUG create_ddl: Exception: {str(e)}\n{traceback.format_exc()}\n") | |
| chat_history[-1] = (message, error_msg) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| elif 'no' in message_lower or 'redo' in message_lower: | |
| response, ddl_code = self.run_ddl_creation() | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| elif current_stage == 'populate': | |
| # Handle "deploy" command when LegitData is ready | |
| if 'deploy' in message_lower: | |
| current_stage = 'deploy' | |
| try: | |
| # Stream deployment progress directly to chat | |
| final_result = None | |
| for progress_update in self.run_deployment_streaming(): | |
| if isinstance(progress_update, tuple): | |
| final_result = progress_update | |
| elif isinstance(progress_update, dict): | |
| current_stage = progress_update.get('stage', current_stage) | |
| chat_history[-1] = (message, progress_update['response']) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| else: | |
| chat_history[-1] = (message, progress_update) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Handle final result | |
| if final_result: | |
| if len(final_result) == 3 and final_result[1] == "auto_ts": | |
| # Auto-continue to ThoughtSpot deployment | |
| deploy_response, _, auto_schema = final_result | |
| chat_history[-1] = (message, deploy_response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Run ThoughtSpot deployment (mirrors 'thoughtspot' handler) | |
| for ts_update in self._run_thoughtspot_deployment(auto_schema, company, use_case): | |
| if isinstance(ts_update, dict): | |
| # Final result | |
| current_stage = ts_update.get('stage', current_stage) | |
| chat_history[-1] = (message, ts_update['response']) | |
| else: | |
| # Progress update | |
| chat_history[-1] = (message, ts_update) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| current_stage = 'complete' | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| else: | |
| deploy_response, next_msg = final_result | |
| chat_history[-1] = (message, deploy_response) | |
| yield chat_history, current_stage, current_model, company, use_case, next_msg | |
| except Exception as e: | |
| # Deployment failed - show error message | |
| error_msg = f"**Deployment Error:** {str(e)}" | |
| chat_history[-1] = (message, error_msg) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Handle population retry if needed | |
| elif 'yes' in message_lower or 'retry' in message_lower: | |
| response, pop_code = self.run_population() | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Auto-proceed to deployment if successful | |
| if "Complete" in response: | |
| current_stage = 'deploy' | |
| try: | |
| # Stream deployment progress | |
| final_result = None | |
| for progress_update in self.run_deployment_streaming(): | |
| if isinstance(progress_update, tuple): | |
| final_result = progress_update | |
| elif isinstance(progress_update, dict): | |
| current_stage = progress_update.get('stage', current_stage) | |
| chat_history.append((None, progress_update['response'])) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| else: | |
| chat_history.append((None, progress_update)) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| if final_result: | |
| if len(final_result) == 3 and final_result[1] == "auto_ts": | |
| # Auto-continue to ThoughtSpot deployment | |
| deploy_response, _, auto_schema = final_result | |
| chat_history.append((None, deploy_response)) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Run ThoughtSpot deployment (mirrors 'thoughtspot' handler) | |
| for ts_update in self._run_thoughtspot_deployment(auto_schema, company, use_case): | |
| if isinstance(ts_update, dict): | |
| # Final result | |
| current_stage = ts_update.get('stage', current_stage) | |
| chat_history.append((None, ts_update['response'])) | |
| else: | |
| # Progress update | |
| chat_history.append((None, ts_update)) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| current_stage = 'complete' | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| else: | |
| deploy_response, next_msg = final_result | |
| chat_history.append((None, deploy_response)) | |
| yield chat_history, current_stage, current_model, company, use_case, next_msg | |
| except Exception as e: | |
| error_msg = str(e) | |
| chat_history.append((None, error_msg)) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| elif current_stage == 'deploy': | |
| # Handle post-deployment commands (thoughtspot, truncate) | |
| if 'thoughtspot' in message_lower: | |
| # User wants to create ThoughtSpot objects - use helper method | |
| schema_name = getattr(self, '_deployed_schema_name', getattr(self, '_last_schema_name', 'UNKNOWN')) | |
| for ts_update in self._run_thoughtspot_deployment(schema_name, company, use_case): | |
| if isinstance(ts_update, dict): | |
| # Final result | |
| current_stage = ts_update.get('stage', current_stage) | |
| chat_history[-1] = (message, ts_update['response']) | |
| else: | |
| # Progress update | |
| chat_history[-1] = (message, ts_update) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Handle deployment errors (usually population failures) | |
| if hasattr(self, '_last_population_error'): | |
| # Handle '1' or 'retry' - retry with same code | |
| if 'retry' in message_lower or message_lower.strip() == '1': | |
| # Retry with same code | |
| chat_history[-1] = (message, "Retrying population...") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| try: | |
| # Check required attributes exist | |
| if not hasattr(self.demo_builder, 'data_population_results') or not self.demo_builder.data_population_results: | |
| response = "❌ **Error:** Population code not found. Please run population again first." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| if not hasattr(self, '_last_schema_name') or not self._last_schema_name: | |
| response = "❌ **Error:** Schema name not found. Please run deployment again first." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| from demo_prep import execute_population_script | |
| is_template = getattr(self.demo_builder, 'population_code_source', 'llm') == 'template' | |
| success, msg = execute_population_script( | |
| self.demo_builder.data_population_results, | |
| self._last_schema_name, | |
| skip_modifications=is_template | |
| ) | |
| if success: | |
| response = f"✅ **Population Successful!**\n\n{msg}\n\nDemo deployed to Snowflake! 🎉" | |
| del self._last_population_error | |
| del self._last_schema_name | |
| else: | |
| response = f"❌ Still failed: {msg[:200]}...\n\nTry 'truncate' (or '2') or 'fix' (or '3')?" | |
| except Exception as e: | |
| import traceback | |
| error_details = traceback.format_exc() | |
| self.log_feedback(f"❌ Retry error: {error_details}") | |
| response = f"❌ **Retry failed with error:**\n\n```\n{str(e)}\n```\n\nPlease try 'truncate' (or '2') to clear tables first." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| elif 'truncate' in message_lower or message_lower.strip() == '2': | |
| # Truncate tables and retry | |
| chat_history[-1] = (message, "Truncating tables and retrying...") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| try: | |
| # Check required attributes exist | |
| if not hasattr(self, '_last_schema_name') or not self._last_schema_name: | |
| response = "❌ **Error:** Schema name not found. Please run deployment again first." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| if not hasattr(self.demo_builder, 'data_population_results') or not self.demo_builder.data_population_results: | |
| response = "❌ **Error:** Population code not found. Please run population again first." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| from cdw_connector import SnowflakeDeployer | |
| from demo_prep import execute_population_script | |
| deployer = SnowflakeDeployer() | |
| deployer.connect() | |
| # Truncate all tables in schema | |
| try: | |
| cursor = deployer.connection.cursor() | |
| cursor.execute(f"USE SCHEMA {self._last_schema_name}") | |
| cursor.execute("SHOW TABLES") | |
| tables = cursor.fetchall() | |
| for table in tables: | |
| table_name = table[1] | |
| self.log_feedback(f"Truncating {table_name}...") | |
| cursor.execute(f"TRUNCATE TABLE {table_name}") | |
| cursor.close() | |
| deployer.disconnect() | |
| self.log_feedback("✅ Tables truncated") | |
| except Exception as e: | |
| self.log_feedback(f"⚠️ Truncate warning: {e}") | |
| if deployer.connection: | |
| deployer.disconnect() | |
| # Retry population | |
| is_template = getattr(self.demo_builder, 'population_code_source', 'llm') == 'template' | |
| success, msg = execute_population_script( | |
| self.demo_builder.data_population_results, | |
| self._last_schema_name, | |
| skip_modifications=is_template | |
| ) | |
| if success: | |
| response = f"✅ **Population Successful!**\n\n{msg}\n\nDemo deployed to Snowflake! 🎉" | |
| del self._last_population_error | |
| del self._last_schema_name | |
| else: | |
| response = f"❌ Still failed: {msg[:200]}...\n\nTry 'fix' (or '3') to let AI correct the code?" | |
| except Exception as e: | |
| import traceback | |
| error_details = traceback.format_exc() | |
| self.log_feedback(f"❌ Truncate/retry error: {error_details}") | |
| response = f"❌ **Truncate/retry failed with error:**\n\n```\n{str(e)}\n```\n\nPlease check the error details above." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| elif 'fix' in message_lower or message_lower.strip() == '3': | |
| # Regenerate the code using the fixed template | |
| chat_history[-1] = (message, "🔧 Regenerating population code with fixed template...") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| try: | |
| # Check required attributes exist | |
| if not hasattr(self.demo_builder, 'schema_generation_results') or not self.demo_builder.schema_generation_results: | |
| response = "❌ **Error:** DDL schema not found. Please run DDL creation again first." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| if not hasattr(self, '_last_schema_name') or not self._last_schema_name: | |
| response = "❌ **Error:** Schema name not found. Please run deployment again first." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| self.log_feedback("🔧 Regenerating population code from scratch...") | |
| # Regenerate using the reliable template | |
| from schema_utils import parse_ddl_schema | |
| schema_info = parse_ddl_schema(self.demo_builder.schema_generation_results) | |
| if not schema_info: | |
| response = "❌ Failed to parse DDL schema. Cannot regenerate." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Generate new code using the template (which includes all fixes) | |
| fixed_code = self.get_fallback_population_code(schema_info) | |
| # Validate it compiles | |
| try: | |
| compile(fixed_code, '<regenerated>', 'exec') | |
| self.log_feedback("✅ Regenerated code validated") | |
| except SyntaxError as e: | |
| response = f"❌ Template generation bug: {e}\n\nPlease contact support." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Update the code and mark as template-generated | |
| self.demo_builder.data_population_results = fixed_code | |
| self.population_code = fixed_code | |
| self.demo_builder.population_code_source = "template" # Mark as template | |
| self.log_feedback("🔧 Code regenerated, retrying deployment...") | |
| # Truncate and retry | |
| from cdw_connector import SnowflakeDeployer | |
| from demo_prep import execute_population_script | |
| deployer = SnowflakeDeployer() | |
| deployer.connect() | |
| try: | |
| cursor = deployer.connection.cursor() | |
| cursor.execute(f"USE SCHEMA {self._last_schema_name}") | |
| cursor.execute("SHOW TABLES") | |
| tables = cursor.fetchall() | |
| for table in tables: | |
| cursor.execute(f"TRUNCATE TABLE {table[1]}") | |
| cursor.close() | |
| deployer.disconnect() | |
| except Exception as e: | |
| self.log_feedback(f"⚠️ Truncate warning: {e}") | |
| if deployer.connection: | |
| deployer.disconnect() | |
| success, msg = execute_population_script( | |
| fixed_code, | |
| self._last_schema_name, | |
| skip_modifications=True # Template code, don't modify | |
| ) | |
| if success: | |
| response = f"✅ **Fixed and Successful!**\n\n{msg}\n\nDemo deployed to Snowflake! 🎉" | |
| del self._last_population_error | |
| del self._last_schema_name | |
| else: | |
| response = f"❌ AI fix didn't work: {msg[:200]}...\n\nTry 'fix' again or 'retry'?" | |
| except Exception as e: | |
| import traceback | |
| error_details = traceback.format_exc() | |
| self.log_feedback(f"❌ Fix/regenerate error: {error_details}") | |
| response = f"❌ **Fix/regenerate failed with error:**\n\n```\n{str(e)}\n```\n\nPlease check the error details above." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| return | |
| elif current_stage == 'outlier_adjustment': | |
| # Handle outlier adjustment stage | |
| if 'done' in message_lower or 'finish' in message_lower or 'complete' in message_lower: | |
| # Close adjuster connection | |
| if hasattr(self, '_adjuster'): | |
| self._adjuster.close() | |
| response = """✅ **Demo Creation Complete!** | |
| Your demo is fully deployed with custom outliers! | |
| **Summary:** | |
| - ✅ Research completed | |
| - ✅ DDL schema created | |
| - ✅ Data populated | |
| - ✅ Deployed to Snowflake | |
| - ✅ ThoughtSpot objects created | |
| - ✅ Custom outliers added | |
| **Access your demo:** | |
| - ThoughtSpot Liveboard: Check your ThoughtSpot instance | |
| - Snowflake Data: Query your schema | |
| 🎉 **Ready to present!**""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Check if waiting for confirmation | |
| if hasattr(self, '_pending_adjustment'): | |
| if 'yes' in message_lower or 'execute' in message_lower or 'confirm' in message_lower: | |
| # Execute the pending adjustment | |
| try: | |
| adjuster = self._adjuster | |
| strategy = self._pending_adjustment['strategy'] | |
| chat_history[-1] = (message, "**Executing SQL...**") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| result = adjuster.execute_sql(strategy['sql']) | |
| if result['success']: | |
| response = f"""✅ **SUCCESS!** | |
| Updated {result['rows_affected']} rows. | |
| **Next steps:** | |
| - 🔄 Refresh your ThoughtSpot liveboard to see changes | |
| - Or make another adjustment | |
| - Type **'done'** when finished | |
| **What else would you like to adjust?**""" | |
| else: | |
| response = f"""❌ **FAILED** | |
| {result['error']} | |
| **Common issues:** | |
| - Number too large for column (try smaller value) | |
| - Database connection issue | |
| Try a different adjustment or type **'done'** to finish.""" | |
| del self._pending_adjustment | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| except Exception as e: | |
| del self._pending_adjustment | |
| response = f"""❌ **Execution Error** | |
| {str(e)} | |
| Try a different adjustment or type **'done'** to finish.""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| elif 'no' in message_lower or 'cancel' in message_lower: | |
| del self._pending_adjustment | |
| response = """❌ **Cancelled** | |
| No changes made. Try another adjustment or type **'done'** to finish.""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Check if adjuster is available | |
| if not hasattr(self, '_adjuster'): | |
| response = """❌ **Adjuster Not Available** | |
| Smart data adjuster was not initialized properly. | |
| Type **'done'** to finish the workflow.""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Process adjustment request | |
| try: | |
| adjuster = self._adjuster | |
| # Show processing message | |
| chat_history[-1] = (message, "🤔 **Analyzing request...**") | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| # Match request to visualization | |
| match = adjuster.match_request_to_viz(message) | |
| if not match: | |
| response = """❌ **Could not understand request** | |
| I couldn't match your request to a visualization. | |
| **Examples:** | |
| - "make 1080p webcam 40B" | |
| - "increase smart watch by 20%" | |
| - "viz 3, increase laptop to 50B" | |
| Try again or type **'done'** to finish.""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Pick metric column (use hint from match if available) | |
| metric_hint = match.get('metric_hint') | |
| metric_column = adjuster._pick_metric_column(metric_hint) | |
| if not metric_column: | |
| response = "❌ Could not identify a metric column in your schema. Try specifying the column name." | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Get current value (new 4-tuple return: value, matched_name, dim_table, fact_table) | |
| entity_type = match.get('entity_type') | |
| current_value, matched_entity, dim_table, fact_table = adjuster.get_current_value( | |
| match['entity_value'], metric_column, entity_type | |
| ) | |
| if current_value == 0 or matched_entity is None: | |
| response = ( | |
| f"❌ **No data found** for `{match['entity_value']}`.\n\n" | |
| f"Check the spelling or try a different entity. Type **'done'** to finish." | |
| ) | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Calculate target | |
| target_value = match.get('target_value') | |
| percentage = match.get('percentage') if match.get('is_percentage') else None | |
| if percentage is not None: | |
| target_value = current_value * (1 + percentage / 100) | |
| match['target_value'] = target_value | |
| # Generate strategy | |
| strategy = adjuster.generate_strategy( | |
| match['entity_value'], | |
| metric_column, | |
| current_value, | |
| target_value=target_value, | |
| percentage=percentage, | |
| entity_type=entity_type, | |
| ) | |
| # Present smart confirmation | |
| confirmation = adjuster.present_smart_confirmation(match, current_value, strategy, metric_column) | |
| # Store for execution if user confirms | |
| self._pending_adjustment = { | |
| 'match': match, | |
| 'strategy': strategy, | |
| 'current_value': current_value | |
| } | |
| response = f"""{confirmation} | |
| **Type 'yes' to execute, 'no' to cancel, or make another adjustment request.**""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| except Exception as e: | |
| import traceback | |
| error_details = traceback.format_exc() | |
| self.log_feedback(f"❌ Adjustment error: {error_details}") | |
| response = f"""❌ **Adjustment Error** | |
| {str(e)} | |
| Try a different request or type **'done'** to finish.""" | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| return | |
| # Fallback to regular message processing | |
| response = self.process_regular_message(message, current_stage, company, use_case) | |
| chat_history[-1] = (message, response) | |
| yield chat_history, current_stage, current_model, company, use_case, "" | |
| def extract_company_from_message(self, message): | |
| """Extract company name from message""" | |
| import re | |
| # Pre-clean the message: fix common typos | |
| # Replace multiple dots with single dot (e.g., "Comscore..com" -> "Comscore.com") | |
| cleaned_message = re.sub(r'\.{2,}', '.', message) | |
| # Remove spaces around dots in URLs (e.g., "Nike . com" -> "Nike.com") | |
| cleaned_message = re.sub(r'\s*\.\s*', '.', cleaned_message) | |
| # Simpler, more robust patterns - capture domain.tld format | |
| # Don't include trailing punctuation in the URL pattern | |
| patterns = [ | |
| # With colon: "company: tinder.com" | |
| r'company:\s*([a-zA-Z0-9-]+\.[a-zA-Z]{2,})', | |
| # Without colon: "for the company tinder.com" or "company tinder.com" | |
| r'for\s+(?:the\s+)?company\s+([a-zA-Z0-9-]+\.[a-zA-Z]{2,})', | |
| r'company\s+([a-zA-Z0-9-]+\.[a-zA-Z]{2,})', | |
| # "demo for tinder.com" | |
| r'demo\s+for\s+([a-zA-Z0-9-]+\.[a-zA-Z]{2,})', | |
| ] | |
| for pattern in patterns: | |
| match = re.search(pattern, cleaned_message, re.IGNORECASE) | |
| if match: | |
| company = match.group(1).strip() | |
| # Clean up trailing dots just in case | |
| company = company.rstrip('.') | |
| return company | |
| # Fallback: any bare domain.tld in the message (e.g. "Nike.com, Retail Sales") | |
| bare_url = re.search(r'\b([a-zA-Z0-9-]+\.[a-zA-Z]{2,})\b', cleaned_message, re.IGNORECASE) | |
| if bare_url: | |
| return bare_url.group(1).rstrip('.') | |
| return None | |
| def extract_use_case_from_message(self, message): | |
| """Extract use case from message""" | |
| import re | |
| # Patterns for extracting use case - ordered by specificity | |
| patterns = [ | |
| r'use\s+case\s+of\s+([^,\n]+?)(?:\s*$)', # "use case of XYZ" | |
| r'use\s+case:\s*([^,\n]+?)(?:\s*$|\s+and)', # "use case: XYZ" | |
| r'(?:the\s+)?use\s+case\s+is\s+([^,\n]+?)(?:\s*$)', # "the use case is XYZ" | |
| r'for\s+(?:the\s+)?use\s+case:\s*([^,\n]+?)(?:\s*$|\s+and)', # "for the use case: XYZ" | |
| r'(?:and|\.)\s+use\s+case\s+(?:of\s+)?([^,\n]+?)(?:\s*$)', # ". use case of XYZ" or "and use case XYZ" | |
| r'with\s+(?:the\s+)?use\s+(?:case\s+)?([^,\n]+?)(?:\s*$)', # "with the use XYZ" or "with the use case XYZ" | |
| r'(?:for|with)\s+(?:the\s+)?use:\s*([^,\n]+?)(?:\s*$)', # "for the use: XYZ" | |
| r'focused on\s+([^,\n]+?)(?:\s*$|\s+and)', # "focused on XYZ" | |
| # "with a X use case" format - capture everything between "with a" and "use case" | |
| r'with\s+(?:a|an)\s+([^,\n]+?)\s+use\s+case', # "with a Subscription use case" | |
| # "with a X" at end of message (no trailing "use case") | |
| r'with\s+(?:a|an)\s+([^,\n]+?)(?:\s*$)', # "with a Subscription Conversion & Pricing Impact" | |
| ] | |
| for pattern in patterns: | |
| match = re.search(pattern, message, re.IGNORECASE) | |
| if match: | |
| use_case = match.group(1).strip() | |
| # Clean up trailing punctuation | |
| use_case = use_case.rstrip('.') | |
| # Don't return if it looks like a company/url (contains .com, .org, etc) | |
| if re.search(r'\.(com|org|net|io|co|ai)\b', use_case, re.IGNORECASE): | |
| continue | |
| return use_case | |
| # Fallback 1: text after a comma or dash following a domain.tld | |
| # Handles: "Nike.com, Retail Sales" / "Nike.com - Supply Chain" | |
| after_domain = re.search( | |
| r'[a-zA-Z0-9-]+\.[a-zA-Z]{2,}[\s]*[,\-–—]\s*(.+)', | |
| message, re.IGNORECASE | |
| ) | |
| if after_domain: | |
| use_case = after_domain.group(1).strip().rstrip('.') | |
| if use_case and not re.search(r'\.(com|org|net|io|co|ai)\b', use_case, re.IGNORECASE): | |
| return use_case | |
| # Fallback 2: text after domain.tld separated only by a space | |
| # Handles: "Caterpillar.com Manufacturing Supply Chain" | |
| after_domain_space = re.search( | |
| r'[a-zA-Z0-9-]+\.[a-zA-Z]{2,}\s+(.+)', | |
| message, re.IGNORECASE | |
| ) | |
| if after_domain_space: | |
| use_case = after_domain_space.group(1).strip().rstrip('.') | |
| if use_case and not re.search(r'\.(com|org|net|io|co|ai)\b', use_case, re.IGNORECASE): | |
| return use_case | |
| return None | |
| def handle_override(self, message): | |
| """Handle /over command to change company or use case""" | |
| parts = message.strip().split(maxsplit=1) | |
| if len(parts) < 2: | |
| return """🔧 **Override Command** | |
| To change settings, use: | |
| - `/over company: [new company]` - Change company | |
| - `/over usecase: [new use case]` - Change use case | |
| - `/over company: [company] usecase: [use case]` - Change both | |
| **Example:** | |
| `/over company: Amazon.com usecase: supply chain` | |
| """ | |
| override_text = parts[1] | |
| new_company = None | |
| new_usecase = None | |
| # Parse override text | |
| if 'company:' in override_text.lower(): | |
| company_part = override_text.lower().split('company:')[1] | |
| if 'usecase:' in company_part: | |
| new_company = company_part.split('usecase:')[0].strip() | |
| else: | |
| new_company = company_part.strip() | |
| if 'usecase:' in override_text.lower(): | |
| usecase_part = override_text.lower().split('usecase:')[1].strip() | |
| new_usecase = usecase_part | |
| if new_company or new_usecase: | |
| response = "✅ **Settings Updated!**\n\n" | |
| if new_company: | |
| response += f"📍 Company: **{new_company}**\n" | |
| if new_usecase: | |
| response += f"🎯 Use Case: **{new_usecase}**\n" | |
| response += "\nWhat would you like to do next?" | |
| return response | |
| return "❌ Could not parse override. Use format: `/over company: [name] usecase: [case]`" | |
| def log_feedback(self, message): | |
| """Add message to AI feedback log and pipeline status""" | |
| import datetime | |
| timestamp = datetime.datetime.now().strftime("%H:%M:%S") | |
| entry = f"[{timestamp}] {message}" | |
| self.ai_feedback_log.append(entry) | |
| # Limit log size to prevent memory issues (keep last 500 entries) | |
| if len(self.ai_feedback_log) > 500: | |
| self.ai_feedback_log = self.ai_feedback_log[-500:] | |
| # Also feed into phase_log so Pipeline Status panel stays in sync | |
| self.phase_log.append(entry) | |
| if len(self.phase_log) > 200: | |
| self.phase_log = self.phase_log[-200:] | |
| print(f"[AI Feedback] {message}") # Also print to console | |
| return "\n".join(self.ai_feedback_log) | |
| def run_research_streaming(self, company, use_case, generic_context=""): | |
| """Run the research phase with streaming updates | |
| Args: | |
| company: Company URL/name | |
| use_case: Use case name | |
| generic_context: Additional context provided by user for generic use cases | |
| """ | |
| _slog = self._session_logger | |
| if _slog: | |
| model_setting = self.settings.get('model', DEFAULT_LLM_MODEL) | |
| try: | |
| provider_name, model_name = map_llm_display_to_provider(model_setting) | |
| resolved_model = f"{provider_name}/{model_name}" | |
| except Exception: | |
| resolved_model = str(model_setting or DEFAULT_LLM_MODEL) | |
| _slog.log( | |
| "run", "run started", | |
| company=company or '', | |
| use_case=use_case or '', | |
| interface=getattr(self, '_run_source', 'chat'), | |
| vertical=getattr(self, 'vertical', '') or '', | |
| line=getattr(self, 'line', '') or '', | |
| function=getattr(self, 'function', '') or '', | |
| is_custom=getattr(self, 'is_generic_use_case', False), | |
| additional_context=(getattr(self, 'generic_use_case_context', '') or '')[:500], | |
| model=resolved_model, | |
| model_setting=model_setting, | |
| payload=self._snapshot_run_payload(), | |
| ) | |
| password_block = self._temporary_password_block_message() | |
| if password_block: | |
| if _slog: | |
| _slog.log("auth", "temporary password blocked pipeline start") | |
| yield password_block | |
| return | |
| _t = _slog.log_start("research") if _slog else None | |
| print(f"\n\n[CACHE DEBUG] === run_research_streaming called ===") | |
| print(f"[CACHE DEBUG] company: {company}") | |
| print(f"[CACHE DEBUG] use_case: {use_case}\n\n") | |
| import time | |
| import os | |
| from main_research import ResultsManager | |
| # Validate that we have actual values | |
| if not company: | |
| yield "❌ **Error:** No company provided. Please specify a company URL." | |
| return | |
| if not use_case: | |
| yield f"❌ **Error:** No use case provided. Please specify what you want to analyze.\n\n**Company provided:** {company}\n**Use case needed:** Tell me what analytics you want!" | |
| return | |
| progress_message = "🔍 **Starting Research**\n\n" | |
| yield progress_message | |
| try: | |
| # Initialize demo builder if needed OR if company/use_case changed | |
| # CRITICAL: Always create fresh DemoBuilder when company/use_case changes | |
| # to avoid persisting prompts/data from previous runs | |
| needs_new_builder = ( | |
| not self.demo_builder or | |
| self.demo_builder.company_url != company or | |
| self.demo_builder.use_case != use_case | |
| ) | |
| if needs_new_builder: | |
| print(f"[Research] Initializing DemoBuilder for {company}", flush=True) | |
| progress_message += "✓ Initializing DemoBuilder...\n" | |
| yield progress_message | |
| self.demo_builder = DemoBuilder( | |
| use_case=use_case, | |
| company_url=company | |
| ) | |
| # Prepare URL - clean up any extra text that might have been captured | |
| # Remove "use case:" and anything after it, and clean whitespace | |
| import re | |
| clean_company = re.sub(r'\s+use\s+case:.*$', '', company, flags=re.IGNORECASE).strip() | |
| clean_company = re.sub(r'\s+and\s+.*$', '', clean_company, flags=re.IGNORECASE).strip() | |
| url = clean_company if clean_company.startswith('http') else f"https://{clean_company}" | |
| # Check for cached research results | |
| domain = url.replace('https://', '').replace('http://', '').replace('www.', '').split('/')[0] | |
| safe_domain = domain.replace('.', '_').replace(':', '_') | |
| # Strip newlines and truncate — use_case can be multi-line; long filenames crash on Linux | |
| use_case_clean = use_case.replace('\n', ' ').replace('\r', ' ').strip() | |
| use_case_safe = use_case_clean.lower().replace(' ', '_').replace('/', '_')[:60] | |
| if generic_context and generic_context.strip(): | |
| import hashlib | |
| context_hash = hashlib.sha256(generic_context.strip().encode("utf-8")).hexdigest()[:8] | |
| use_case_safe = f"{use_case_safe}_{context_hash}" | |
| # Try new format first (with use case) | |
| # Use absolute path to ensure we find cache regardless of CWD | |
| script_dir = os.path.dirname(os.path.abspath(__file__)) | |
| results_dir = os.path.join(script_dir, "results") | |
| cache_filename = f"{safe_domain}_{use_case_safe}.json" | |
| cache_filepath = os.path.join(results_dir, cache_filename) | |
| # If exact match doesn't exist, try fuzzy matching for similar use cases | |
| if not os.path.exists(cache_filepath): | |
| import glob | |
| print(f"[CACHE DEBUG] Current working directory: {os.getcwd()}") | |
| print(f"[CACHE DEBUG] Script directory: {script_dir}") | |
| print(f"[CACHE DEBUG] Results directory: {results_dir}") | |
| similar_files = glob.glob(os.path.join(results_dir, f"{safe_domain}_*.json")) | |
| print(f"[CACHE DEBUG] Exact file {cache_filepath} not found") | |
| print(f"[CACHE DEBUG] Glob pattern: {results_dir}/{safe_domain}_*.json") | |
| print(f"[CACHE DEBUG] Similar files found: {similar_files}") | |
| if similar_files: | |
| # Found similar cache files for this company | |
| cache_filepath = similar_files[0] # Use the first one found | |
| cache_filename = os.path.basename(cache_filepath) | |
| print(f"[CACHE DEBUG] Using similar file: {cache_filename}") | |
| self.log_feedback(f"📋 Found similar cache file: {cache_filename}") | |
| elif not os.path.exists(cache_filepath): | |
| # Try old format (without use case) | |
| old_cache_filename = f"research_{safe_domain}.json" | |
| old_cache_filepath = os.path.join(results_dir, old_cache_filename) | |
| if os.path.exists(old_cache_filepath): | |
| cache_filename = old_cache_filename | |
| cache_filepath = old_cache_filepath | |
| cached_results = None | |
| cache_age_hours = None | |
| # Check for cached research and use automatically if valid | |
| print(f"[CACHE DEBUG] Final cache_filepath: {cache_filepath}, exists: {os.path.exists(cache_filepath)}") | |
| if os.path.exists(cache_filepath): | |
| try: | |
| # Check cache age (5 day expiry) | |
| cache_mtime = os.path.getmtime(cache_filepath) | |
| cache_age = time.time() - cache_mtime | |
| cache_age_hours = cache_age / 3600 # Convert to hours | |
| if cache_age_hours <= 120: # Cache valid for 5 days (120 hours) | |
| self.log_feedback(f"📋 Using cached research (age: {cache_age_hours:.1f} hours)") | |
| progress_message += f"📋 **Using Cached Research** ({cache_age_hours:.1f} hours old)\n\n" | |
| # Load cached results automatically | |
| with open(cache_filepath, 'r') as f: | |
| cached_data = json.load(f) | |
| self.demo_builder.company_analysis_results = cached_data.get('company_summary', '') | |
| self.demo_builder.industry_research_results = cached_data.get('research_paper', '') | |
| self.demo_builder.combined_research_results = self.demo_builder.get_research_context() | |
| self.demo_builder.company_url = cached_data.get('url', url) | |
| self.demo_builder.advance_stage() | |
| progress_message += "✅ **Research loaded from cache!**" | |
| self.log_feedback("✅ Research loaded from cache") | |
| yield progress_message | |
| if _slog and _t: | |
| _slog.log_end("research", _t) | |
| return | |
| else: | |
| self.log_feedback(f"📋 Cache too old ({cache_age_hours:.1f} hours), running fresh research") | |
| progress_message += f"📋 Cache expired ({cache_age_hours:.1f} hours old), running fresh research...\n" | |
| yield progress_message | |
| except Exception as e: | |
| self.log_feedback(f"⚠️ Could not load cache: {str(e)}") | |
| progress_message += f"⚠️ Could not load cache, running fresh research...\n" | |
| yield progress_message | |
| else: | |
| progress_message += "📋 No cached research found, running fresh research...\n" | |
| yield progress_message | |
| # No valid cache, proceed with fresh research | |
| yield from self._run_fresh_research( | |
| company, | |
| use_case, | |
| url, | |
| progress_message, | |
| cache_filename, | |
| results_dir, | |
| generic_context, | |
| ) | |
| if _slog and _t: | |
| _slog.log_end("research", _t) | |
| except Exception as e: | |
| import traceback | |
| trace = traceback.format_exc() | |
| error_msg = f"❌ Research failed: {str(e)}\n{trace}" | |
| self.log_feedback(error_msg) | |
| yield f"❌ Research failed: {str(e)}" | |
| if _slog and _t: | |
| _slog.log_end("research", _t, error=str(e), traceback=trace) | |
| def _run_fresh_research( | |
| self, | |
| company, | |
| use_case, | |
| url, | |
| progress_message, | |
| cache_filename, | |
| results_dir, | |
| generic_context="", | |
| ): | |
| """Run fresh research (no cache) | |
| Args: | |
| results_dir: Absolute path to the research cache directory. | |
| generic_context: Additional context for generic use cases | |
| """ | |
| import os | |
| from datetime import datetime | |
| from main_research import ResultsManager | |
| _slog = self._session_logger | |
| # Extract website content (silent — failure is handled gracefully) | |
| website = Website(url) | |
| self.demo_builder.website_data = website | |
| if not website.text or len(website.text) == 0: | |
| print(f"[Research] Website unavailable for {url}, using internet research", flush=True) | |
| website.text = f"Website content unavailable. Company URL: {url}. Please research this company using general knowledge and the use case context." | |
| website.title = company | |
| website.css_links = [] | |
| website.logo_candidates = [] | |
| # Get LLM provider | |
| model = self.settings.get('model', DEFAULT_LLM_MODEL) | |
| provider_name, model_name = map_llm_display_to_provider(model) | |
| if _slog: | |
| _slog.log('pipeline', 'model resolved', model=f"{provider_name}/{model_name}", model_setting=model) | |
| progress_message += f"✓ Using {provider_name}/{model_name}\n\n" | |
| yield progress_message | |
| # Initialize researcher | |
| researcher = MultiLLMResearcher(provider=provider_name, model=model_name) | |
| # Company Analysis | |
| progress_message += "🔍 **Phase 1: Company Analysis**\n" | |
| yield progress_message | |
| self.log_feedback("Analyzing company...") | |
| system_prompt, user_prompt = build_company_analysis_prompt( | |
| use_case, | |
| website.title, | |
| url, | |
| website.text, | |
| len(website.css_links), | |
| website.logo_candidates | |
| ) | |
| # Inject generic use case context if provided | |
| if generic_context: | |
| user_prompt += f"\n\n**Additional Context from User:**\n{generic_context}" | |
| messages = [ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": user_prompt} | |
| ] | |
| response = researcher.make_request(messages, temperature=0.3, max_tokens=4000, stream=True) | |
| company_analysis = "" | |
| chunk_count = 0 | |
| for chunk in response: | |
| chunk_text = researcher.extract_chunk_content(chunk) | |
| if chunk_text: | |
| company_analysis += chunk_text | |
| chunk_count += 1 | |
| if chunk_count % 5 == 0: # Update every 5 chunks | |
| progress_message_temp = progress_message + f"Analyzing... ({len(company_analysis)} chars)\n" | |
| yield progress_message_temp | |
| # Log the prompt/response | |
| from prompt_logger import log_researcher_call | |
| log_researcher_call("research_company", researcher, messages, company_analysis, logger=self._prompt_logger) | |
| self.demo_builder.company_analysis_results = company_analysis | |
| progress_message += f"✅ Company analysis complete\n\n" | |
| yield progress_message | |
| # Industry Research | |
| progress_message += "🔍 **Phase 2: Industry Research**\n" | |
| yield progress_message | |
| system_prompt, user_prompt = build_industry_research_prompt(use_case, company_analysis) | |
| # Inject generic use case context if provided | |
| if generic_context: | |
| user_prompt += f"\n\n**Additional Context from User:**\n{generic_context}" | |
| messages = [ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": user_prompt} | |
| ] | |
| # Stream the response | |
| response = researcher.make_request(messages, temperature=0.4, max_tokens=4000, stream=True) | |
| industry_research = "" | |
| chunk_count = 0 | |
| for chunk in response: | |
| chunk_text = researcher.extract_chunk_content(chunk) | |
| if chunk_text: | |
| industry_research += chunk_text | |
| chunk_count += 1 | |
| if chunk_count % 5 == 0: # Update every 5 chunks | |
| progress_message_temp = progress_message + f"Researching... ({len(industry_research)} chars)\n" | |
| yield progress_message_temp | |
| # Log the prompt/response | |
| log_researcher_call("research_industry", researcher, messages, industry_research, logger=self._prompt_logger) | |
| self.demo_builder.industry_research_results = industry_research | |
| self.demo_builder.combined_research_results = self.demo_builder.get_research_context() | |
| self.log_feedback("✅ Industry research complete!") | |
| progress_message += f"✅ Industry research complete ({len(industry_research)} chars)\n\n" | |
| yield progress_message | |
| # Save to cache | |
| try: | |
| self.log_feedback("💾 Saving research to cache...") | |
| research_results = { | |
| 'company_summary': company_analysis, | |
| 'research_paper': industry_research, | |
| 'url': url, | |
| 'use_case': use_case, | |
| 'generated_at': datetime.now().isoformat(), | |
| } | |
| os.makedirs(results_dir, exist_ok=True) | |
| ResultsManager.save_results(research_results, cache_filename, results_dir) | |
| except Exception as e: | |
| # Cache failure is non-fatal — log internally but don't surface to UI | |
| print(f"[CACHE] Could not save research cache: {e}", flush=True) | |
| if _slog: | |
| _slog.log_verbose("research", "cache write failed", error=str(e)) | |
| # Update stage | |
| self.demo_builder.advance_stage() | |
| self.demo_builder.set_ready() | |
| self.research_results = { | |
| 'company': company, | |
| 'use_case': use_case, | |
| 'completed': True | |
| } | |
| # Generate synopsis | |
| company_name = self.demo_builder.extract_company_name() | |
| synopsis = f"""✅ **Research Complete!** | |
| **Company:** {company_name} | |
| **Use Case:** {use_case} | |
| **What I learned:** | |
| - Analyzed company website and business model | |
| - Researched industry best practices for {use_case} | |
| - Generated context for building realistic demo data | |
| """ | |
| yield synopsis | |
| def _load_cached_research(self, cache_filepath, company, use_case): | |
| """Load research from cache""" | |
| from main_research import ResultsManager | |
| try: | |
| cached_results = ResultsManager.load_results(cache_filepath) | |
| if not isinstance(cached_results, dict): | |
| return None | |
| # Store in demo_builder | |
| self.demo_builder.company_analysis_results = cached_results.get('company_summary', '') | |
| self.demo_builder.industry_research_results = cached_results.get('research_paper', '') | |
| # Also need to get website data for company name extraction | |
| url = company if company.startswith('http') else f"https://{company}" | |
| self.demo_builder.website_data = Website(url) | |
| # Update stage | |
| self.demo_builder.advance_stage() | |
| self.demo_builder.set_ready() | |
| self.research_results = { | |
| 'company': company, | |
| 'use_case': use_case, | |
| 'completed': True | |
| } | |
| return True | |
| except Exception as e: | |
| self.log_feedback(f"❌ Error loading cache: {str(e)}") | |
| return None | |
| def _generate_spotter_questions(self, use_case: str, ddl_code: str) -> list: | |
| """Generate use-case specific Spotter questions based on the schema. | |
| Priority order: | |
| 1. liveboard_questions.spotter_qs from the vertical×function config | |
| 2. FUNCTIONS[fn].spotter_templates from the config | |
| 3. Hardcoded fallbacks per use case | |
| 4. Generic questions | |
| """ | |
| import re | |
| # Priority 1: Use spotter_qs from liveboard_questions in use case config | |
| try: | |
| uc_config = self.use_case_config or get_use_case_config( | |
| self.vertical or "Generic", self.function or "Generic" | |
| ) | |
| lq = uc_config.get("liveboard_questions", []) | |
| configured_questions = [] | |
| for q in lq: | |
| for sq in q.get('spotter_qs', []): | |
| configured_questions.append({ | |
| 'question': sq, | |
| 'purpose': f'Reveals {q["title"]} pattern' | |
| }) | |
| if configured_questions: | |
| v = self.vertical or "Generic" | |
| f = self.function or "Generic" | |
| self.log_feedback(f"📋 Using {len(configured_questions)} Spotter questions from {v}×{f} config") | |
| return configured_questions[:8] | |
| except Exception as e: | |
| self.log_feedback(f"⚠️ Spotter questions not available: {e}") | |
| # Priority 2: Try FUNCTIONS config for spotter_templates | |
| try: | |
| uc_config = self.use_case_config or get_use_case_config( | |
| self.vertical or "Generic", self.function or "Generic" | |
| ) | |
| spotter_templates = uc_config.get('spotter_templates', []) | |
| if spotter_templates: | |
| template_questions = [{'question': t, 'purpose': 'From use case config'} for t in spotter_templates] | |
| self.log_feedback(f"📋 Using {len(template_questions)} Spotter templates from config") | |
| return template_questions[:8] | |
| except: | |
| pass | |
| # Priority 3+4: Fallback to DDL-based and hardcoded questions | |
| # Extract column names from DDL | |
| columns = [] | |
| if ddl_code: | |
| # Find column definitions (word followed by data type) | |
| col_pattern = r'^\s+(\w+)\s+(VARCHAR|NUMBER|DATE|BOOLEAN|INT|DECIMAL|FLOAT|TIMESTAMP)' | |
| for match in re.finditer(col_pattern, ddl_code, re.MULTILINE | re.IGNORECASE): | |
| col_name = match.group(1).lower().replace('_', ' ') | |
| columns.append(col_name) | |
| # Find likely metrics (columns with revenue, sales, cost, amount, count, total, etc.) | |
| metrics = [c for c in columns if any(m in c for m in ['revenue', 'sales', 'cost', 'amount', 'count', 'total', 'spend', 'clicks', 'impressions', 'conversions', 'rate', 'roi', 'ctr', 'cpm', 'cpc'])] | |
| # Find likely dimensions (columns with name, type, category, region, channel, segment, etc.) | |
| dimensions = [c for c in columns if any(d in c for d in ['name', 'type', 'category', 'region', 'channel', 'segment', 'campaign', 'audience', 'product', 'brand', 'status'])] | |
| # Use-case specific question templates (legacy fallback) | |
| use_case_questions = { | |
| 'Marketing Analytics': [ | |
| {'question': 'What is total spend by channel this quarter?', 'purpose': 'Shows channel performance analysis'}, | |
| {'question': 'Which campaigns have the highest ROI?', 'purpose': 'Shows ranking and efficiency metrics'}, | |
| {'question': 'How have conversions changed month over month?', 'purpose': 'Shows trend analysis and change detection'}, | |
| {'question': 'What is the conversion rate by audience segment?', 'purpose': 'Shows segmentation analysis'}, | |
| ], | |
| 'Sales Analytics': [ | |
| {'question': 'What is total revenue by region this quarter?', 'purpose': 'Shows geographic performance'}, | |
| {'question': 'Who are the top 10 sales reps by revenue?', 'purpose': 'Shows ranking capabilities'}, | |
| {'question': 'How has pipeline changed compared to last month?', 'purpose': 'Shows change detection'}, | |
| {'question': 'What is win rate by product category?', 'purpose': 'Shows conversion analysis'}, | |
| ], | |
| 'Demand/Inventory Planning': [ | |
| {'question': 'What is current inventory by product category?', 'purpose': 'Shows inventory overview'}, | |
| {'question': 'Which products are at risk of stockout?', 'purpose': 'Shows proactive alerting'}, | |
| {'question': 'How has demand changed compared to forecast?', 'purpose': 'Shows variance analysis'}, | |
| {'question': 'What is days of supply by warehouse?', 'purpose': 'Shows operational metrics'}, | |
| ], | |
| 'Merchandising': [ | |
| {'question': 'What is sales performance by product category?', 'purpose': 'Shows category analysis'}, | |
| {'question': 'Which products have the highest margin?', 'purpose': 'Shows profitability ranking'}, | |
| {'question': 'How has sell-through rate changed this month?', 'purpose': 'Shows trend analysis'}, | |
| {'question': 'What is inventory turnover by store?', 'purpose': 'Shows store-level metrics'}, | |
| ], | |
| 'Loss Prevention Analytics': [ | |
| {'question': 'What is total shrinkage by store?', 'purpose': 'Shows loss by location'}, | |
| {'question': 'Which stores have the highest shrinkage rate?', 'purpose': 'Shows risk ranking'}, | |
| {'question': 'How has shrinkage trended over the past 6 months?', 'purpose': 'Shows trend analysis'}, | |
| {'question': 'What are the top causes of loss?', 'purpose': 'Shows root cause analysis'}, | |
| ], | |
| } | |
| # Get questions for this use case, or fall back to generic | |
| questions = use_case_questions.get(use_case, [ | |
| {'question': 'What are the key metrics this month?', 'purpose': 'Shows summary view'}, | |
| {'question': 'What changed compared to last period?', 'purpose': 'Shows change detection'}, | |
| {'question': 'What are the top performers?', 'purpose': 'Shows ranking capabilities'}, | |
| {'question': 'How have trends changed over time?', 'purpose': 'Shows time-series analysis'}, | |
| ]) | |
| # If we found actual metrics/dimensions in DDL, try to make questions more specific | |
| if metrics and dimensions: | |
| metric = metrics[0].title() | |
| dimension = dimensions[0].title() | |
| questions[0] = {'question': f'What is total {metric} by {dimension}?', 'purpose': 'Shows core metric breakdown'} | |
| return questions | |
| def _get_demo_tips(self, use_case: str) -> str: | |
| """Get use-case specific demo tips""" | |
| tips = { | |
| 'Marketing Analytics': """- **Lead with ROI**: Marketing leaders care about efficiency, show spend vs. results | |
| - **Highlight attribution**: Show how ThoughtSpot can break down performance by channel/campaign | |
| - **Show real-time**: Marketing moves fast - emphasize live data and quick answers | |
| - **Monitor setup**: Demo alerting for campaign performance thresholds""", | |
| 'Sales Analytics': """- **Focus on pipeline**: Sales leaders want to see deal flow and forecasting | |
| - **Show rep performance**: Ranking and leaderboards resonate with sales teams | |
| - **Highlight forecasting**: Show how AI can predict outcomes | |
| - **Territory analysis**: Geographic breakdowns are always compelling""", | |
| 'Demand/Inventory Planning': """- **Lead with stockouts**: Show how to prevent lost sales | |
| - **Forecast vs. actual**: Variance analysis is key for planners | |
| - **Supplier performance**: Show lead time and reliability metrics | |
| - **Seasonal patterns**: Highlight time-series capabilities""", | |
| 'Merchandising': """- **Category performance**: Start with what's selling | |
| - **Margin analysis**: Profitability is always top of mind | |
| - **Assortment optimization**: Show breadth vs. depth analysis | |
| - **Store comparisons**: Regional and store-level drill-down""", | |
| 'Loss Prevention Analytics': """- **Risk scoring**: Show high-risk locations or categories | |
| - **Trend detection**: Highlight anomaly detection capabilities | |
| - **Root cause**: Drill into why losses are occurring | |
| - **ROI of prevention**: Connect to business impact""", | |
| } | |
| return tips.get(use_case, """- **Start broad**: Begin with executive summary metrics | |
| - **Then drill down**: Show the ability to explore details | |
| - **Ask questions**: Let the AI demonstrate natural language | |
| - **End with action**: Show how insights lead to decisions""") | |
| def _generate_ai_spotter_story(self, company_name: str, use_case: str, | |
| model_name: str = None, model_url: str = None, | |
| liveboard_name: str = None, | |
| actual_columns: list = None) -> str: | |
| """Pure AI-generated Spotter Viz story — no matrix reference. | |
| AI decides what a compelling liveboard story looks like for this company + use case. | |
| """ | |
| from prompts import build_prompt | |
| from demo_personas import parse_use_case | |
| v, f = parse_use_case(use_case or '') | |
| vertical = v or "Generic" | |
| function = f or "Generic" | |
| company_context = f"Company: {company_name}\nUse Case: {use_case}" | |
| if model_name: | |
| company_context += f"\nData Source/Model: {model_name}" | |
| if model_url: | |
| company_context += f"\nModel URL: {model_url}" | |
| if liveboard_name: | |
| company_context += f"\nLiveboard Name: {liveboard_name}" | |
| if hasattr(self, 'demo_builder') and self.demo_builder: | |
| research = getattr(self.demo_builder, 'company_summary', '') or '' | |
| if research: | |
| company_context += f"\n\nCompany Research:\n{research[:1500]}" | |
| if actual_columns: | |
| company_context += ( | |
| f"\n\nACTUAL columns in the deployed data model (use these exact names " | |
| f"when referencing metrics or dimensions in Spotter prompts — do not invent column names " | |
| f"that don't appear in this list):\n{', '.join(actual_columns)}" | |
| ) | |
| try: | |
| prompt = build_prompt( | |
| stage="spotter_viz_story", | |
| vertical=vertical, | |
| function=function, | |
| company_context=company_context, | |
| ) | |
| llm_model = self.settings.get('model', DEFAULT_LLM_MODEL) | |
| self.log_feedback(f"🎬 Generating AI Spotter Viz story ({llm_model})...") | |
| provider_name, model_name_str = map_llm_display_to_provider(llm_model) | |
| researcher = MultiLLMResearcher(provider=provider_name, model=model_name_str) | |
| messages = [{"role": "user", "content": prompt}] | |
| result = researcher.make_request(messages, max_tokens=2000, temperature=0.7) | |
| from prompt_logger import log_researcher_call | |
| log_researcher_call("spotter_viz_story_ai", researcher, messages, result or "", logger=self._prompt_logger) | |
| return result | |
| except Exception as e: | |
| self.log_feedback(f"⚠️ AI Spotter Viz story generation failed: {e}") | |
| return f"*(Generation failed: {e})*" | |
| def _generate_matrix_spotter_story(self, company_name: str, use_case: str, | |
| model_name: str = None, model_url: str = None, | |
| liveboard_name: str = None, | |
| actual_columns: list = None) -> str: | |
| """Matrix/ThoughtSpot-recommended Spotter Viz story. | |
| Builds from the vertical×function matrix (KPIs, liveboard_questions, story controls, persona). | |
| AI writes it — adds narrative — but every step comes from the matrix. | |
| actual_columns: real column names from the deployed DDL — used to ground metric references. | |
| """ | |
| from prompts import build_prompt | |
| from demo_personas import parse_use_case, get_use_case_config | |
| v, f = parse_use_case(use_case or '') | |
| uc_cfg = get_use_case_config(v or "Generic", f or "Generic") | |
| data_source = model_name or f"{company_name} model" | |
| # Build rich matrix context for the prompt | |
| kpis = uc_cfg.get("kpis", []) | |
| lq = uc_cfg.get("liveboard_questions", []) | |
| story_controls = uc_cfg.get("story_controls", {}) | |
| persona = uc_cfg.get("persona", "") | |
| business_problem = uc_cfg.get("business_problem", "") | |
| use_case_name = uc_cfg.get("use_case_name", use_case) | |
| matrix_context = f"Company: {company_name}\nUse Case: {use_case_name}\nData Source/Model: {data_source}" | |
| if model_url: | |
| matrix_context += f"\nModel URL: {model_url}" | |
| if liveboard_name: | |
| matrix_context += f"\nLiveboard Name: {liveboard_name}" | |
| if actual_columns: | |
| matrix_context += ( | |
| f"\n\nACTUAL columns in the deployed data model (map matrix KPIs to the closest " | |
| f"matching actual column name — do not reference columns that don't appear in this list):\n" | |
| f"{', '.join(actual_columns)}" | |
| ) | |
| if persona: | |
| matrix_context += f"\nTarget Persona: {persona}" | |
| if business_problem: | |
| matrix_context += f"\nBusiness Problem: {business_problem}" | |
| if kpis: | |
| kpi_lines = "\n".join( | |
| f" - {k['name']}: {k.get('definition', '')}" if isinstance(k, dict) else f" - {k}" | |
| for k in kpis | |
| ) | |
| matrix_context += f"\n\nKPIs (from ThoughtSpot matrix):\n{kpi_lines}" | |
| if lq: | |
| matrix_context += "\n\nLiveboard Questions (in order):" | |
| for q in lq: | |
| req = " [required]" if q.get("required") else "" | |
| matrix_context += f"\n - {q['title']} ({q.get('viz_type','chart')}){req}: {q['viz_question']}" | |
| if q.get('insight'): | |
| matrix_context += f"\n Insight: {q['insight']}" | |
| if story_controls: | |
| dims = story_controls.get("dimensions", []) | |
| if dims: | |
| matrix_context += f"\n\nKey Dimensions: {', '.join(dims)}" | |
| seasonal = story_controls.get("seasonal_strength") or story_controls.get("seasonal") | |
| if seasonal: | |
| matrix_context += f"\nSeasonal pattern: {seasonal}" | |
| try: | |
| prompt = build_prompt( | |
| stage="spotter_viz_story_matrix", | |
| vertical=v or "Generic", | |
| function=f or "Generic", | |
| company_context=matrix_context, | |
| ) | |
| llm_model = self.settings.get('model', DEFAULT_LLM_MODEL) | |
| self.log_feedback(f"🎬 Generating Matrix Spotter Viz story ({llm_model})...") | |
| provider_name, model_name_str = map_llm_display_to_provider(llm_model) | |
| researcher = MultiLLMResearcher(provider=provider_name, model=model_name_str) | |
| messages = [{"role": "user", "content": prompt}] | |
| result = researcher.make_request(messages, max_tokens=2000, temperature=0.6) | |
| from prompt_logger import log_researcher_call | |
| log_researcher_call("spotter_viz_story_matrix", researcher, messages, result or "", logger=self._prompt_logger) | |
| return result | |
| except Exception as e: | |
| self.log_feedback(f"⚠️ Matrix Spotter Viz story generation failed: {e}") | |
| return f"*(Generation failed: {e})*" | |
| def _build_fallback_spotter_story(self, company_name: str, use_case: str, | |
| model_name: str = None) -> str: | |
| """Build a basic Spotter Viz story without LLM, using available context.""" | |
| data_source = model_name or f"{company_name} model" | |
| # Get spotter questions from use case config | |
| spotter_qs = [] | |
| try: | |
| from demo_personas import parse_use_case, get_use_case_config | |
| v, f = parse_use_case(use_case or '') | |
| uc_cfg = get_use_case_config(v or "Generic", f or "Generic") | |
| for q in uc_cfg.get("liveboard_questions", []): | |
| if q.get("required") and q.get("spotter_qs"): | |
| spotter_qs.append(q["spotter_qs"][0]) | |
| except: | |
| pass | |
| story = f"""# Spotter Viz Story: {company_name} | |
| ## {use_case} | |
| *Copy these prompts into ThoughtSpot Spotter Viz to build this liveboard interactively.* | |
| --- | |
| ### Step 1: Set Context | |
| > "Create a new liveboard for {company_name} {use_case} using the {data_source} data source." | |
| **Expected result:** Empty liveboard created with the correct data source connected. | |
| ### Step 2: Add Key KPIs | |
| > "Add KPI cards showing the main metrics with weekly sparklines." | |
| **Expected result:** KPI tiles with sparkline trends at the top of the liveboard. | |
| ### Step 3: Add Trend Analysis | |
| > "Add a line chart showing how the primary metric has trended over the last 12 months." | |
| **Expected result:** Time-series visualization showing monthly trends. | |
| ### Step 4: Add Category Breakdown | |
| > "Show a bar chart breaking down performance by the main dimension." | |
| **Expected result:** Categorical breakdown chart. | |
| ### Step 5: Add Comparison | |
| > "Add a comparison showing this period vs. last period." | |
| **Expected result:** Period-over-period comparison visualization. | |
| """ | |
| if spotter_qs: | |
| story += "\n### Step 6: Explore with Spotter Questions\n" | |
| for i, q in enumerate(spotter_qs[:3]): | |
| story += f'> "{q}"\n\n' | |
| story += """ | |
| --- | |
| *Refine the liveboard further by asking Spotter Viz to adjust colors, reorganize tiles, or add filters.* | |
| """ | |
| return story | |
| def run_research(self, company, use_case): | |
| """Run the research phase""" | |
| import time | |
| self.log_feedback(f"🔍 Starting research for {company} - {use_case}") | |
| try: | |
| # Initialize demo builder if needed OR if company/use_case changed | |
| # CRITICAL: Always create fresh DemoBuilder when company/use_case changes | |
| # to avoid persisting prompts/data from previous runs | |
| needs_new_builder = ( | |
| not self.demo_builder or | |
| self.demo_builder.company_url != company or | |
| self.demo_builder.use_case != use_case | |
| ) | |
| if needs_new_builder: | |
| if self.demo_builder: | |
| self.log_feedback(f"🔄 Company/use case changed - creating fresh DemoBuilder (was: {self.demo_builder.company_url}/{self.demo_builder.use_case})") | |
| else: | |
| self.log_feedback("Initializing DemoBuilder...") | |
| self.demo_builder = DemoBuilder( | |
| use_case=use_case, | |
| company_url=company | |
| ) | |
| # Extract website content | |
| url = company if company.startswith('http') else f"https://{company}" | |
| website = Website(url) | |
| self.demo_builder.website_data = website | |
| # Check if website extraction failed — silent, use fallback | |
| if not website.text or len(website.text) == 0: | |
| print(f"[Research] Website unavailable for {url}, using internet research", flush=True) | |
| website.text = f"Website content unavailable. Company URL: {url}. Please research this company using general knowledge and the use case context." | |
| website.title = company | |
| website.css_links = [] | |
| website.logo_candidates = [] | |
| # Get LLM provider | |
| model = self.settings.get('model', DEFAULT_LLM_MODEL) | |
| provider_name, model_name = map_llm_display_to_provider(model) | |
| # Initialize researcher | |
| researcher = MultiLLMResearcher(provider=provider_name, model=model_name) | |
| # Company Analysis | |
| system_prompt, user_prompt = build_company_analysis_prompt( | |
| use_case, | |
| website.title, | |
| url, | |
| website.text, | |
| len(website.css_links), | |
| website.logo_candidates | |
| ) | |
| messages = [ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": user_prompt} | |
| ] | |
| # Stream the response | |
| response = researcher.make_request(messages, temperature=0.3, max_tokens=4000, stream=True) | |
| company_analysis = "" | |
| for chunk in response: | |
| chunk_text = researcher.extract_chunk_content(chunk) | |
| if chunk_text: | |
| company_analysis += chunk_text | |
| from prompt_logger import log_researcher_call | |
| log_researcher_call("research_company", researcher, messages, company_analysis, logger=self._prompt_logger) | |
| self.demo_builder.company_analysis_results = company_analysis | |
| self.log_feedback("✅ Company analysis complete!") | |
| # Industry Research | |
| self.log_feedback("Researching industry best practices...") | |
| system_prompt, user_prompt = build_industry_research_prompt(use_case, company_analysis) | |
| messages = [ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": user_prompt} | |
| ] | |
| # Stream the response | |
| response = researcher.make_request(messages, temperature=0.4, max_tokens=4000, stream=True) | |
| industry_research = "" | |
| for chunk in response: | |
| chunk_text = researcher.extract_chunk_content(chunk) | |
| if chunk_text: | |
| industry_research += chunk_text | |
| log_researcher_call("research_industry", researcher, messages, industry_research, logger=self._prompt_logger) | |
| self.demo_builder.industry_research_results = industry_research | |
| self.log_feedback("✅ Industry research complete!") | |
| # Update stage | |
| self.demo_builder.advance_stage() | |
| self.demo_builder.set_ready() | |
| self.research_results = { | |
| 'company': company, | |
| 'use_case': use_case, | |
| 'completed': True | |
| } | |
| # Generate synopsis | |
| company_name = self.demo_builder.extract_company_name() | |
| synopsis = f"""✅ **Research Complete!** | |
| **Company:** {company_name} | |
| **Use Case:** {use_case} | |
| **What I learned:** | |
| - Analyzed company website and business model | |
| - Researched industry best practices for {use_case} | |
| - Generated context for building realistic demo data | |
| """ | |
| return synopsis | |
| except Exception as e: | |
| import traceback | |
| error_msg = f"❌ Research failed: {str(e)}\n{traceback.format_exc()}" | |
| self.log_feedback(error_msg) | |
| return f"❌ Research failed: {str(e)}" | |
| def run_ddl_creation(self): | |
| """Run DDL creation""" | |
| self.log_feedback("📝 Starting DDL creation...") | |
| _slog = self._session_logger | |
| _t_ddl = _slog.log_start("ddl") if _slog else None | |
| try: | |
| # Get timestamp for schema naming | |
| from datetime import datetime | |
| now = datetime.now() | |
| yymmdd = now.strftime("%y%m%d") | |
| hhmmss = now.strftime("%H%M%S") | |
| # Clean company and use case names (5 chars company, 3 chars use case) | |
| company_clean = self.demo_builder.extract_company_name().replace(" ", "").replace(".", "")[:5] | |
| usecase_clean = self.demo_builder.use_case.replace(" ", "").replace("-", "")[:3] | |
| schema_name = f"DM{yymmdd}_{hhmmss}_{company_clean}_{usecase_clean}" | |
| self._demo_bundle = None | |
| from demoprep_app.pipeline.build_demo import build_demo | |
| row_count_guidance = int(self.settings.get("fact_table_size", 5000) or 5000) | |
| company_name = self.demo_builder.extract_company_name() | |
| build = build_demo( | |
| company_name=company_name, | |
| company_url=self.demo_builder.company_url, | |
| use_case=self.demo_builder.use_case, | |
| vertical=self.vertical, | |
| function=self.function, | |
| row_count_guidance=row_count_guidance, | |
| research_context="\n\n".join( | |
| part for part in [ | |
| getattr(self.demo_builder, "combined_research_results", "") or "", | |
| getattr(self, "generic_use_case_context", "") or "", | |
| ] | |
| if part | |
| ), | |
| user_request=getattr(self, "generic_use_case_context", "") or self.demo_builder.use_case, | |
| llm_model=self.settings.get("model", DEFAULT_LLM_MODEL), | |
| progress_callback=self.log_feedback, | |
| prompt_logger=self._prompt_logger, | |
| ) | |
| if not build.ddl or "CREATE TABLE" not in build.ddl.upper(): | |
| raise RuntimeError( | |
| "Blueprint pipeline produced empty DDL — the blueprint likely had no fact tables or dimensions. " | |
| "Check the warnings above and retry." | |
| ) | |
| self._demo_bundle = build.dataset | |
| self.demo_builder.schema_generation_results = build.ddl | |
| self.ddl_code = build.ddl | |
| for warning in build.warnings: | |
| self.log_feedback(f"⚠️ {warning}") | |
| self.demo_builder.advance_stage() | |
| if _slog: | |
| _slog.log_end( | |
| "ddl", | |
| _t_ddl, | |
| table_count=build.ddl.upper().count("CREATE TABLE"), | |
| generation_mode="blueprint", | |
| scenario_type=build.scenario.scenario_type, | |
| fact_grain=build.scenario.fact_grain, | |
| contract_source=build.scenario.metadata.get("contract_source"), | |
| business_domain=build.scenario.metadata.get("business_domain"), | |
| table_names=[table.name for table in build.dataset.tables], | |
| ) | |
| self.log_feedback("✅ Schema created successfully!") | |
| insight_lines = "" | |
| if build.blueprint and build.blueprint.insights: | |
| headlines = [f" • {i.headline}" for i in build.blueprint.insights] | |
| insight_lines = "\n\n**Planted Demo Insights:**\n" + "\n".join(headlines) | |
| response = f"""✅ **Schema Creation Complete!** | |
| **Schema:** {schema_name} | |
| **Scenario:** {build.scenario.scenario_type} | |
| **Fact grain:** {build.scenario.fact_grain} | |
| **Generated rows:** {sum(len(table.rows) for table in build.dataset.tables):,}{insight_lines}""" | |
| return response, self.ddl_code | |
| except Exception as e: | |
| import traceback | |
| error_msg = f"❌ DDL creation failed: {str(e)}\n{traceback.format_exc()}" | |
| if _slog: _slog.log_end("ddl", _t_ddl, error=str(e)) | |
| self.log_feedback(error_msg) | |
| self.demo_builder.schema_generation_results = "" | |
| self.ddl_code = "" | |
| return error_msg, "" | |
| def get_fallback_population_code(self, schema_info, fact_rows=10000, dim_rows=100): | |
| """Generate a simple, reliable fallback population code using plain strings | |
| Args: | |
| schema_info: Dict of table definitions | |
| fact_rows: Number of rows for fact tables (default: 10000) | |
| dim_rows: Number of rows for dimension tables (default: 100) | |
| """ | |
| # schema_info is a dict: {'table_name': {'columns': [...], 'raw_definition': '...'}} | |
| def is_fact_table(table_name, columns): | |
| """Detect if table is likely a fact table (has measures/metrics)""" | |
| table_lower = table_name.lower() | |
| # FIRST: Explicitly classify dimension tables (these are NEVER facts) | |
| dimension_keywords = ['customer', 'product', 'seller', 'vendor', 'user', 'employee', | |
| 'center', 'warehouse', 'store', 'location', 'region', 'category', | |
| 'fulfillment', 'supplier', 'account', 'item_master', 'channel'] | |
| if any(keyword in table_lower for keyword in dimension_keywords): | |
| return False | |
| # SECOND: Common fact table name patterns | |
| fact_keywords = ['transaction', 'order', 'sale', 'event', 'log', 'activity', | |
| 'purchase', 'payment', 'shipment', 'invoice', 'fact', 'line'] | |
| if any(keyword in table_lower for keyword in fact_keywords): | |
| return True | |
| # THIRD: Check for numeric/measure columns (amount, quantity, price, etc.) | |
| measure_count = 0 | |
| for col in columns: | |
| col_name = col['name'].lower() | |
| col_type = col.get('type', '').upper() | |
| if any(word in col_name for word in ['amount', 'quantity', 'price', 'total', 'cost', 'revenue']): | |
| measure_count += 1 | |
| if 'DECIMAL' in col_type or 'FLOAT' in col_type or 'DOUBLE' in col_type: | |
| measure_count += 1 | |
| # If has 3+ measure-like columns, likely a fact table (raised threshold) | |
| return measure_count >= 3 | |
| code_parts = [] | |
| # Header | |
| code_parts.append("from dotenv import load_dotenv") | |
| code_parts.append("import os") | |
| code_parts.append("import snowflake.connector") | |
| code_parts.append("from faker import Faker") | |
| code_parts.append("import random") | |
| code_parts.append("from datetime import datetime, timedelta") | |
| code_parts.append("") | |
| code_parts.append("load_dotenv()") | |
| code_parts.append("") | |
| code_parts.append("from snowflake_auth import get_snowflake_connection_params") | |
| code_parts.append("") | |
| # Build populate functions | |
| table_names = [] | |
| for table_name, table_data in schema_info.items(): | |
| columns = table_data['columns'] | |
| # Build column lists (skip auto-increment IDs and malformed names) | |
| col_names = [] | |
| col_types = [] # Track types for each valid column | |
| for col in columns: | |
| col_name = col['name'] | |
| col_type = col.get('type', 'VARCHAR').upper() | |
| # Skip IDs (let database auto-generate) | |
| if col_name.lower() in ['id', table_name.lower() + '_id']: | |
| continue | |
| if 'IDENTITY' in col_type or 'AUTOINCREMENT' in col_type: | |
| continue | |
| # Skip malformed column names (numbers, special chars, etc) | |
| if not col_name.replace('_', '').replace(' ', '').isalnum(): | |
| continue | |
| if col_name.isdigit(): | |
| continue | |
| if any(char in col_name for char in ['(', ')', ',', ';']): | |
| continue | |
| col_names.append(col_name) | |
| col_types.append(col_type) | |
| if not col_names: | |
| continue # Skip tables with no insertable columns | |
| # Only add to table_names if we're actually creating a function for it | |
| # Determine row count based on table type | |
| is_fact = is_fact_table(table_name, columns) | |
| row_count = fact_rows if is_fact else dim_rows | |
| table_names.append((table_name, row_count)) | |
| placeholders = ', '.join(['%s'] * len(col_names)) | |
| col_list = ', '.join(col_names) | |
| # Build fake data generation (ONLY for valid columns in col_names) | |
| fake_values = [] | |
| for i, col_name in enumerate(col_names): | |
| col_type = col_types[i] # Use the col_type we saved earlier | |
| # Check for special business columns FIRST (by name pattern) | |
| col_name_upper = col_name.upper() | |
| # FLAG columns | |
| if 'FLAG' in col_name_upper or col_name_upper.startswith('IS_'): | |
| if 'INT' in col_type or 'NUMBER' in col_type: | |
| fake_values.append("random.choice([0, 1])") | |
| else: | |
| fake_values.append("random.choice(['Y', 'N'])") | |
| # QUARTER columns - Q1, Q2, Q3, Q4 | |
| elif 'QUARTER' in col_name_upper or col_name_upper == 'QTR': | |
| if 'INT' in col_type or 'NUMBER' in col_type: | |
| fake_values.append("random.randint(1, 4)") | |
| else: | |
| fake_values.append("random.choice(['Q1', 'Q2', 'Q3', 'Q4'])") | |
| # MONTH columns | |
| elif 'MONTH' in col_name_upper: | |
| if 'INT' in col_type or 'NUMBER' in col_type: | |
| fake_values.append("random.randint(1, 12)") | |
| else: | |
| fake_values.append("random.choice(['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])") | |
| # YEAR columns | |
| elif 'YEAR' in col_name_upper: | |
| fake_values.append("random.randint(2020, 2024)") | |
| # STATUS columns (short codes) | |
| elif 'STATUS' in col_name_upper and 'CHAR' in col_type: | |
| fake_values.append("random.choice(['ACTIVE', 'PENDING', 'CLOSED'])") | |
| # TYPE columns (short codes) - but NOT CATEGORY (handled below with full names) | |
| elif 'TYPE' in col_name_upper and 'CHAR' in col_type and 'CATEGORY' not in col_name_upper: | |
| fake_values.append("random.choice(['A', 'B', 'C'])") | |
| elif 'VARCHAR' in col_type or 'TEXT' in col_type or 'STRING' in col_type or 'CHAR' in col_type: | |
| # Extract VARCHAR length - always truncate generated values to fit | |
| import re | |
| length_match = re.search(r'\((\d+)\)', col_type) | |
| varchar_length = int(length_match.group(1)) if length_match else 255 | |
| # Generate domain-specific realistic data based on column name, then truncate to fit | |
| base_value = None | |
| if 'NAME' in col_name_upper and 'COMPANY' not in col_name_upper: | |
| # Check domain-specific name columns BEFORE falling back to fake.name() | |
| if 'DRUG' in col_name_upper or 'MEDICATION' in col_name_upper or 'THERAPEUTIC' in col_name_upper: | |
| base_value = "random.choice(['Lipitor', 'Humira', 'Eliquis', 'Keytruda', 'Revlimid', 'Opdivo', 'Ozempic', 'Dupixent', 'Trulicity', 'Entresto', 'Metformin', 'Atorvastatin', 'Lisinopril', 'Amlodipine', 'Metoprolol', 'Omeprazole', 'Simvastatin', 'Losartan', 'Albuterol', 'Gabapentin'])" | |
| elif 'PRODUCT' in col_name_upper: | |
| base_value = "random.choice(['Laptop Pro 15', 'Wireless Mouse 2.4GHz', 'USB-C Cable 6ft', 'Monitor Stand Adjustable', 'Mechanical Keyboard RGB', 'Noise Canceling Headphones', '1080p Webcam', 'Portable SSD 1TB', 'Power Bank 20000mAh', 'Tablet 10 inch', 'Smart Watch', 'Bluetooth Speaker', 'Gaming Mouse Pad', 'Phone Case', 'Screen Protector', 'Charging Cable', 'Desk Lamp LED', 'Laptop Bag', 'Wireless Earbuds', 'USB Hub'])" | |
| elif 'CUSTOMER' in col_name_upper or 'USER' in col_name_upper: | |
| base_value = "fake.name()" | |
| elif 'SELLER' in col_name_upper or 'VENDOR' in col_name_upper: | |
| base_value = "random.choice(['Amazon', 'Best Buy', 'Walmart', 'Target', 'Costco', 'Home Depot', 'Lowes', 'Macys', 'Nordstrom', 'Kohls'])" | |
| else: | |
| base_value = "fake.name()" | |
| elif 'CATEGORY' in col_name_upper: | |
| base_value = "random.choice(['Electronics', 'Home & Kitchen', 'Books', 'Clothing', 'Sports', 'Toys', 'Beauty', 'Automotive'])" | |
| elif 'BRAND' in col_name_upper: | |
| base_value = "random.choice(['Samsung', 'Apple', 'Sony', 'LG', 'Dell', 'HP', 'Lenovo', 'Amazon Basics', 'Anker', 'Logitech'])" | |
| elif 'CHANNEL' in col_name_upper or 'SOURCE' in col_name_upper: | |
| # Marketing channels for lead generation / call tracking | |
| base_value = "random.choice(['Google Ads Search', 'Bing Ads', 'Facebook Ads', 'LinkedIn Ads', 'Instagram Ads', 'Twitter Ads', 'Display Network', 'Programmatic Display', 'Retargeting', 'TV Commercial', 'Radio Ads', 'Billboard', 'Print Ads', 'Direct Mail', 'Email Newsletter', 'Organic Search', 'Social Media Organic', 'Google My Business', 'Referral', 'Affiliate Marketing', 'Content Marketing', 'Webinar', 'Podcast Sponsorship'])" | |
| elif 'CAMPAIGN' in col_name_upper and ('NAME' in col_name_upper or col_name_upper == 'CAMPAIGN_NAME'): | |
| # Marketing campaign names (usually reference the channel) | |
| base_value = "random.choice(['Google Ads Q4 Lead Gen', 'Facebook Black Friday Promo', 'LinkedIn Spring Campaign', 'Instagram New Product Launch', 'Email Brand Awareness', 'Display Holiday Special', 'Google Ads Summer Sale', 'Facebook Back to School', 'LinkedIn Valentine Promo', 'Google Shopping Cyber Monday', 'Email Free Trial Offer', 'Webinar Registration Q3', 'Email Nurture Series', 'Display Retargeting Q3', 'Google Ads Demo Request', 'Referral Rewards Program', 'Google Ads Year End Sale', 'Facebook New Year Campaign', 'Instagram Flash Sale', 'Email Limited Time Offer', 'Google Ads Early Bird', 'LinkedIn VIP Member Drive', 'Facebook Product Teaser', 'Display Conference Promo', 'Email Partner Campaign', 'Google Ads Seasonal', 'Facebook Customer Appreciation', 'Email Win Back Campaign', 'LinkedIn Upsell Drive', 'Display Cross-Sell Q4'])" | |
| elif ('CENTER' in col_name_upper and 'NAME' in col_name_upper) or ('CALL' in col_name_upper and 'CENTER' in col_name_upper): | |
| # Call center names | |
| base_value = "random.choice(['New York Contact Center', 'Los Angeles Support Hub', 'Chicago Call Center', 'Dallas Operations Center', 'Phoenix Customer Care', 'Philadelphia Service Center', 'San Diego Support Center', 'Miami Contact Hub', 'Atlanta Operations', 'Denver Call Center', 'Seattle Support Center', 'Boston Customer Service', 'Portland Contact Center', 'Austin Operations Hub', 'Las Vegas Call Center', 'Toronto Support Center', 'Offshore Manila Center', 'Offshore Bangalore Hub', 'Remote East Coast Team', 'Remote West Coast Team', 'Central Support Center', 'National Call Center', 'Regional North Hub', 'Regional South Hub', 'Enterprise Support Center'])" | |
| elif 'DESCRIPTION' in col_name_upper or 'DESC' in col_name_upper: | |
| base_value = "random.choice(['High quality product', 'Best seller', 'Customer favorite', 'New arrival', 'Limited edition', 'Premium quality'])" | |
| elif 'EMAIL' in col_name_upper: | |
| base_value = "fake.email()" | |
| elif 'PHONE' in col_name_upper: | |
| base_value = "f'{random.randint(200, 999)}-{random.randint(200, 999)}-{random.randint(1000, 9999)}'" | |
| elif 'ADDRESS' in col_name_upper or 'STREET' in col_name_upper: | |
| base_value = "f'{random.randint(1, 9999)} {random.choice([\"Main\", \"Oak\", \"Park\", \"Maple\", \"Cedar\", \"Elm\", \"Washington\", \"Lake\", \"Hill\", \"Broadway\"])} {random.choice([\"St\", \"Ave\", \"Blvd\", \"Dr\", \"Ln\"])}'" | |
| elif 'CITY' in col_name_upper: | |
| base_value = "random.choice(['New York', 'Los Angeles', 'Chicago', 'Houston', 'Phoenix', 'Philadelphia', 'San Antonio', 'San Diego', 'Dallas', 'San Jose', 'Austin', 'Seattle', 'Denver', 'Boston', 'Portland', 'Miami', 'Atlanta', 'Detroit', 'Las Vegas', 'Toronto'])" | |
| elif 'STATE' in col_name_upper or 'PROVINCE' in col_name_upper: | |
| base_value = "random.choice(['California', 'Texas', 'New York', 'Florida', 'Illinois', 'Ohio', 'Georgia', 'Washington', 'Virginia', 'Arizona', 'Colorado', 'Oregon', 'Nevada', 'Utah', 'Iowa'])" | |
| elif 'COUNTRY' in col_name_upper: | |
| base_value = "random.choice(['USA', 'Canada', 'UK', 'Germany', 'France', 'Japan', 'Australia', 'India', 'China', 'Brazil', 'Mexico', 'Spain', 'Italy', 'Netherlands', 'Sweden'])" | |
| elif 'ZIP' in col_name_upper or 'POSTAL' in col_name_upper: | |
| base_value = "random.choice(['10001', '90210', '60601', '77001', '85001', '19101', '78201', '92101', '75201', '95101', '78701', '98101', '80201', '02101', '97201'])" | |
| elif 'COMPANY' in col_name_upper: | |
| base_value = "random.choice(['Amazon', 'Microsoft', 'Apple Inc', 'Google LLC', 'Meta', 'Tesla Inc', 'Netflix', 'Adobe Inc', 'Oracle Corp', 'Salesforce', 'IBM Corp', 'Intel Corp', 'Cisco Systems', 'Dell Technologies', 'HP Inc'])" | |
| # --- People names (columns without NAME in them) --- | |
| elif any(kw in col_name_upper for kw in ['SALES_REP', 'SALESREP', 'ACCOUNT_REP', 'REP_ID']): | |
| base_value = "fake.name()" | |
| elif col_name_upper in ('REP', 'AGENT', 'REPRESENTATIVE'): | |
| base_value = "fake.name()" | |
| elif any(kw in col_name_upper for kw in ['MANAGER', 'SUPERVISOR', 'DIRECTOR', 'OWNER']): | |
| base_value = "fake.name()" | |
| elif any(kw in col_name_upper for kw in ['EMPLOYEE', 'EMP_', 'STAFF', 'ASSOCIATE']): | |
| base_value = "fake.name()" | |
| elif any(kw in col_name_upper for kw in ['CONTACT', 'ASSIGNED_TO', 'CREATED_BY', 'UPDATED_BY', 'APPROVED_BY']): | |
| base_value = "fake.name()" | |
| elif any(kw in col_name_upper for kw in ['PHYSICIAN', 'DOCTOR', 'PROVIDER', 'PRESCRIBER', 'HCP']): | |
| base_value = "fake.name()" | |
| elif any(kw in col_name_upper for kw in ['PATIENT', 'MEMBER', 'SUBSCRIBER']): | |
| base_value = "fake.name()" | |
| # --- Geographic / organizational --- | |
| elif 'REGION' in col_name_upper: | |
| base_value = "random.choice(['Northeast', 'Southeast', 'Midwest', 'Southwest', 'West', 'Pacific Northwest', 'Mid-Atlantic', 'New England', 'South Central', 'Mountain West', 'Great Lakes', 'Gulf Coast'])" | |
| elif 'TERRITORY' in col_name_upper: | |
| base_value = "random.choice(['Northeast Territory', 'Southeast Territory', 'Central Territory', 'Western Territory', 'Pacific Territory', 'Mountain Territory', 'Great Lakes Territory', 'Southern Territory', 'Mid-Atlantic Territory', 'Texas Territory'])" | |
| elif 'DEPARTMENT' in col_name_upper or 'DEPT' in col_name_upper: | |
| base_value = "random.choice(['Sales', 'Marketing', 'Finance', 'Operations', 'Engineering', 'Human Resources', 'Customer Success', 'Legal', 'Product', 'IT', 'Supply Chain', 'Research'])" | |
| elif 'SEGMENT' in col_name_upper: | |
| base_value = "random.choice(['Enterprise', 'Mid-Market', 'SMB', 'Consumer', 'Government', 'Education', 'Healthcare', 'Premium', 'Standard', 'Basic'])" | |
| elif 'TIER' in col_name_upper: | |
| base_value = "random.choice(['Platinum', 'Gold', 'Silver', 'Bronze', 'Premium', 'Standard', 'Basic'])" | |
| elif 'PRIORITY' in col_name_upper: | |
| base_value = "random.choice(['Critical', 'High', 'Medium', 'Low', 'Urgent'])" | |
| # --- Business domain --- | |
| elif 'PAYMENT' in col_name_upper and ('METHOD' in col_name_upper or 'TYPE' in col_name_upper): | |
| base_value = "random.choice(['Credit Card', 'Debit Card', 'ACH Transfer', 'Wire Transfer', 'Check', 'PayPal', 'Apple Pay', 'Google Pay'])" | |
| elif 'SHIPPING' in col_name_upper or 'SHIP_METHOD' in col_name_upper: | |
| base_value = "random.choice(['Standard Ground', 'Express 2-Day', 'Next Day Air', 'Economy', 'Freight', 'Same Day', 'International Standard', 'Priority Mail'])" | |
| elif 'WAREHOUSE' in col_name_upper or 'FULFILLMENT' in col_name_upper: | |
| base_value = "random.choice(['East Coast DC', 'West Coast DC', 'Central Hub', 'Southeast Warehouse', 'Pacific Distribution', 'Northeast Fulfillment', 'Texas DC', 'Midwest Hub', 'Mountain West DC', 'Southern Distribution'])" | |
| elif 'STORE' in col_name_upper or 'LOCATION' in col_name_upper: | |
| base_value = "random.choice(['Downtown Flagship', 'Mall of America', 'Northgate Plaza', 'Southside Center', 'Airport Terminal', 'University District', 'Waterfront Promenade', 'Tech Park', 'Suburban Commons', 'Metro Center', 'Eastside Galleria', 'Westfield Mall'])" | |
| elif 'TITLE' in col_name_upper or 'JOB' in col_name_upper or 'ROLE' in col_name_upper or 'POSITION' in col_name_upper: | |
| base_value = "random.choice(['VP Sales', 'Account Executive', 'Sales Manager', 'Director of Marketing', 'Product Manager', 'Data Analyst', 'Regional Manager', 'CFO', 'Operations Lead', 'Supply Chain Manager', 'Customer Success Manager', 'Business Analyst'])" | |
| elif 'SUBCATEGORY' in col_name_upper or 'SUB_CATEGORY' in col_name_upper: | |
| base_value = "random.choice(['Laptops', 'Smartphones', 'Headphones', 'Monitors', 'Tablets', 'Accessories', 'Networking', 'Storage', 'Printers', 'Cameras', 'Wearables', 'Audio'])" | |
| elif 'CURRENCY' in col_name_upper: | |
| base_value = "random.choice(['USD', 'EUR', 'GBP', 'CAD', 'JPY', 'AUD', 'MXN'])" | |
| elif 'REASON' in col_name_upper: | |
| base_value = "random.choice(['Price', 'Quality', 'Delivery Delay', 'Wrong Item', 'Defective', 'Changed Mind', 'Better Alternative', 'Budget Cut', 'Not as Described'])" | |
| elif 'OUTCOME' in col_name_upper or 'RESULT' in col_name_upper or 'DISPOSITION' in col_name_upper: | |
| base_value = "random.choice(['Won', 'Lost', 'Pending', 'Qualified', 'Disqualified', 'No Decision', 'Deferred'])" | |
| elif 'STAGE' in col_name_upper or 'PHASE' in col_name_upper: | |
| base_value = "random.choice(['Prospecting', 'Qualification', 'Proposal', 'Negotiation', 'Closed Won', 'Closed Lost', 'Discovery', 'Demo', 'Contract Review'])" | |
| elif 'RATING' in col_name_upper or 'GRADE' in col_name_upper or 'SCORE' in col_name_upper: | |
| base_value = "random.choice(['A', 'B', 'C', 'D', 'Excellent', 'Good', 'Average', 'Below Average'])" | |
| elif 'COLOR' in col_name_upper: | |
| base_value = "random.choice(['Black', 'White', 'Navy', 'Red', 'Blue', 'Gray', 'Green', 'Brown', 'Beige', 'Olive'])" | |
| elif 'SIZE' in col_name_upper: | |
| base_value = "random.choice(['XS', 'S', 'M', 'L', 'XL', 'XXL', 'One Size'])" | |
| elif 'GENDER' in col_name_upper or 'SEX' in col_name_upper: | |
| base_value = "random.choice(['Male', 'Female', 'Non-Binary', 'Prefer Not to Say'])" | |
| elif 'INDUSTRY' in col_name_upper or 'VERTICAL' in col_name_upper: | |
| base_value = "random.choice(['Technology', 'Healthcare', 'Financial Services', 'Retail', 'Manufacturing', 'Education', 'Government', 'Media', 'Energy', 'Real Estate'])" | |
| elif 'SPECIALTY' in col_name_upper or 'SPECIALIZATION' in col_name_upper: | |
| base_value = "random.choice(['Cardiology', 'Oncology', 'Orthopedics', 'Neurology', 'Pediatrics', 'Dermatology', 'Internal Medicine', 'Family Medicine', 'Psychiatry', 'Radiology'])" | |
| elif 'DRUG' in col_name_upper or 'MEDICATION' in col_name_upper or 'THERAPEUTIC' in col_name_upper: | |
| base_value = "random.choice(['Lipitor', 'Humira', 'Eliquis', 'Keytruda', 'Revlimid', 'Opdivo', 'Ozempic', 'Dupixent', 'Trulicity', 'Entresto'])" | |
| elif 'SKU' in col_name_upper or 'ITEM_CODE' in col_name_upper or 'PRODUCT_CODE' in col_name_upper: | |
| base_value = "f'SKU-{random.randint(10000, 99999)}'" | |
| elif 'ORDER_NUMBER' in col_name_upper or 'ORDER_NUM' in col_name_upper or 'INVOICE_NUM' in col_name_upper: | |
| base_value = "f'ORD-{random.randint(100000, 999999)}'" | |
| elif 'URL' in col_name_upper or 'WEBSITE' in col_name_upper: | |
| base_value = "fake.url()" | |
| elif 'NOTES' in col_name_upper or 'COMMENT' in col_name_upper: | |
| base_value = "random.choice(['Follow up next week', 'Priority customer', 'Pending review', 'No issues', 'Escalated', 'Resolved', 'Awaiting approval', 'On track'])" | |
| else: | |
| # Default: use faker word as last resort | |
| base_value = "fake.word()" | |
| # Always truncate to VARCHAR length - simple and works for all cases | |
| fake_values.append(f"({base_value})[:{varchar_length}]") | |
| elif 'INT' in col_type or 'NUMBER' in col_type or 'BIGINT' in col_type: | |
| fake_values.append("random.randint(1, 1000)") | |
| elif 'DECIMAL' in col_type or 'FLOAT' in col_type or 'DOUBLE' in col_type or 'NUMERIC' in col_type: | |
| # Extract precision and scale from DECIMAL(p,s) | |
| import re | |
| decimal_match = re.search(r'\((\d+),\s*(\d+)\)', col_type) | |
| if decimal_match: | |
| precision = int(decimal_match.group(1)) | |
| scale = int(decimal_match.group(2)) | |
| # Max value is 10^(precision-scale) - 1, with 'scale' decimal places | |
| # E.g., DECIMAL(3,2) -> max is 9.99, DECIMAL(5,2) -> max is 999.99 | |
| max_val = (10 ** (precision - scale)) - 1 | |
| fake_values.append(f"round(random.uniform(0, {max_val}), {scale})") | |
| else: | |
| # No precision specified, use safe defaults | |
| fake_values.append("round(random.uniform(10, 1000), 2)") | |
| elif 'DATE' in col_type and 'TIME' not in col_type: # DATE but not DATETIME/TIMESTAMP | |
| fake_values.append("fake.date_between(start_date='-2y', end_date='today')") | |
| elif 'TIMESTAMP' in col_type or 'DATETIME' in col_type: | |
| fake_values.append("fake.date_time_between(start_date='-2y', end_date='now')") | |
| elif 'BOOL' in col_type or 'BOOLEAN' in col_type: | |
| # For Snowflake BOOLEAN, use True/False which will be converted to SQL TRUE/FALSE | |
| fake_values.append("random.choice([True, False])") | |
| else: | |
| # Unknown type - default to string | |
| fake_values.append("str(fake.word())") | |
| fake_values_str = ', '.join(fake_values) if fake_values else "fake.word()" | |
| # Build function | |
| code_parts.append(f"def populate_{table_name.lower()}(cursor, fake):") | |
| code_parts.append(f' """Populate {table_name} table ({"fact" if is_fact else "dimension"})"""') | |
| code_parts.append(" data = []") | |
| code_parts.append(f" for _ in range({row_count}):") | |
| code_parts.append(f" data.append(({fake_values_str},))") | |
| code_parts.append("") | |
| code_parts.append(" cursor.executemany(") | |
| code_parts.append(f' "INSERT INTO {table_name} ({col_list}) VALUES ({placeholders})",') | |
| code_parts.append(" data") | |
| code_parts.append(" )") | |
| code_parts.append(f' print(f"✅ Inserted {{len(data)}} rows into {table_name}")') | |
| code_parts.append("") | |
| # Safety check - ensure we have at least one table | |
| if not table_names: | |
| raise Exception("No valid tables found in schema. All tables were skipped due to having no insertable columns.") | |
| # Main function | |
| code_parts.append("def main():") | |
| code_parts.append(" conn_params = get_snowflake_connection_params()") | |
| code_parts.append(" conn_params.pop('schema', None) # Remove to avoid duplicate") | |
| code_parts.append(" conn = snowflake.connector.connect(**conn_params, schema=os.getenv('SNOWFLAKE_SCHEMA'), autocommit=False)") | |
| code_parts.append("") | |
| code_parts.append(" try:") | |
| code_parts.append(" cursor = conn.cursor()") | |
| code_parts.append(" fake = Faker()") | |
| code_parts.append("") | |
| # Add function calls | |
| for table_name, row_count in table_names: | |
| code_parts.append(f" populate_{table_name.lower()}(cursor, fake)") | |
| code_parts.append("") | |
| code_parts.append(" conn.commit()") | |
| code_parts.append(' print("✅ All data committed successfully")') | |
| code_parts.append(" except Exception as e:") | |
| code_parts.append(" conn.rollback()") | |
| code_parts.append(' print(f"❌ Error: {str(e)}")') | |
| code_parts.append(" raise") | |
| code_parts.append(" finally:") | |
| code_parts.append(" cursor.close()") | |
| code_parts.append(" conn.close()") | |
| code_parts.append("") | |
| code_parts.append('if __name__ == "__main__":') | |
| code_parts.append(" main()") | |
| code = '\n'.join(code_parts) | |
| # Validate the generated code compiles | |
| try: | |
| compile(code, '<template_generation>', 'exec') | |
| self.log_feedback("✅ Template validated successfully before return") | |
| except SyntaxError as e: | |
| self.log_feedback(f"❌ TEMPLATE GENERATION BUG: {e}") | |
| self.log_feedback(f" Error at line {e.lineno}: {e.msg}") | |
| # Show the problematic lines | |
| lines = code.split('\n') | |
| if e.lineno: | |
| start = max(0, e.lineno - 3) | |
| end = min(len(lines), e.lineno + 2) | |
| self.log_feedback(f"\n Context:") | |
| for i in range(start, end): | |
| marker = ">>> " if i == e.lineno - 1 else " " | |
| self.log_feedback(f"{marker}{i+1:3}: {lines[i]}") | |
| raise Exception(f"Template generation has a bug: {e}") | |
| # Debug: Log first 1000 chars | |
| self.log_feedback(f"Generated template preview:\n{code[:1000]}") | |
| return code | |
| def run_population(self): | |
| """Run data population code generation""" | |
| self.log_feedback("🔢 Starting data population...") | |
| try: | |
| from schema_utils import parse_ddl_schema, generate_schema_constrained_prompt | |
| import re | |
| # Parse use case into vertical and function | |
| vertical, function = parse_use_case(self.demo_builder.use_case) | |
| config = get_use_case_config(vertical or "Generic", function or "Generic") | |
| # Build business context for population | |
| # Handle both new config structure and backward compatibility | |
| target_persona = config.get('target_persona', 'Business Leader') | |
| business_problem = config.get('business_problem', 'Need for faster, data-driven decisions') | |
| demo_objectives = config.get('demo_objectives', 'Show self-service analytics and business insights') | |
| # For generic cases, use the use_case_name | |
| use_case_display = config.get('use_case_name', self.demo_builder.use_case) | |
| business_context = f""" | |
| BUSINESS CONTEXT: | |
| - Use Case: {use_case_display} | |
| - Target Persona: {target_persona} | |
| - Business Problem: {business_problem} | |
| - Demo Objectives: {demo_objectives} | |
| MANDATORY CONNECTION CODE (MUST BE COMPLETE): | |
| ```python | |
| from dotenv import load_dotenv | |
| import os | |
| import snowflake.connector | |
| from faker import Faker | |
| import random | |
| from datetime import datetime, timedelta | |
| load_dotenv() | |
| from snowflake_auth import get_snowflake_connection_params | |
| def main(): | |
| conn_params = get_snowflake_connection_params() | |
| conn = snowflake.connector.connect(**conn_params, schema=os.getenv('SNOWFLAKE_SCHEMA'), autocommit=False) | |
| try: | |
| cursor = conn.cursor() | |
| fake = Faker() | |
| # [YOUR POPULATION CODE HERE - populate tables] | |
| conn.commit() | |
| print("✅ All data committed successfully") | |
| except Exception as e: | |
| conn.rollback() | |
| print(f"❌ Error: {{str(e)}}") | |
| raise | |
| finally: | |
| cursor.close() | |
| conn.close() | |
| if __name__ == "__main__": | |
| main() | |
| ``` | |
| CRITICAL REQUIREMENTS: | |
| 1. **COMPLETE try/except/finally blocks** - NO incomplete blocks | |
| 2. Use cursor.executemany() for batch inserts with %s placeholders (NOT ?) | |
| 3. Create baseline normal data (1000+ rows per table) | |
| 4. Include strategic outliers with structured comments | |
| 5. NO explanatory text, just executable Python code | |
| 6. **DO NOT leave try blocks incomplete** - always include except and finally | |
| 7. Use Faker library for realistic data generation | |
| 8. **PROPER INDENTATION** - Use 4 spaces per indent level, NO TABS | |
| 9. **SYNTAX CHECK** - Ensure all code is valid Python with correct indentation | |
| RETURN FORMAT: | |
| - Return ONLY the complete Python code | |
| - Start with imports, end with if __name__ == "__main__" | |
| - NO markdown, NO explanations, NO comments outside the code | |
| - All code must be properly indented and executable | |
| """ | |
| # Parse schema and generate prompt | |
| self.log_feedback("Parsing DDL schema...") | |
| # Validate DDL exists | |
| if not self.demo_builder.schema_generation_results: | |
| raise Exception("❌ DDL is missing. Please create DDL first.") | |
| self.log_feedback(f"DDL length: {len(self.demo_builder.schema_generation_results)} characters") | |
| schema_info = parse_ddl_schema(self.demo_builder.schema_generation_results) | |
| if not schema_info: | |
| raise Exception("❌ Failed to parse DDL schema. DDL may be malformed.") | |
| self.log_feedback(f"Parsed {len(schema_info)} tables from DDL") | |
| self.log_feedback("Using LegitData for data generation...") | |
| self.demo_builder.data_population_results = "LEGITDATA" | |
| self.demo_builder.population_code_source = "legitdata" | |
| self.population_code = "# Data generated by LegitData" | |
| self.demo_builder.advance_stage() | |
| self.log_feedback("✅ Ready for deployment with LegitData!") | |
| response = f"""✅ **Data Population Ready!** | |
| LegitData will generate realistic, AI-powered data. | |
| **When you're ready, type 'deploy'** to: | |
| - Create Snowflake schema & tables | |
| - Populate with generated data | |
| - Create ThoughtSpot model & liveboard | |
| ⏱️ *This takes 2-5 minutes - watch the terminal for progress.*""" | |
| return response, self.population_code | |
| except Exception as e: | |
| import traceback | |
| error_msg = f"❌ Population failed: {str(e)}\n\n{traceback.format_exc()}\n\n**Would you like to retry?** (Type 'yes' to retry)" | |
| self.log_feedback(error_msg) | |
| return error_msg, self.population_code if hasattr(self, 'population_code') else "" | |
| def run_deployment(self): | |
| """Run deployment to Snowflake using LegitData (non-streaming version)""" | |
| # Consume the streaming version and return final result | |
| result = None | |
| for update in self.run_deployment_streaming(): | |
| result = update | |
| return result | |
| def run_deployment_streaming(self): | |
| """Run deployment to Snowflake using LegitData - yields progress updates""" | |
| _slog = self._session_logger | |
| _t_deploy = _slog.log_start("deploy") if _slog else None | |
| _deploy_error = None | |
| _deploy_meta = {} | |
| progress = "" | |
| schema_name = None | |
| company_name = None | |
| # Clear and initialize live progress for Snowflake deployment | |
| self.live_progress_log = ["=" * 60, "SNOWFLAKE DEPLOYMENT STARTING", "=" * 60, ""] | |
| self._dq_warnings = [] # reset data-quality gate warnings for this run | |
| def log_progress(msg): | |
| """Log to live progress tab only — not pipeline status""" | |
| print(f"[Deploy] {msg}", flush=True) | |
| self.live_progress_log.append(msg) | |
| try: | |
| # Ensure deploy-time modules that still use os.getenv() see Supabase admin settings. | |
| inject_admin_settings_to_env() | |
| from cdw_connector import SnowflakeDeployer | |
| # Step 1: Connect | |
| progress = "**Step 1/3: Connecting to Snowflake...**" | |
| yield progress | |
| log_progress("Connecting to Snowflake...") | |
| deployer = SnowflakeDeployer() | |
| success, message = deployer.connect() | |
| if not success: | |
| if _slog: _slog.log("deploy", "snowflake connect failed", error=message) | |
| raise Exception(f"Snowflake connection failed: {message}") | |
| if _slog: _slog.log_verbose("deploy", "snowflake connected") | |
| progress += f"\n[OK] {message}" | |
| log_progress(f"[OK] {message}") | |
| yield progress | |
| # Step 2: Create schema and tables | |
| company_name = self.demo_builder.extract_company_name() | |
| self._slack_deployment_company = company_name | |
| from slack_notifier import notify_deployment_event | |
| notify_deployment_event( | |
| "DemoPrep deployment", | |
| "Started", | |
| [ | |
| ("Company", company_name), | |
| ("Use case", self.demo_builder.use_case), | |
| ], | |
| ) | |
| progress += f"\n\n**Step 2/3: Creating schema and tables...**" | |
| progress += f"\n Company: {company_name}" | |
| progress += f"\n Use Case: {self.demo_builder.use_case}" | |
| yield progress | |
| log_progress(f"Creating schema and deploying DDL for {company_name}...") | |
| # Generate base name for schema | |
| from demo_prep import generate_demo_base_name | |
| naming_prefix = self.settings.get('object_naming_prefix', '') | |
| base_name = generate_demo_base_name(naming_prefix, company_name) | |
| success, schema_name, deploy_message = deployer.create_demo_schema_and_deploy( | |
| base_name, | |
| self.demo_builder.schema_generation_results | |
| ) | |
| if not success: | |
| if _slog: _slog.log("deploy", "ddl push failed", error=deploy_message) | |
| log_progress(f"[ERROR] DDL Deployment failed!") | |
| raise Exception(f"Schema deployment failed: {deploy_message}") | |
| if _slog: _slog.log_verbose("deploy", "ddl pushed", schema=schema_name, schema_name=schema_name) | |
| progress += f"\n[OK] Schema created: {schema_name}" | |
| progress += f"\n[OK] Tables created" | |
| log_progress(f"[OK] Schema created: {schema_name}") | |
| log_progress(f"[OK] Tables created successfully") | |
| yield progress | |
| # Step 3: Populate Snowflake from the generated dataset bundle | |
| import threading | |
| import time as time_module | |
| # Determine size from settings | |
| fact_rows = int(self.settings.get('fact_table_size', 5000)) | |
| if fact_rows <= 100: | |
| size = "small" | |
| elif fact_rows <= 1000: | |
| size = "medium" | |
| elif fact_rows <= 5000: | |
| size = "standard" | |
| elif fact_rows <= 10000: | |
| size = "large" | |
| else: | |
| size = "xl" | |
| # Show size details | |
| size_details = { | |
| "small": "~500 rows total", | |
| "medium": "~1,500 rows total", | |
| "standard": "~5,500 rows total", | |
| "large": "~15,000 rows total", | |
| "xl": "~50,000+ rows total" | |
| } | |
| progress += f"\n\n**Step 3/3: Populating tables with data...**" | |
| progress += f"\n Size: {size} ({size_details.get(size, '')})" | |
| progress += f"\n Loading dataset into Snowflake... (typically under 1 minute)" | |
| progress += f"\n *(detailed progress in Live Progress tab)*" | |
| yield progress | |
| log_progress("") | |
| log_progress("Loading dataset into Snowflake...") | |
| log_progress(f" Size preset: {size}") | |
| log_progress(f" Company URL: {self.demo_builder.company_url}") | |
| # Track population progress | |
| pop_messages = [] | |
| def pop_callback(msg): | |
| log_progress(msg) | |
| pop_messages.append(msg) | |
| if _slog: | |
| _slog.log("deploy", f"population progress: {str(msg)[:120]}") | |
| # Run populate_dataset_bundle in a background thread so we can yield progress | |
| pop_result = {"success": None, "message": None, "results": None, "done": False} | |
| def run_population(): | |
| try: | |
| from demoprep_app.integrations.snowflake import populate_dataset_bundle | |
| pop_callback("Loading dataset into Snowflake...") | |
| results = populate_dataset_bundle( | |
| deployer.connection, | |
| schema_name, | |
| self._demo_bundle, | |
| progress_callback=pop_callback, | |
| ) | |
| total_rows = sum(results.values()) | |
| pop_callback(f"Load complete: {total_rows:,} rows inserted") | |
| pop_result["success"] = True | |
| pop_result["message"] = f"Load complete: {total_rows:,} rows inserted" | |
| pop_result["results"] = results | |
| except Exception as e: | |
| pop_result["success"] = False | |
| pop_result["message"] = str(e) | |
| pop_result["results"] = None | |
| finally: | |
| pop_result["done"] = True | |
| pop_thread = threading.Thread(target=run_population) | |
| pop_thread.start() | |
| # Yield progress updates while LegitData runs | |
| POP_TIMEOUT = 900 | |
| spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] | |
| spinner_idx = 0 | |
| start_time = time_module.time() | |
| last_msg_count = 0 | |
| last_heartbeat = start_time | |
| while not pop_result["done"]: | |
| time_module.sleep(2) | |
| elapsed = time_module.time() - start_time | |
| # Hard timeout — break out so UI doesn't freeze forever | |
| if elapsed > POP_TIMEOUT: | |
| timeout_err = f"Data generation timed out after {int(POP_TIMEOUT/60)} minutes" | |
| pop_result["done"] = True | |
| pop_result["success"] = False | |
| pop_result["message"] = timeout_err | |
| if _slog: | |
| _slog.log("deploy", "LegitData timeout", error=timeout_err) | |
| break | |
| # Heartbeat every 60 seconds so we can tell live vs. stuck | |
| if _slog and (time_module.time() - last_heartbeat) >= 60: | |
| _slog.log("deploy", f"dataset loader heartbeat - {int(elapsed/60)}m {int(elapsed%60)}s elapsed") | |
| last_heartbeat = time_module.time() | |
| mins = int(elapsed) // 60 | |
| secs = int(elapsed) % 60 | |
| spinner_char = spinner[spinner_idx % len(spinner)] | |
| spinner_idx += 1 | |
| progress_update = progress + f"\n\n{spinner_char} **LegitData running...** ({mins}:{secs:02d} elapsed)" | |
| # Show recent progress messages from callback | |
| if len(pop_messages) > last_msg_count: | |
| recent_msgs = pop_messages[last_msg_count:last_msg_count + 3] | |
| for msg in recent_msgs: | |
| if msg.strip(): | |
| progress_update += f"\n • {msg[:60]}..." | |
| last_msg_count = len(pop_messages) | |
| yield progress_update | |
| # Get results from thread | |
| pop_success = pop_result["success"] | |
| pop_message = pop_result["message"] | |
| results = pop_result["results"] | |
| if not pop_success: | |
| self._last_population_error = pop_message | |
| self._last_schema_name = schema_name | |
| log_progress(f"[ERROR] Population error: {pop_message}") | |
| raise Exception(f"Population failed: {pop_message[:200]}") | |
| progress += f"\n[OK] Data populated" | |
| log_progress(f"[OK] {pop_message}") | |
| # Data-quality gate: after every load, profile the MEASURE columns | |
| # that were actually written to Snowflake and fail loudly if any is | |
| # entirely zero/null (recurring zero-measure defect — derived | |
| # measures like TOTAL_*_USD loading as all zeros → blank tiles). | |
| from demoprep_app.integrations.snowflake import ( | |
| ZeroMeasureError, | |
| run_measure_quality_gate, | |
| ) | |
| progress += f"\n\n**Verifying data quality (measure columns)...**" | |
| yield progress | |
| log_progress("") | |
| log_progress("Running data-quality gate on measure columns...") | |
| try: | |
| dq_profile = run_measure_quality_gate( | |
| deployer.connection, schema_name, progress_callback=log_progress | |
| ) | |
| except ZeroMeasureError as e: | |
| self._last_population_error = str(e) | |
| self._last_schema_name = schema_name | |
| _deploy_meta = { | |
| "schema_name": schema_name, | |
| "data_quality": { | |
| "failures": e.failures, | |
| "warnings": e.profile.get("warnings", []), | |
| }, | |
| } | |
| log_progress(f"[ERROR] {e}") | |
| log_progress(f" Schema {schema_name} left in place for inspection.") | |
| if _slog: | |
| _slog.log("deploy", "data quality gate failed", error=str(e)) | |
| raise Exception(str(e)) | |
| dq_warnings = dq_profile.get("warnings", []) | |
| self._dq_warnings = dq_warnings | |
| progress += ( | |
| f"\n[OK] Data quality gate passed " | |
| f"({dq_profile['measure_columns_checked']} measure columns " | |
| f"across {dq_profile['tables_checked']} tables)" | |
| ) | |
| for _dq_w in dq_warnings: | |
| progress += f"\n[WARN] {_dq_w}" | |
| if _slog: | |
| _slog.log_verbose( | |
| "deploy", "data quality gate passed", | |
| measure_columns_checked=dq_profile["measure_columns_checked"], | |
| tables_checked=dq_profile["tables_checked"], | |
| dq_warnings=dq_warnings, | |
| ) | |
| self._deployed_schema_name = schema_name | |
| _deploy_meta = { | |
| "schema_name": schema_name, | |
| "generation_mode": "blueprint", | |
| "table_names": [table.name for table in self._demo_bundle.tables] if self._demo_bundle else [], | |
| "row_counts": {table.name: len(table.rows) for table in self._demo_bundle.tables} if self._demo_bundle else {}, | |
| "data_quality": { | |
| "measure_columns_checked": dq_profile["measure_columns_checked"], | |
| "tables_checked": dq_profile["tables_checked"], | |
| "warnings": dq_warnings, | |
| }, | |
| } | |
| log_progress("") | |
| log_progress("=" * 60) | |
| log_progress("SNOWFLAKE DEPLOYMENT COMPLETE") | |
| log_progress("=" * 60) | |
| # Check validation_mode setting to decide whether to auto-continue | |
| from supabase_client import load_gradio_settings, get_admin_setting | |
| settings = load_gradio_settings(self._get_effective_user_email()) | |
| validation_mode = settings.get('validation_mode', 'Off') | |
| if validation_mode == 'On': | |
| # Validation mode - show message asking to type 'thoughtspot' | |
| final_response = f"""{progress} | |
| **Deployment Complete** | |
| Schema: **{schema_name}** | |
| Tables: Created and populated | |
| Status: Ready for ThoughtSpot | |
| **Next:** Type **'thoughtspot'** to create ThoughtSpot objects""" | |
| notify_deployment_event( | |
| "DemoPrep deployment", | |
| "Ready for ThoughtSpot", | |
| [ | |
| ("Company", company_name), | |
| ("Use case", self.demo_builder.use_case), | |
| ("Schema", schema_name), | |
| ], | |
| ) | |
| if _slog: | |
| _slog.log( | |
| "run", | |
| "run waiting for user", | |
| checkpoint="thoughtspot", | |
| reason="validation mode requested manual ThoughtSpot start", | |
| schema_name=schema_name, | |
| ) | |
| yield (final_response, "thoughtspot") | |
| else: | |
| # Auto-continue to ThoughtSpot deployment (default behavior) | |
| final_response = f"""{progress} | |
| **Deployment Complete** | |
| Schema: **{schema_name}** | |
| Tables: Created and populated | |
| **Auto-continuing to ThoughtSpot deployment...**""" | |
| log_progress("Auto-continuing to ThoughtSpot deployment...") | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"ABOUT TO YIELD auto_ts tuple, schema={schema_name}\n") | |
| yield (final_response, "auto_ts", schema_name) | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"YIELD COMPLETED for auto_ts\n") | |
| except Exception as e: | |
| import traceback | |
| error_msg = str(e) | |
| _deploy_error = error_msg | |
| log_progress(f"[ERROR] {error_msg}") | |
| try: | |
| from slack_notifier import notify_deployment_event | |
| notify_deployment_event( | |
| "DemoPrep deployment", | |
| "Failed", | |
| [ | |
| ("Company", company_name), | |
| ("Use case", getattr(self.demo_builder, "use_case", None)), | |
| ("Schema", schema_name), | |
| ("Error", error_msg[:500]), | |
| ], | |
| ) | |
| except Exception: | |
| pass | |
| raise Exception(f"Deployment failed: {error_msg}") | |
| finally: | |
| if _slog and _t_deploy: | |
| _slog.log_end("deploy", _t_deploy, error=_deploy_error, **_deploy_meta) | |
| def _run_thoughtspot_deployment(self, schema_name, company, use_case, on_progress=None): | |
| """ | |
| Generator that runs ThoughtSpot deployment and yields progress updates. | |
| Yields: | |
| str: Progress messages during deployment | |
| dict: Final result with 'response' and 'stage' keys | |
| """ | |
| # Debug log at entry | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"_run_thoughtspot_deployment: ENTERED, schema={schema_name}\n") | |
| import os | |
| from thoughtspot_deployer import ThoughtSpotDeployer | |
| from cdw_connector import SnowflakeDeployer | |
| from supabase_client import load_gradio_settings, get_admin_setting | |
| from demo_prep import generate_demo_base_name | |
| from slack_notifier import notify_deployment_event | |
| _slog = self._session_logger | |
| _t_ts = _slog.log_start("thoughtspot") if _slog else None | |
| _ts_error = None | |
| _ts_meta = {} | |
| slack_company = getattr(self, "_slack_deployment_company", None) or company | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"_run_thoughtspot_deployment: imports done, about to yield first message\n") | |
| yield "**Starting ThoughtSpot Deployment...**\n\n" | |
| with open('/tmp/demoprep_debug.log', 'a') as f: | |
| f.write(f"_run_thoughtspot_deployment: first yield complete\n") | |
| self.log_feedback("Deploying to ThoughtSpot...") | |
| # Check if using existing model mode | |
| use_existing_model = self.settings.get('use_existing_model', False) | |
| existing_model_guid = self.settings.get('existing_model_guid', '') | |
| if use_existing_model and existing_model_guid: | |
| # Skip all table creation - go directly to liveboard creation | |
| yield f"**Using Existing Model Mode**\n\nModel GUID: `{existing_model_guid}`\n\nSkipping table/model creation...\n\n" | |
| self.log_feedback(f"Using existing model: {existing_model_guid}") | |
| try: | |
| from liveboard_creator import create_liveboard_from_model_mcp | |
| from thoughtspot_deployer import ThoughtSpotDeployer | |
| # Get ThoughtSpot settings | |
| ts_url = (self.settings.get('thoughtspot_url') or '').strip() | |
| ts_secret = (self.settings.get('thoughtspot_trusted_auth_key') or '').strip() | |
| if not ts_url or not ts_secret: | |
| raise ValueError("ThoughtSpot environment not set — select a TS environment from the dropdown") | |
| ts_user = self._get_effective_user_email() | |
| # Clean company name for display (strip .com, .org, etc) | |
| clean_company = company.split('.')[0].title() if '.' in company else company | |
| liveboard_name = self.settings.get('liveboard_name', '') or f"{clean_company} - {use_case}" | |
| # Get company data for liveboard | |
| company_data = { | |
| 'name': clean_company, | |
| 'url': getattr(self.demo_builder, 'company_url', company), | |
| 'logo_url': getattr(self.demo_builder, 'logo_url', None), | |
| 'primary_color': getattr(self.demo_builder, 'primary_color', '#3498db'), | |
| 'secondary_color': getattr(self.demo_builder, 'secondary_color', '#2c3e50'), | |
| 'additional_context': getattr(self, 'generic_use_case_context', '') or '', | |
| } | |
| yield f"**Creating Liveboard from Existing Model**\n\nModel: `{existing_model_guid}`\n\n" | |
| # Auth a deployer so we can pass ts_client to MCP | |
| ts_client = ThoughtSpotDeployer(ts_url, ts_user, ts_secret) | |
| if not ts_client.authenticate(): | |
| raise ValueError("ThoughtSpot authentication failed.") | |
| llm_model = self.settings.get('model', DEFAULT_LLM_MODEL) | |
| # Create liveboard through the supported MCP path. | |
| liveboard_result = create_liveboard_from_model_mcp( | |
| ts_client=ts_client, | |
| model_id=existing_model_guid, | |
| model_name="Existing Model", | |
| company_data=company_data, | |
| use_case=use_case, | |
| num_visualizations=8, | |
| liveboard_name=liveboard_name, | |
| llm_model=llm_model, | |
| prompt_logger=self._prompt_logger, | |
| ) | |
| if liveboard_result.get('success'): | |
| liveboard_url = liveboard_result.get('liveboard_url', '') | |
| yield { | |
| 'response': f"""✅ **Liveboard Created from Existing Model** | |
| **Model GUID:** `{existing_model_guid}` | |
| **Liveboard:** [{liveboard_name}]({liveboard_url}) | |
| The liveboard was created using the existing model. No new tables or models were created.""", | |
| 'stage': 'complete' | |
| } | |
| return | |
| else: | |
| error = liveboard_result.get('error', 'Unknown error') | |
| yield { | |
| 'response': f"""❌ **Liveboard Creation Failed** | |
| **Model GUID:** `{existing_model_guid}` | |
| **Error:** {error} | |
| Please verify the model GUID is correct and you have access to it.""", | |
| 'stage': 'thoughtspot' | |
| } | |
| return | |
| except Exception as e: | |
| import traceback | |
| yield { | |
| 'response': f"""❌ **Error Using Existing Model** | |
| **Error:** {str(e)} | |
| **Details:** | |
| ``` | |
| {traceback.format_exc()} | |
| ```""", | |
| 'stage': 'thoughtspot' | |
| } | |
| return | |
| try: | |
| # FIRST: Verify schema exists in Snowflake | |
| yield "**Starting ThoughtSpot Deployment...**\n\nVerifying Snowflake schema..." | |
| sf_deployer = SnowflakeDeployer() | |
| sf_deployer.connect() | |
| cursor = sf_deployer.connection.cursor() | |
| # Get DDL and database info | |
| ddl = self.demo_builder.schema_generation_results | |
| from snowflake_auth import get_demo_database | |
| database = get_demo_database() | |
| # Check if schema exists | |
| cursor.execute(f"USE DATABASE {database}") | |
| cursor.execute(f"SHOW SCHEMAS LIKE '{schema_name}'") | |
| schemas = cursor.fetchall() | |
| if not schemas: | |
| yield { | |
| 'response': f"""❌ **Schema Not Found in Snowflake** | |
| The schema `{database}.{schema_name}` doesn't exist in Snowflake. | |
| **Did the deployment complete successfully?** | |
| If not, try these options: | |
| 1. Type **'deploy'** - Deploy to Snowflake first | |
| 2. Check if tables were actually created in Snowflake | |
| Cannot deploy to ThoughtSpot without valid Snowflake schema.""", | |
| 'stage': 'deploy' | |
| } | |
| return | |
| # Check if tables exist | |
| cursor.execute(f'USE SCHEMA "{schema_name}"') | |
| cursor.execute(f"SHOW TABLES") | |
| tables = cursor.fetchall() | |
| if not tables: | |
| yield { | |
| 'response': f"""❌ **No Tables Found** | |
| The schema `{database}.{schema_name}` exists but has no tables! | |
| **What happened:** | |
| - Schema was created but table creation may have failed | |
| - Or tables were dropped/truncated | |
| **Next steps:** | |
| 1. Type **'deploy'** - Re-run the full Snowflake deployment | |
| 2. Or check Snowflake manually to see what's there | |
| Cannot deploy to ThoughtSpot without tables.""", | |
| 'stage': 'deploy' | |
| } | |
| return | |
| sf_deployer.connection.close() | |
| yield f"**Starting ThoughtSpot Deployment...**\n\nSchema verified: {database}.{schema_name}\nFound {len(tables)} tables\n\n" | |
| # Create deployer — prefer session-selected env (from TS env dropdown), | |
| # TS env must be selected from dropdown — no admin fallback. | |
| ts_url = (self.settings.get('thoughtspot_url') or '').strip() | |
| ts_secret = (self.settings.get('thoughtspot_trusted_auth_key') or '').strip() | |
| if not ts_url or not ts_secret: | |
| raise ValueError("ThoughtSpot environment not set — select a TS environment from the dropdown") | |
| ts_user = self._get_effective_user_email() | |
| deployer = ThoughtSpotDeployer( | |
| base_url=ts_url, | |
| username=ts_user, | |
| secret_key=ts_secret | |
| ) | |
| # Apply column naming style from settings | |
| deployer.column_naming_style = self.settings.get('column_naming_style', 'Regular Case') | |
| deployer.prompt_logger = self._prompt_logger | |
| # Clear and prepare progress capture | |
| self.live_progress_log = ["=" * 60, "THOUGHTSPOT DEPLOYMENT STARTING", "=" * 60, ""] | |
| progress_messages = [] | |
| def progress_callback(msg): | |
| progress_messages.append(msg) | |
| self.live_progress_log.append(msg) | |
| self.log_feedback(msg) | |
| safe_print(msg, flush=True) | |
| # Optional external hook (e.g. the MCP status updater): fires for | |
| # every deploy line so a live status can track real progress instead | |
| # of freezing between the generator's sparse yields. | |
| if on_progress is not None: | |
| try: | |
| on_progress(msg) | |
| except Exception: | |
| pass | |
| # Show initial message | |
| yield { | |
| 'stage': 'thoughtspot', | |
| 'response': """**Starting ThoughtSpot Deployment...** | |
| Authenticating with ThoughtSpot... | |
| **This takes 2-5 minutes.** | |
| **Switch to the "Live Progress" tab** to watch real-time progress. | |
| Steps: | |
| 1. Schema creation | |
| 2. Data generation | |
| 3. Model creation | |
| 4. Liveboard creation | |
| This chat will update when complete.""" | |
| } | |
| safe_print("\n" + "="*60, flush=True) | |
| safe_print("THOUGHTSPOT DEPLOYMENT STARTING", flush=True) | |
| safe_print("="*60, flush=True) | |
| safe_print("Watch Live Progress tab for real-time updates...\n", flush=True) | |
| # Load settings | |
| settings = load_gradio_settings(self._get_effective_user_email()) | |
| liveboard_name = self.settings.get('liveboard_name', '') | |
| # Default liveboard name to company name (without .com) if blank | |
| if not liveboard_name: | |
| liveboard_name = company.replace('.com', '').replace('.', ' ').strip().title() | |
| llm_model = settings.get('default_llm', self.settings.get('model', DEFAULT_LLM_MODEL)) | |
| tag_name_value = self.settings.get('tag_name') or settings.get('tag_name') | |
| naming_prefix = self.settings.get('object_naming_prefix') or settings.get('object_naming_prefix', '') | |
| # Extract base_name from schema_name (e.g., BLA_01311648_EAX_sch -> BLA_01311648_EAX) | |
| # DO NOT regenerate - must match what Snowflake deployment used | |
| base_name = schema_name.replace('_sch', '') if schema_name.endswith('_sch') else schema_name | |
| print(f"🔍 DEBUG: tag_name='{tag_name_value}'") | |
| # Run deployment in a thread with progress spinner (like LegitData) | |
| import threading | |
| import time as time_module | |
| ts_result = {"done": False, "results": None, "error": None} | |
| def run_ts_deployment(): | |
| try: | |
| ts_result["results"] = deployer.deploy_all( | |
| ddl=ddl, | |
| database=database, | |
| schema=schema_name, | |
| base_name=base_name, | |
| company_name=company, | |
| use_case=use_case, | |
| liveboard_name=liveboard_name, | |
| llm_model=llm_model, | |
| tag_name=tag_name_value, | |
| share_with=self.settings.get('share_with', '').strip() or None, | |
| company_research=self.demo_builder.get_research_context() if self.demo_builder else None, | |
| additional_context=getattr(self, 'generic_use_case_context', '') or '', | |
| vertical=getattr(self, 'vertical', None), | |
| line=getattr(self, 'line', None), | |
| function=getattr(self, 'function', None), | |
| progress_callback=progress_callback, | |
| session_logger=_slog, | |
| ) | |
| except Exception as e: | |
| ts_result["error"] = str(e) | |
| finally: | |
| ts_result["done"] = True | |
| ts_thread = threading.Thread(target=run_ts_deployment) | |
| ts_thread.start() | |
| # Yield progress updates with spinner while ThoughtSpot deployment runs. | |
| # ThoughtSpot metadata imports can outlive the 300s gateway timeout; do | |
| # not mark the run failed while the worker is still running. | |
| TS_SOFT_WARNING_SECONDS = 2700 | |
| spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] | |
| spinner_idx = 0 | |
| start_time = time_module.time() | |
| last_msg_count = 0 | |
| last_heartbeat = start_time | |
| soft_warning_logged = False | |
| base_progress = """**ThoughtSpot Deployment in Progress...** | |
| Steps: | |
| 1. ✅ Authenticating | |
| 2. 🔄 Creating connection & tables | |
| 3. ⏳ Creating model | |
| 4. ⏳ Creating liveboard | |
| **Watch "Live Progress" tab for real-time details.**""" | |
| while not ts_result["done"]: | |
| time_module.sleep(2) | |
| elapsed = time_module.time() - start_time | |
| if elapsed > TS_SOFT_WARNING_SECONDS and not soft_warning_logged: | |
| warning = ( | |
| f"ThoughtSpot deployment still running after " | |
| f"{int(TS_SOFT_WARNING_SECONDS/60)} minutes" | |
| ) | |
| if _slog: | |
| _slog.log("thoughtspot", "TS deployment still running", elapsed_s=round(elapsed, 1)) | |
| progress_messages.append(f"⚠️ {warning}; continuing to wait for the worker to finish") | |
| soft_warning_logged = True | |
| # Heartbeat every 60 seconds | |
| if _slog and (time_module.time() - last_heartbeat) >= 60: | |
| _slog.log_verbose("thoughtspot", f"TS heartbeat - {int(elapsed/60)}m {int(elapsed%60)}s elapsed") | |
| last_heartbeat = time_module.time() | |
| mins = int(elapsed) // 60 | |
| secs = int(elapsed) % 60 | |
| spinner_char = spinner[spinner_idx % len(spinner)] | |
| spinner_idx += 1 | |
| progress_update = base_progress + f"\n\n{spinner_char} **ThoughtSpot deploying...** ({mins}:{secs:02d} elapsed)" | |
| # Show recent progress messages from callback | |
| if len(progress_messages) > last_msg_count: | |
| recent_msgs = progress_messages[-3:] | |
| progress_update += "\n\n**Recent:**" | |
| for msg in recent_msgs: | |
| if msg.strip(): | |
| clean_msg = msg.replace('[ThoughtSpot]', '').replace('[AI Feedback]', '').strip() | |
| if clean_msg: | |
| progress_update += f"\n • {clean_msg[:70]}" | |
| last_msg_count = len(progress_messages) | |
| yield progress_update | |
| # Get results from thread | |
| if ts_result["error"]: | |
| raise Exception(ts_result["error"]) | |
| results = ts_result["results"] | |
| self.record_deployment_completion(results, database, schema_name, use_case) | |
| _ts_meta = { | |
| "schema_name": schema_name, | |
| "connection": results.get("connection"), | |
| "model_guid": results.get("model_guid"), | |
| "liveboard_guid": results.get("liveboard_guid") or results.get("liveboard_id"), | |
| "liveboard_url": results.get("liveboard_url"), | |
| "table_names": results.get("tables", []), | |
| "success": results.get("success"), | |
| "warnings": results.get("warnings", []), | |
| } | |
| safe_print("\n" + "="*60, flush=True) | |
| if results.get('success'): | |
| safe_print("DEPLOYMENT COMPLETE", flush=True) | |
| self.live_progress_log.extend(["", "=" * 60, "DEPLOYMENT COMPLETE", "=" * 60]) | |
| # Capture any non-fatal errors (e.g. enhance failure) so they reach session_logs | |
| if results.get('errors'): | |
| _ts_error = '; '.join(results['errors']) | |
| else: | |
| safe_print("DEPLOYMENT FAILED", flush=True) | |
| safe_print(f"Errors: {results.get('errors', [])}", flush=True) | |
| self.live_progress_log.extend(["", "DEPLOYMENT FAILED", f"Errors: {results.get('errors', [])}"]) | |
| if results.get('errors'): | |
| _ts_error = '; '.join(results['errors']) | |
| safe_print("="*60 + "\n", flush=True) | |
| notify_deployment_event( | |
| "DemoPrep deployment", | |
| "Complete" if results.get("success") else "Failed", | |
| [ | |
| ("Company", slack_company), | |
| ("Use case", use_case), | |
| ("Schema", f"{database}.{schema_name}"), | |
| ("Connection", results.get("connection")), | |
| ("Model GUID", results.get("model_guid")), | |
| ("Liveboard", results.get("liveboard_url") or results.get("liveboard_guid") or results.get("liveboard_id")), | |
| ("Errors", "; ".join(results.get("errors", []))[:500] if results.get("errors") else None), | |
| ], | |
| ) | |
| progress_log = '\n'.join(progress_messages) if progress_messages else 'No progress messages captured' | |
| # Generate Demo Pack on success | |
| if results.get('success'): | |
| try: | |
| company_name = company.replace('.com', '').replace('.', ' ').title() | |
| # Generate use-case specific Spotter questions | |
| spotter_questions = self._generate_spotter_questions(use_case, self.ddl_code) | |
| spotter_section = "\n".join([f'{i+1}. **"{q["question"]}"** - {q["purpose"]}' | |
| for i, q in enumerate(spotter_questions)]) | |
| # Use-case specific demo tips | |
| demo_tips = self._get_demo_tips(use_case) | |
| # Build URLs for demo pack | |
| ts_base = (self.settings.get('thoughtspot_url') or '').rstrip('/') | |
| pack_model_url = f"{ts_base}/#/data/tables/{results.get('model_guid', '')}" if results.get('model_guid') and ts_base else results.get('model', 'N/A') | |
| pack_lb_url = results.get('liveboard_url') or (f"{ts_base}/#/pinboard/{liveboard_guid}" if liveboard_guid and ts_base else results.get('liveboard', 'N/A')) | |
| self.demo_pack_content = f"""# {company_name} Demo Pack | |
| ## {use_case} | |
| *Generated: {__import__('datetime').datetime.now().strftime('%Y-%m-%d %H:%M')}* | |
| --- | |
| ## Deployment Summary | |
| - **Liveboard:** {pack_lb_url} | |
| - **Model:** {pack_model_url} | |
| - **Tables:** {len(results.get('tables', []))} imported | |
| --- | |
| ## Suggested Spotter Questions | |
| Ask these questions to showcase ThoughtSpot's AI capabilities: | |
| {spotter_section} | |
| --- | |
| ## Demo Flow | |
| 1. Start with the overview liveboard to set context | |
| 2. Ask a Spotter question to show natural language search | |
| 3. Drill into an interesting metric to show interactivity | |
| 4. Show Monitor for proactive alerting | |
| --- | |
| ## Tips for {use_case} | |
| {demo_tips} | |
| --- | |
| *Pro tip: Use "what changed" questions to show ThoughtSpot's change detection!* | |
| """ | |
| safe_print("Demo Pack generated - check the Demo Pack tab.", flush=True) | |
| self.live_progress_log.append("Demo Pack generated") | |
| except Exception as e: | |
| safe_print(f"Could not generate demo pack: {e}", flush=True) | |
| self.demo_pack_content = f"*Demo pack generation failed: {e}*" | |
| # Generate Spotter Viz Stories (both matrix and AI versions) | |
| try: | |
| _model_name = results.get('model', None) | |
| _model_guid = results.get('model_guid', None) | |
| _ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/') | |
| if _model_guid and _ts_url: | |
| _model_url = f"{_ts_url}/#/data/tables/{_model_guid}" | |
| else: | |
| _model_url = None | |
| # Extract actual column names from the deployed DDL so the story | |
| # generator uses real column names instead of idealized matrix ones. | |
| _actual_columns = [] | |
| if self.ddl_code: | |
| import re as _re | |
| _skip = {'CREATE', 'TABLE', 'IF', 'NOT', 'EXISTS', 'PRIMARY', 'FOREIGN', | |
| 'KEY', 'REFERENCES', 'UNIQUE', 'INDEX', 'CONSTRAINT', 'DEFAULT', | |
| 'NULL', 'AUTO_INCREMENT', 'IDENTITY'} | |
| for _m in _re.finditer(r'^\s+"?([A-Za-z_]\w*)"?\s+\w', self.ddl_code, _re.MULTILINE): | |
| _col = _m.group(1) | |
| if _col.upper() not in _skip: | |
| _actual_columns.append(_col) | |
| # Deduplicate while preserving order | |
| _seen = set() | |
| _actual_columns = [c for c in _actual_columns if not (_seen.add(c) or c in _seen)] | |
| _story_args = dict( | |
| company_name=company_name, | |
| use_case=use_case, | |
| model_name=_model_name, | |
| model_url=_model_url, | |
| liveboard_name=results.get('liveboard', None), | |
| actual_columns=_actual_columns[:80], # cap to avoid prompt bloat | |
| ) | |
| self.spotter_story_matrix = self._generate_matrix_spotter_story(**_story_args) | |
| self.spotter_story_ai = self._generate_ai_spotter_story(**_story_args) | |
| safe_print("Spotter Viz Stories generated - check the Spotter Viz Story tab.", flush=True) | |
| self.live_progress_log.append("Spotter Viz Stories generated") | |
| except Exception as e: | |
| safe_print(f"Could not generate Spotter Viz story: {e}", flush=True) | |
| _err = f"*(Generation failed: {e})*" | |
| self.spotter_story_matrix = _err | |
| self.spotter_story_ai = _err | |
| # Build final response | |
| if results.get('success'): | |
| # Safely extract GUIDs - handle None, 'None', 'N/A', empty string | |
| liveboard_guid = results.get('liveboard_guid') or results.get('liveboard_id') | |
| if not liveboard_guid or liveboard_guid in ('None', 'N/A', ''): | |
| liveboard_guid = None | |
| liveboard_name_result = results.get('liveboard', 'N/A') | |
| self._liveboard_guid = liveboard_guid | |
| self._liveboard_name = liveboard_name_result | |
| final_stage = 'deploy' | |
| # Try to load adjuster for outliers stage | |
| if liveboard_guid: | |
| try: | |
| from smart_data_adjuster import SmartDataAdjuster | |
| # Pass selected LLM model + session-selected TS env to adjuster | |
| llm_model = self.settings.get('model', DEFAULT_LLM_MODEL) | |
| adjuster = SmartDataAdjuster( | |
| database, schema_name, liveboard_guid, | |
| llm_model=llm_model, | |
| ts_url=self.settings.get('thoughtspot_url') or None, | |
| ts_secret=self.settings.get('thoughtspot_trusted_auth_key') or None, | |
| username=self._get_effective_user_email(), | |
| prompt_logger=self._prompt_logger, | |
| ) | |
| adjuster.connect() | |
| if adjuster.load_liveboard_context(): | |
| self._adjuster = adjuster | |
| viz_list = "\n".join([ | |
| f" [{i+1}] {v['name']}" | |
| for i, v in enumerate(adjuster.visualizations) | |
| ]) | |
| # Build clickable links - validate GUIDs before creating URLs | |
| ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/') | |
| model_guid = results.get('model_guid') or '' | |
| # Only create liveboard URL if we have a valid GUID | |
| lb_url = results.get('liveboard_url', '') | |
| if not lb_url and liveboard_guid and ts_url: | |
| lb_url = f"{ts_url}/#/pinboard/{liveboard_guid}" | |
| # Format table names | |
| table_names = results.get('tables', []) | |
| tables_list = ', '.join(table_names) if table_names else 'N/A' | |
| # Build URLs for easy Slack pasting | |
| model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else None | |
| response = f"""**ThoughtSpot Deployment Complete** | |
| **Created:** | |
| - Connection: {results.get('connection', 'N/A')} | |
| - Tables: {tables_list} | |
| - Model: {model_url if model_url else results.get('model', 'N/A')} | |
| - Liveboard: {lb_url if lb_url else liveboard_name_result} | |
| **Your demo is ready!** 🎉 | |
| --- | |
| **🎯 Ready for Outlier Adjustments!** | |
| I've loaded your liveboard context. Here are the visualizations: | |
| {viz_list} | |
| **What you can do:** | |
| - Naturally request changes: "make 1080p webcam 40B" | |
| - Adjust by percentage: "increase smart watch by 20%" | |
| - Reference by viz number: "viz 3, increase laptop to 50B" | |
| **Try an adjustment now, or type 'done' to finish!**""" | |
| final_stage = 'outlier_adjustment' | |
| else: | |
| ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/') | |
| model_guid = results.get('model_guid') or '' | |
| lb_url = results.get('liveboard_url', '') | |
| if not lb_url and liveboard_guid and ts_url: | |
| lb_url = f"{ts_url}/#/pinboard/{liveboard_guid}" | |
| table_names = results.get('tables', []) | |
| tables_list = ', '.join(table_names) if table_names else 'N/A' | |
| # Build URLs for easy Slack pasting | |
| model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else None | |
| response = f"""**ThoughtSpot Deployment Complete** | |
| **Created:** | |
| - Connection: {results.get('connection', 'N/A')} | |
| - Tables: {tables_list} | |
| - Model: {model_url if model_url else results.get('model', 'N/A')} | |
| - Liveboard: {lb_url if lb_url else liveboard_name_result} | |
| Your demo is ready! | |
| Note: Could not load liveboard context for adjustments. | |
| Type **'done'** to finish.""" | |
| except Exception as e: | |
| self.log_feedback(f"Failed to load liveboard context: {e}") | |
| ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/') | |
| model_guid = results.get('model_guid') or '' | |
| lb_url = results.get('liveboard_url', '') | |
| if not lb_url and liveboard_guid and ts_url: | |
| lb_url = f"{ts_url}/#/pinboard/{liveboard_guid}" | |
| table_names = results.get('tables', []) | |
| tables_list = ', '.join(table_names) if table_names else 'N/A' | |
| # Build URLs for easy Slack pasting | |
| model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else None | |
| response = f"""**ThoughtSpot Deployment Complete** | |
| **Created:** | |
| - Connection: {results.get('connection', 'N/A')} | |
| - Tables: {tables_list} | |
| - Model: {model_url if model_url else results.get('model', 'N/A')} | |
| - Liveboard: {lb_url if lb_url else liveboard_name_result} | |
| Your demo is ready! | |
| Note: Could not load liveboard context for adjustments: {str(e)} | |
| Type **'done'** to finish.""" | |
| else: | |
| # Deployed OK but no liveboard GUID returned — treat as partial success | |
| ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/') | |
| model_guid = results.get('model_guid') or '' | |
| table_names = results.get('tables', []) | |
| tables_list = ', '.join(table_names) if table_names else 'N/A' | |
| model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else results.get('model', 'N/A') | |
| response = f"""⚠️ **Partial Success — Dataset & Model Created** | |
| Your Snowflake data and ThoughtSpot model were deployed successfully. | |
| The liveboard GUID couldn't be retrieved — it may still have been created in ThoughtSpot. | |
| **Created:** | |
| - Connection: {results.get('connection', 'N/A')} | |
| - Tables: {tables_list} | |
| - Model: {model_url} | |
| **What you can do:** | |
| - Check the **Spotter Viz Story** tab to recreate the liveboard manually | |
| - Log into ThoughtSpot to verify whether the liveboard was created | |
| - Type **'retry liveboard'** to try building it again | |
| Type **'done'** to finish.""" | |
| # Surface data-quality warnings from the load gate in the completion panel | |
| _dq_warnings = getattr(self, '_dq_warnings', None) | |
| if _dq_warnings: | |
| response += "\n\n---\n\n**⚠️ Data quality warnings:**\n" + \ | |
| "\n".join(f"- {w}" for w in _dq_warnings) | |
| yield {'response': response, 'stage': final_stage} | |
| else: | |
| errors = results.get('errors', ['Unknown error']) | |
| error_details = '\n'.join(errors) | |
| # Check for partial success: Snowflake + model deployed OK but liveboard failed | |
| model_ok = bool(results.get('model_guid')) | |
| liveboard_errors = [e for e in errors if 'liveboard' in e.lower()] | |
| non_liveboard_errors = [e for e in errors if 'liveboard' not in e.lower()] | |
| if model_ok and liveboard_errors and not non_liveboard_errors: | |
| # Partial success — dataset and model are live, only liveboard failed | |
| ts_url = (self.settings.get('thoughtspot_url') or '').rstrip('/') | |
| model_guid = results.get('model_guid') or '' | |
| table_names = results.get('tables', []) | |
| tables_list = ', '.join(table_names) if table_names else 'N/A' | |
| model_url = f"{ts_url}/#/data/tables/{model_guid}" if model_guid and ts_url else results.get('model', 'N/A') | |
| lb_error = liveboard_errors[0] | |
| yield { | |
| 'response': f"""⚠️ **Partial Success — Dataset & Model Created** | |
| Your Snowflake data and ThoughtSpot model were deployed successfully. | |
| The liveboard couldn't be built automatically. | |
| **Created:** | |
| - Connection: {results.get('connection', 'N/A')} | |
| - Tables: {tables_list} | |
| - Model: {model_url} | |
| **Liveboard error:** | |
| ``` | |
| {lb_error} | |
| ``` | |
| **What you can do:** | |
| - Check the **Spotter Viz Story** tab — use that sequence to recreate the liveboard manually in Spotter Viz | |
| - Type **'retry liveboard'** to try building it again | |
| - Or continue in ThoughtSpot using your model directly""", | |
| 'stage': 'deploy' | |
| } | |
| else: | |
| if 'schema validation' in error_details.lower() or 'schema' in error_details.lower(): | |
| guidance = "**Root Cause:** The model TML has validation errors." | |
| elif 'connection' in error_details.lower(): | |
| guidance = "**Root Cause:** Connection issue with ThoughtSpot or Snowflake" | |
| elif 'authenticate' in error_details.lower() or 'auth' in error_details.lower(): | |
| guidance = "**Root Cause:** Authentication failed" | |
| else: | |
| guidance = "**Check the progress log above for details.**" | |
| yield { | |
| 'response': f"""❌ **ThoughtSpot Deployment Failed** | |
| **Error Details:** | |
| ``` | |
| {error_details} | |
| ``` | |
| **Progress Log:** | |
| ``` | |
| {progress_log} | |
| ``` | |
| {guidance} | |
| **Next Steps:** | |
| - Type **'retry'** to try again | |
| - Or fix the issues above first""", | |
| 'stage': 'deploy' | |
| } | |
| except Exception as e: | |
| import traceback | |
| error_details = traceback.format_exc() | |
| _ts_error = str(e) | |
| self.log_feedback(f"❌ ThoughtSpot deployment error: {error_details}") | |
| try: | |
| from slack_notifier import notify_deployment_event | |
| notify_deployment_event( | |
| "DemoPrep deployment", | |
| "Failed", | |
| [ | |
| ("Company", slack_company), | |
| ("Use case", use_case), | |
| ("Schema", schema_name), | |
| ("Error", str(e)[:500]), | |
| ], | |
| ) | |
| except Exception: | |
| pass | |
| yield { | |
| 'response': f"""❌ **ThoughtSpot Deployment Error** | |
| **Error:** {str(e)} | |
| **Full Details:** | |
| ``` | |
| {error_details} | |
| ``` | |
| **Common Causes:** | |
| - Missing ThoughtSpot credentials in .env | |
| - Snowflake connection issues | |
| - Invalid schema or table names | |
| **Next Steps:** | |
| - Verify credentials are correct | |
| - Type **'retry'** to try again""", | |
| 'stage': 'deploy' | |
| } | |
| finally: | |
| if _slog and _t_ts: | |
| _slog.log_end("thoughtspot", _t_ts, error=_ts_error, **_ts_meta) | |
| def process_regular_message(self, message, current_stage, company, use_case): | |
| """Process regular chat messages""" | |
| message_lower = message.lower() | |
| # Check if message contains company and use case info | |
| if "creating a demo for" in message_lower or "create a demo for" in message_lower: | |
| # Extract company and use case from message | |
| extracted_company = self.extract_company_from_message(message) | |
| extracted_use_case = self.extract_use_case_from_message(message) | |
| if extracted_company: | |
| company = extracted_company | |
| if extracted_use_case: | |
| # Use vertical × function system to resolve use case | |
| v, f = parse_use_case(extracted_use_case) | |
| config = get_use_case_config(v or "Generic", f or "Generic") | |
| use_case = config.get('use_case_name', extracted_use_case) | |
| # Automatically trigger research | |
| return f"""✅ **Got it!** | |
| **Company:** {company} | |
| **Use Case:** {use_case} | |
| 🔍 **Starting Research...** | |
| I'll analyze {company} and research {use_case} best practices. | |
| This will take about 2-3 minutes. | |
| (Type 'stop' if you want to change anything)""" | |
| # Simple intent detection (Phase 1) | |
| if any(word in message_lower for word in ['research', 'start', 'begin', 'analyze']): | |
| # Just confirm and tell them to use the proper format | |
| return f"""To start research, please use this format: | |
| ``` | |
| I'm creating a demo for company: {company} use case: {use_case} | |
| ``` | |
| This will automatically begin the research process!""" | |
| elif any(word in message_lower for word in ['configure', 'settings', 'change']): | |
| return """⚙️ **Configuration Options** | |
| You can change: | |
| - **Company**: `/over company: [new company]` | |
| - **Use Case**: `/over usecase: [new use case]` | |
| - **AI Model**: Use the dropdown on the right → | |
| What would you like to adjust?""" | |
| elif 'help' in message_lower: | |
| return """💡 **How to Use This Interface** | |
| **Commands:** | |
| - `/over company: [name]` - Change company | |
| - `/over usecase: [case]` - Change use case | |
| **What You Can Say:** | |
| - "Start research" - Begin demo creation | |
| - "Configure settings" - Adjust parameters | |
| - "What stage are we at?" - Check progress | |
| - Ask any question naturally! | |
| **Current Stage:** You can see it on the right side → | |
| **AI Model:** Editable in the dropdown on the right → | |
| What would you like to do?""" | |
| elif any(word in message_lower for word in ['stage', 'progress', 'status', 'where']): | |
| return f"""📊 **Current Status** | |
| **Stage:** {current_stage.replace('_', ' ').title()} | |
| **Company:** {company} | |
| **Use Case:** {use_case} | |
| **What's Next:** | |
| Tell me what you'd like to do, and I'll guide you through the process!""" | |
| else: | |
| # Generic helpful response - shouldn't normally reach here | |
| return f"""I'm ready to work on **{use_case}** for **{company}**. | |
| To get started, just say: | |
| ``` | |
| I'm creating a demo for company: {company} use case: {use_case} | |
| ``` | |
| Or if you want to change something, use `/over` to adjust.""" | |
| def create_chat_interface(): | |
| """ | |
| Create the new chat-based demo builder interface | |
| IMPORTANT: Each browser session gets its own ChatDemoInterface instance | |
| via gr.State() to enable multi-user concurrent access. | |
| """ | |
| # Bootstrap defaults before authenticated user-specific settings are loaded | |
| default_settings = { | |
| "company": "", | |
| "use_case": "", | |
| "model": DEFAULT_LLM_MODEL, | |
| "stage": "initialization", | |
| } | |
| _remember_me_js = """ | |
| (function() { | |
| var KEY = 'demoprep_remembered_user'; | |
| var _done = false; | |
| function injectIntoForm(loginRoot) { | |
| if (_done || !loginRoot) return; | |
| var formDiv = loginRoot.querySelector('div.form'); | |
| if (!formDiv) return; | |
| var inputs = formDiv.querySelectorAll('input, textarea'); | |
| var uInput = null; | |
| for (var i = 0; i < inputs.length; i++) { | |
| var tp = (inputs[i].type || inputs[i].tagName).toLowerCase(); | |
| if (tp !== 'password' && tp !== 'submit' && tp !== 'checkbox' && tp !== 'hidden') { | |
| uInput = inputs[i]; break; | |
| } | |
| } | |
| if (!uInput || loginRoot.querySelector('#dp-rmb')) return; | |
| var saved = localStorage.getItem(KEY); | |
| if (saved && !uInput.value) { | |
| try { | |
| var desc = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value') || | |
| Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value'); | |
| if (desc && desc.set) desc.set.call(uInput, saved); | |
| else uInput.value = saved; | |
| uInput.dispatchEvent(new Event('input', { bubbles: true })); | |
| } catch(e) { uInput.value = saved; } | |
| var pwInput = formDiv.querySelector('input[type="password"]'); | |
| if (pwInput) setTimeout(function() { pwInput.focus(); }, 50); | |
| } | |
| var lbl = document.createElement('label'); | |
| lbl.style.cssText = 'display:flex;align-items:center;gap:6px;font-size:13px;margin:8px 0 12px;cursor:pointer;color:#374151;'; | |
| var cb = document.createElement('input'); | |
| cb.type = 'checkbox'; cb.id = 'dp-rmb'; cb.checked = !!saved; cb.style.cursor = 'pointer'; | |
| lbl.appendChild(cb); | |
| lbl.appendChild(document.createTextNode(' Remember me')); | |
| var btn = loginRoot.querySelector('button'); | |
| if (btn && btn.parentNode) btn.parentNode.insertBefore(lbl, btn); | |
| else loginRoot.appendChild(lbl); | |
| function save() { | |
| if (cb.checked && uInput.value) localStorage.setItem(KEY, uInput.value); | |
| else localStorage.removeItem(KEY); | |
| } | |
| if (btn) btn.addEventListener('click', save); | |
| _done = true; | |
| console.log('[DemoPrep] Remember me injected'); | |
| } | |
| function tryInject() { | |
| if (_done) return; | |
| if (!window.gradio_config || !window.gradio_config.auth_required) return; | |
| var wrap = document.querySelector('div.wrap'); | |
| if (wrap) { injectIntoForm(wrap); if (_done) return; } | |
| document.querySelectorAll('div.form').forEach(function(f) { | |
| if (!_done) injectIntoForm(f.parentElement); | |
| }); | |
| } | |
| var t = setInterval(function() { tryInject(); if (_done) clearInterval(t); }, 200); | |
| setTimeout(function() { clearInterval(t); }, 10000); | |
| })(); | |
| """ | |
| with gr.Blocks( | |
| title=f"{'[TEST] ' if IS_TEST else ''}ThoughtSpot Demo Builder - Chat", | |
| theme=gr.themes.Soft(primary_hue="blue", secondary_hue="cyan"), | |
| css=""" | |
| .tabitem { padding-top: 6px !important; } | |
| /* Remove chatbot box — let content sit directly on page */ | |
| #main-chatbot { border: none !important; background: transparent !important; box-shadow: none !important; padding: 0 !important; } | |
| #main-chatbot > div { border: none !important; background: transparent !important; box-shadow: none !important; } | |
| #main-chatbot .bubble-wrap { background: transparent !important; padding-top: 0 !important; padding-bottom: 0 !important; } | |
| #main-chatbot .wrap { background: transparent !important; border: none !important; } | |
| #env-banner { background: #92400e; color: #fef3c7; padding: 8px 16px; text-align: center; font-weight: 600; font-size: 14px; border-radius: 6px; margin-bottom: 8px; } | |
| """, | |
| ) as interface: | |
| # SESSION STATE: Each user gets their own ChatDemoInterface instance | |
| # This is the key fix for multi-user support! | |
| chat_controller_state = gr.State(None) # Initialized on first interaction | |
| # State variables | |
| current_stage = gr.State(default_settings['stage']) | |
| current_model = gr.State(default_settings['model']) | |
| current_company = gr.State(default_settings['company']) | |
| current_usecase = gr.State(default_settings['use_case']) | |
| current_liveboard_name = gr.State(default_settings.get('liveboard_name', '')) | |
| # Test environment banner | |
| if IS_TEST: | |
| gr.HTML('<div id="env-banner">⚠️ TEST ENVIRONMENT — thoughtspot-dp-test-demoprep.hf.space</div>') | |
| # Header with logout link | |
| with gr.Row(equal_height=True): | |
| gr.Markdown(""" | |
| # 💬 ThoughtSpot Demo Builder | |
| ### AI-Powered Conversational Demo Creation | |
| """) | |
| gr.HTML(""" | |
| <div style="display:flex; align-items:center; justify-content:flex-end; padding:8px 0;"> | |
| <a href="/logout" style=" | |
| color: #6b7280; | |
| text-decoration: none; | |
| font-size: 14px; | |
| padding: 6px 14px; | |
| border: 1px solid #d1d5db; | |
| border-radius: 6px; | |
| white-space: nowrap; | |
| "> | |
| Sign Out → | |
| </a> | |
| </div> | |
| """) | |
| # Additional state for new tabs | |
| ai_feedback_state = gr.State("") | |
| ddl_code_state = gr.State("") | |
| population_code_state = gr.State("") | |
| live_progress_state = gr.State("") | |
| demo_pack_state = gr.State("") | |
| with gr.Group(visible=False) as password_gate: | |
| gr.Markdown(""" | |
| ## Change Password Required | |
| You are signed in with a temporary password. Set a new password before using DemoPrep. | |
| """) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| gate_current_password = gr.Textbox( | |
| label="Temporary Password", | |
| type="password", | |
| placeholder="Enter the password you just used to sign in", | |
| ) | |
| gate_new_password = gr.Textbox( | |
| label="New Password", | |
| type="password", | |
| placeholder="At least 8 characters", | |
| ) | |
| gate_confirm_password = gr.Textbox( | |
| label="Confirm New Password", | |
| type="password", | |
| placeholder="Repeat new password", | |
| ) | |
| gate_change_password_btn = gr.Button("Change Password", variant="primary") | |
| gate_password_status = gr.Markdown("") | |
| with gr.Tabs(visible=True) as main_tabs: | |
| with gr.Tab("📱 App"): | |
| chat_components = create_chat_tab( | |
| chat_controller_state, default_settings, current_stage, current_model, | |
| current_company, current_usecase, | |
| ai_feedback_state, ddl_code_state, population_code_state | |
| ) | |
| with gr.Tab("🤖 AI Feedback"): | |
| gr.Markdown("### AI Processing") | |
| gr.Markdown("*Research, LLM calls, and AI decisions*") | |
| ai_feedback_display = gr.TextArea( | |
| label="AI Processing Log", | |
| value="", | |
| lines=15, | |
| max_lines=15, | |
| interactive=False, | |
| show_label=False | |
| ) | |
| gr.Markdown("### Deploy Log") | |
| gr.Markdown("*Snowflake + ThoughtSpot deployment steps*") | |
| live_progress_display = gr.TextArea( | |
| label="Deploy Log", | |
| value="Waiting for deployment to start...\n\nGo to Chat tab and type 'thoughtspot' or 'deploy' to begin.", | |
| lines=15, | |
| max_lines=15, | |
| interactive=False, | |
| show_label=False, | |
| elem_classes=["live-progress-area"] | |
| ) | |
| # Timer-based refresh for live progress (every 2 seconds) | |
| # INACTIVE by default - activated when deployment starts, deactivated when done | |
| # Otherwise idle tabs consume concurrency slots and block new sessions | |
| live_progress_timer = gr.Timer(value=2, active=False) | |
| def refresh_live_progress(controller): | |
| """Poll and return current progress log""" | |
| if controller is None: | |
| return "Waiting for deployment to start...\n\nGo to Chat tab and type 'thoughtspot' or 'deploy' to begin." | |
| live_progress = getattr(controller, 'live_progress_log', []) | |
| if live_progress: | |
| return "\n".join(live_progress) | |
| return "Waiting for deployment to start...\n\nGo to Chat tab and type 'thoughtspot' or 'deploy' to begin." | |
| live_progress_timer.tick( | |
| fn=refresh_live_progress, | |
| inputs=[chat_controller_state], | |
| outputs=[live_progress_display] | |
| ) | |
| with gr.Tab("🎬 Demo Assets"): | |
| _spotter_default = "Story will be generated after liveboard creation.\n\n**What is Spotter Viz?** An AI agent in ThoughtSpot that builds Liveboards through natural language — type a request, it creates and refines the dashboard step by step." | |
| with gr.Tabs(): | |
| with gr.Tab("🗺️ SpotterViz TS"): | |
| gr.Markdown("*Built from the ThoughtSpot-recommended KPIs and visualizations for this vertical × function.*") | |
| spotter_matrix_display = gr.Markdown( | |
| value=_spotter_default, | |
| elem_classes=["spotter-viz-story-content"] | |
| ) | |
| with gr.Tab("✨ SpotterViz AI"): | |
| gr.Markdown("*Pure AI — no matrix constraints. What the AI thinks would make a compelling liveboard story.*") | |
| spotter_ai_display = gr.Markdown( | |
| value=_spotter_default, | |
| elem_classes=["spotter-viz-story-content"] | |
| ) | |
| with gr.Tab("📋 Demo Pack"): | |
| gr.Markdown("### Demo Pack - Talking Points & Spotter Questions") | |
| gr.Markdown("*Generated automatically after deployment completes*") | |
| demo_pack_display = gr.Markdown( | |
| value="Demo pack will be generated after deployment completes.\n\nThis will include:\n- Key insights/outliers\n- Spotter questions to ask\n- Talking points for the demo", | |
| elem_classes=["demo-pack-content"] | |
| ) | |
| with gr.Tab("⚙️ Settings"): | |
| settings_components = create_settings_tab() | |
| with gr.Tab("📄 DDL Code"): | |
| ddl_display = gr.Code( | |
| label="Generated DDL", | |
| language="sql", | |
| value="-- DDL will appear here after generation", | |
| lines=30, | |
| interactive=False | |
| ) | |
| with gr.Tab("👑 Admin", visible=False) as admin_tab: | |
| with gr.Tabs(): | |
| with gr.Tab("👤 User Management") as user_mgmt_tab: | |
| gr.Markdown("### User Management (Admin Only)") | |
| gr.Markdown("*Add, deactivate, or reset passwords for DemoPrep users.*") | |
| with gr.Row(): | |
| with gr.Column(scale=2): | |
| gr.Markdown("#### Current Users") | |
| user_list_display = gr.Dataframe( | |
| headers=["Email", "Display Name", "Admin", "Active", "Must Change PW", "Last Login EST"], | |
| datatype=["str", "str", "bool", "bool", "bool", "str"], | |
| interactive=False, | |
| label="Users" | |
| ) | |
| refresh_users_btn = gr.Button("🔄 Refresh User List", size="sm") | |
| with gr.Column(scale=1): | |
| gr.Markdown("#### Add New User") | |
| new_user_email = gr.Textbox(label="Email", placeholder="user@company.com") | |
| new_user_password = gr.Textbox( | |
| label="Temporary Password (optional)", | |
| type="password", | |
| placeholder="Leave blank to generate one" | |
| ) | |
| new_user_display = gr.Textbox(label="Display Name", placeholder="Jane Doe") | |
| new_user_admin = gr.Checkbox(label="Admin?", value=False) | |
| add_user_btn = gr.Button("➕ Add User", variant="primary") | |
| invite_message = gr.Textbox( | |
| label="Slack invite message", | |
| lines=9, | |
| interactive=True, | |
| show_copy_button=True, | |
| placeholder="Add a user to generate the message to send in Slack." | |
| ) | |
| gr.Markdown("---") | |
| gr.Markdown("#### User Actions") | |
| action_email = gr.Textbox(label="User Email (for actions below)") | |
| with gr.Row(): | |
| deactivate_btn = gr.Button("🚫 Deactivate", size="sm") | |
| activate_btn = gr.Button("✅ Activate", size="sm") | |
| new_password = gr.Textbox(label="New Password", type="password") | |
| reset_pw_btn = gr.Button("🔑 Reset Password", size="sm") | |
| user_mgmt_status = gr.Textbox(label="Status", interactive=False) | |
| def load_user_list(): | |
| """Load user list from Supabase.""" | |
| try: | |
| from supabase_client import UserManager | |
| from zoneinfo import ZoneInfo | |
| um = UserManager() | |
| users = um.list_users() | |
| users = sorted(users, key=lambda u: u.get('last_login') or '', reverse=True) | |
| eastern = ZoneInfo('America/New_York') | |
| rows = [] | |
| for u in users: | |
| raw_login = u.get('last_login') | |
| if raw_login: | |
| try: | |
| from datetime import datetime | |
| dt = datetime.fromisoformat(str(raw_login).replace('Z', '+00:00')) | |
| last_login_str = dt.astimezone(eastern).strftime('%Y-%m-%d %H:%M') | |
| except Exception: | |
| last_login_str = str(raw_login)[:19] | |
| else: | |
| last_login_str = 'Never' | |
| rows.append([ | |
| u.get('email', ''), | |
| u.get('display_name', ''), | |
| u.get('is_admin', False), | |
| u.get('is_active', True), | |
| u.get('must_change_password', False), | |
| last_login_str | |
| ]) | |
| return rows | |
| except Exception as e: | |
| return [[f"Error: {e}", "", False, False, False, ""]] | |
| def add_user_handler(email, password, display_name, is_admin, request: gr.Request = None): | |
| """Add a new user.""" | |
| if not email: | |
| return load_user_list(), "Email is required.", "" | |
| try: | |
| from supabase_client import UserManager | |
| um = UserManager() | |
| temp_password = password or um.generate_temp_password() | |
| success = um.add_user( | |
| email, | |
| temp_password, | |
| display_name, | |
| is_admin, | |
| must_change_password=True, | |
| ) | |
| if success: | |
| clean_email = email.lower().strip() | |
| display = (display_name or clean_email.split("@")[0]).strip() | |
| app_url = resolve_app_url_for_invite(request).rstrip("/") + "/" | |
| invite = ( | |
| f"Hi {display}, welcome to DemoPrep.\n\n" | |
| f"The app is located here: {app_url}\n" | |
| "The quick start guide and other documentation are here: " | |
| "https://thoughtspot-dp-demoprep-doc.static.hf.space/index.html\n\n" | |
| f"Username: {clean_email}\n" | |
| f"Temporary password: {temp_password}\n\n" | |
| "When you first sign in, DemoPrep will ask you to choose a new password. " | |
| "After that, you can start building demos from the main screen.\n\n" | |
| "If you have any problems, please Slack Mike Boone." | |
| ) | |
| if not um._supports_must_change_password(): | |
| invite += ( | |
| "\n\nAdmin note: temporary-password enforcement is not active " | |
| "until the demoprep_users.must_change_password migration is applied." | |
| ) | |
| return load_user_list(), f"User {clean_email} added. Slack invite generated below.", invite | |
| else: | |
| return load_user_list(), f"Failed to add user {email}.", "" | |
| except Exception as e: | |
| return load_user_list(), f"Error: {e}", "" | |
| def deactivate_handler(email): | |
| if not email: | |
| return load_user_list(), "Enter an email first." | |
| try: | |
| from supabase_client import UserManager | |
| um = UserManager() | |
| um.deactivate_user(email) | |
| return load_user_list(), f"User {email} deactivated." | |
| except Exception as e: | |
| return load_user_list(), f"Error: {e}" | |
| def activate_handler(email): | |
| if not email: | |
| return load_user_list(), "Enter an email first." | |
| try: | |
| from supabase_client import UserManager | |
| um = UserManager() | |
| um.activate_user(email) | |
| return load_user_list(), f"User {email} activated." | |
| except Exception as e: | |
| return load_user_list(), f"Error: {e}" | |
| def reset_password_handler(email, new_pw): | |
| if not email or not new_pw: | |
| return "Enter email and new password." | |
| try: | |
| from supabase_client import UserManager | |
| um = UserManager() | |
| um.reset_password(email, new_pw, must_change_password=True) | |
| return f"Password reset for {email}." | |
| except Exception as e: | |
| return f"Error: {e}" | |
| refresh_users_btn.click(fn=load_user_list, inputs=[], outputs=[user_list_display]) | |
| add_user_btn.click(fn=add_user_handler, inputs=[new_user_email, new_user_password, new_user_display, new_user_admin], outputs=[user_list_display, user_mgmt_status, invite_message]) | |
| deactivate_btn.click(fn=deactivate_handler, inputs=[action_email], outputs=[user_list_display, user_mgmt_status]) | |
| activate_btn.click(fn=activate_handler, inputs=[action_email], outputs=[user_list_display, user_mgmt_status]) | |
| reset_pw_btn.click(fn=reset_password_handler, inputs=[action_email, new_password], outputs=[user_mgmt_status]) | |
| interface.load(fn=load_user_list, inputs=[], outputs=[user_list_display]) | |
| with gr.Tab("⚙️ Admin Settings") as admin_settings_tab: | |
| gr.Markdown("### System-Wide Settings") | |
| gr.Markdown("These settings apply to all users. Only admins can view and edit.") | |
| # Hidden fields — values still saved/loaded but not shown in UI | |
| admin_ts_url = gr.Textbox(visible=False) | |
| admin_openai_key = gr.Textbox(visible=False) | |
| admin_google_key = gr.Textbox(visible=False) | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("#### ThoughtSpot Connection") | |
| admin_share_with = gr.Textbox( | |
| label="Default Share With (User or Group)", | |
| placeholder="user@company.com or group-name", | |
| info="System-wide default: model + liveboard shared here after every build" | |
| ) | |
| admin_log_level = gr.Dropdown( | |
| label="Log Level", | |
| choices=["off", "regular", "verbose"], | |
| value="regular", | |
| info="off = no Supabase logging · regular = stage start/end + TS milestones · verbose = full sub-step detail" | |
| ) | |
| with gr.Column(): | |
| gr.Markdown("#### Snowflake Connection") | |
| admin_sf_account = gr.Textbox(label="Snowflake Account") | |
| admin_sf_kp_user = gr.Textbox(label="Key Pair User") | |
| admin_sf_kp_pk = gr.Textbox(label="Private Key (PEM)", lines=3, type="password") | |
| admin_sf_kp_pass = gr.Textbox(label="Private Key Passphrase", type="password") | |
| admin_sf_role = gr.Textbox(label="Role") | |
| admin_sf_warehouse = gr.Textbox(label="Warehouse") | |
| admin_sf_database = gr.Textbox(label="Database") | |
| admin_sf_sso_user = gr.Textbox(label="SSO User (for Snowflake browser auth)") | |
| with gr.Row(): | |
| load_admin_btn = gr.Button("🔄 Load Current Settings", size="sm") | |
| save_admin_btn = gr.Button("💾 Save Admin Settings", variant="primary", size="sm") | |
| admin_settings_status = gr.Textbox(label="Status", interactive=False) | |
| admin_fields = [ | |
| admin_ts_url, | |
| admin_openai_key, admin_google_key, | |
| admin_sf_account, admin_sf_kp_user, admin_sf_kp_pk, | |
| admin_sf_kp_pass, admin_sf_role, admin_sf_warehouse, | |
| admin_sf_database, admin_sf_sso_user, | |
| admin_share_with, | |
| admin_log_level, | |
| ] | |
| admin_keys_order = [ | |
| "THOUGHTSPOT_URL", | |
| "OPENAI_API_KEY", "GOOGLE_API_KEY", | |
| "SNOWFLAKE_ACCOUNT", "SNOWFLAKE_KP_USER", "SNOWFLAKE_KP_PK", | |
| "SNOWFLAKE_KP_PASSPHRASE", "SNOWFLAKE_ROLE", "SNOWFLAKE_WAREHOUSE", | |
| "SNOWFLAKE_DATABASE", "SNOWFLAKE_SSO_USER", | |
| "SHARE_WITH", | |
| "LOG_LEVEL", | |
| ] | |
| def load_admin_settings_handler(): | |
| try: | |
| from supabase_client import load_admin_settings | |
| settings = load_admin_settings(force_refresh=True) | |
| values = [settings.get(k, "") for k in admin_keys_order] | |
| return values + ["Settings loaded from Supabase."] | |
| except Exception as e: | |
| return [""] * len(admin_keys_order) + [f"Error loading: {e}"] | |
| def save_admin_settings_handler(*field_values): | |
| try: | |
| from supabase_client import save_admin_settings | |
| settings_dict = {key: (val or "") for key, val in zip(admin_keys_order, field_values)} | |
| success = save_admin_settings(settings_dict) | |
| return "✅ Admin settings saved to Supabase and applied." if success else "❌ Failed to save some admin settings." | |
| except Exception as e: | |
| return f"❌ Error saving: {e}" | |
| load_admin_btn.click(fn=load_admin_settings_handler, inputs=[], outputs=admin_fields + [admin_settings_status]) | |
| save_admin_btn.click(fn=save_admin_settings_handler, inputs=admin_fields, outputs=[admin_settings_status]) | |
| interface.load(fn=load_admin_settings_handler, inputs=[], outputs=admin_fields + [admin_settings_status]) | |
| # --- Session Log Viewer --- | |
| gr.Markdown("---") | |
| gr.Markdown("### 📋 Session Logs") | |
| with gr.Row(): | |
| log_user_filter = gr.Textbox(label="Filter by user (email, blank=all)", scale=2) | |
| log_limit = gr.Dropdown(label="Show", choices=["25", "50", "100"], value="50", scale=1) | |
| log_refresh_btn = gr.Button("🔄 Refresh", scale=1) | |
| session_log_display = gr.Dataframe( | |
| headers=["Time", "User", "Stage", "Event", "Duration (ms)", "Error"], | |
| label="Recent Sessions", | |
| interactive=False, | |
| wrap=True, | |
| ) | |
| def load_session_logs(user_filter, limit): | |
| try: | |
| from supabase_client import SupabaseSettings | |
| ss = SupabaseSettings() | |
| if not ss.is_enabled(): | |
| return [["Supabase not configured", "", "", "", "", ""]] | |
| query = ss.client.table("session_logs").select( | |
| "ts,user_email,stage,event,duration_ms,error" | |
| ).order("ts", desc=True).limit(int(limit)) | |
| if user_filter and user_filter.strip(): | |
| query = query.ilike("user_email", f"%{user_filter.strip()}%") | |
| result = query.execute() | |
| rows = [] | |
| for r in result.data: | |
| ts = r.get("ts", "")[:19].replace("T", " ") | |
| rows.append([ts, r.get("user_email", ""), r.get("stage", ""), r.get("event", ""), str(r.get("duration_ms", "") or ""), r.get("error", "") or ""]) | |
| return rows if rows else [["No logs found", "", "", "", "", ""]] | |
| except Exception as e: | |
| return [[f"Error: {e}", "", "", "", "", ""]] | |
| log_refresh_btn.click(fn=load_session_logs, inputs=[log_user_filter, log_limit], outputs=[session_log_display]) | |
| with gr.Tab("📝 Prompt Log") as prompt_log_tab: | |
| gr.Markdown("### Prompt Log — What We Send to LLMs") | |
| gr.Markdown("*Every prompt and response is logged here for review. Updates every 5 seconds.*") | |
| prompt_log_display = gr.Markdown( | |
| value="No prompts logged yet. Start a demo build to see LLM calls here.", | |
| elem_classes=["prompt-log-content"] | |
| ) | |
| prompt_log_timer = gr.Timer(value=5, active=True) | |
| def refresh_prompt_log(controller): | |
| try: | |
| if controller is not None and controller._prompt_logger is not None: | |
| return controller._prompt_logger.get_summary() | |
| except Exception: | |
| pass | |
| return "No prompts logged yet." | |
| prompt_log_timer.tick(fn=refresh_prompt_log, inputs=[chat_controller_state], outputs=[prompt_log_display]) | |
| with gr.Tab("🧩 Matrix"): | |
| matrix_components = create_matrix_tab(interface) | |
| with gr.Tab("📊 Run History"): | |
| gr.Markdown("### Pipeline Run History") | |
| gr.Markdown("*Every pipeline run — who ran it, whether it succeeded, and where it failed.*") | |
| with gr.Row(): | |
| run_history_refresh_btn = gr.Button("🔄 Refresh", size="sm") | |
| run_history_email_filter = gr.Textbox(label="Filter by email", placeholder="user@company.com", scale=2) | |
| run_history_show_tests = gr.Checkbox( | |
| label="Show test runs", | |
| value=False, | |
| scale=1, | |
| ) | |
| run_history_limit = gr.Dropdown( | |
| label="Show", | |
| choices=["10", "50", "100", "All"], | |
| value="10", | |
| scale=1, | |
| ) | |
| run_history_display = gr.Dataframe( | |
| headers=["Time (UTC)", "User", "Company", "Use Case", "Interface", "Status", "Failed At", "Duration"], | |
| datatype=["str", "str", "str", "str", "str", "str", "str", "str"], | |
| column_widths=["130px", "200px", "130px", "150px", "100px", "90px", "200px", "80px"], | |
| interactive=False, | |
| label="Runs", | |
| wrap=False, | |
| ) | |
| full_use_cases_state = gr.State([]) | |
| use_case_detail = gr.Textbox( | |
| label="Full Use Case", | |
| lines=4, | |
| interactive=False, | |
| visible=False, | |
| ) | |
| def load_run_history(email_filter="", show_tests=False, limit_choice="10"): | |
| try: | |
| from supabase_client import SupabaseSettings | |
| from datetime import datetime as _dt, timezone as _timezone | |
| ss = SupabaseSettings() | |
| if not ss.is_enabled(): | |
| return [["Supabase not configured", "", "", "", "", "", ""]] | |
| display_limit = None if limit_choice == "All" else int(limit_choice) | |
| # Fetch enough raw rows to aggregate into desired number of sessions | |
| fetch_limit = 2000 if limit_choice == "All" else max(500, (display_limit or 10) * 20) | |
| query = ss.client.table("session_logs") \ | |
| .select("session_id,user_email,ts,stage,event,duration_ms,error,meta") \ | |
| .order("ts", desc=True) \ | |
| .limit(fetch_limit) | |
| if email_filter and email_filter.strip(): | |
| query = query.eq("user_email", email_filter.strip()) | |
| elif not show_tests: | |
| query = query.neq("user_email", "testrunner@thoughtspot.com") | |
| result = query.execute() | |
| rows = result.data or [] | |
| # Group by session_id | |
| sessions = {} | |
| for row in rows: | |
| sid = row.get('session_id', '') | |
| if not sid: | |
| continue | |
| if sid not in sessions: | |
| sessions[sid] = { | |
| 'user': row.get('user_email', ''), | |
| 'events': [], | |
| 'start_ts': row.get('ts', ''), | |
| 'end_ts': row.get('ts', ''), | |
| 'errors': [], | |
| 'stages': [], | |
| 'meta': {}, | |
| 'last_stage': row.get('stage', ''), | |
| 'last_event': row.get('event', ''), | |
| 'row_count': 0, | |
| } | |
| s = sessions[sid] | |
| s['row_count'] += 1 | |
| s['events'].append(row.get('event', '')) | |
| ts = row.get('ts', '') | |
| if ts and ts < s['start_ts']: | |
| s['start_ts'] = ts | |
| if ts and ts > s['end_ts']: | |
| s['end_ts'] = ts | |
| s['last_stage'] = row.get('stage', '') | |
| s['last_event'] = row.get('event', '') | |
| if row.get('error'): | |
| s['errors'].append(f"{row.get('stage','?')}: {row.get('error','')[:100]}") | |
| stage = row.get('stage') | |
| if stage and stage not in s['stages']: | |
| s['stages'].append(stage) | |
| if row.get('meta'): | |
| s['meta'].update(row.get('meta') or {}) | |
| sorted_sessions = sorted(sessions.items(), key=lambda x: x[1]['start_ts'], reverse=True) | |
| if display_limit: | |
| sorted_sessions = sorted_sessions[:display_limit] | |
| display_rows = [] | |
| full_use_cases = [] | |
| now_utc = _dt.now(_timezone.utc) | |
| for sid, s in sorted_sessions: | |
| company = s['meta'].get('company', '') or s['meta'].get('company_name', '') | |
| use_case = s['meta'].get('use_case', '') | |
| interface = s['meta'].get('interface', '') | |
| terminal_events = set(e or "" for e in s['events']) | |
| has_explicit_failure = any(e.endswith('failed') for e in s['events']) | |
| has_errors = len(s['errors']) > 0 | |
| has_final_complete = 'thoughtspot completed' in s['events'] | |
| has_run_complete = 'run completed' in terminal_events | |
| has_waiting = 'run waiting for user' in terminal_events | |
| has_interrupted = 'run interrupted' in terminal_events | |
| has_no_input = ( | |
| not company | |
| and not use_case | |
| and s['row_count'] <= 2 | |
| and all((e or '') == 'run started' for e in s['events']) | |
| ) | |
| try: | |
| end_for_age = _dt.fromisoformat(s['end_ts'].replace('Z', '+00:00')) | |
| if end_for_age.tzinfo is None: | |
| end_for_age = end_for_age.replace(tzinfo=_timezone.utc) | |
| age_s = int((now_utc - end_for_age).total_seconds()) | |
| except Exception: | |
| age_s = 0 | |
| if has_final_complete and has_errors: | |
| status = '⚠️ Partial Success' | |
| failed_at = s['errors'][0][:80] | |
| elif has_final_complete or has_run_complete: | |
| status = '✅ Success' | |
| failed_at = '' | |
| elif has_waiting: | |
| status = '⏸ Waiting for User' | |
| failed_at = s.get('last_event') or '' | |
| elif has_no_input: | |
| status = '⚪ No Run Started' | |
| failed_at = 'No company/use case submitted' | |
| elif has_interrupted: | |
| status = '⚠️ Interrupted' | |
| failed_at = s['errors'][0][:80] if s['errors'] else (s.get('last_event') or '') | |
| elif has_explicit_failure or has_errors: | |
| status = '❌ Failed' | |
| if s['errors']: | |
| failed_at = s['errors'][0][:80] | |
| else: | |
| failed_at = next((e for e in s['events'] if e.endswith('failed')), 'unknown') | |
| elif age_s > 30 * 60: | |
| status = '⚠️ Stale / Interrupted' | |
| failed_at = f"Last event: {s.get('last_stage')}: {s.get('last_event')}" | |
| else: | |
| status = '⏳ In Progress' | |
| failed_at = '' | |
| try: | |
| start = _dt.fromisoformat(s['start_ts'].replace('Z', '+00:00')) | |
| end = _dt.fromisoformat(s['end_ts'].replace('Z', '+00:00')) | |
| dur_s = int((end - start).total_seconds()) | |
| dur_str = f"{dur_s//60}m {dur_s%60}s" if dur_s >= 60 else f"{dur_s}s" | |
| except Exception: | |
| dur_str = '' | |
| use_case_display = (use_case[:60] + '…') if len(use_case) > 60 else use_case | |
| full_use_cases.append(use_case) | |
| display_rows.append([ | |
| s['start_ts'][:16].replace('T', ' '), | |
| s['user'], | |
| company, | |
| use_case_display, | |
| interface, | |
| status, | |
| failed_at, | |
| dur_str, | |
| ]) | |
| if not display_rows: | |
| return [["No runs found", "", "", "", "", "", ""]], [] | |
| return display_rows, full_use_cases | |
| except Exception as e: | |
| return [[f"Error: {e}", "", "", "", "", "", ""]], [] | |
| def show_full_use_case(evt: gr.SelectData, full_use_cases): | |
| row_idx = evt.index[0] | |
| if row_idx < len(full_use_cases) and len(full_use_cases[row_idx]) > 60: | |
| return gr.Textbox(value=full_use_cases[row_idx], visible=True) | |
| return gr.Textbox(visible=False) | |
| run_history_refresh_btn.click( | |
| fn=load_run_history, | |
| inputs=[run_history_email_filter, run_history_show_tests, run_history_limit], | |
| outputs=[run_history_display, full_use_cases_state] | |
| ) | |
| interface.load(fn=load_run_history, inputs=[], outputs=[run_history_display, full_use_cases_state]) | |
| run_history_display.select( | |
| fn=show_full_use_case, | |
| inputs=[full_use_cases_state], | |
| outputs=[use_case_detail] | |
| ) | |
| with gr.Tab("🧾 TS Table Calls"): | |
| gr.Markdown("### ThoughtSpot Table TML Calls") | |
| gr.Markdown("*Every table TML import request and response persisted in session_logs.*") | |
| with gr.Row(): | |
| table_calls_refresh_btn = gr.Button("🔄 Refresh", size="sm") | |
| table_calls_session_filter = gr.Textbox(label="Session ID", placeholder="Leave blank for latest table-call run", scale=3) | |
| table_calls_email_filter = gr.Textbox(label="Filter by email", placeholder="user@company.com", scale=2) | |
| table_calls_limit = gr.Dropdown(label="Show", choices=["25", "50", "100", "250"], value="100", scale=1) | |
| table_calls_display = gr.Dataframe( | |
| headers=["#", "Phase", "Table(s)", "Request UTC", "Response UTC", "Status", "Seconds", "Payload", "Session", "User", "Environment", "Error"], | |
| datatype=["number", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "str"], | |
| column_widths=["50px", "220px", "180px", "145px", "145px", "70px", "70px", "85px", "210px", "200px", "260px", "320px"], | |
| interactive=False, | |
| label="Table Calls", | |
| wrap=False, | |
| ) | |
| def load_table_tml_calls(session_filter="", email_filter="", limit_choice="100"): | |
| try: | |
| from supabase_client import SupabaseSettings | |
| ss = SupabaseSettings() | |
| if not ss.is_enabled(): | |
| return [["", "Supabase not configured", "", "", "", "", "", "", "", "", "", ""]] | |
| fetch_limit = max(500, int(limit_choice) * 6) | |
| query = ( | |
| ss.client.table("session_logs") | |
| .select("session_id,user_email,ts,event,error,meta") | |
| .order("ts", desc=True) | |
| .limit(fetch_limit) | |
| ) | |
| if session_filter and session_filter.strip(): | |
| query = query.eq("session_id", session_filter.strip()) | |
| elif email_filter and email_filter.strip(): | |
| query = query.eq("user_email", email_filter.strip()) | |
| raw_rows = list(reversed(query.execute().data or [])) | |
| if not session_filter or not session_filter.strip(): | |
| latest_sid = "" | |
| for row in reversed(raw_rows): | |
| event = row.get("event") or "" | |
| meta = row.get("meta") or {} | |
| if event in { | |
| "table tml import request started", | |
| "table tml import response received", | |
| "table tml import exception", | |
| "tml import HTTP error", | |
| } or meta.get("table_names"): | |
| latest_sid = row.get("session_id") or "" | |
| break | |
| if latest_sid: | |
| raw_rows = [r for r in raw_rows if r.get("session_id") == latest_sid] | |
| calls = [] | |
| for row in raw_rows: | |
| meta = row.get("meta") or {} | |
| event = row.get("event") or "" | |
| phase = meta.get("phase") or "" | |
| table_names = ", ".join(meta.get("table_names") or []) | |
| if event == "table tml import request started": | |
| calls.append({ | |
| "session": row.get("session_id") or "", | |
| "user": row.get("user_email") or "", | |
| "environment": meta.get("ts_environment") or meta.get("ts_env") or meta.get("ts_url") or "", | |
| "phase": phase, | |
| "tables": table_names, | |
| "request": (row.get("ts") or "")[:19].replace("T", " "), | |
| "response": "", | |
| "status": "", | |
| "seconds": "", | |
| "payload": str(meta.get("payload_bytes") or ""), | |
| "error": "", | |
| }) | |
| elif event in {"table tml import response received", "table tml import exception", "tml import HTTP error"}: | |
| match = None | |
| for call in reversed(calls): | |
| if call["phase"] == phase and call["tables"] == table_names and not call["response"]: | |
| match = call | |
| break | |
| if match is None: | |
| match = { | |
| "session": row.get("session_id") or "", | |
| "user": row.get("user_email") or "", | |
| "environment": meta.get("ts_environment") or meta.get("ts_env") or meta.get("ts_url") or "", | |
| "phase": phase, | |
| "tables": table_names, | |
| "request": "", | |
| "response": "", | |
| "status": "", | |
| "seconds": "", | |
| "payload": str(meta.get("payload_bytes") or ""), | |
| "error": "", | |
| } | |
| calls.append(match) | |
| if not match.get("environment"): | |
| match["environment"] = meta.get("ts_environment") or meta.get("ts_env") or meta.get("ts_url") or "" | |
| match["response"] = (row.get("ts") or "")[:19].replace("T", " ") | |
| match["status"] = "EXCEPTION" if event == "table tml import exception" else str(meta.get("status_code") or "") | |
| match["seconds"] = str(meta.get("elapsed_s") or "") | |
| match["payload"] = str(meta.get("payload_bytes") or match["payload"]) | |
| match["error"] = (row.get("error") or meta.get("response_text") or "")[:500] | |
| calls = calls[-int(limit_choice):] | |
| rows = [ | |
| [ | |
| idx, | |
| call["phase"], | |
| call["tables"], | |
| call["request"], | |
| call["response"] or "open", | |
| call["status"] or "-", | |
| call["seconds"] or "-", | |
| call["payload"], | |
| call["session"], | |
| call["user"], | |
| call.get("environment") or "", | |
| call["error"], | |
| ] | |
| for idx, call in enumerate(calls, start=1) | |
| ] | |
| return rows if rows else [["", "No table calls found", "", "", "", "", "", "", "", "", "", ""]] | |
| except Exception as e: | |
| return [["", f"Error: {e}", "", "", "", "", "", "", "", "", "", ""]] | |
| table_calls_refresh_btn.click( | |
| fn=load_table_tml_calls, | |
| inputs=[table_calls_session_filter, table_calls_email_filter, table_calls_limit], | |
| outputs=[table_calls_display], | |
| ) | |
| interface.load(fn=load_table_tml_calls, inputs=[], outputs=[table_calls_display]) | |
| # Check admin status and toggle admin-only settings visibility | |
| def check_admin_visibility(request: gr.Request): | |
| """Check if logged-in user is admin and toggle settings visibility.""" | |
| username = getattr(request, 'username', None) or '' | |
| is_admin = False | |
| try: | |
| from supabase_client import UserManager | |
| um = UserManager() | |
| if um.enabled and username: | |
| is_admin = um.is_admin(username) | |
| else: | |
| is_admin = True # Local dev mode - show everything | |
| except Exception: | |
| is_admin = True # If check fails, show everything | |
| return ( | |
| gr.update(visible=is_admin), # admin_ai_accordion | |
| gr.update(visible=is_admin), # admin_db_accordion | |
| gr.update(visible=is_admin), # admin_tab | |
| ) | |
| admin_outputs = [ | |
| settings_components['_admin_ai_accordion'], | |
| settings_components['_admin_db_accordion'], | |
| admin_tab, | |
| ] | |
| interface.load( | |
| fn=check_admin_visibility, | |
| inputs=[], | |
| outputs=admin_outputs | |
| ) | |
| def check_password_gate(request: gr.Request = None): | |
| """Show only the forced password-change panel for temp-password users.""" | |
| try: | |
| user_email = require_authenticated_email(request) | |
| from supabase_client import UserManager | |
| um = UserManager() | |
| must_change = um.enabled and um.must_change_password(user_email) | |
| return gr.update(visible=must_change), gr.update(visible=not must_change) | |
| except Exception as e: | |
| print(f"[LOAD] password gate check skipped: {e}") | |
| return gr.update(visible=False), gr.update(visible=True) | |
| def complete_required_password_change(current, new_pw, confirm, request: gr.Request = None): | |
| if not current or not new_pw or not confirm: | |
| return "❌ All fields are required.", gr.update(), gr.update() | |
| if new_pw != confirm: | |
| return "❌ New passwords don't match.", gr.update(), gr.update() | |
| try: | |
| user_email = require_authenticated_email(request) | |
| from supabase_client import UserManager | |
| um = UserManager() | |
| if not um.enabled: | |
| return "⚠️ Supabase not configured — password change unavailable.", gr.update(), gr.update() | |
| if not um.authenticate(user_email, current): | |
| return "❌ Temporary password is incorrect.", gr.update(), gr.update() | |
| if not um.reset_password(user_email, new_pw, must_change_password=False): | |
| return "❌ Password change failed. Please try again or ask an admin to reset it.", gr.update(), gr.update() | |
| um.clear_must_change_password(user_email) | |
| return ( | |
| "✅ Password changed. DemoPrep is ready.", | |
| gr.update(visible=False), | |
| gr.update(visible=True), | |
| ) | |
| except Exception as e: | |
| return f"❌ Error: {e}", gr.update(), gr.update() | |
| interface.load( | |
| fn=check_password_gate, | |
| inputs=[], | |
| outputs=[password_gate, main_tabs], | |
| ) | |
| gate_change_password_btn.click( | |
| fn=complete_required_password_change, | |
| inputs=[gate_current_password, gate_new_password, gate_confirm_password], | |
| outputs=[gate_password_status, password_gate, main_tabs], | |
| ) | |
| # Create update function for tabs | |
| _spotter_waiting = "Story will be generated after liveboard creation." | |
| def update_all_tabs(controller): | |
| if controller is None: | |
| return ( | |
| "", | |
| "-- DDL will appear here after generation", | |
| "Progress will appear here during deployment...", | |
| "Demo pack will be generated after deployment completes.\n\nThis will include:\n- Key insights/outliers\n- Spotter questions to ask\n- Talking points for the demo", | |
| _spotter_waiting, | |
| _spotter_waiting, | |
| ) | |
| live_progress = getattr(controller, 'live_progress_log', []) | |
| live_progress_text = "\n".join(live_progress) if live_progress else "Progress will appear here during deployment..." | |
| demo_pack = getattr(controller, 'demo_pack_content', '') | |
| demo_pack_text = demo_pack if demo_pack else "Demo pack will be generated after deployment completes.\n\nThis will include:\n- Key insights/outliers\n- Spotter questions to ask\n- Talking points for the demo" | |
| matrix_story = getattr(controller, 'spotter_story_matrix', '') or _spotter_waiting | |
| ai_story = getattr(controller, 'spotter_story_ai', '') or _spotter_waiting | |
| return ( | |
| "\n".join(controller.ai_feedback_log), | |
| controller.ddl_code if controller.ddl_code else "-- DDL will appear here after generation", | |
| live_progress_text, | |
| demo_pack_text, | |
| matrix_story, | |
| ai_story, | |
| ) | |
| # Wire up tab updates on chat interactions | |
| chat_components['chatbot'].change( | |
| fn=update_all_tabs, | |
| inputs=[chat_controller_state], | |
| outputs=[ai_feedback_display, ddl_display, live_progress_display, demo_pack_display, | |
| spotter_matrix_display, spotter_ai_display] | |
| ) | |
| # Load settings from Supabase on startup (uses SETTINGS_SCHEMA) | |
| def load_settings_on_startup(request: gr.Request = None): | |
| """Load saved settings from Supabase - uses schema-driven helper""" | |
| try: | |
| user_email = require_authenticated_email(request) | |
| print(f"[LOAD] load_settings_on_startup for {user_email}") | |
| settings = load_gradio_settings(user_email) | |
| result = load_settings_values(settings, user_email) | |
| print(f"[LOAD] load_settings_on_startup OK — {len(result)} values") | |
| return result | |
| except Exception as e: | |
| print(f"[LOAD] No authenticated user; using local default settings: {e}") | |
| # Return schema-length list of defaults so Gradio doesn't crash | |
| return [default for _, _, default, _ in SETTINGS_SCHEMA] | |
| def load_session_state_on_startup(request: gr.Request = None): | |
| """Initialize chat/session state from the same authenticated user settings.""" | |
| try: | |
| user_email = require_authenticated_email(request) | |
| print(f"[LOAD] load_session_state_on_startup for {user_email}") | |
| except Exception as e: | |
| print(f"[LOAD] No authenticated user; using local default session state: {e}") | |
| _ts_choices = get_ts_environments() | |
| return ( | |
| "initialization", DEFAULT_LLM_MODEL, "", "", "", | |
| gr.update(value=DEFAULT_LLM_MODEL), gr.update(value=""), | |
| gr.update(value="Small"), gr.update(value="USA Only"), | |
| gr.update(value=""), gr.update(value="Regular Case"), | |
| gr.update(value=""), gr.update(value=""), "", | |
| gr.update(value=_ts_choices[0]) if _ts_choices else gr.update(), | |
| gr.update(), gr.update(), gr.update(), gr.update(), | |
| ) | |
| try: | |
| settings = load_gradio_settings(user_email) | |
| model = (str(settings.get("default_llm", "")).strip() or DEFAULT_LLM_MODEL) | |
| if model not in UI_MODEL_CHOICES: | |
| model = DEFAULT_LLM_MODEL | |
| liveboard_name = str(settings.get("liveboard_name", "")).strip() | |
| # Data Size — prefer new field, fall back to legacy fact_table_size | |
| saved_size = str(settings.get("default_data_size", "")).strip() | |
| if not saved_size: | |
| _ft_to_size = {"1000": "Small", "10000": "Medium"} | |
| saved_size = _ft_to_size.get(str(settings.get("fact_table_size", "1000")), "Small") | |
| saved_geo = str(settings.get("geo_scope", "USA Only")).strip() or "USA Only" | |
| saved_tag = str(settings.get("tag_name", "")).strip() | |
| saved_col_naming = str(settings.get("column_naming_style", "Regular Case")).strip() or "Regular Case" | |
| saved_obj_prefix = str(settings.get("object_naming_prefix", "")).strip() | |
| saved_share_with = str(settings.get("share_with", "")).strip() | |
| # TS Environment | |
| _ts_choices = get_ts_environments() | |
| saved_ts_env = str(settings.get("default_ts_env", "")).strip() | |
| if saved_ts_env not in _ts_choices: | |
| saved_ts_env = _ts_choices[0] if _ts_choices else "" | |
| # Optional run input defaults | |
| use_defaults = settings.get("use_default_inputs", False) | |
| if isinstance(use_defaults, str): | |
| use_defaults = use_defaults.lower() == "true" | |
| def_vertical = str(settings.get("default_vertical", "")).strip() | |
| def_line = str(settings.get("default_line", "")).strip() | |
| def_function = str(settings.get("default_function", "")).strip() | |
| def_company_url = str(settings.get("default_company_url", "")).strip() | |
| # Use legacy default_company_url / default_use_case for state | |
| company = def_company_url if use_defaults else "" | |
| use_case = str(settings.get("default_use_case", "")).strip() | |
| print(f"[LOAD] load_session_state_on_startup OK — model={model}, ts_env={saved_ts_env}, use_defaults={use_defaults}") | |
| return ( | |
| "initialization", | |
| model, | |
| company, | |
| use_case, | |
| liveboard_name, | |
| gr.update(value=model), | |
| gr.update(value=liveboard_name), | |
| gr.update(value=saved_size), | |
| gr.update(value=saved_geo), | |
| gr.update(value=saved_tag), | |
| gr.update(value=saved_col_naming), | |
| gr.update(value=saved_obj_prefix), | |
| gr.update(value=saved_share_with), | |
| "", | |
| gr.update(value=saved_ts_env) if saved_ts_env else gr.update(), | |
| gr.update(value=def_vertical) if (use_defaults and def_vertical) else gr.update(), | |
| gr.update(value=def_line) if (use_defaults and def_line) else gr.update(), | |
| gr.update(value=def_function) if (use_defaults and def_function) else gr.update(), | |
| gr.update(value=def_company_url) if (use_defaults and def_company_url) else gr.update(), | |
| ) | |
| except Exception as e: | |
| import traceback | |
| print(f"[LOAD ERROR] load_session_state_on_startup failed: {e}\n{traceback.format_exc()}") | |
| return ( | |
| "initialization", DEFAULT_LLM_MODEL, "", "", "", | |
| gr.update(value=DEFAULT_LLM_MODEL), gr.update(value=""), | |
| gr.update(value="Small"), gr.update(value="USA Only"), | |
| gr.update(value=""), gr.update(value="Regular Case"), | |
| gr.update(value=""), gr.update(value=""), "", | |
| gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), | |
| ) | |
| # Wire up load handler - outputs follow SETTINGS_SCHEMA order | |
| interface.load( | |
| fn=load_settings_on_startup, | |
| inputs=[], | |
| outputs=[settings_components[key] for key, _, _, _ in SETTINGS_SCHEMA] | |
| ) | |
| interface.load( | |
| fn=load_session_state_on_startup, | |
| inputs=[], | |
| outputs=[ | |
| current_stage, | |
| current_model, | |
| current_company, | |
| current_usecase, | |
| current_liveboard_name, | |
| chat_components["model_dropdown"], | |
| chat_components["liveboard_name_input"], | |
| chat_components["data_size_dropdown"], | |
| chat_components["geo_scope_dropdown"], | |
| chat_components["tag_name_input"], | |
| chat_components["column_naming_dropdown"], | |
| chat_components["object_prefix_input"], | |
| chat_components["share_with_input"], | |
| chat_components["msg"], | |
| chat_components["ts_env_dropdown"], | |
| chat_components["vertical_dd"], | |
| chat_components["line_dd"], | |
| chat_components["function_dd"], | |
| chat_components["url_input"], | |
| ] | |
| ) | |
| return interface | |
| def create_chat_tab(chat_controller_state, settings, current_stage, current_model, current_company, current_usecase, | |
| ai_feedback_state=None, ddl_code_state=None, population_code_state=None): | |
| """Create the main chat interface tab | |
| IMPORTANT: chat_controller_state is a gr.State that holds each session's | |
| ChatDemoInterface instance. This enables multi-user concurrent access. | |
| """ | |
| # Create initial welcome message (before any session exists) | |
| initial_controller = ChatDemoInterface() | |
| initial_welcome = initial_controller.format_welcome_message( | |
| settings['company'], | |
| settings['use_case'] | |
| ) | |
| with gr.Row(): | |
| # Left column - App form + Chat | |
| with gr.Column(scale=5): | |
| with gr.Tabs(): | |
| # ── App tab: clean form, no chatbot ────────────────────── | |
| with gr.Tab("App"): | |
| _vertical_choices = list(VERTICAL_LINES.keys()) | |
| _first_vertical = _vertical_choices[0] | |
| _first_lines = VERTICAL_LINES[_first_vertical] | |
| vertical_dd = gr.Dropdown( | |
| label="Vertical", | |
| choices=_vertical_choices, | |
| value=_first_vertical, | |
| interactive=True, | |
| ) | |
| line_dd = gr.Dropdown( | |
| label="Line", | |
| choices=_first_lines, | |
| value=_first_lines[0] if _first_lines else None, | |
| interactive=True, | |
| ) | |
| function_dd = gr.Dropdown( | |
| label="Function", | |
| choices=DEMO_FUNCTIONS, | |
| value=DEMO_FUNCTIONS[0], | |
| interactive=True, | |
| ) | |
| with gr.Row(): | |
| url_input = gr.Textbox( | |
| label="Company URL", | |
| placeholder="e.g. Amazon.com", | |
| lines=1, | |
| scale=4, | |
| interactive=True, | |
| ) | |
| use_url_cb = gr.Checkbox( | |
| label="Use URL", | |
| value=True, | |
| scale=1, | |
| interactive=True, | |
| ) | |
| additional_info_input = gr.Textbox( | |
| label="Context", | |
| placeholder="Any extra context for the demo...", | |
| lines=2, | |
| interactive=True, | |
| ) | |
| go_btn = gr.Button("→ GO", variant="primary") | |
| # ── Chat tab: full conversational view ─────────────────── | |
| with gr.Tab("Chat"): | |
| welcome_md = gr.Markdown(value=initial_welcome, visible=True) | |
| chatbot = gr.Chatbot( | |
| value=[], | |
| height=400, | |
| label="Demo Builder Assistant", | |
| show_label=False, | |
| avatar_images=None, | |
| type='tuples', | |
| elem_id="main-chatbot", | |
| visible=False, | |
| ) | |
| with gr.Row(): | |
| msg = gr.Textbox( | |
| label="Your message", | |
| value="", | |
| placeholder="e.g. Amazon.com Retail Sales — or continue a conversation here", | |
| lines=1, | |
| max_lines=1, | |
| scale=5, | |
| show_label=False, | |
| interactive=True | |
| ) | |
| send_btn = gr.Button("Send", variant="primary", scale=1) | |
| # Right column - Status & Settings | |
| with gr.Column(scale=2): | |
| # TS Environment selector (always visible) | |
| ts_env_choices = get_ts_environments() | |
| _saved_ts_env = str(settings.get('default_ts_env', '')).strip() | |
| _init_ts_env = _saved_ts_env if _saved_ts_env in ts_env_choices else (ts_env_choices[0] if ts_env_choices else None) | |
| ts_env_dropdown = gr.Dropdown( | |
| label="TS Environment", | |
| choices=ts_env_choices, | |
| value=_init_ts_env, | |
| interactive=True, | |
| ) | |
| # AI Model selector (always visible) | |
| model_dropdown = gr.Dropdown( | |
| label="AI Model", | |
| choices=list(UI_MODEL_CHOICES), | |
| value=settings['model'], | |
| interactive=True, | |
| allow_custom_value=False, | |
| info="Temporary model failover active: using claude-sonnet-4-6." | |
| ) | |
| # Run-time settings — collapsible, all pipeline knobs here | |
| settings_accordion = gr.Accordion("⚙️ Settings", open=False) | |
| with settings_accordion: | |
| liveboard_name_input = gr.Textbox( | |
| label="Liveboard Name", | |
| placeholder="Auto from company URL if blank", | |
| value=settings.get('liveboard_name', ''), | |
| lines=1, | |
| interactive=True, | |
| ) | |
| # Seed Data Size from default_data_size; fall back to fact_table_size for old records | |
| _init_data_size = str(settings.get('default_data_size', '')).strip() | |
| if not _init_data_size: | |
| _ft_to_size = {"1000": "Small", "10000": "Medium"} | |
| _init_data_size = _ft_to_size.get(str(settings.get('fact_table_size', '1000')), "Small") | |
| data_size_dropdown = gr.Dropdown( | |
| label="Data Size", | |
| choices=["Small", "Medium"], | |
| value=_init_data_size, | |
| interactive=True, | |
| info="Small 1k rows · Medium 10k rows", | |
| ) | |
| geo_scope_dropdown = gr.Dropdown( | |
| label="Geographic Scope", | |
| choices=["USA Only", "International"], | |
| value=settings.get('geo_scope', 'USA Only'), | |
| interactive=True, | |
| ) | |
| tag_name_input = gr.Textbox( | |
| label="Tag Name", | |
| placeholder="e.g. Sales_Demo (blank = no tag)", | |
| value=settings.get('tag_name', ''), | |
| lines=1, | |
| interactive=True, | |
| ) | |
| column_naming_dropdown = gr.Dropdown( | |
| label="Column Naming Style", | |
| choices=["Regular Case", "snake_case", "camelCase", "PascalCase", "UPPER_CASE", "original"], | |
| value=settings.get('column_naming_style', 'Regular Case'), | |
| interactive=True, | |
| ) | |
| object_prefix_input = gr.Textbox( | |
| label="Object Naming Prefix", | |
| placeholder="e.g. ACME_ (blank = none)", | |
| value=settings.get('object_naming_prefix', ''), | |
| lines=1, | |
| interactive=True, | |
| ) | |
| share_with_input = gr.Textbox( | |
| label="Share With", | |
| placeholder="user@company.com or group-name (blank = no share)", | |
| value=settings.get('share_with', ''), | |
| lines=1, | |
| interactive=True, | |
| ) | |
| gr.Markdown("### 📈 Progress") | |
| # Stage order used to determine done/current/upcoming | |
| _STAGE_ORDER = [ | |
| 'initialization', 'awaiting_context', 'research', | |
| 'create_ddl', 'deploy', 'populate', | |
| 'thoughtspot', 'outlier_adjustment', 'complete', | |
| ] | |
| # Each display step: (label, [stage keys that map to it]) | |
| _PROGRESS_STEPS = [ | |
| ('Init', ['initialization']), | |
| ('Research', ['awaiting_context', 'research']), | |
| ('DDL', ['create_ddl']), | |
| ('Data', ['deploy', 'populate']), | |
| ('ThoughtSpot', ['thoughtspot']), | |
| ('Data Adjuster',['outlier_adjustment']), | |
| ('Complete', ['complete']), | |
| ] | |
| def get_progress_html(stage): | |
| """Generate progress HTML showing done/current/upcoming states.""" | |
| try: | |
| current_idx = _STAGE_ORDER.index(stage) | |
| except ValueError: | |
| current_idx = 0 | |
| # Find which display step is current | |
| current_step = None | |
| for step_label, step_keys in _PROGRESS_STEPS: | |
| for k in step_keys: | |
| if stage == k: | |
| current_step = step_label | |
| break | |
| # Build ordered list of display steps that are reached | |
| reached = set() | |
| for step_label, step_keys in _PROGRESS_STEPS: | |
| for k in step_keys: | |
| try: | |
| if _STAGE_ORDER.index(k) <= current_idx: | |
| reached.add(step_label) | |
| except ValueError: | |
| pass | |
| html = "<div style='padding:8px 4px; font-size:13px; line-height:1.8;'>" | |
| for step_label, _ in _PROGRESS_STEPS: | |
| # Skip Data Adjuster unless it's active | |
| if step_label == 'Data Adjuster' and step_label not in reached: | |
| continue | |
| if step_label == current_step: | |
| html += (f"<div style='margin:3px 0; color:#3b82f6; font-weight:bold;'>" | |
| f"▶ {step_label}</div>") | |
| elif step_label in reached: | |
| html += (f"<div style='margin:3px 0; color:#22c55e;'>" | |
| f"✓ {step_label}</div>") | |
| else: | |
| html += (f"<div style='margin:3px 0; color:#9ca3af;'>" | |
| f"○ {step_label}</div>") | |
| html += "</div>" | |
| return html | |
| progress_html = gr.HTML(get_progress_html('initialization')) | |
| # Phase log stream — scrolling status updates from controller.phase_log | |
| phase_log_display = gr.Textbox( | |
| label="Pipeline Status", | |
| value="", | |
| lines=6, | |
| max_lines=6, | |
| interactive=False, | |
| placeholder="Pipeline status will appear here when the GO button is pressed...", | |
| elem_classes=["phase-log-stream"], | |
| ) | |
| deployment_links_panel = gr.HTML(value="", visible=False) | |
| # Timer polls controller.phase_log every 2 seconds (activated when GO is pressed) | |
| phase_log_timer = gr.Timer(value=2, active=True) | |
| # Event handlers - each creates/uses session-specific controller | |
| def send_message(controller, message, history, stage, model, company, usecase, env_label=None, liveboard_name_ui=None, request: gr.Request = None): | |
| """Handle sending a message - creates controller if needed""" | |
| import traceback | |
| username = getattr(request, 'username', None) if request else None | |
| if controller is None: | |
| controller = ChatDemoInterface(user_email=username) | |
| print(f"[SESSION] Created new ChatDemoInterface for {username or 'anonymous'}") | |
| elif username: | |
| controller.user_email = username | |
| # Apply the selected TS environment on every message, not just on the | |
| # first — so a changed env dropdown re-targets the deploy on a reused session. | |
| if env_label: | |
| _url = get_ts_env_url(env_label) | |
| _key_value = get_ts_env_auth_key(env_label) | |
| if _url: | |
| controller.settings['thoughtspot_url'] = _url | |
| if _key_value: | |
| controller.settings['thoughtspot_trusted_auth_key'] = _key_value | |
| # Always use the current UI values — take priority over DB-loaded defaults | |
| if liveboard_name_ui is not None: | |
| controller.settings['liveboard_name'] = liveboard_name_ui | |
| if model: | |
| controller.settings['model'] = model | |
| hide_welcome = gr.update(visible=False) | |
| try: | |
| for result in controller.process_chat_message( | |
| message, history, stage, model, company, usecase | |
| ): | |
| new_stage = result[1] if len(result) > 1 else stage | |
| progress = get_progress_html(new_stage) | |
| chatbot_update = gr.update(value=result[0], visible=True) | |
| yield (controller, chatbot_update) + result[1:] + ( | |
| progress, | |
| hide_welcome, | |
| controller.render_deployment_completion_html(), | |
| ) | |
| except Exception as e: | |
| err_tb = traceback.format_exc() | |
| print(f"[ERROR] send_message unhandled exception:\n{err_tb}") | |
| err_msg = ( | |
| f"❌ **An unexpected error occurred**\n\n" | |
| f"`{type(e).__name__}: {e}`\n\n" | |
| f"The pipeline has been interrupted. You can try again or start a new session." | |
| ) | |
| history = history or [] | |
| history.append((message, err_msg)) | |
| completion_update = controller.render_deployment_completion_html() if controller else gr.update(value="", visible=False) | |
| yield ( | |
| controller, | |
| gr.update(value=history, visible=True), | |
| stage, | |
| model, | |
| company, | |
| usecase, | |
| "", | |
| get_progress_html(stage), | |
| hide_welcome, | |
| completion_update, | |
| ) | |
| # Wire up send button and enter key | |
| _send_inputs = [chat_controller_state, msg, chatbot, current_stage, current_model, current_company, current_usecase, ts_env_dropdown, liveboard_name_input] | |
| _send_outputs = [ | |
| chat_controller_state, chatbot, current_stage, current_model, | |
| current_company, current_usecase, msg, progress_html, welcome_md, | |
| deployment_links_panel, | |
| ] | |
| msg.submit(fn=send_message, inputs=_send_inputs, outputs=_send_outputs) | |
| send_btn.click(fn=send_message, inputs=_send_inputs, outputs=_send_outputs) | |
| # App tab: vertical → line + function cascade | |
| def update_line_on_vertical(vertical): | |
| lines = VERTICAL_LINES.get(vertical, []) | |
| if vertical == "* CUSTOM *" or not lines: | |
| return ( | |
| gr.Dropdown(choices=["— not used —"], value="— not used —", interactive=False, label="Line (n/a for custom)"), | |
| gr.Dropdown(choices=["— not used —"], value="— not used —", interactive=False, label="Function (n/a for custom)"), | |
| gr.Textbox(label="Context *", placeholder="Describe your use case, industry, and key metrics...", interactive=True), | |
| ) | |
| return ( | |
| gr.Dropdown(choices=lines, value=lines[0], interactive=True, label="Line"), | |
| gr.Dropdown(choices=DEMO_FUNCTIONS, value=DEMO_FUNCTIONS[0], interactive=True, label="Function"), | |
| gr.Textbox(label="Context", placeholder="Any extra context for the demo...", interactive=True), | |
| ) | |
| vertical_dd.change( | |
| fn=update_line_on_vertical, | |
| inputs=[vertical_dd], | |
| outputs=[line_dd, function_dd, additional_info_input] | |
| ) | |
| # Defined tab: GO button handler | |
| def defined_go(controller, vertical, line, function, url, use_url, additional_info, | |
| history, stage, model, company, usecase, env_label, lb_name, | |
| data_size, geo_scope, tag_name, col_naming, obj_prefix, share_with, | |
| request: gr.Request = None): | |
| import traceback | |
| function_clean = (function or "").strip() | |
| username = getattr(request, 'username', None) if request else None | |
| if controller is None: | |
| controller = ChatDemoInterface(user_email=username) | |
| print(f"[SESSION] Created new ChatDemoInterface for {username or 'anonymous'}") | |
| elif username: | |
| controller.user_email = username | |
| # Apply the selected TS environment on EVERY GO, not only when a new | |
| # controller is created. A reused session (e.g. the e2e runs all 8 demos | |
| # on one login) otherwise keeps the first/default env and silently ignores | |
| # a changed dropdown — deploying to the wrong ThoughtSpot instance. | |
| if env_label: | |
| _url = get_ts_env_url(env_label) | |
| _key_value = get_ts_env_auth_key(env_label) | |
| if _url: | |
| controller.settings['thoughtspot_url'] = _url | |
| if _key_value: | |
| controller.settings['thoughtspot_trusted_auth_key'] = _key_value | |
| if lb_name is not None: | |
| controller.settings['liveboard_name'] = lb_name | |
| if model: | |
| controller.settings['model'] = model | |
| if data_size: | |
| _size_map = { | |
| "Small": ("1000", "50"), | |
| "Medium": ("10000", "500"), | |
| } | |
| _ft, _dt = _size_map.get(data_size, ("1000", "100")) | |
| controller.settings['fact_table_size'] = _ft | |
| controller.settings['dim_table_size'] = _dt | |
| if geo_scope: | |
| controller.settings['geo_scope'] = geo_scope | |
| if tag_name is not None: | |
| controller.settings['tag_name'] = tag_name | |
| if col_naming: | |
| controller.settings['column_naming_style'] = col_naming | |
| if obj_prefix is not None: | |
| controller.settings['object_naming_prefix'] = obj_prefix | |
| if share_with is not None: | |
| controller.settings['share_with'] = share_with | |
| # Capture the raw GO form payload so the run logger records exactly | |
| # what was submitted (logged secret-redacted via _snapshot_run_payload) | |
| controller._run_payload = { | |
| 'vertical': vertical, | |
| 'line': line, | |
| 'function': function_clean, | |
| 'url': url, | |
| 'use_url': use_url, | |
| 'additional_info': additional_info, | |
| 'model': model, | |
| 'ts_environment': env_label, | |
| 'liveboard_name': lb_name, | |
| 'data_size': data_size, | |
| 'geo_scope': geo_scope, | |
| 'tag_name': tag_name, | |
| 'column_naming': col_naming, | |
| 'object_prefix': obj_prefix, | |
| 'share_with': share_with, | |
| } | |
| # Derive company name from URL or line+function label | |
| if use_url and url.strip(): | |
| raw_company = url.strip() | |
| else: | |
| raw_company = f"{line or vertical} Demo" | |
| is_custom = (vertical == "* CUSTOM *") | |
| hide_welcome = gr.update(visible=False) | |
| password_block = controller._temporary_password_block_message() | |
| if password_block: | |
| history = history or [] | |
| history.append(("GO", password_block)) | |
| controller.phase_log = ["Password change required before starting pipeline."] | |
| yield ( | |
| controller, | |
| gr.update(value=history, visible=True), | |
| stage, | |
| model, | |
| company, | |
| usecase, | |
| get_progress_html(stage), | |
| hide_welcome, | |
| gr.update(open=False), | |
| gr.update(value="", visible=False), | |
| ) | |
| return | |
| if is_custom: | |
| # Custom: context field drives the use case | |
| use_case_str = (additional_info or "").strip() or "Custom Demo" | |
| controller.vertical = None | |
| controller.line = None | |
| controller.function = None | |
| controller.use_case_config = get_use_case_config("Generic", "Generic") | |
| controller.is_generic_use_case = True | |
| controller.generic_use_case_context = use_case_str | |
| else: | |
| # Use line + function as the use case (vertical is context only) | |
| use_case_str = f"{line} {function_clean}" if line else f"{vertical} {function_clean}" | |
| controller.vertical = vertical | |
| controller.line = line | |
| controller.function = function_clean | |
| controller.use_case_config = get_use_case_config( | |
| line or vertical or "Generic", | |
| function_clean or "Generic", | |
| vertical_fallback=vertical if line else None, | |
| ) | |
| is_known = (line and function_clean | |
| and not controller.use_case_config.get('is_generic')) | |
| controller.is_generic_use_case = not is_known | |
| controller.generic_use_case_context = additional_info.strip() if additional_info else "" | |
| controller.pending_generic_company = raw_company | |
| # For Custom, the display name IS the context text; for standard, use config name | |
| if is_custom: | |
| use_case_display = use_case_str | |
| else: | |
| use_case_display = controller.use_case_config.get('use_case_name', use_case_str) | |
| controller.pending_generic_use_case = use_case_display | |
| # Clear phase log for a fresh run | |
| controller.phase_log = [f"→ GO received — preparing {raw_company} · {use_case_display}"] | |
| controller.clear_deployment_completion() | |
| # Tag the run source so session logging can record which interface was used | |
| controller._run_source = 'app_custom' if is_custom else 'app_defined' | |
| # Drive straight to awaiting_context → 'proceed' to skip confirmation dialog | |
| # For Custom, pass the context as the message so process_chat_message stores it; | |
| # for standard, use 'proceed' (context already set above). | |
| proceed_msg = use_case_str if is_custom else "proceed" | |
| history = history or [] | |
| try: | |
| for result in controller.process_chat_message( | |
| proceed_msg, history, 'awaiting_context', model, | |
| raw_company, use_case_display | |
| ): | |
| new_stage = result[1] if len(result) > 1 else stage | |
| progress = get_progress_html(new_stage) | |
| chatbot_update = gr.update(value=result[0], visible=True) | |
| yield (controller, chatbot_update) + result[1:5] + ( | |
| progress, | |
| hide_welcome, | |
| gr.update(open=False), | |
| controller.render_deployment_completion_html(), | |
| ) | |
| except Exception as e: | |
| err_tb = traceback.format_exc() | |
| print(f"[ERROR] defined_go unhandled exception:\n{err_tb}") | |
| controller.phase_log.append(f"❌ GO failed before pipeline start: {type(e).__name__}: {e}") | |
| err_msg = ( | |
| f"❌ **An unexpected error occurred**\n\n" | |
| f"`{type(e).__name__}: {e}`\n\n" | |
| f"The pipeline has been interrupted. You can try again or start a new session." | |
| ) | |
| history.append((f"GO: {use_case_str}", err_msg)) | |
| yield ( | |
| controller, | |
| gr.update(value=history, visible=True), | |
| stage, | |
| model, | |
| company, | |
| usecase, | |
| get_progress_html(stage), | |
| hide_welcome, | |
| gr.update(open=False), | |
| controller.render_deployment_completion_html(), | |
| ) | |
| _go_inputs = [ | |
| chat_controller_state, vertical_dd, line_dd, function_dd, url_input, use_url_cb, | |
| additional_info_input, chatbot, current_stage, current_model, | |
| current_company, current_usecase, ts_env_dropdown, liveboard_name_input, | |
| data_size_dropdown, geo_scope_dropdown, tag_name_input, | |
| column_naming_dropdown, object_prefix_input, share_with_input, | |
| ] | |
| _go_outputs = [ | |
| chat_controller_state, chatbot, current_stage, current_model, | |
| current_company, current_usecase, progress_html, welcome_md, | |
| settings_accordion, deployment_links_panel, | |
| ] | |
| go_btn.click(fn=defined_go, inputs=_go_inputs, outputs=_go_outputs) | |
| # Phase log timer — poll controller.phase_log and render as newline-joined text | |
| def refresh_phase_log(controller): | |
| if controller is None: | |
| return gr.update() | |
| log = getattr(controller, 'phase_log', []) | |
| return "\n".join(log) if log else "" | |
| phase_log_timer.tick( | |
| fn=refresh_phase_log, | |
| inputs=[chat_controller_state], | |
| outputs=[phase_log_display], | |
| trigger_mode="always_last", | |
| ) | |
| # Model dropdown change | |
| def update_model(new_model, controller, history): | |
| if controller is not None: | |
| controller.settings['model'] = new_model | |
| return new_model, history | |
| model_dropdown.change( | |
| fn=update_model, | |
| inputs=[model_dropdown, chat_controller_state, chatbot], | |
| outputs=[current_model, chatbot] | |
| ) | |
| # Liveboard name blur — update controller settings when user leaves the field. | |
| # Using blur (not change) avoids per-keystroke queue events that cause the | |
| # Gradio progress spinner/timer to appear on this component while a deploy runs. | |
| # No output needed — the value is already in the textbox; we only update the dict. | |
| def update_liveboard_name(name, controller): | |
| if controller is not None: | |
| controller.settings['liveboard_name'] = name | |
| liveboard_name_input.blur( | |
| fn=update_liveboard_name, | |
| inputs=[liveboard_name_input, chat_controller_state], | |
| outputs=[] | |
| ) | |
| # TS environment change — update controller settings in real-time | |
| def update_ts_env(label, controller): | |
| url = get_ts_env_url(label) | |
| auth_key_value = get_ts_env_auth_key(label) | |
| if controller is not None: | |
| if url: | |
| controller.settings['thoughtspot_url'] = url | |
| if auth_key_value: | |
| controller.settings['thoughtspot_trusted_auth_key'] = auth_key_value | |
| return label | |
| ts_env_dropdown.change( | |
| fn=update_ts_env, | |
| inputs=[ts_env_dropdown, chat_controller_state], | |
| outputs=[] | |
| ) | |
| # Return components for external access | |
| return { | |
| 'chatbot': chatbot, | |
| 'msg': msg, | |
| 'model_dropdown': model_dropdown, | |
| 'send_btn': send_btn, | |
| 'ts_env_dropdown': ts_env_dropdown, | |
| 'liveboard_name_input': liveboard_name_input, | |
| 'data_size_dropdown': data_size_dropdown, | |
| 'geo_scope_dropdown': geo_scope_dropdown, | |
| 'tag_name_input': tag_name_input, | |
| 'column_naming_dropdown': column_naming_dropdown, | |
| 'object_prefix_input': object_prefix_input, | |
| 'share_with_input': share_with_input, | |
| 'progress_html': progress_html, | |
| 'phase_log_display': phase_log_display, | |
| 'phase_log_timer': phase_log_timer, | |
| # App tab form inputs — seeded from default run inputs if enabled | |
| 'vertical_dd': vertical_dd, | |
| 'line_dd': line_dd, | |
| 'function_dd': function_dd, | |
| 'url_input': url_input, | |
| } | |
| def create_matrix_tab(interface): | |
| """Matrix editor tab — view and edit Vertical × Function cell configs.""" | |
| def _cell_data(vertical, function): | |
| """Compute all display values for a given cell.""" | |
| config = get_use_case_config(vertical, function) | |
| has_override = (vertical, function) in MATRIX_OVERRIDES | |
| # Coverage line | |
| if has_override: | |
| coverage = "✅ **Override** — enriched persona, custom questions, tuned story controls" | |
| else: | |
| coverage = "🔵 **Base merge** — vertical + function defaults combined" | |
| # Persona / problem (overrides only) | |
| persona = config.get('target_persona', '') | |
| problem = config.get('business_problem', '') | |
| persona_md = "" | |
| if persona: | |
| persona_md += f"**Target persona:** {persona} \n" | |
| if problem: | |
| persona_md += f"**Business problem:** {problem}" | |
| # KPIs | |
| kpis = config.get('kpis', []) | |
| kpi_defs = config.get('kpi_definitions', {}) | |
| lines = [] | |
| for k in kpis: | |
| d = kpi_defs.get(k, '') | |
| lines.append(f"**{k}** — {d}" if d else f"**{k}**") | |
| kpi_md = " \n".join(lines) if lines else "_None defined_" | |
| # Liveboard questions → dataframe rows | |
| questions = config.get('liveboard_questions', []) | |
| rows = [] | |
| for q in questions: | |
| spotter = ", ".join(q.get('spotter_qs', [])) | |
| rows.append([ | |
| q.get('title', ''), | |
| q.get('viz_type', ''), | |
| q.get('required', False), | |
| q.get('viz_question', ''), | |
| q.get('insight', ''), | |
| spotter, | |
| ]) | |
| # Story controls as formatted text | |
| sc = config.get('story_controls', {}) | |
| sc_lines = [] | |
| for k, v in sc.items(): | |
| if isinstance(v, dict): | |
| sc_lines.append(f"**{k}:** {v}") | |
| elif isinstance(v, list): | |
| sc_lines.append(f"**{k}:** {', '.join(str(x) for x in v)}") | |
| elif isinstance(v, float): | |
| sc_lines.append(f"**{k}:** {v:.4g}") | |
| else: | |
| sc_lines.append(f"**{k}:** {v}") | |
| sc_md = " \n".join(sc_lines) if sc_lines else "_None_" | |
| return coverage, persona_md, kpi_md, rows, sc_md | |
| # --- Initial values --- | |
| init_v = list(VERTICALS.keys())[0] | |
| init_f = list(FUNCTIONS.keys())[0] | |
| init_cov, init_persona, init_kpi, init_rows, init_sc = _cell_data(init_v, init_f) | |
| gr.Markdown("## 🧩 Matrix Editor") | |
| gr.Markdown( | |
| "The matrix defines what gets built for every Vertical × Function combination — " | |
| "KPIs, visualizations, story controls, and persona. \n" | |
| "Changes here are **in-session only** for now; persistence via Supabase is coming next." | |
| ) | |
| # --- Selectors --- | |
| with gr.Row(): | |
| m_vertical = gr.Dropdown( | |
| label="Vertical", | |
| choices=list(VERTICALS.keys()), | |
| value=init_v, | |
| interactive=True, | |
| scale=1, | |
| ) | |
| m_function = gr.Dropdown( | |
| label="Function", | |
| choices=list(FUNCTIONS.keys()), | |
| value=init_f, | |
| interactive=True, | |
| scale=1, | |
| ) | |
| coverage_md = gr.Markdown(value=init_cov) | |
| persona_md = gr.Markdown(value=init_persona) | |
| gr.Markdown("---") | |
| # --- KPIs --- | |
| with gr.Accordion("📊 KPIs", open=True): | |
| kpi_md = gr.Markdown(value=init_kpi) | |
| # --- Liveboard Questions --- | |
| with gr.Accordion("📋 Liveboard Questions", open=True): | |
| gr.Markdown( | |
| "*Edit cells directly. Add rows with the + button. " | |
| "Changes affect this session until persistence is wired up.*" | |
| ) | |
| questions_df = gr.Dataframe( | |
| value=init_rows, | |
| headers=["Title", "Viz Type", "Required", "Question", "Insight", "Spotter Questions"], | |
| datatype=["str", "str", "bool", "str", "str", "str"], | |
| interactive=True, | |
| row_count=(len(init_rows), "dynamic"), | |
| col_count=(6, "fixed"), | |
| wrap=True, | |
| ) | |
| # --- Story Controls --- | |
| with gr.Accordion("⚙️ Story Controls", open=False): | |
| sc_md = gr.Markdown(value=init_sc) | |
| # --- Wire dropdowns --- | |
| def on_cell_change(vertical, function): | |
| cov, persona, kpi, rows, sc = _cell_data(vertical, function) | |
| return cov, persona, kpi, rows, sc | |
| _outputs = [coverage_md, persona_md, kpi_md, questions_df, sc_md] | |
| m_vertical.change(fn=on_cell_change, inputs=[m_vertical, m_function], outputs=_outputs) | |
| m_function.change(fn=on_cell_change, inputs=[m_vertical, m_function], outputs=_outputs) | |
| # Populate on tab load | |
| interface.load( | |
| fn=on_cell_change, | |
| inputs=[m_vertical, m_function], | |
| outputs=_outputs, | |
| ) | |
| return {'vertical': m_vertical, 'function': m_function, 'questions_df': questions_df} | |
| def create_settings_tab(): | |
| """Create the settings configuration tab - returns components for loading | |
| Uses module-level SETTINGS_SCHEMA for consistency with load/save functions. | |
| """ | |
| gr.Markdown("## ⚙️ Configuration Settings") | |
| gr.Markdown("Configure your demo builder preferences") | |
| # ── Default Settings — mirrors App tab right panel ──────────────────────── | |
| gr.Markdown("### ⭐ Default Settings") | |
| gr.Markdown("*These seed the App tab right panel on every page load.*") | |
| default_ai_model = gr.Dropdown( | |
| label="Default AI Model", | |
| choices=list(UI_MODEL_CHOICES), | |
| value=DEFAULT_LLM_MODEL, | |
| info="Temporary model failover active: using claude-sonnet-4-6.", | |
| allow_custom_value=False, | |
| ) | |
| _ts_env_choices = get_ts_environments() | |
| default_ts_env = gr.Dropdown( | |
| label="Default TS Environment", | |
| choices=_ts_env_choices, | |
| value=_ts_env_choices[0] if _ts_env_choices else None, | |
| info="Which ThoughtSpot environment to select on load", | |
| allow_custom_value=True, | |
| ) | |
| liveboard_name = gr.Textbox( | |
| label="Default Liveboard Name", | |
| placeholder="Auto from company URL if blank", | |
| value="", | |
| info="Leave blank to auto-derive from company URL", | |
| ) | |
| default_data_size = gr.Dropdown( | |
| label="Default Data Size", | |
| choices=["Small", "Medium"], | |
| value="Small", | |
| info="Small = 1k rows · Medium = 10k rows", | |
| ) | |
| geo_scope = gr.Dropdown( | |
| label="Default Geographic Scope", | |
| choices=["USA Only", "International"], | |
| value="USA Only", | |
| info="USA Only = US states/cities/USD · International = global", | |
| ) | |
| tag_name = gr.Textbox( | |
| label="Default Tag Name", | |
| placeholder="e.g. Sales_Demo (blank = no tag)", | |
| value="", | |
| info="Tag applied to all TS objects after each build", | |
| ) | |
| column_naming_style = gr.Dropdown( | |
| label="Default Column Naming Style", | |
| choices=["Regular Case", "snake_case", "camelCase", "PascalCase", "UPPER_CASE", "original"], | |
| value="Regular Case", | |
| info="Regular Case = State Id, Total Revenue", | |
| ) | |
| object_naming_prefix = gr.Textbox( | |
| label="Default Object Naming Prefix", | |
| placeholder="e.g. ACME_ (blank = none)", | |
| value="", | |
| ) | |
| share_with = gr.Textbox( | |
| label="Default Share With", | |
| placeholder="user@company.com or group-name (blank = no share)", | |
| value="", | |
| info="Model + liveboard shared after every build", | |
| ) | |
| # ── Optional default run inputs ──────────────────────────────────────────── | |
| gr.Markdown("---") | |
| gr.Markdown("### 🏁 Default Run Inputs *(optional)*") | |
| gr.Markdown("*When enabled, these pre-fill Vertical, Line, Function, and Company URL on the App tab at load time.*") | |
| use_default_inputs = gr.Checkbox( | |
| label="Pre-fill App tab on load", | |
| value=False, | |
| ) | |
| _vert_choices = list(VERTICAL_LINES.keys()) | |
| default_vertical = gr.Dropdown( | |
| label="Default Vertical", | |
| choices=_vert_choices, | |
| value=_vert_choices[0] if _vert_choices else None, | |
| allow_custom_value=True, | |
| ) | |
| _first_lines = VERTICAL_LINES.get(_vert_choices[0], []) if _vert_choices else [] | |
| default_line = gr.Dropdown( | |
| label="Default Line", | |
| choices=_first_lines, | |
| value=_first_lines[0] if _first_lines else None, | |
| allow_custom_value=True, | |
| info="Updates when Default Vertical changes", | |
| ) | |
| default_function = gr.Dropdown( | |
| label="Default Function", | |
| choices=DEMO_FUNCTIONS, | |
| value=DEMO_FUNCTIONS[0] if DEMO_FUNCTIONS else None, | |
| allow_custom_value=True, | |
| ) | |
| default_company_url = gr.Textbox( | |
| label="Default Company URL", | |
| placeholder="e.g. Nike.com", | |
| value="", | |
| ) | |
| # Wire vertical → line cascade | |
| def _update_default_line(vertical): | |
| lines = VERTICAL_LINES.get(vertical, []) | |
| return gr.update(choices=lines, value=lines[0] if lines else None) | |
| default_vertical.change(fn=_update_default_line, inputs=[default_vertical], outputs=[default_line]) | |
| # ── Other settings ───────────────────────────────────────────────────────── | |
| gr.Markdown("---") | |
| gr.Markdown("### 🌍 Other Settings") | |
| validation_mode = gr.Radio( | |
| label="Validation Mode", | |
| choices=["On", "Off"], | |
| value="Off", | |
| info="On: Pause at DDL & TS checkpoints. Off: Auto-run entire pipeline after context question.", | |
| ) | |
| # Hidden legacy fields — kept in schema for backward compat | |
| default_use_case = gr.Textbox(visible=False, value="Sales Analytics") | |
| fact_table_size = gr.Textbox(visible=False, value="1000") | |
| dim_table_size = gr.Textbox(visible=False, value="100") | |
| use_existing_model = gr.Checkbox(visible=False, value=False) | |
| existing_model_guid = gr.Textbox(visible=False, value="") | |
| # Advanced AI Settings (Admin Only) | |
| admin_ai_accordion = gr.Accordion("🤖 Advanced AI Settings (Admin)", open=False, visible=True) | |
| with admin_ai_accordion: | |
| with gr.Row(): | |
| temperature_slider = gr.Slider( | |
| minimum=0.0, | |
| maximum=1.0, | |
| value=0.3, | |
| step=0.1, | |
| label="Temperature", | |
| info="Controls randomness (lower = more focused)" | |
| ) | |
| max_tokens = gr.Number( | |
| value=4000, | |
| label="Max Tokens", | |
| info="Maximum tokens for AI responses" | |
| ) | |
| with gr.Row(): | |
| batch_size = gr.Slider( | |
| minimum=1000, | |
| maximum=50000, | |
| value=5000, | |
| step=1000, | |
| label="Batch Size", | |
| info="Rows per batch for bulk operations" | |
| ) | |
| thread_count = gr.Slider( | |
| minimum=1, | |
| maximum=16, | |
| value=4, | |
| step=1, | |
| label="Thread Count", | |
| info="Parallel threads for data generation" | |
| ) | |
| # Database Connections (Admin Only) | |
| admin_db_accordion = gr.Accordion("💾 Database Connections (Admin)", open=False, visible=True) | |
| with admin_db_accordion: | |
| gr.Markdown(""" | |
| **⚠️ Note:** These fields are **legacy placeholders** and are **not used by deploy runtime**. | |
| Active deployment credentials come from **Admin Settings** (system-wide `__admin__` values in Supabase). | |
| This section is for reference/future use only. | |
| """) | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("### ❄️ Snowflake Connection") | |
| sf_account = gr.Textbox( | |
| label="Snowflake Account", | |
| placeholder="xy12345.us-east-1", | |
| info="Your Snowflake account identifier" | |
| ) | |
| sf_user = gr.Textbox( | |
| label="Snowflake User", | |
| placeholder="your_username", | |
| info="Snowflake username (password in .env)" | |
| ) | |
| sf_role = gr.Textbox( | |
| label="Snowflake Role", | |
| value="ACCOUNTADMIN", | |
| info="Default role for connections" | |
| ) | |
| default_warehouse = gr.Textbox( | |
| label="Default Warehouse", | |
| value="COMPUTE_WH", | |
| info="Snowflake warehouse for demos" | |
| ) | |
| default_database = gr.Textbox( | |
| label="Default Database", | |
| value="DEMO_DB", | |
| info="Snowflake database for demos" | |
| ) | |
| default_schema = gr.Textbox( | |
| label="Default Schema", | |
| value="PUBLIC", | |
| info="Default schema for demos" | |
| ) | |
| with gr.Column(): | |
| gr.Markdown("### 📊 ThoughtSpot Settings") | |
| # ts_instance_url removed — replaced by TS Environment dropdown on front page | |
| ts_instance_url = gr.Textbox(visible=False) | |
| ts_username = gr.Textbox( | |
| label="ThoughtSpot Username", | |
| placeholder="your.email@company.com", | |
| info="Your ThoughtSpot login" | |
| ) | |
| # ts_password removed — auth is handled by TS Environment dropdown (API key), | |
| # not username/password. This field was never wired to SETTINGS_SCHEMA. | |
| gr.Markdown("---") | |
| gr.Markdown("### 🔧 Data Adjuster") | |
| gr.Markdown("*Jump straight to data adjustment on an existing liveboard — skips the build pipeline entirely.*") | |
| with gr.Row(): | |
| with gr.Column(): | |
| data_adjuster_url = gr.Textbox( | |
| label="Liveboard URL", | |
| placeholder="https://your-instance.thoughtspot.cloud/#/pinboard/guid", | |
| value="", | |
| info="Paste a ThoughtSpot liveboard URL to open it directly in Data Adjuster" | |
| ) | |
| gr.Markdown("---") | |
| with gr.Row(): | |
| save_settings_btn = gr.Button("💾 Save Settings", variant="primary", size="lg") | |
| reset_settings_btn = gr.Button("🔄 Reset to Defaults", size="lg") | |
| settings_status = gr.Markdown("") | |
| def save_settings_handler(request: gr.Request, *args): | |
| """Save all settings to Supabase - uses schema-driven helper""" | |
| try: | |
| from supabase_client import SupabaseSettings | |
| user_email = require_authenticated_email(request) | |
| settings_client = SupabaseSettings() | |
| if not settings_client.is_enabled(): | |
| return "⚠️ Supabase not configured. Settings saved locally only." | |
| # Build save dict from args using schema | |
| settings_to_save = build_settings_save_dict(list(args)) | |
| success = settings_client.save_all_settings(user_email, settings_to_save) | |
| if success: | |
| return f"✅ **Settings saved successfully!**\n\nSaved for user: `{user_email}`" | |
| else: | |
| return "❌ Error saving some settings. Check console for details." | |
| except Exception as e: | |
| return f"❌ Error saving settings: {str(e)}" | |
| def reset_settings_handler(): | |
| return "🔄 Settings reset to defaults! (Refresh page to see defaults)" | |
| # Build components dict matching SETTINGS_SCHEMA order | |
| # This is the single mapping from schema keys to component variables | |
| all_components = { | |
| # Panel defaults (mirrors App tab right panel) | |
| 'default_ai_model': default_ai_model, | |
| 'default_ts_env': default_ts_env, | |
| 'liveboard_name': liveboard_name, | |
| 'default_data_size': default_data_size, | |
| 'geo_scope': geo_scope, | |
| 'tag_name': tag_name, | |
| 'column_naming_style': column_naming_style, | |
| 'object_naming_prefix': object_naming_prefix, | |
| 'share_with': share_with, | |
| # Optional run input defaults | |
| 'use_default_inputs': use_default_inputs, | |
| 'default_vertical': default_vertical, | |
| 'default_line': default_line, | |
| 'default_function': default_function, | |
| 'default_company_url': default_company_url, | |
| # Other settings | |
| 'validation_mode': validation_mode, | |
| # Legacy hidden fields | |
| 'default_use_case': default_use_case, | |
| 'fact_table_size': fact_table_size, | |
| 'dim_table_size': dim_table_size, | |
| 'use_existing_model': use_existing_model, | |
| 'existing_model_guid': existing_model_guid, | |
| # Advanced AI Settings | |
| 'temperature_slider': temperature_slider, | |
| 'max_tokens': max_tokens, | |
| 'batch_size': batch_size, | |
| 'thread_count': thread_count, | |
| # Database Connection Settings | |
| 'sf_account': sf_account, | |
| 'sf_user': sf_user, | |
| 'sf_role': sf_role, | |
| 'default_warehouse': default_warehouse, | |
| 'default_database': default_database, | |
| 'default_schema': default_schema, | |
| 'ts_instance_url': ts_instance_url, | |
| 'ts_username': ts_username, | |
| 'data_adjuster_url': data_adjuster_url, | |
| # Status | |
| 'settings_status': settings_status, | |
| # Admin-only visibility toggles | |
| '_admin_ai_accordion': admin_ai_accordion, | |
| '_admin_db_accordion': admin_db_accordion, | |
| } | |
| # Get inputs in schema order (exclude settings_status which is output only) | |
| save_inputs = [all_components[key] for key, storage_key, _, _ in SETTINGS_SCHEMA if storage_key is not None] | |
| save_settings_btn.click( | |
| fn=save_settings_handler, | |
| inputs=save_inputs, | |
| outputs=[settings_status] | |
| ) | |
| reset_settings_btn.click( | |
| fn=reset_settings_handler, | |
| inputs=[], | |
| outputs=[settings_status] | |
| ) | |
| # ----------------------------------------------------------------------- | |
| # Change Password | |
| # ----------------------------------------------------------------------- | |
| gr.Markdown("---") | |
| with gr.Accordion("🔒 Change Password", open=False): | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| cp_current = gr.Textbox(label="Current Password", type="password", placeholder="Your current password") | |
| cp_new = gr.Textbox(label="New Password", type="password", placeholder="New password") | |
| cp_confirm = gr.Textbox(label="Confirm New Password", type="password", placeholder="Repeat new password") | |
| cp_btn = gr.Button("Change Password", variant="primary") | |
| cp_status = gr.Markdown("") | |
| with gr.Column(scale=1): | |
| gr.Markdown(""" | |
| **Password requirements:** | |
| - Current password required to confirm identity | |
| *Forgot your password? Ask an admin to reset it via the Admin tab, then change it here after signing in.* | |
| """) | |
| def change_password_handler(current, new_pw, confirm, request: gr.Request = None): | |
| if not current or not new_pw or not confirm: | |
| return "❌ All fields are required." | |
| if new_pw != confirm: | |
| return "❌ New passwords don't match." | |
| try: | |
| user_email = require_authenticated_email(request) | |
| from supabase_client import UserManager | |
| um = UserManager() | |
| if not um.enabled: | |
| return "⚠️ Supabase not configured — password change unavailable." | |
| if not um.authenticate(user_email, current): | |
| return "❌ Current password is incorrect." | |
| um.reset_password(user_email, new_pw) | |
| um.clear_must_change_password(user_email) | |
| return "✅ Password changed successfully. Use your new password next time you sign in." | |
| except Exception as e: | |
| return f"❌ Error: {e}" | |
| cp_btn.click( | |
| fn=change_password_handler, | |
| inputs=[cp_current, cp_new, cp_confirm], | |
| outputs=[cp_status] | |
| ) | |
| # Return components dict for loading (follows schema order) | |
| return all_components | |
| def authenticate_user(username: str, password: str) -> bool: | |
| """ | |
| Gradio auth callback — validates against Supabase demoprep_users table. | |
| Falls back to no-auth if Supabase is not configured (local dev). | |
| """ | |
| try: | |
| from supabase_client import UserManager | |
| um = UserManager() | |
| if not um.enabled: | |
| # Supabase not configured — allow anyone (local dev mode) | |
| print(f"[Auth] Supabase not configured, allowing login for: {username}") | |
| return True | |
| user = um.authenticate(username, password) | |
| if user: | |
| # Keep env-backed modules aligned with Supabase admin settings after login. | |
| inject_admin_settings_to_env() | |
| print(f"[Auth] Login successful: {username} (admin={user.get('is_admin', False)})") | |
| return True | |
| else: | |
| print(f"[Auth] Login failed: {username}") | |
| return False | |
| except Exception as e: | |
| print(f"[Auth] Error during authentication: {e}") | |
| # If auth system is broken, don't lock everyone out | |
| return False | |
| _REMEMBER_ME_SCRIPT = """ | |
| \t\t\t<script> | |
| \t\t\t/* DemoPrep: Remember Me — Gradio 4.44.1 uses div.form, not <form>, no shadow DOM */ | |
| \t\t\t(function() { | |
| \t\t\t\tvar KEY = 'demoprep_remembered_user'; | |
| \t\t\t\tvar _done = false; | |
| \t\t\t\tfunction injectIntoForm(loginRoot) { | |
| \t\t\t\t\tif (_done || !loginRoot) return; | |
| \t\t\t\t\tvar formDiv = loginRoot.querySelector('div.form'); | |
| \t\t\t\t\tif (!formDiv) return; | |
| \t\t\t\t\tvar inputs = formDiv.querySelectorAll('input, textarea'); | |
| \t\t\t\t\tvar uInput = null; | |
| \t\t\t\t\tfor (var i = 0; i < inputs.length; i++) { | |
| \t\t\t\t\t\tvar tp = (inputs[i].type || inputs[i].tagName).toLowerCase(); | |
| \t\t\t\t\t\tif (tp !== 'password' && tp !== 'submit' && tp !== 'checkbox' && tp !== 'hidden') { | |
| \t\t\t\t\t\t\tuInput = inputs[i]; break; | |
| \t\t\t\t\t\t} | |
| \t\t\t\t\t} | |
| \t\t\t\t\tif (!uInput || loginRoot.querySelector('#dp-rmb')) return; | |
| \t\t\t\t\tvar saved = localStorage.getItem(KEY); | |
| \t\t\t\t\tif (saved && !uInput.value) { | |
| \t\t\t\t\t\ttry { | |
| \t\t\t\t\t\t\tvar desc = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value') || | |
| \t\t\t\t\t\t\t Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value'); | |
| \t\t\t\t\t\t\tif (desc && desc.set) desc.set.call(uInput, saved); | |
| \t\t\t\t\t\t\telse uInput.value = saved; | |
| \t\t\t\t\t\t\tuInput.dispatchEvent(new Event('input', { bubbles: true })); | |
| \t\t\t\t\t\t} catch(e) { uInput.value = saved; } | |
| \t\t\t\t\t\tvar pwInput = formDiv.querySelector('input[type="password"]'); | |
| \t\t\t\t\t\tif (pwInput) setTimeout(function() { pwInput.focus(); }, 50); | |
| \t\t\t\t\t} | |
| \t\t\t\t\tvar lbl = document.createElement('label'); | |
| \t\t\t\t\tlbl.style.cssText = 'display:flex;align-items:center;gap:6px;font-size:13px;margin:8px 0 12px;cursor:pointer;color:#374151;'; | |
| \t\t\t\t\tvar cb = document.createElement('input'); | |
| \t\t\t\t\tcb.type = 'checkbox'; cb.id = 'dp-rmb'; cb.checked = !!saved; cb.style.cursor = 'pointer'; | |
| \t\t\t\t\tlbl.appendChild(cb); | |
| \t\t\t\t\tlbl.appendChild(document.createTextNode(' Remember me')); | |
| \t\t\t\t\tvar btn = loginRoot.querySelector('button'); | |
| \t\t\t\t\tif (btn && btn.parentNode) btn.parentNode.insertBefore(lbl, btn); | |
| \t\t\t\t\telse loginRoot.appendChild(lbl); | |
| \t\t\t\t\tfunction save() { | |
| \t\t\t\t\t\tif (cb.checked && uInput.value) localStorage.setItem(KEY, uInput.value); | |
| \t\t\t\t\t\telse localStorage.removeItem(KEY); | |
| \t\t\t\t\t} | |
| \t\t\t\t\tif (btn) btn.addEventListener('click', save); | |
| \t\t\t\t\t_done = true; | |
| \t\t\t\t\tconsole.log('[DemoPrep] Remember me injected'); | |
| \t\t\t\t} | |
| \t\t\t\tfunction tryInject() { | |
| \t\t\t\t\tif (_done) return; | |
| \t\t\t\t\tif (!window.gradio_config || !window.gradio_config.auth_required) return; | |
| \t\t\t\t\tvar wrap = document.querySelector('div.wrap'); | |
| \t\t\t\t\tif (wrap) { injectIntoForm(wrap); if (_done) return; } | |
| \t\t\t\t\tdocument.querySelectorAll('div.form').forEach(function(f) { | |
| \t\t\t\t\t\tif (!_done) injectIntoForm(f.parentElement); | |
| \t\t\t\t\t}); | |
| \t\t\t\t} | |
| \t\t\t\tvar t = setInterval(function() { tryInject(); if (_done) clearInterval(t); }, 200); | |
| \t\t\t\tsetTimeout(function() { clearInterval(t); }, 10000); | |
| \t\t\t})(); | |
| \t\t\t</script>""" | |
| _REMEMBER_ME_MARKER = "/* DemoPrep: Remember Me" | |
| _REMEMBER_ME_ANCHOR = '</head>' | |
| def _patch_gradio_template(): | |
| """Inject the Remember Me script into Gradio's index.html template. | |
| Runs at startup — idempotent, works on any machine including HuggingFace. | |
| """ | |
| import gradio | |
| template_path = os.path.join( | |
| os.path.dirname(gradio.__file__), | |
| "templates", "frontend", "index.html" | |
| ) | |
| try: | |
| content = open(template_path, encoding="utf-8").read() | |
| if _REMEMBER_ME_MARKER in content: | |
| print("[DemoPrep] Gradio template already patched — skipping") | |
| return | |
| if _REMEMBER_ME_ANCHOR not in content: | |
| print("[DemoPrep] WARNING: Could not find anchor in Gradio template — Remember Me not injected") | |
| return | |
| patched = content.replace(_REMEMBER_ME_ANCHOR, _REMEMBER_ME_SCRIPT + _REMEMBER_ME_ANCHOR) | |
| open(template_path, "w", encoding="utf-8").write(patched) | |
| print(f"[DemoPrep] Gradio template patched with Remember Me script") | |
| except Exception as e: | |
| print(f"[DemoPrep] WARNING: Could not patch Gradio template: {e}") | |
| if __name__ == "__main__": | |
| """Launch the chat interface standalone""" | |
| print("Starting Chat-Based Demo Builder...") | |
| _patch_gradio_template() | |
| app = create_chat_interface() | |
| # Enable queue with concurrency to handle multiple requests | |
| app.queue( | |
| default_concurrency_limit=10, # Allow up to 10 concurrent requests | |
| api_open=False | |
| ) | |
| # Bypass Gradio localhost accessibility check (httpx 0.28 compatibility) | |
| import gradio.networking as _gn | |
| _gn.url_ok = lambda url: True | |
| # Determine auth mode | |
| # If DEMOPREP_NO_AUTH=true, skip login (local dev override) | |
| no_auth = os.getenv('DEMOPREP_NO_AUTH', 'false').lower() in ('true', '1', 'yes') | |
| auth_fn = None if no_auth else authenticate_user | |
| app.launch( | |
| server_name="0.0.0.0", | |
| server_port=7863, # Different port from main app (7860) and old chat (7861) | |
| share=False, | |
| inbrowser=False, | |
| debug=True, | |
| auth=auth_fn, | |
| max_threads=20 # Allow multiple threads for concurrent requests | |
| ) | |