feat: Improve Liveboard creation, add outlier documentation, and fix table join updates

- Refactored model column fetching to use TML export API for better reliability
- Added structured outlier documentation format to data generation prompts (DEMO_OUTLIER comments)
- Fixed table GUID handling when updating tables with joins in Phase 2
- Enhanced error handling and logging for model column retrieval

These changes improve demo quality by:
1. Making liveboard generation more robust with proper column metadata
2. Guiding AI to create intentional data patterns with demo talking points
3. Ensuring table relationships update correctly without creating duplicates

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

liveboard_creator.py

@@ -31,22 +31,40 @@ class LiveboardCreator:
         self.openai_client = OpenAI(api_key=os.getenv('OPENAI_API_KEY'))
 
     def _fetch_model_columns(self) -> List[Dict]:
-        """Get available columns from the model for search query generation"""
+        """Get available columns from the model via TML export"""
         try:
-            response = self.ts_client.session.get(
-                f"{self.ts_client.base_url}/api/rest/2.0/metadata/details",
+            import yaml
+
+            response = self.ts_client.session.post(
+                f"{self.ts_client.base_url}/api/rest/2.0/metadata/tml/export",
                 headers=self.ts_client.headers,
-                params={"id": self.model_id}
+                json={"metadata": [{"identifier": self.model_id}]}
             )
 
             if response.status_code == 200:
-                data = response.json()
-                return data.get("storables", [{}])[0].get("columns", [])
+                result = response.json()
+                if isinstance(result, list) and len(result) > 0:
+                    edoc = result[0].get('edoc')
+                    if edoc:
+                        tml = yaml.safe_load(edoc)
+                        # Get columns from model TML
+                        if 'model' in tml:
+                            columns = tml['model'].get('columns', [])
+                            # Convert to simpler format
+                            return [{
+                                'name': col.get('name'),
+                                'type': col.get('properties', {}).get('column_type', 'ATTRIBUTE'),
+                                'column_id': col.get('column_id')
+                            } for col in columns]
+                print(f"Warning: Could not parse model columns")
+                return []
             else:
-                print(f"Warning: Could not fetch model columns: {response.text}")
+                print(f"Warning: Could not fetch model TML: {response.status_code}")
                 return []
         except Exception as e:
             print(f"Error fetching model columns: {e}")
+            import traceback
+            traceback.print_exc()
             return []
 
     def load_reference_tml(self, reference_type: str = "simple") -> Dict:
@@ -55,16 +73,16 @@ class LiveboardCreator:
 
         Args:
             reference_type: 'simple' or 'demogold'
-                - simple: dev_notes/liveboard_simple/Global Retail Apparel Sales.liveboard.tml
-                - demogold: dev_notes/liveboard_demogold/rl test.liveboard.tml
+                - simple: dev_notes/tml_examples/liveboard_simple/Global Retail Apparel Sales.liveboard.tml
+                - demogold: dev_notes/tml_examples/liveboard_demogold/rl test.liveboard.tml
 
         Returns:
             Parsed TML dictionary
         """
         if reference_type == "simple":
-            tml_path = "dev_notes/liveboard_simple/Global Retail Apparel Sales.liveboard.tml"
+            tml_path = "dev_notes/tml_examples/liveboard_simple/Global Retail Apparel Sales.liveboard.tml"
         else:
-            tml_path = "dev_notes/liveboard_demogold/rl test.liveboard.tml"
+            tml_path = "dev_notes/tml_examples/liveboard_demogold/rl test.liveboard.tml"
 
         try:
             with open(tml_path, 'r') as f:
@@ -324,32 +342,34 @@ Return ONLY a valid JSON object with structure:
         Returns:
             Complete Liveboard TML as YAML string
         """
-        # Generate visualization configs using AI
-        viz_configs = self.generate_visualizations_from_research(
-            company_data,
-            use_case,
-            num_visualizations
-        )
-
-        # Create visualization TML objects
+        # Generate visualization configs using AI (skip if 0 visualizations)
         visualizations = []
-        for i, viz_config in enumerate(viz_configs):
-            viz_config['id'] = f'Viz_{i+1}'
-            viz_tml = self.create_visualization_tml(viz_config)
-            visualizations.append(viz_tml)
-
-        # Create simple grid layout (2 columns)
         tiles = []
-        for i, viz in enumerate(visualizations):
-            col = i % 2
-            row = i // 2
-            tiles.append({
-                'visualization_id': viz['id'],
-                'x': col * 6,  # 12-column grid system
-                'y': row * 5,
-                'height': 5,
-                'width': 6
-            })
+
+        if num_visualizations > 0:
+            viz_configs = self.generate_visualizations_from_research(
+                company_data,
+                use_case,
+                num_visualizations
+            )
+
+            # Create visualization TML objects
+            for i, viz_config in enumerate(viz_configs):
+                viz_config['id'] = f'Viz_{i+1}'
+                viz_tml = self.create_visualization_tml(viz_config)
+                visualizations.append(viz_tml)
+
+            # Create simple grid layout (2 columns)
+            for i, viz in enumerate(visualizations):
+                col = i % 2
+                row = i // 2
+                tiles.append({
+                    'visualization_id': viz['id'],
+                    'x': col * 6,  # 12-column grid system
+                    'y': row * 5,
+                    'height': 5,
+                    'width': 6
+                })
 
         # Assemble complete Liveboard TML
         liveboard_tml = {
@@ -404,7 +424,10 @@ Return ONLY a valid JSON object with structure:
 
                 # Extract liveboard ID from response
                 liveboard_id = None
-                if result.get('object') and len(result['object']) > 0:
+                # Handle both dict and list responses
+                if isinstance(result, list) and len(result) > 0:
+                    liveboard_id = result[0].get('response', {}).get('header', {}).get('id_guid')
+                elif isinstance(result, dict) and result.get('object') and len(result['object']) > 0:
                     liveboard_id = result['object'][0].get('response', {}).get('header', {}).get('id')
 
                 return {

schema_utils.py

@@ -174,6 +174,18 @@ REQUIREMENTS:
 5. Include realistic business scenarios and edge cases
 6. Use proper data types and constraints
 7. Include error handling for connection issues
+8. **IMPORTANT**: Document strategic outliers with structured comments for demo purposes
+
+OUTLIER DOCUMENTATION FORMAT:
+For each strategic outlier or interesting pattern you inject into the data, add structured comments ABOVE the code that injects it:
+
+# DEMO_OUTLIER: [Brief title - e.g., "High-Value Customers at Risk"]
+# INSIGHT: [What pattern/anomaly exists - e.g., "Top 5 customers (>$50K LTV) showing declining satisfaction"]
+# SHOW_ME: [ThoughtSpot query to find it - e.g., "Show customers where lifetime_value > 50000 and satisfaction < 3"]
+# IMPACT: [Business impact - e.g., "$250K annual revenue at risk if these customers churn"]
+# TALKING_POINT: [What sales person should say - e.g., "Notice how ThoughtSpot instantly surfaces your most valuable at-risk accounts"]
+
+Create 3-5 strategic outliers that would make compelling demo talking points. Place these comments immediately BEFORE the code that injects each outlier.
 
 CONNECTION TEMPLATE:
 ```python

thoughtspot_deployer.py

@@ -236,9 +236,13 @@ class ThoughtSpotDeployer:
                     print(f"   ❌ Relationship API call failed: {response.status_code}")
                     print(f"   📋 Response: {response.text}")
 
-    def create_table_tml(self, table_name: str, columns: List, connection_name: str, 
-                        database: str, schema: str, all_tables: Dict = None) -> str:
-        """Generate table TML matching working example structure"""
+    def create_table_tml(self, table_name: str, columns: List, connection_name: str,
+                        database: str, schema: str, all_tables: Dict = None, table_guid: str = None) -> str:
+        """Generate table TML matching working example structure
+
+        Args:
+            table_guid: If provided, use this GUID (for updating existing tables with joins)
+        """
         tml_columns = []
         
         # Generate columns with proper typing
@@ -280,7 +284,7 @@ class ThoughtSpotDeployer:
             tml_columns.append(column_def)
         
         table_tml = {
-            'guid': None,
+            'guid': table_guid,  # Use provided GUID or None for new tables
             'table': {
                 'name': table_name.upper(),
                 'db': database,
@@ -1304,13 +1308,20 @@ class ThoughtSpotDeployer:
             log_progress("\n   📋 Phase 2: Adding joins to tables...")
             for table_name, columns in tables.items():
                 # Only add joins if the table was created successfully in Phase 1
-                if table_name.upper() not in table_guids:
-                    log_progress(f"   ⏭️  Skipping joins for {table_name.upper()} (table creation failed)")
+                table_name_upper = table_name.upper()
+                if table_name_upper not in table_guids:
+                    log_progress(f"   ⏭️  Skipping joins for {table_name_upper} (table creation failed)")
                     continue
 
-                log_progress(f"   🔗 Adding joins to: {table_name.upper()}...")
-                # Create table TML WITH joins_with section
-                table_tml = self.create_table_tml(table_name, columns, connection_name, database, schema, all_tables=tables)
+                # Get the GUID for this table
+                table_guid = table_guids[table_name_upper]
+
+                log_progress(f"   🔗 Adding joins to: {table_name_upper}...")
+                # Create table TML WITH joins_with section AND the table GUID
+                table_tml = self.create_table_tml(
+                    table_name, columns, connection_name, database, schema,
+                    all_tables=tables, table_guid=table_guid
+                )
 
                 response = self.session.post(
                     f"{self.base_url}/api/rest/2.0/metadata/tml/import",