Update app.py

app.py

@@ -732,10 +732,296 @@ def create_entropy_geometry_plot(phi: np.ndarray):
     
     return fig
 
+def create_2d_projection_plot(df_filtered, lens_selected, color_scheme):
+    """Create 2D projection plot."""
+    alpha_col = f"diag_alpha_{lens_selected}"
+    srl_col = f"diag_srl_{lens_selected}"
+    
+    # Color mapping
+    if color_scheme == "Species":
+        color_values = [1 if s == "Human" else 0 for s in df_filtered['source']]
+        colorscale = [[0, '#1f77b4'], [1, '#ff7f0e']]
+        colorbar_title = "Species"
+    elif color_scheme == "Emotion":
+        unique_emotions = df_filtered['label'].unique()
+        emotion_map = {emotion: i for i, emotion in enumerate(unique_emotions)}
+        color_values = [emotion_map[label] for label in df_filtered['label']]
+        colorscale = 'Viridis'
+        colorbar_title = "Emotional State"
+    else:
+        color_values = df_filtered['cluster'].values
+        colorscale = 'Plotly3'
+        colorbar_title = "Cluster"
+    
+    fig = go.Figure()
+    
+    fig.add_trace(go.Scatter(
+        x=df_filtered['x'],
+        y=df_filtered['y'],
+        mode='markers',
+        marker=dict(
+            size=8,
+            color=color_values,
+            colorscale=colorscale,
+            showscale=True,
+            colorbar=dict(title=colorbar_title),
+            opacity=0.7,
+            line=dict(width=0.5, color='rgba(50,50,50,0.5)')
+        ),
+        text=[f"{row['source']}: {row['label']}" for _, row in df_filtered.iterrows()],
+        name='Communications'
+    ))
+    
+    fig.update_layout(
+        title="2D Manifold Projection",
+        xaxis_title='Dimension 1',
+        yaxis_title='Dimension 2',
+        margin=dict(l=0, r=0, b=0, t=40)
+    )
+    
+    return fig
+
+def create_density_heatmap(df_filtered):
+    """Create density heatmap."""
+    from scipy.stats import gaussian_kde
+    
+    if len(df_filtered) < 10:
+        return go.Figure(layout={"title": "Insufficient data for density plot"})
+    
+    x = df_filtered['x'].values
+    y = df_filtered['y'].values
+    
+    # Create density estimation
+    try:
+        kde = gaussian_kde(np.vstack([x, y]))
+        
+        # Create grid
+        x_range = np.linspace(x.min(), x.max(), 50)
+        y_range = np.linspace(y.min(), y.max(), 50)
+        X, Y = np.meshgrid(x_range, y_range)
+        positions = np.vstack([X.ravel(), Y.ravel()])
+        Z = kde(positions).reshape(X.shape)
+        
+        fig = go.Figure(data=go.Heatmap(
+            x=x_range,
+            y=y_range,
+            z=Z,
+            colorscale='Viridis',
+            showscale=True
+        ))
+        
+        # Add scatter points
+        fig.add_trace(go.Scatter(
+            x=x, y=y,
+            mode='markers',
+            marker=dict(size=4, color='white', opacity=0.8),
+            name='Data Points'
+        ))
+        
+        fig.update_layout(
+            title="Communication Density Landscape",
+            xaxis_title='Dimension 1',
+            yaxis_title='Dimension 2',
+            margin=dict(l=0, r=0, b=0, t=40)
+        )
+        
+        return fig
+    except:
+        return go.Figure(layout={"title": "Could not create density plot"})
+
+def create_feature_distributions(df_filtered, lens_selected):
+    """Create feature distribution plots."""
+    alpha_col = f"diag_alpha_{lens_selected}"
+    srl_col = f"diag_srl_{lens_selected}"
+    
+    fig = make_subplots(
+        rows=2, cols=2,
+        subplot_titles=(
+            f"Alpha Distribution ({lens_selected})",
+            f"SRL Distribution ({lens_selected})",
+            "Species Distribution",
+            "Emotion Distribution"
+        ),
+        specs=[[{"type": "histogram"}, {"type": "histogram"}],
+               [{"type": "bar"}, {"type": "bar"}]]
+    )
+    
+    # Alpha distribution
+    if alpha_col in df_filtered.columns:
+        fig.add_trace(go.Histogram(
+            x=df_filtered[alpha_col],
+            name="Alpha",
+            nbinsx=30,
+            marker_color='lightblue'
+        ), row=1, col=1)
+    
+    # SRL distribution
+    if srl_col in df_filtered.columns:
+        fig.add_trace(go.Histogram(
+            x=df_filtered[srl_col],
+            name="SRL",
+            nbinsx=30,
+            marker_color='lightgreen'
+        ), row=1, col=2)
+    
+    # Species distribution
+    species_counts = df_filtered['source'].value_counts()
+    fig.add_trace(go.Bar(
+        x=species_counts.index,
+        y=species_counts.values,
+        name="Species",
+        marker_color=['#1f77b4', '#ff7f0e']
+    ), row=2, col=1)
+    
+    # Emotion distribution
+    emotion_counts = df_filtered['label'].value_counts().head(10)
+    fig.add_trace(go.Bar(
+        x=emotion_counts.index,
+        y=emotion_counts.values,
+        name="Emotions",
+        marker_color='lightcoral'
+    ), row=2, col=2)
+    
+    fig.update_layout(
+        title_text="Statistical Distributions",
+        showlegend=False,
+        margin=dict(l=0, r=0, b=0, t=60)
+    )
+    
+    return fig
+
+def create_cluster_analysis(df_filtered):
+    """Create cluster analysis visualization."""
+    fig = make_subplots(
+        rows=1, cols=2,
+        subplot_titles=("Cluster Distribution", "Cluster Composition"),
+        specs=[[{"type": "bar"}, {"type": "bar"}]]
+    )
+    
+    # Cluster distribution
+    cluster_counts = df_filtered['cluster'].value_counts().sort_index()
+    fig.add_trace(go.Bar(
+        x=[f"C{i}" for i in cluster_counts.index],
+        y=cluster_counts.values,
+        name="Cluster Size",
+        marker_color='skyblue'
+    ), row=1, col=1)
+    
+    # Species composition per cluster
+    cluster_species = df_filtered.groupby(['cluster', 'source']).size().unstack(fill_value=0)
+    if len(cluster_species.columns) > 0:
+        for species in cluster_species.columns:
+            fig.add_trace(go.Bar(
+                x=[f"C{i}" for i in cluster_species.index],
+                y=cluster_species[species],
+                name=species,
+                marker_color='#1f77b4' if species == 'Human' else '#ff7f0e'
+            ), row=1, col=2)
+    
+    fig.update_layout(
+        title_text="Cluster Analysis",
+        margin=dict(l=0, r=0, b=0, t=60)
+    )
+    
+    return fig
+
+def create_similarity_matrix(df_filtered, lens_selected):
+    """Create species similarity matrix."""
+    alpha_col = f"diag_alpha_{lens_selected}"
+    srl_col = f"diag_srl_{lens_selected}"
+    
+    # Calculate mean values for each species-emotion combination
+    similarity_data = []
+    
+    for species in df_filtered['source'].unique():
+        for emotion in df_filtered['label'].unique():
+            subset = df_filtered[(df_filtered['source'] == species) & (df_filtered['label'] == emotion)]
+            if len(subset) > 0:
+                alpha_mean = subset[alpha_col].mean() if alpha_col in subset.columns else 0
+                srl_mean = subset[srl_col].mean() if srl_col in subset.columns else 0
+                similarity_data.append({
+                    'species': species,
+                    'emotion': emotion,
+                    'alpha': alpha_mean,
+                    'srl': srl_mean,
+                    'combined': alpha_mean + srl_mean
+                })
+    
+    if not similarity_data:
+        return go.Figure(layout={"title": "No data for similarity matrix"})
+    
+    similarity_df = pd.DataFrame(similarity_data)
+    pivot_table = similarity_df.pivot(index='emotion', columns='species', values='combined')
+    
+    fig = go.Figure(data=go.Heatmap(
+        z=pivot_table.values,
+        x=pivot_table.columns,
+        y=pivot_table.index,
+        colorscale='RdYlBu_r',
+        showscale=True,
+        colorbar=dict(title="Similarity Score")
+    ))
+    
+    fig.update_layout(
+        title="Cross-Species Similarity Matrix",
+        margin=dict(l=0, r=0, b=0, t=40)
+    )
+    
+    return fig
+
+def calculate_live_statistics(df_filtered, lens_selected):
+    """Calculate live statistics for the dataset."""
+    alpha_col = f"diag_alpha_{lens_selected}"
+    srl_col = f"diag_srl_{lens_selected}"
+    
+    stats = {
+        'total_samples': len(df_filtered),
+        'species_counts': df_filtered['source'].value_counts().to_dict(),
+        'emotion_counts': len(df_filtered['label'].unique()),
+        'cluster_count': len(df_filtered['cluster'].unique())
+    }
+    
+    if alpha_col in df_filtered.columns:
+        stats['alpha_mean'] = df_filtered[alpha_col].mean()
+        stats['alpha_std'] = df_filtered[alpha_col].std()
+    
+    if srl_col in df_filtered.columns:
+        stats['srl_mean'] = df_filtered[srl_col].mean()
+        stats['srl_std'] = df_filtered[srl_col].std()
+    
+    # Format as HTML
+    html_content = f"""
+    <div style="padding: 10px; background-color: #f0f8ff; border-radius: 8px;">
+        <h4>📊 Live Dataset Statistics</h4>
+        <p><strong>Total Samples:</strong> {stats['total_samples']}</p>
+        <p><strong>Species:</strong> 
+           {' | '.join([f"{k}: {v}" for k, v in stats['species_counts'].items()])}
+        </p>
+        <p><strong>Emotions:</strong> {stats['emotion_counts']}</p>
+        <p><strong>Clusters:</strong> {stats['cluster_count']}</p>
+    """
+    
+    if 'alpha_mean' in stats:
+        html_content += f"""
+        <p><strong>Alpha ({lens_selected}):</strong> 
+           μ={stats['alpha_mean']:.3f}, σ={stats['alpha_std']:.3f}
+        </p>
+        """
+    
+    if 'srl_mean' in stats:
+        html_content += f"""
+        <p><strong>SRL ({lens_selected}):</strong> 
+           μ={stats['srl_mean']:.3f}, σ={stats['srl_std']:.3f}
+        </p>
+        """
+    
+    html_content += "</div>"
+    return html_content
+
 def update_manifold_visualization(species_selection, emotion_selection, lens_selection, 
                                 alpha_min, alpha_max, srl_min, srl_max, 
                                 point_size, show_boundary, show_trajectories, color_scheme):
-    """Update manifold visualization with filters."""
+    """Update all manifold visualizations with filters."""
     
     df_filtered = df_combined.copy()
     
@@ -765,28 +1051,102 @@ def update_manifold_visualization(species_selection, emotion_selection, lens_sel
             text="No data points match the current filters", 
             xref="paper", yref="paper", x=0.5, y=0.5, showarrow=False
         )
-        return empty_fig
+        empty_stats = "<p>No data available</p>"
+        return empty_fig, empty_fig, empty_fig, empty_fig, empty_fig, empty_fig, empty_stats
     
-    return create_enhanced_manifold_plot(
+    # Create all visualizations
+    main_plot = create_enhanced_manifold_plot(
         df_filtered, lens_selection, color_scheme, point_size, 
         show_boundary, show_trajectories
     )
+    
+    projection_2d = create_2d_projection_plot(df_filtered, lens_selection, color_scheme)
+    density_plot = create_density_heatmap(df_filtered)
+    feature_dists = create_feature_distributions(df_filtered, lens_selection)
+    cluster_plot = create_cluster_analysis(df_filtered)
+    similarity_plot = create_similarity_matrix(df_filtered, lens_selection)
+    stats_html = calculate_live_statistics(df_filtered, lens_selection)
+    
+    return main_plot, projection_2d, density_plot, feature_dists, cluster_plot, similarity_plot, stats_html
+
+def export_filtered_data(species_selection, emotion_selection, lens_selection, 
+                        alpha_min, alpha_max, srl_min, srl_max):
+    """Export filtered dataset for analysis."""
+    import tempfile
+    import json
+    
+    df_filtered = df_combined.copy()
+    
+    if species_selection:
+        df_filtered = df_filtered[df_filtered['source'].isin(species_selection)]
+    
+    if emotion_selection:
+        df_filtered = df_filtered[df_filtered['label'].isin(emotion_selection)]
+    
+    alpha_col = f"diag_alpha_{lens_selection}"
+    srl_col = f"diag_srl_{lens_selection}"
+    
+    if alpha_col in df_filtered.columns:
+        df_filtered = df_filtered[
+            (df_filtered[alpha_col] >= alpha_min) & 
+            (df_filtered[alpha_col] <= alpha_max)
+        ]
+    
+    if srl_col in df_filtered.columns:
+        df_filtered = df_filtered[
+            (df_filtered[srl_col] >= srl_min) & 
+            (df_filtered[srl_col] <= srl_max)
+        ]
+    
+    if len(df_filtered) == 0:
+        return "<p style='color: red;'>❌ No data to export with current filters</p>"
+    
+    # Create export summary
+    export_summary = {
+        "export_timestamp": pd.Timestamp.now().isoformat(),
+        "total_samples": len(df_filtered),
+        "species_counts": df_filtered['source'].value_counts().to_dict(),
+        "emotion_types": df_filtered['label'].unique().tolist(),
+        "lens_used": lens_selection,
+        "filters_applied": {
+            "species": species_selection,
+            "emotions": emotion_selection,
+            "alpha_range": [alpha_min, alpha_max],
+            "srl_range": [srl_min, srl_max]
+        }
+    }
+    
+    summary_html = f"""
+    <div style="padding: 10px; background-color: #e8f5e8; border-radius: 8px; margin-top: 10px;">
+        <h4>✅ Export Ready</h4>
+        <p><strong>Samples:</strong> {export_summary['total_samples']}</p>
+        <p><strong>Species:</strong> {', '.join([f"{k}({v})" for k, v in export_summary['species_counts'].items()])}</p>
+        <p><strong>Emotions:</strong> {len(export_summary['emotion_types'])} types</p>
+        <p><strong>Lens:</strong> {lens_selection}</p>
+        <p><em>Data ready for download via your browser's dev tools or notebook integration.</em></p>
+    </div>
+    """
+    
+    return summary_html
 
 # ---------------------------------------------------------------
 # Gradio Interface
 # ---------------------------------------------------------------
 with gr.Blocks(theme=gr.themes.Soft(primary_hue="teal", secondary_hue="cyan")) as demo:
     gr.Markdown("""
-    # 🌟 **CMT Holographic Information Geometry Engine**
-    *Full-featured visualization suite with mathematical rigor*
-    
-    **Features:**
-    - Complete holographic field reconstruction
-    - Cross-species communication mapping
-    - Interactive 3D manifold exploration
-    - Entropy and phase analysis
-    - Side-by-side comparison capabilities
-    - Automatic neighbor finding for grammar mapping
+    # 🌟 **CMT Holographic Information Geometry Engine v5.0**
+    *Revolutionary interspecies communication analysis platform*
+    
+    **🚀 Enhanced Features:**
+    - **🌌 Universal Manifold Explorer**: Multi-dimensional visualization suite with live statistics
+    - **🔬 Interactive Holography**: Cross-species communication mapping with mathematical precision
+    - **📊 Real-time Analytics**: Dynamic filtering, clustering, and similarity analysis
+    - **🎨 Rich Visualizations**: 2D/3D plots, density heatmaps, feature distributions
+    - **💾 Data Export**: Export filtered datasets for external analysis
+    - **⚡ Auto-loading**: Manifold visualizations load automatically on startup
+    
+    ---
+    **🎯 Goal**: Map the geometric structure of communication to reveal universal patterns across species
     """)
     
     with gr.Tabs():
@@ -830,9 +1190,26 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="teal", secondary_hue="cyan")) a
                             choices=["Species", "Emotion", "CMT_Alpha", "CMT_SRL", "Cluster"],
                             value="Species"
                         )
+                    
+                    with gr.Accordion("📊 Live Statistics", open=True):
+                        stats_html = gr.HTML(label="Dataset Statistics")
+                        similarity_matrix = gr.Plot(label="Species Similarity Matrix")
+                    
+                    with gr.Accordion("💾 Data Export", open=False):
+                        gr.Markdown("**Export filtered dataset for further analysis**")
+                        export_button = gr.Button("📥 Export Filtered Data", variant="secondary")
+                        export_status = gr.HTML("")
                 
                 with gr.Column(scale=3):
                     manifold_plot = gr.Plot(label="Universal Communication Manifold")
+                    
+                    with gr.Row():
+                        projection_2d = gr.Plot(label="2D Projection")
+                        density_plot = gr.Plot(label="Density Heatmap")
+                    
+                    with gr.Row():
+                        feature_distributions = gr.Plot(label="Feature Distributions")
+                        cluster_analysis = gr.Plot(label="Cluster Analysis")
             
             # Wire up events
             manifold_inputs = [
@@ -841,12 +1218,27 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="teal", secondary_hue="cyan")) a
                 point_size, show_species_boundary, show_trajectories, color_scheme
             ]
             
+            manifold_outputs = [
+                manifold_plot, projection_2d, density_plot, 
+                feature_distributions, cluster_analysis, similarity_matrix, stats_html
+            ]
+            
             for component in manifold_inputs:
                 component.change(
                     update_manifold_visualization,
                     inputs=manifold_inputs,
-                    outputs=[manifold_plot]
+                    outputs=manifold_outputs
                 )
+            
+            # Wire up export button
+            export_button.click(
+                export_filtered_data,
+                inputs=[
+                    species_filter, emotion_filter, lens_selector,
+                    alpha_min, alpha_max, srl_min, srl_max
+                ],
+                outputs=[export_status]
+            )
         
         with gr.TabItem("🔬 Interactive Holography"):
             with gr.Row():
@@ -1004,6 +1396,25 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="teal", secondary_hue="cyan")) a
                     outputs=cross_species_outputs
                 )
 
+    # Auto-load manifold visualizations on startup
+    demo.load(
+        update_manifold_visualization,
+        inputs=[
+            gr.State(["Human", "Dog"]),  # species_filter default
+            gr.State(list(df_combined['label'].unique())),  # emotion_filter default
+            gr.State("gamma"),  # lens_selector default
+            gr.State(0),  # alpha_min default
+            gr.State(5),  # alpha_max default
+            gr.State(0),  # srl_min default
+            gr.State(100),  # srl_max default
+            gr.State(6),  # point_size default
+            gr.State(True),  # show_species_boundary default
+            gr.State(False),  # show_trajectories default
+            gr.State("Species")  # color_scheme default
+        ],
+        outputs=manifold_outputs
+    )
+
 print("✅ CMT Holographic Visualization Suite Ready!")
 
 if __name__ == "__main__":