Update streamlit_app.py

streamlit_app.py

@@ -416,239 +416,88 @@ def main():
         )
         st.stop()
 
-    # Metric selection
-    st.sidebar.header("📊 Metrics")
-
-    # Dynamic metric categorization based on common patterns
-    def categorize_metrics(metrics):
-        """Dynamically categorize metrics based on naming patterns"""
-        categories = {"All": metrics}  # Always include all metrics
-
-        # Common patterns to look for
-        patterns = {
-            "Length": ["length", "byte", "word", "token", "char"],
-            "Readability": ["readability", "flesch", "standard"],
-            "Compression": ["lzw", "compression"],
-            "Language Model": ["ll_", "rll_", "logprob"],
-            "Working Memory": ["wm_"],
-            "Discourse": ["discourse"],
-            "Evaluation": ["rubric", "evaluation", "stealth"],
-            "Distribution": ["zipf", "type_token"],
-            "Coherence": ["coherence"],
-            "Entity": ["entity", "entities"],
-            "Cognitive": ["cognitive", "load"],
-        }
-
-        # Categorize metrics
-        for category, keywords in patterns.items():
-            matching_metrics = [
-                m for m in metrics if any(keyword in m.lower() for keyword in keywords)
-            ]
-            if matching_metrics:
-                categories[category] = matching_metrics
-
-        # Find uncategorized metrics
-        categorized = set()
-        for cat_metrics in categories.values():
-            if cat_metrics != metrics:  # Skip "All" category
-                categorized.update(cat_metrics)
-
-        uncategorized = [m for m in metrics if m not in categorized]
-        if uncategorized:
-            categories["Other"] = uncategorized
-
-        return categories
-
-    metric_categories = categorize_metrics(available_metrics)
-
-    # Metric selection interface
-    selection_mode = st.sidebar.radio(
-        "Selection Mode",
-        ["By Category", "Search/Filter", "Select All"],
-        help="Choose how to select metrics",
-    )
-
-    if selection_mode == "By Category":
-        selected_category = st.sidebar.selectbox(
-            "Metric Category",
-            options=list(metric_categories.keys()),
-            help=f"Found {len(metric_categories)} categories",
-        )
-
-        available_in_category = metric_categories[selected_category]
-        default_selection = (
-            available_in_category[:5]
-            if len(available_in_category) > 5
-            else available_in_category
-        )
-
-        # Add select all button for category
-        col1, col2 = st.sidebar.columns(2)
-        with col1:
-            if st.button("Select All", key="select_all_category"):
-                st.session_state.selected_metrics_category = available_in_category
-        with col2:
-            if st.button("Clear All", key="clear_all_category"):
-                st.session_state.selected_metrics_category = []
-
-        # Use session state for persistence
-        if "selected_metrics_category" not in st.session_state:
-            st.session_state.selected_metrics_category = default_selection
-
-        selected_metrics = st.sidebar.multiselect(
-            f"Select Metrics ({len(available_in_category)} available)",
-            options=available_in_category,
-            default=st.session_state.selected_metrics_category,
-            key="metrics_multiselect_category",
-            help="Choose metrics to visualize",
-        )
-
-    elif selection_mode == "Search/Filter":
-        search_term = st.sidebar.text_input(
-            "Search Metrics",
-            placeholder="Enter keywords to filter metrics...",
-            help="Search for metrics containing specific terms",
-        )
-
-        if search_term:
-            filtered_metrics = [
-                m for m in available_metrics if search_term.lower() in m.lower()
-            ]
-        else:
-            filtered_metrics = available_metrics
-
-        st.sidebar.write(f"Found {len(filtered_metrics)} metrics")
-
-        # Add select all button for search results
-        col1, col2 = st.sidebar.columns(2)
-        with col1:
-            if st.button("Select All", key="select_all_search"):
-                st.session_state.selected_metrics_search = filtered_metrics
-        with col2:
-            if st.button("Clear All", key="clear_all_search"):
-                st.session_state.selected_metrics_search = []
-
-        # Use session state for persistence
-        if "selected_metrics_search" not in st.session_state:
-            st.session_state.selected_metrics_search = (
-                filtered_metrics[:5]
-                if len(filtered_metrics) > 5
-                else filtered_metrics[:3]
-            )
-
-        selected_metrics = st.sidebar.multiselect(
-            "Select Metrics",
-            options=filtered_metrics,
-            default=st.session_state.selected_metrics_search,
-            key="metrics_multiselect_search",
-            help="Choose metrics to visualize",
-        )
-
-    else:  # Select All
-        # Add select all button for all metrics
-        col1, col2 = st.sidebar.columns(2)
-        with col1:
-            if st.button("Select All", key="select_all_all"):
-                st.session_state.selected_metrics_all = available_metrics
-        with col2:
-            if st.button("Clear All", key="clear_all_all"):
-                st.session_state.selected_metrics_all = []
-
-        # Use session state for persistence
-        if "selected_metrics_all" not in st.session_state:
-            st.session_state.selected_metrics_all = available_metrics[
-                :10
-            ]  # Limit default to first 10 for performance
-
-        selected_metrics = st.sidebar.multiselect(
-            f"All Metrics ({len(available_metrics)} total)",
-            options=available_metrics,
-            default=st.session_state.selected_metrics_all,
-            key="metrics_multiselect_all",
-            help="All available metrics - be careful with performance for large selections",
-        )
-
-    # Show selection summary
-    if selected_metrics:
-        st.sidebar.success(f"Selected {len(selected_metrics)} metrics")
-
-        # Performance warning for large selections
-        if len(selected_metrics) > 20:
-            st.sidebar.warning(
-                f"⚠️ Large selection ({len(selected_metrics)} metrics) may impact performance"
-            )
-        elif len(selected_metrics) > 50:
-            st.sidebar.error(
-                f"🚨 Very large selection ({len(selected_metrics)} metrics) - consider reducing for better performance"
-            )
-    else:
-        st.sidebar.warning("No metrics selected")
-
-    # Metric info expander
-    with st.sidebar.expander("ℹ️ Metric Information", expanded=False):
-        st.write(f"**Total Available Metrics:** {len(available_metrics)}")
-        st.write(f"**Categories Found:** {len(metric_categories)}")
-
-        if st.checkbox("Show all metric names", key="show_all_metrics"):
-            st.write("**All Available Metrics:**")
-            for i, metric in enumerate(available_metrics, 1):
-                st.write(f"{i}. `{metric}`")
-
     # Main content tabs
     tab1, tab2, tab3, tab4, tab5 = st.tabs(
         [
             "📊 Distributions",
-            "🔗 Correlations",
+            "🔗 Correlations", 
             "📈 Comparisons",
             "🔍 Conversation",
             "🎯 Details",
         ]
     )
 
+    # Make available metrics accessible to all tabs
+    available_metrics_for_analysis = available_metrics
+
     with tab1:
         st.header("Distribution Analysis")
-
-        if not selected_metrics:
-            st.warning("Please select at least one metric to visualize.")
-            return
-
-        # Create buttons for each metric to prevent loading all at once
-        st.info(
-            f"📊 Select a metric to plot its distribution ({len(selected_metrics)} metrics available)"
+        
+        # Simple metric selection - just show all metrics with checkboxes
+        st.subheader("📊 Select Metrics to Plot")
+        st.info(f"**{len(available_metrics)} metrics available** - Check the boxes below to plot their distributions")
+        
+        # Optional: Add search functionality to help users find metrics
+        search_term = st.text_input(
+            "🔍 Search metrics (optional)",
+            placeholder="Enter keywords to filter metrics...",
+            help="Search for metrics containing specific terms"
         )
+        
+        if search_term:
+            filtered_metrics = [
+                m for m in available_metrics if search_term.lower() in m.lower()
+            ]
+            st.write(f"**{len(filtered_metrics)} metrics** match your search")
+        else:
+            filtered_metrics = available_metrics
 
-        # Organize buttons in columns for better layout
-        cols_per_row = 3
-        for i in range(0, len(selected_metrics), cols_per_row):
-            cols = st.columns(cols_per_row)
-            for j, metric in enumerate(selected_metrics[i : i + cols_per_row]):
-                with cols[j]:
-                    friendly_name = get_human_friendly_metric_name(metric)
-                    # Truncate button text if too long
-                    button_text = (
-                        friendly_name[:25] + "..."
-                        if len(friendly_name) > 25
-                        else friendly_name
-                    )
-
-                    if st.button(
-                        f"📈 {button_text}",
-                        key=f"plot_{metric}",
-                        help=f"Plot distribution for {friendly_name}",
-                    ):
-                        render_metric_distribution(
-                            metric, filtered_df_exploded, selected_types
+        # Create checkboxes for each metric to allow multiple selections
+        if not filtered_metrics:
+            st.warning("No metrics found. Try adjusting your search.")
+        else:
+            # Organize checkboxes in columns for better layout
+            cols_per_row = 3
+            selected_for_plotting = []
+            
+            for i in range(0, len(filtered_metrics), cols_per_row):
+                cols = st.columns(cols_per_row)
+                for j, metric in enumerate(filtered_metrics[i : i + cols_per_row]):
+                    with cols[j]:
+                        friendly_name = get_human_friendly_metric_name(metric)
+                        # Truncate checkbox text if too long
+                        checkbox_text = (
+                            friendly_name[:25] + "..."
+                            if len(friendly_name) > 25
+                            else friendly_name
                         )
 
+                        if st.checkbox(
+                            f"📈 {checkbox_text}",
+                            key=f"plot_{metric}",
+                            help=f"Plot distribution for {friendly_name}",
+                        ):
+                            selected_for_plotting.append(metric)
+            
+            # Render selected metrics
+            if selected_for_plotting:
+                st.success(f"Plotting {len(selected_for_plotting)} selected metrics...")
+                for metric in selected_for_plotting:
+                    render_metric_distribution(
+                        metric, filtered_df_exploded, selected_types
+                    )
+            else:
+                st.info("👆 Check the boxes above to plot metric distributions")
+
     with tab2:
         st.header("Correlation Analysis")
 
-        if len(selected_metrics) < 2:
+        if len(available_metrics_for_analysis) < 2:
             st.warning("Please select at least 2 metrics for correlation analysis.")
         else:
             # Add button to trigger correlation analysis
             st.info(
-                f"🔗 Ready to analyze correlations between {len(selected_metrics)} metrics"
+                f"🔗 Ready to analyze correlations between {len(available_metrics_for_analysis)} metrics"
             )
 
             col1, col2 = st.columns([1, 3])
@@ -658,15 +507,15 @@ def main():
                     help="Calculate and display correlation matrix and scatter plots",
                 )
             with col2:
-                if len(selected_metrics) > 10:
+                if len(available_metrics_for_analysis) > 10:
                     st.warning(
-                        f"⚠️ Large analysis ({len(selected_metrics)} metrics) - may take some time"
+                        f"⚠️ Large analysis ({len(available_metrics_for_analysis)} metrics) - may take some time"
                     )
 
             if run_correlation:
                 with st.spinner("Calculating correlations..."):
                     # Prepare correlation data
-                    corr_columns = [f"turn.turn_metrics.{m}" for m in selected_metrics]
+                    corr_columns = [f"turn.turn_metrics.{m}" for m in available_metrics_for_analysis]
                     corr_data = filtered_df_exploded[corr_columns + ["type"]].copy()
 
                     # Clean column names for display
@@ -739,11 +588,11 @@ def main():
     with tab3:
         st.header("Type Comparisons")
 
-        if not selected_metrics:
+        if not available_metrics_for_analysis:
             st.warning("Please select at least one metric to compare.")
         else:
             # Box plots for each metric
-            for metric in selected_metrics:
+            for metric in available_metrics_for_analysis:
                 full_metric_name = f"turn.turn_metrics.{metric}"
 
                 if full_metric_name not in filtered_df_exploded.columns:
@@ -777,53 +626,45 @@ def main():
         # Conversation selector
         st.subheader("🔍 Select Conversation")
 
-        # Get unique conversations with some metadata
-        conversation_info = []
-        for idx, row in filtered_df.iterrows():
-            conv_type = row["type"]
-            # Get basic info about the conversation
-            conv_turns = len(row.get("conversation", []))
-            conversation_info.append(
-                {
-                    "index": idx,
-                    "type": conv_type,
-                    "turns": conv_turns,
-                    "display": f"Conversation {idx} ({conv_type}) - {conv_turns} turns",
-                }
-            )
-
-        # Sort by type and number of turns for better organization
-        conversation_info = sorted(
-            conversation_info, key=lambda x: (x["type"], -x["turns"])
-        )
+        # Get total number of conversations and basic info
+        total_conversations = len(filtered_df)
+        available_indices = list(filtered_df.index)
+        
+        st.info(f"📊 Dataset contains {total_conversations:,} conversations (indices: {min(available_indices)} to {max(available_indices)})")
 
-        # Conversation selection
-        col1, col2 = st.columns([3, 1])
+        # Conversation selection with number input
+        col1, col2, col3 = st.columns([2, 1, 1])
 
         with col1:
-            selected_conv_display = st.selectbox(
-                "Choose a conversation to analyze",
-                options=[conv["display"] for conv in conversation_info],
-                help="Select a conversation to view detailed metrics and content",
+            selected_idx = st.number_input(
+                "Conversation Index",
+                min_value=min(available_indices),
+                max_value=max(available_indices),
+                value=available_indices[0],  # Default to first available
+                step=1,
+                help=f"Enter a conversation index between {min(available_indices)} and {max(available_indices)}"
             )
 
         with col2:
             if st.button("🎲 Random", help="Select a random conversation"):
                 import random
-
-                selected_conv_display = random.choice(
-                    [conv["display"] for conv in conversation_info]
-                )
+                selected_idx = random.choice(available_indices)
                 st.rerun()
+        
+        with col3:
+            if st.button("ℹ️ Info", help="Show conversation preview"):
+                if selected_idx in available_indices:
+                    preview_row = filtered_df.loc[selected_idx]
+                    st.info(f"**Type:** {preview_row['type']} | **Turns:** {len(preview_row.get('conversation', []))}")
+                else:
+                    st.error("Invalid conversation index")
 
-        # Get the selected conversation data
-        selected_conv_info = next(
-            conv
-            for conv in conversation_info
-            if conv["display"] == selected_conv_display
-        )
-        selected_idx = selected_conv_info["index"]
-        selected_conversation = filtered_df.iloc[selected_idx]
+        # Validate and get the selected conversation data
+        if selected_idx not in available_indices:
+            st.error(f"❌ Conversation index {selected_idx} not found in filtered dataset. Available range: {min(available_indices)} to {max(available_indices)}")
+            st.stop()
+        
+        selected_conversation = filtered_df.loc[selected_idx]
 
         # Display conversation metadata
         st.subheader("📋 Conversation Overview")
@@ -979,7 +820,7 @@ def main():
             st.subheader("💬 Conversation with Metrics")
 
             # Get actual turn-level data for this conversation
-            turn_metric_columns = [f"turn.turn_metrics.{m}" for m in selected_metrics]
+            turn_metric_columns = [f"turn.turn_metrics.{m}" for m in available_metrics_for_analysis]
             available_columns = [
                 col
                 for col in turn_metric_columns
@@ -1200,7 +1041,7 @@ def main():
                         "No numeric metric data available to plot for this conversation type."
                     )
 
-            elif selected_metrics:
+            elif available_metrics_for_analysis:
                 st.info(
                     "Select metrics that are available in the dataset to see turn-level analysis."
                 )
@@ -1223,7 +1064,7 @@ def main():
                 help="Generate comprehensive dataset overview and metric analysis",
             )
         with col2:
-            if len(selected_metrics) > 20:
+            if len(available_metrics_for_analysis) > 20:
                 st.warning("⚠️ Large metric selection - analysis may take some time")
 
         if show_details:
@@ -1263,7 +1104,7 @@ def main():
                 if st.checkbox("Show raw data sample"):
                     sample_cols = ["type"] + [
                         f"turn.turn_metrics.{m}"
-                        for m in selected_metrics
+                        for m in available_metrics_for_analysis
                         if f"turn.turn_metrics.{m}" in filtered_df_exploded.columns
                     ]
                     sample_data = filtered_df_exploded[sample_cols].head(100)
@@ -1273,7 +1114,7 @@ def main():
                 st.subheader("📊 Metric Availability")
 
                 metric_completeness = {}
-                for metric in selected_metrics:
+                for metric in available_metrics_for_analysis:
                     full_metric_name = f"turn.turn_metrics.{metric}"
                     if full_metric_name in filtered_df_exploded.columns:
                         completeness = (