Update leaderboard display

app.py

@@ -153,143 +153,90 @@ def filter_and_process_results(results, n_versions, only_actual_versions):
     """Filter results by version and process them for display."""
     if not results or "items" not in results:
         return pd.DataFrame(), [], [], []
-    
-    all_items = results["items"]
-    last_version_str = results.get("last_version", "1.0")
-    last_version = version.parse(last_version_str)
 
-    print(f"Last version: {last_version_str}")
+    all_items = results["items"]
     
-    # Group items by model_name
-    model_groups = {}
+    # Get all versions and sort them
+    all_versions_sorted = sorted([version.parse(v_str) for v_str in all_items.keys()], reverse=True)
+    
+    # Filter versions to consider based on n_versions slider
+    versions_to_consider = all_versions_sorted[:n_versions]
+    versions_to_consider_str = {str(v) for v in versions_to_consider}
 
+    rows = []
     for version_str, version_items in all_items.items():
-        version_obj = version.parse(version_str)
-        for item_id, item in version_items.items():
-            model_name = item.get("model_name", "Unknown")
+        if version_str not in versions_to_consider_str:
+            continue
             
-            if model_name not in model_groups:
-                model_groups[model_name] = []
+        for guid, item in version_items.items():
+            config = item.get("config", {})
+            model_name = item.get("model_name", "N/A")
+            metrics = item.get("metrics", {})
+            judge_metrics = metrics.get("judge", {})
             
-            # Add version info to the item (both as string and as parsed version object for comparison)
-            item["version_str"] = version_str
-            item["version_obj"] = version_obj
-            model_groups[model_name].append(item)
-    
-    rows = []
-    for model_name, items in model_groups.items():
-        # Sort items by version (newest first)
-        items.sort(key=lambda x: x["version_obj"], reverse=True)
-        
-        # Filter versions based on selection
-        filtered_items = []
-        
-        if only_actual_versions:
-            # Get the n most recent actual dataset versions
-            all_versions = sorted([version.parse(v_str) for v_str in all_items.keys()], reverse=True)
-            # Take at most n_versions
-            versions_to_consider = all_versions[:n_versions] if all_versions else []
+            row = {
+                'Model': f"{model_name} ({guid[:6]})",
+                'Embeddings': config.get('embedding_model', 'N/A'),
+                'Top-K': config.get('retrieval_config', {}).get('top_k', 'N/A'),
+                'Judge': round(judge_metrics.get("judge_total_score", 0.0) / 2, 4),
+                'Version': version_str,
+                'Last Updated': item.get("timestamp", ""),
+                'guid': guid
+            }
             
-            # Filter items that match those versions
-            filtered_items = [item for item in items if any(item["version_obj"] == v for v in versions_to_consider)]
-        else:
-            # Consider n_versions most recent items for this model
-            filtered_items = items[:n_versions]
-        
-        if not filtered_items:
-            continue
-        
-        config = filtered_items[0]["config"]  # Use config from most recent version
-        
-        # Create row with basic info
-        row = {
-            'Model': model_name,
-            'Embeddings': config.get('embedding_model', 'N/A'),
-            'Retriever': config.get('retriever_type', 'N/A'),
-            'Top-K': config.get('retrieval_config', {}).get('top_k', 'N/A'),
-            'Versions': ", ".join([item["version_str"] for item in filtered_items]),
-            'Last Updated': filtered_items[0].get("timestamp", "")
-        }
-        
-        # Format timestamp if available
-        if row['Last Updated']:
-            try:
-                dt = datetime.fromisoformat(row['Last Updated'].replace('Z', '+00:00'))
-                row['Last Updated'] = dt.strftime("%Y-%m-%d")
-            except:
-                pass
-        
-        # Process metrics based on categories
-        category_metrics = {
-            category: {
-                metric_type: {
-                    "avg": 0.0,
-                    "count": 0
-                } for metric_type in METRIC_TYPES
-            } for category in QUESTION_CATEGORIES
-        }
-        
-        # Collect metrics by category
-        for item in filtered_items:
-            metrics = item.get("metrics", {})
+            if row['Last Updated']:
+                try:
+                    dt = datetime.fromisoformat(row['Last Updated'].replace('Z', '+00:00'))
+                    row['Last Updated'] = dt.strftime("%Y-%m-%d")
+                except (ValueError, TypeError):
+                    pass
+            
+            category_sums = {mtype: 0.0 for mtype in METRIC_TYPES}
+            category_counts = {mtype: 0 for mtype in METRIC_TYPES}
+
             for category in QUESTION_CATEGORIES:
                 if category in metrics:
                     for metric_type in METRIC_TYPES:
                         if metric_type in metrics[category]:
                             metric_values = metrics[category][metric_type]
-                            avg_value = sum(metric_values.values()) / len(metric_values)
-                            
-                            # Add to the running sum for this category and metric type
-                            category_metrics[category][metric_type]["avg"] += avg_value
-                            category_metrics[category][metric_type]["count"] += 1
-        
-        # Calculate averages and add to row
-        for category in QUESTION_CATEGORIES:
-            for metric_type in METRIC_TYPES:
-                metric_data = category_metrics[category][metric_type]
-                if metric_data["count"] > 0:
-                    avg_value = metric_data["avg"] / metric_data["count"]
-                    # Add to row with appropriate column name
-                    col_name = f"{category}_{metric_type}"
-                    row[col_name] = round(avg_value, 4)
-        
-        # Calculate overall averages for each metric type
-        for metric_type in METRIC_TYPES:
-            total_sum = 0
-            total_count = 0
+                            if metric_values and len(metric_values) > 0:
+                                avg_value = sum(metric_values.values()) / len(metric_values)
+                                col_name = f"{category}_{metric_type}"
+                                row[col_name] = round(avg_value, 4)
+                                category_sums[metric_type] += avg_value
+                                category_counts[metric_type] += 1
             
-            for category in QUESTION_CATEGORIES:
-                metric_data = category_metrics[category][metric_type]
-                if metric_data["count"] > 0:
-                    total_sum += metric_data["avg"]
-                    total_count += metric_data["count"]
+            for metric_type in METRIC_TYPES:
+                if category_counts[metric_type] > 0:
+                    avg = category_sums[metric_type] / category_counts[metric_type]
+                    row[f"{metric_type}_avg"] = round(avg, 4)
             
-            if total_count > 0:
-                row[f"{metric_type}_avg"] = round(total_sum / total_count, 4)
-        
-        rows.append(row)
-    
-    # Create DataFrame
+            rows.append(row)
+
     df = pd.DataFrame(rows)
     
     # Get lists of metrics for each category
     category_metrics = []
-    for category in QUESTION_CATEGORIES:
-        metrics = []
-        for metric_type in METRIC_TYPES:
-            col_name = f"{category}_{metric_type}"
-            if col_name in df.columns:
-                metrics.append(col_name)
-        if metrics:
-            category_metrics.append((category, metrics))
-    
+    if not df.empty:
+        for category in QUESTION_CATEGORIES:
+            metrics_list = []
+            for metric_type in METRIC_TYPES:
+                col_name = f"{category}_{metric_type}"
+                if col_name in df.columns:
+                    metrics_list.append(col_name)
+            if metrics_list:
+                category_metrics.append((category, metrics_list))
+
     # Define retrieval and generation columns for radar charts
-    retrieval_metrics = [f"{category}_retrieval" for category in QUESTION_CATEGORIES if f"{category}_retrieval" in df.columns]
-    generation_metrics = [f"{category}_generation" for category in QUESTION_CATEGORIES if f"{category}_generation" in df.columns]
+    retrieval_metrics = []
+    generation_metrics = []
+    if not df.empty:
+        retrieval_metrics = [f"{category}_retrieval" for category, _ in category_metrics if f"{category}_retrieval" in df.columns]
+        generation_metrics = [f"{category}_generation" for category, _ in category_metrics if f"{category}_generation" in df.columns]
     
     return df, retrieval_metrics, generation_metrics, category_metrics
 
-def create_radar_chart(df, selected_models, metrics, title):
+def create_radar_chart(df, selected_models, metrics, title, name_col="Model"):
     """Create a radar chart for the selected models and metrics."""
     if not metrics or len(selected_models) == 0:
         # Return empty figure if no metrics or models selected
@@ -335,7 +282,7 @@ def create_radar_chart(df, selected_models, metrics, title):
         categories_loop = categories + [categories[0]]
         
         fig.add_trace(go.Scatterpolar(
-            name=row['Model'],
+            name=row[name_col],
             r=values,
             theta=categories_loop,
             showlegend=True,
@@ -398,21 +345,23 @@ def create_summary_df(df, retrieval_metrics, generation_metrics):
         generation_avg = summary_df[generation_metrics].mean(axis=1).round(4)
         summary_df['Generation (avg)'] = generation_avg
     
-    # Add total score if both averages exist
-    if 'Retrieval (avg)' in summary_df.columns and 'Generation (avg)' in summary_df.columns:
-        summary_df['Total Score'] = summary_df['Retrieval (avg)'] + summary_df['Generation (avg)']
+    # Add total score if all three columns exist
+    if 'Retrieval (avg)' in summary_df.columns and 'Generation (avg)' in summary_df.columns and 'Judge' in summary_df.columns:
+        summary_df['Total Score'] = summary_df[['Retrieval (avg)', 'Generation (avg)', 'Judge']].mean(axis=1).round(4)
         summary_df = summary_df.sort_values('Total Score', ascending=False)
     
     # Select columns for display
-    summary_cols = ['Model', 'Embeddings', 'Retriever', 'Top-K']
+    summary_cols = ['Model', 'Embeddings', 'Top-K']
+    if 'Judge' in summary_df.columns:
+        summary_cols.append('Judge')
     if 'Retrieval (avg)' in summary_df.columns:
         summary_cols.append('Retrieval (avg)')
     if 'Generation (avg)' in summary_df.columns:
         summary_cols.append('Generation (avg)')
     if 'Total Score' in summary_df.columns:
         summary_cols.append('Total Score')
-    if 'Versions' in summary_df.columns:
-        summary_cols.append('Versions')
+    if 'Version' in summary_df.columns:
+        summary_cols.append('Version')
     if 'Last Updated' in summary_df.columns:
         summary_cols.append('Last Updated')
     
@@ -432,7 +381,7 @@ def create_category_df(df, category, retrieval_col, generation_col):
     category_df = category_df.sort_values(f'Score', ascending=False)
     
     # Select columns for display
-    category_cols = ['Model', 'Embeddings', 'Retriever', retrieval_col, generation_col, f'Score']
+    category_cols = ['Model', 'Embeddings', retrieval_col, generation_col, f'Score']
     
     # Rename columns for display
     category_df = category_df[category_cols].rename(columns={
@@ -458,7 +407,7 @@ default_models = df['Model'].head(5).tolist() if not df.empty else []
 initial_gen_chart_title = LANGUAGES[DEFAULT_LANG]["radar_gen_title"]
 initial_ret_chart_title = LANGUAGES[DEFAULT_LANG]["radar_ret_title"]
 initial_gen_chart = create_radar_chart(df, default_models, generation_metrics, initial_gen_chart_title)
-initial_ret_chart = create_radar_chart(df, default_models, retrieval_metrics, initial_ret_chart_title)
+initial_ret_chart = create_radar_chart(df, default_models, retrieval_metrics, initial_ret_chart_title, name_col='Embeddings')
 
 # Create summary dataframe
 summary_df = create_summary_df(df, retrieval_metrics, generation_metrics)
@@ -526,7 +475,7 @@ with gr.Blocks(css="""
     }
     .info-text {
         font-size: 0.9em;
-        font-style: italic;
+        font-style: italic; 
         color: #666;
         margin-top: 5px;
     }
@@ -771,7 +720,7 @@ with gr.Blocks(css="""
                  print(f"IndexError: row_idx {row_idx} out of bounds for the component's data.")
                  # Potentially return current state without changes
                  gen_chart = create_radar_chart(current_df, selected_models, current_gen_metrics, LANGUAGES[current_lang]["radar_gen_title"])
-                 ret_chart = create_radar_chart(current_df, selected_models, current_ret_metrics, LANGUAGES[current_lang]["radar_ret_title"])
+                 ret_chart = create_radar_chart(current_df, selected_models, current_ret_metrics, LANGUAGES[current_lang]["radar_ret_title"], name_col='Embeddings')
                  return selected_models, gen_chart, ret_chart
             except Exception as e:
                 print(f"Error extracting model name: {e}")
@@ -802,7 +751,7 @@ with gr.Blocks(css="""
 
             # Create radar charts using the current dataframe and metrics
             gen_chart = create_radar_chart(current_df, selected_models, current_gen_metrics, LANGUAGES[current_lang]["radar_gen_title"])
-            ret_chart = create_radar_chart(current_df, selected_models, current_ret_metrics, LANGUAGES[current_lang]["radar_ret_title"])
+            ret_chart = create_radar_chart(current_df, selected_models, current_ret_metrics, LANGUAGES[current_lang]["radar_ret_title"], name_col='Embeddings')
 
             return selected_models, gen_chart, ret_chart
         except Exception as e:
@@ -810,7 +759,7 @@ with gr.Blocks(css="""
             print(traceback.format_exc())
             # Return potentially existing chart values if error occurs
             current_gen_chart = create_radar_chart(df, selected_models, generation_metrics, LANGUAGES[current_lang]["radar_gen_title"])
-            current_ret_chart = create_radar_chart(df, selected_models, retrieval_metrics, LANGUAGES[current_lang]["radar_ret_title"])
+            current_ret_chart = create_radar_chart(df, selected_models, retrieval_metrics, LANGUAGES[current_lang]["radar_ret_title"], name_col='Embeddings')
             return selected_models, current_gen_chart, current_ret_chart
 
 
@@ -852,7 +801,7 @@ with gr.Blocks(css="""
 
             # Create charts with localized titles
             gen_chart_val = create_radar_chart(df, filtered_selected_models, generation_metrics, LANGUAGES[current_lang]["radar_gen_title"])
-            ret_chart_val = create_radar_chart(df, filtered_selected_models, retrieval_metrics, LANGUAGES[current_lang]["radar_ret_title"])
+            ret_chart_val = create_radar_chart(df, filtered_selected_models, retrieval_metrics, LANGUAGES[current_lang]["radar_ret_title"], name_col='Embeddings')
 
             # Create summary dataframe
             summary_df_val = create_summary_df(df, retrieval_metrics, generation_metrics)
@@ -916,7 +865,7 @@ with gr.Blocks(css="""
         empty_models = []
         # Create empty charts with localized titles
         empty_gen_chart = create_radar_chart(df, empty_models, generation_metrics, LANGUAGES[current_lang]["radar_gen_title"])
-        empty_ret_chart = create_radar_chart(df, empty_models, retrieval_metrics, LANGUAGES[current_lang]["radar_ret_title"])
+        empty_ret_chart = create_radar_chart(df, empty_models, retrieval_metrics, LANGUAGES[current_lang]["radar_ret_title"], name_col='Embeddings')
         return empty_models, empty_gen_chart, empty_ret_chart
 
     # Connect clear charts button
@@ -960,7 +909,7 @@ with gr.Blocks(css="""
                for category, title_md in category_title_mds.items()},
              # Update chart titles dynamically by re-plotting (needed if chart titles change)
              generation_chart: create_radar_chart(df, selected_models.value, generation_metrics, lang_dict["radar_gen_title"]),
-             retrieval_chart: create_radar_chart(df, selected_models.value, retrieval_metrics, lang_dict["radar_ret_title"])
+             retrieval_chart: create_radar_chart(df, selected_models.value, retrieval_metrics, lang_dict["radar_ret_title"], name_col='Embeddings')
         }
 
         # Return updates in the correct order based on outputs list below