Update src/leaderboard.py

src/leaderboard.py

@@ -5,438 +5,633 @@ import json
 import datetime
 from typing import Dict, List, Optional, Tuple
 import os
-from config import LEADERBOARD_DATASET, HF_TOKEN, ALL_UG40_LANGUAGES, GOOGLE_SUPPORTED_LANGUAGES
-from src.utils import create_submission_id, sanitize_model_name, get_all_language_pairs, get_google_comparable_pairs
+import numpy as np
+from config import (
+    LEADERBOARD_DATASET,
+    HF_TOKEN,
+    EVALUATION_TRACKS,
+    MODEL_CATEGORIES,
+    STATISTICAL_CONFIG,
+    METRICS_CONFIG,
+)
+from src.utils import create_submission_id, sanitize_model_name
+
+
+def initialize_scientific_leaderboard() -> pd.DataFrame:
+    """Initialize empty scientific leaderboard DataFrame with all required columns."""
 
-def initialize_leaderboard() -> pd.DataFrame:
-    """Initialize empty leaderboard DataFrame."""
-    
     columns = {
-        'submission_id': [],
-        'model_name': [],
-        'author': [],
-        'submission_date': [],
-        'model_type': [],
-        'description': [],
-        
-        # Primary metrics
-        'quality_score': [],
-        'bleu': [],
-        'chrf': [],
-        
-        # Secondary metrics  
-        'rouge1': [],
-        'rouge2': [],
-        'rougeL': [],
-        'cer': [],
-        'wer': [],
-        'len_ratio': [],
-        
-        # Google comparable metrics
-        'google_quality_score': [],
-        'google_bleu': [],
-        'google_chrf': [],
-        
-        # Coverage info
-        'total_samples': [],
-        'language_pairs_covered': [],
-        'google_pairs_covered': [],
-        'coverage_rate': [],
-        
-        # Detailed results
-        'detailed_metrics': [],  # JSON string
-        'validation_report': [],
-        
+        # Basic information
+        "submission_id": [],
+        "model_name": [],
+        "author": [],
+        "submission_date": [],
+        "model_category": [],
+        "description": [],
+        # Track-specific quality scores
+        "google_comparable_quality": [],
+        "ug40_complete_quality": [],
+        "language_pair_matrix_quality": [],
+        # Track-specific BLEU scores
+        "google_comparable_bleu": [],
+        "ug40_complete_bleu": [],
+        "language_pair_matrix_bleu": [],
+        # Track-specific ChrF scores
+        "google_comparable_chrf": [],
+        "ug40_complete_chrf": [],
+        "language_pair_matrix_chrf": [],
+        # Statistical metadata
+        "google_comparable_ci_lower": [],
+        "google_comparable_ci_upper": [],
+        "ug40_complete_ci_lower": [],
+        "ug40_complete_ci_upper": [],
+        "language_pair_matrix_ci_lower": [],
+        "language_pair_matrix_ci_upper": [],
+        # Coverage information
+        "google_comparable_samples": [],
+        "ug40_complete_samples": [],
+        "language_pair_matrix_samples": [],
+        "google_comparable_pairs": [],
+        "ug40_complete_pairs": [],
+        "language_pair_matrix_pairs": [],
+        # Statistical adequacy flags
+        "google_comparable_adequate": [],
+        "ug40_complete_adequate": [],
+        "language_pair_matrix_adequate": [],
+        # Detailed results (JSON strings)
+        "detailed_google_comparable": [],
+        "detailed_ug40_complete": [],
+        "detailed_language_pair_matrix": [],
+        "cross_track_analysis": [],
         # Metadata
-        'evaluation_date': [],
-        'leaderboard_version': []
+        "evaluation_date": [],
+        "leaderboard_version": [],
+        "scientific_adequacy_score": [],
     }
-    
+
     return pd.DataFrame(columns)
 
-def load_leaderboard() -> pd.DataFrame:
-    """Load current leaderboard from HuggingFace dataset."""
-    
+
+def load_scientific_leaderboard() -> pd.DataFrame:
+    """Load current scientific leaderboard from HuggingFace dataset."""
+
     try:
-        print("Loading leaderboard...")
-        dataset = load_dataset(LEADERBOARD_DATASET, split='train')
+        print("📥 Loading scientific leaderboard...")
+        dataset = load_dataset(LEADERBOARD_DATASET + "-scientific", split="train")
         df = dataset.to_pandas()
-        
+
         # Ensure all required columns exist
-        required_columns = list(initialize_leaderboard().columns)
+        required_columns = list(initialize_scientific_leaderboard().columns)
         for col in required_columns:
             if col not in df.columns:
-                if col in ['quality_score', 'bleu', 'chrf', 'rouge1', 'rouge2', 'rougeL', 
-                          'cer', 'wer', 'len_ratio', 'google_quality_score', 'google_bleu', 
-                          'google_chrf', 'total_samples', 'language_pairs_covered', 
-                          'google_pairs_covered', 'coverage_rate']:
+                if "quality" in col or "bleu" in col or "chrf" in col or "ci_" in col:
+                    df[col] = 0.0
+                elif "samples" in col or "pairs" in col:
+                    df[col] = 0
+                elif "adequate" in col:
+                    df[col] = False
+                elif col == "scientific_adequacy_score":
                     df[col] = 0.0
-                elif col in ['leaderboard_version']:
-                    df[col] = 1
+                elif col == "leaderboard_version":
+                    df[col] = 2  # Scientific version
                 else:
-                    df[col] = ''
-        
-        print(f"Loaded leaderboard with {len(df)} entries")
+                    df[col] = ""
+
+        print(f"✅ Loaded scientific leaderboard with {len(df)} entries")
         return df
-        
+
     except Exception as e:
-        print(f"Could not load leaderboard: {e}")
-        print("Initializing empty leaderboard...")
-        return initialize_leaderboard()
+        print(f"⚠️ Could not load scientific leaderboard: {e}")
+        print("🔄 Initializing empty scientific leaderboard...")
+        return initialize_scientific_leaderboard()
+
+
+def save_scientific_leaderboard(df: pd.DataFrame) -> bool:
+    """Save scientific leaderboard to HuggingFace dataset."""
 
-def save_leaderboard(df: pd.DataFrame) -> bool:
-    """Save leaderboard to HuggingFace dataset."""
-    
     try:
         # Clean data before saving
         df_clean = df.copy()
-        
+
         # Ensure numeric columns are proper types
-        numeric_columns = ['quality_score', 'bleu', 'chrf', 'rouge1', 'rouge2', 'rougeL',
-                          'cer', 'wer', 'len_ratio', 'google_quality_score', 'google_bleu',
-                          'google_chrf', 'total_samples', 'language_pairs_covered',
-                          'google_pairs_covered', 'coverage_rate', 'leaderboard_version']
-        
+        numeric_columns = [
+            col
+            for col in df_clean.columns
+            if any(
+                x in col
+                for x in [
+                    "quality",
+                    "bleu",
+                    "chrf",
+                    "ci_",
+                    "samples",
+                    "pairs",
+                    "adequacy",
+                ]
+            )
+        ]
+
         for col in numeric_columns:
             if col in df_clean.columns:
-                df_clean[col] = pd.to_numeric(df_clean[col], errors='coerce').fillna(0.0)
-        
+                if "adequate" in col:
+                    df_clean[col] = df_clean[col].astype(bool)
+                else:
+                    df_clean[col] = pd.to_numeric(
+                        df_clean[col], errors="coerce"
+                    ).fillna(0.0)
+
         # Convert to dataset
         dataset = Dataset.from_pandas(df_clean)
-        
+
         # Push to hub
         dataset.push_to_hub(
-            LEADERBOARD_DATASET,
+            LEADERBOARD_DATASET + "-scientific",
             token=HF_TOKEN,
-            commit_message=f"Update leaderboard - {datetime.datetime.now().isoformat()[:19]}"
+            commit_message=f"Update scientific leaderboard - {datetime.datetime.now().isoformat()[:19]}",
         )
-        
-        print("Leaderboard saved successfully!")
+
+        print("✅ Scientific leaderboard saved successfully!")
         return True
-        
+
     except Exception as e:
-        print(f"Error saving leaderboard: {e}")
+        print(f"❌ Error saving scientific leaderboard: {e}")
         return False
 
-def add_model_to_leaderboard(
+
+def add_model_to_scientific_leaderboard(
     model_name: str,
     author: str,
     evaluation_results: Dict,
-    validation_info: Dict,
-    model_type: str = "",
-    description: str = ""
+    model_category: str = "community",
+    description: str = "",
 ) -> pd.DataFrame:
-    """
-    Add new model results to leaderboard, with JSON-safe detailed_metrics.
-    """
+    """Add new model results to scientific leaderboard."""
+
     # Load current leaderboard
-    df = load_leaderboard()
+    df = load_scientific_leaderboard()
 
     # Remove existing entry if present
-    existing_mask = df['model_name'] == model_name
+    existing_mask = df["model_name"] == model_name
     if existing_mask.any():
         df = df[~existing_mask]
 
-    # Safely serialize evaluation_results by dropping non-JSON types
-    safe_results = evaluation_results.copy()
-    # Remove sample_metrics DataFrame which isn't JSON serializable
-    if 'sample_metrics' in safe_results:
-        safe_results.pop('sample_metrics')
+    # Extract track results
+    tracks = evaluation_results.get("tracks", {})
+    cross_track = evaluation_results.get("cross_track_analysis", {})
 
-    detailed_json = json.dumps(safe_results)
-
-    # Extract metrics
-    averages = evaluation_results.get('averages', {})
-    google_averages = evaluation_results.get('google_comparable_averages', {})
-    summary = evaluation_results.get('summary', {})
+    # Calculate scientific adequacy score
+    adequacy_score = calculate_scientific_adequacy_score(evaluation_results)
 
     # Prepare new entry
     new_entry = {
-        'submission_id': create_submission_id(),
-        'model_name': sanitize_model_name(model_name),
-        'author': author[:100] if author else 'Anonymous',
-        'submission_date': datetime.datetime.now().isoformat(),
-        'model_type': model_type[:50] if model_type else 'unknown',
-        'description': description[:500] if description else '',
-
-        # Primary metrics
-        'quality_score': float(averages.get('quality_score', 0.0)),
-        'bleu': float(averages.get('bleu', 0.0)),
-        'chrf': float(averages.get('chrf', 0.0)),
-
-        # Secondary metrics
-        'rouge1': float(averages.get('rouge1', 0.0)),
-        'rouge2': float(averages.get('rouge2', 0.0)),
-        'rougeL': float(averages.get('rougeL', 0.0)),
-        'cer': float(averages.get('cer', 0.0)),
-        'wer': float(averages.get('wer', 0.0)),
-        'len_ratio': float(averages.get('len_ratio', 0.0)),
-
-        # Google comparable metrics
-        'google_quality_score': float(google_averages.get('quality_score', 0.0)),
-        'google_bleu': float(google_averages.get('bleu', 0.0)),
-        'google_chrf': float(google_averages.get('chrf', 0.0)),
-
-        # Coverage info
-        'total_samples': int(summary.get('total_samples', 0)),
-        'language_pairs_covered': int(summary.get('language_pairs_covered', 0)),
-        'google_pairs_covered': int(summary.get('google_comparable_pairs', 0)),
-        'coverage_rate': float(validation_info.get('coverage', 0.0)),
-
-        # Detailed results (JSON string)
-        'detailed_metrics': detailed_json,
-        'validation_report': validation_info.get('report', ''),
-
+        "submission_id": create_submission_id(),
+        "model_name": sanitize_model_name(model_name),
+        "author": author[:100] if author else "Anonymous",
+        "submission_date": datetime.datetime.now().isoformat(),
+        "model_category": (
+            model_category if model_category in MODEL_CATEGORIES else "community"
+        ),
+        "description": description[:500] if description else "",
+        # Extract track-specific metrics
+        **extract_track_metrics(tracks),
+        # Statistical metadata
+        **extract_statistical_metadata(tracks),
+        # Coverage information
+        **extract_coverage_information(tracks),
+        # Adequacy flags
+        **extract_adequacy_flags(tracks),
+        # Detailed results (JSON strings)
+        **serialize_detailed_results(tracks, cross_track),
         # Metadata
-        'evaluation_date': datetime.datetime.now().isoformat(),
-        'leaderboard_version': 1
+        "evaluation_date": datetime.datetime.now().isoformat(),
+        "leaderboard_version": 2,
+        "scientific_adequacy_score": adequacy_score,
     }
 
     # Convert to DataFrame and append
     new_row_df = pd.DataFrame([new_entry])
     updated_df = pd.concat([df, new_row_df], ignore_index=True)
-    updated_df = updated_df.sort_values('quality_score', ascending=False).reset_index(drop=True)
 
     # Save to hub
-    save_leaderboard(updated_df)
+    save_scientific_leaderboard(updated_df)
 
     return updated_df
 
-def prepare_leaderboard_display(df: pd.DataFrame) -> pd.DataFrame:
-    """Prepare leaderboard for display by formatting and selecting appropriate columns."""
-    
+
+def extract_track_metrics(tracks: Dict) -> Dict:
+    """Extract primary metrics from each track."""
+
+    metrics = {}
+
+    for track_name in EVALUATION_TRACKS.keys():
+        track_data = tracks.get(track_name, {})
+        track_averages = track_data.get("track_averages", {})
+
+        # Quality score
+        metrics[f"{track_name}_quality"] = float(
+            track_averages.get("quality_score", 0.0)
+        )
+
+        # BLEU score
+        metrics[f"{track_name}_bleu"] = float(track_averages.get("bleu", 0.0))
+
+        # ChrF score
+        metrics[f"{track_name}_chrf"] = float(track_averages.get("chrf", 0.0))
+
+    return metrics
+
+
+def extract_statistical_metadata(tracks: Dict) -> Dict:
+    """Extract confidence intervals from each track."""
+
+    metadata = {}
+
+    for track_name in EVALUATION_TRACKS.keys():
+        track_data = tracks.get(track_name, {})
+        track_statistics = track_data.get("track_statistics", {})
+
+        quality_stats = track_statistics.get("quality_score", {})
+        metadata[f"{track_name}_ci_lower"] = float(quality_stats.get("ci_lower", 0.0))
+        metadata[f"{track_name}_ci_upper"] = float(quality_stats.get("ci_upper", 0.0))
+
+    return metadata
+
+
+def extract_coverage_information(tracks: Dict) -> Dict:
+    """Extract coverage information from each track."""
+
+    coverage = {}
+
+    for track_name in EVALUATION_TRACKS.keys():
+        track_data = tracks.get(track_name, {})
+        summary = track_data.get("summary", {})
+
+        coverage[f"{track_name}_samples"] = int(summary.get("total_samples", 0))
+        coverage[f"{track_name}_pairs"] = int(
+            summary.get("language_pairs_evaluated", 0)
+        )
+
+    return coverage
+
+
+def extract_adequacy_flags(tracks: Dict) -> Dict:
+    """Extract statistical adequacy flags for each track."""
+
+    adequacy = {}
+
+    for track_name in EVALUATION_TRACKS.keys():
+        track_data = tracks.get(track_name, {})
+        summary = track_data.get("summary", {})
+
+        min_required = EVALUATION_TRACKS[track_name][
+            "min_samples_per_pair"
+        ] * summary.get("language_pairs_evaluated", 0)
+        is_adequate = summary.get("total_samples", 0) >= min_required
+
+        adequacy[f"{track_name}_adequate"] = bool(is_adequate)
+
+    return adequacy
+
+
+def serialize_detailed_results(tracks: Dict, cross_track: Dict) -> Dict:
+    """Serialize detailed results for storage."""
+
+    detailed = {}
+
+    for track_name in EVALUATION_TRACKS.keys():
+        track_data = tracks.get(track_name, {})
+
+        # Remove non-serializable data
+        safe_track_data = {}
+        for key, value in track_data.items():
+            if key != "sample_metrics":  # Skip large DataFrames
+                safe_track_data[key] = value
+
+        detailed[f"detailed_{track_name}"] = json.dumps(safe_track_data)
+
+    detailed["cross_track_analysis"] = json.dumps(cross_track)
+
+    return detailed
+
+
+def calculate_scientific_adequacy_score(evaluation_results: Dict) -> float:
+    """Calculate overall scientific adequacy score (0-1)."""
+
+    tracks = evaluation_results.get("tracks", {})
+
+    adequacy_scores = []
+
+    for track_name in EVALUATION_TRACKS.keys():
+        track_data = tracks.get(track_name, {})
+        summary = track_data.get("summary", {})
+
+        if track_data.get("error"):
+            adequacy_scores.append(0.0)
+            continue
+
+        # Sample size adequacy
+        min_required = EVALUATION_TRACKS[track_name][
+            "min_samples_per_pair"
+        ] * summary.get("language_pairs_evaluated", 0)
+        sample_adequacy = min(
+            summary.get("total_samples", 0) / max(min_required, 1), 1.0
+        )
+
+        # Coverage adequacy
+        total_possible_pairs = len(EVALUATION_TRACKS[track_name]["languages"]) * (
+            len(EVALUATION_TRACKS[track_name]["languages"]) - 1
+        )
+        coverage_adequacy = summary.get("language_pairs_evaluated", 0) / max(
+            total_possible_pairs, 1
+        )
+
+        # Track adequacy
+        track_adequacy = (sample_adequacy + coverage_adequacy) / 2
+        adequacy_scores.append(track_adequacy)
+
+    return float(np.mean(adequacy_scores))
+
+
+def get_track_leaderboard(
+    df: pd.DataFrame,
+    track: str,
+    metric: str = "quality",
+    category_filter: str = "all",
+    min_adequacy: float = 0.0,
+) -> pd.DataFrame:
+    """Get leaderboard for a specific track with filtering."""
+
+    if df.empty:
+        return df
+
+    track_quality_col = f"{track}_{metric}"
+    track_adequate_col = f"{track}_adequate"
+
+    # Filter by adequacy
+    if min_adequacy > 0:
+        adequacy_mask = df["scientific_adequacy_score"] >= min_adequacy
+        df = df[adequacy_mask]
+
+    # Filter by category
+    if category_filter != "all":
+        df = df[df["model_category"] == category_filter]
+
+    # Filter to models that have this track
+    valid_mask = (df[track_quality_col] > 0) & df[track_adequate_col]
+    df = df[valid_mask]
+
+    if df.empty:
+        return df
+
+    # Sort by track-specific metric
+    df = df.sort_values(track_quality_col, ascending=False).reset_index(drop=True)
+
+    return df
+
+
+def prepare_track_leaderboard_display(df: pd.DataFrame, track: str) -> pd.DataFrame:
+    """Prepare track-specific leaderboard for display."""
+
     if df.empty:
         return df
-    
-    # Select columns for display (exclude detailed_metrics and validation_report)
-    display_columns = [
-        'model_name', 'author', 'submission_date', 'model_type',
-        'quality_score', 'bleu', 'chrf', 
-        'rouge1', 'rougeL',
-        'total_samples', 'language_pairs_covered', 'google_pairs_covered',
-        'coverage_rate'
+
+    # Select relevant columns for this track
+    base_columns = ["model_name", "author", "submission_date", "model_category"]
+
+    track_columns = [
+        f"{track}_quality",
+        f"{track}_bleu",
+        f"{track}_chrf",
+        f"{track}_ci_lower",
+        f"{track}_ci_upper",
+        f"{track}_samples",
+        f"{track}_pairs",
+        f"{track}_adequate",
     ]
-    
+
     # Only include columns that exist
-    available_columns = [col for col in display_columns if col in df.columns]
+    available_columns = [
+        col for col in base_columns + track_columns if col in df.columns
+    ]
     display_df = df[available_columns].copy()
-    
+
     # Format numeric columns
     numeric_format = {
-        'quality_score': '{:.4f}',
-        'bleu': '{:.2f}',
-        'chrf': '{:.4f}',
-        'rouge1': '{:.4f}',
-        'rougeL': '{:.4f}',
-        'coverage_rate': '{:.1%}',
+        f"{track}_quality": "{:.4f}",
+        f"{track}_bleu": "{:.2f}",
+        f"{track}_chrf": "{:.4f}",
+        f"{track}_ci_lower": "{:.4f}",
+        f"{track}_ci_upper": "{:.4f}",
     }
-    
+
     for col, fmt in numeric_format.items():
         if col in display_df.columns:
-            display_df[col] = display_df[col].apply(lambda x: fmt.format(float(x)) if pd.notnull(x) else "0.0000")
-    
+            display_df[col] = display_df[col].apply(
+                lambda x: fmt.format(float(x)) if pd.notnull(x) else "0.0000"
+            )
+
+    # Format confidence intervals
+    if (
+        f"{track}_ci_lower" in display_df.columns
+        and f"{track}_ci_upper" in display_df.columns
+    ):
+        display_df[f"{track}_confidence_interval"] = (
+            "["
+            + display_df[f"{track}_ci_lower"]
+            + ", "
+            + display_df[f"{track}_ci_upper"]
+            + "]"
+        )
+        # Remove individual CI columns for cleaner display
+        display_df = display_df.drop(columns=[f"{track}_ci_lower", f"{track}_ci_upper"])
+
     # Format submission date
-    if 'submission_date' in display_df.columns:
-        display_df['submission_date'] = pd.to_datetime(display_df['submission_date']).dt.strftime('%Y-%m-%d %H:%M')
-    
+    if "submission_date" in display_df.columns:
+        display_df["submission_date"] = pd.to_datetime(
+            display_df["submission_date"]
+        ).dt.strftime("%Y-%m-%d")
+
     # Rename columns for better display
+    track_name = EVALUATION_TRACKS[track]["name"].split()[0]  # First word
     column_renames = {
-        'model_name': 'Model Name',
-        'author': 'Author', 
-        'submission_date': 'Submitted',
-        'model_type': 'Type',
-        'quality_score': 'Quality Score',
-        'bleu': 'BLEU',
-        'chrf': 'ChrF',
-        'rouge1': 'ROUGE-1',
-        'rougeL': 'ROUGE-L',
-        'total_samples': 'Samples',
-        'language_pairs_covered': 'Lang Pairs',
-        'google_pairs_covered': 'Google Pairs',
-        'coverage_rate': 'Coverage'
+        "model_name": "Model Name",
+        "author": "Author",
+        "submission_date": "Submitted",
+        "model_category": "Category",
+        f"{track}_quality": f"{track_name} Quality",
+        f"{track}_bleu": f"{track_name} BLEU",
+        f"{track}_chrf": f"{track_name} ChrF",
+        f"{track}_confidence_interval": "95% CI",
+        f"{track}_samples": "Samples",
+        f"{track}_pairs": "Pairs",
+        f"{track}_adequate": "Adequate",
     }
-    
+
     display_df = display_df.rename(columns=column_renames)
-    
+
     return display_df
 
-def get_leaderboard_stats(df: pd.DataFrame) -> Dict:
-    """Get summary statistics for the leaderboard."""
-    
+
+def get_scientific_leaderboard_stats(df: pd.DataFrame, track: str = None) -> Dict:
+    """Get comprehensive statistics for the scientific leaderboard."""
+
     if df.empty:
         return {
-            'total_models': 0,
-            'avg_quality_score': 0.0,
-            'best_model': None,
-            'latest_submission': None,
-            'google_comparable_models': 0,
-            'coverage_distribution': {},
-            'language_pair_coverage': {}
+            "total_models": 0,
+            "models_by_category": {},
+            "track_statistics": {},
+            "adequacy_distribution": {},
+            "best_models_by_track": {},
         }
-    
-    # Basic stats
+
     stats = {
-        'total_models': len(df),
-        'avg_quality_score': float(df['quality_score'].mean()),
-        'best_model': {
-            'name': df.iloc[0]['model_name'],
-            'score': float(df.iloc[0]['quality_score']),
-            'author': df.iloc[0]['author']
-        } if len(df) > 0 else None,
-        'latest_submission': df['submission_date'].max() if len(df) > 0 else None
+        "total_models": len(df),
+        "models_by_category": df["model_category"].value_counts().to_dict(),
+        "adequacy_distribution": {},
+        "track_statistics": {},
+        "best_models_by_track": {},
     }
-    
-    # Google comparable models
-    stats['google_comparable_models'] = int((df['google_pairs_covered'] > 0).sum())
-    
-    # Coverage distribution
-    coverage_bins = pd.cut(df['coverage_rate'], bins=[0, 0.5, 0.8, 0.95, 1.0], 
-                          labels=['<50%', '50-80%', '80-95%', '95-100%'])
-    stats['coverage_distribution'] = coverage_bins.value_counts().to_dict()
-    
-    # Language pair coverage
-    if len(df) > 0:
-        stats['avg_pairs_covered'] = float(df['language_pairs_covered'].mean())
-        stats['max_pairs_covered'] = int(df['language_pairs_covered'].max())
-        stats['total_possible_pairs'] = len(get_all_language_pairs())
-    
+
+    # Adequacy distribution
+    adequacy_bins = pd.cut(
+        df["scientific_adequacy_score"],
+        bins=[0, 0.3, 0.6, 0.8, 1.0],
+        labels=["Poor", "Fair", "Good", "Excellent"],
+    )
+    stats["adequacy_distribution"] = adequacy_bins.value_counts().to_dict()
+
+    # Track-specific statistics
+    for track_name in EVALUATION_TRACKS.keys():
+        quality_col = f"{track_name}_quality"
+        adequate_col = f"{track_name}_adequate"
+
+        if quality_col in df.columns and adequate_col in df.columns:
+            track_models = df[df[adequate_col] & (df[quality_col] > 0)]
+
+            if len(track_models) > 0:
+                stats["track_statistics"][track_name] = {
+                    "participating_models": len(track_models),
+                    "avg_quality": float(track_models[quality_col].mean()),
+                    "std_quality": float(track_models[quality_col].std()),
+                    "best_quality": float(track_models[quality_col].max()),
+                }
+
+                # Best model for this track
+                best_model = track_models.loc[track_models[quality_col].idxmax()]
+                stats["best_models_by_track"][track_name] = {
+                    "name": best_model["model_name"],
+                    "category": best_model["model_category"],
+                    "quality": float(best_model[quality_col]),
+                }
+
     return stats
 
-def filter_leaderboard(
-    df: pd.DataFrame,
-    search_query: str = "",
-    model_type: str = "",
-    min_coverage: float = 0.0,
-    google_comparable_only: bool = False,
-    top_n: int = None
-) -> pd.DataFrame:
-    """Filter leaderboard based on various criteria."""
-    
-    filtered_df = df.copy()
-    
-    # Text search
-    if search_query:
-        query_lower = search_query.lower()
-        mask = (
-            filtered_df['model_name'].str.lower().str.contains(query_lower, na=False) |
-            filtered_df['author'].str.lower().str.contains(query_lower, na=False) |
-            filtered_df['description'].str.lower().str.contains(query_lower, na=False)
-        )
-        filtered_df = filtered_df[mask]
-    
-    # Model type filter
-    if model_type and model_type != "all":
-        filtered_df = filtered_df[filtered_df['model_type'] == model_type]
-    
-    # Coverage filter
-    if min_coverage > 0:
-        filtered_df = filtered_df[filtered_df['coverage_rate'] >= min_coverage]
-    
-    # Google comparable filter
-    if google_comparable_only:
-        filtered_df = filtered_df[filtered_df['google_pairs_covered'] > 0]
-    
-    # Top N filter
-    if top_n:
-        filtered_df = filtered_df.head(top_n)
-    
-    return filtered_df
-
-def get_model_comparison(df: pd.DataFrame, model_names: List[str]) -> Dict:
-    """Get detailed comparison between specific models."""
-    
-    models = df[df['model_name'].isin(model_names)]
-    
+
+def perform_fair_comparison(
+    df: pd.DataFrame, model_names: List[str], shared_pairs_only: bool = True
+) -> Dict:
+    """Perform fair comparison between models using only shared language pairs."""
+
+    models = df[df["model_name"].isin(model_names)]
+
     if len(models) == 0:
-        return {'error': 'No models found'}
-    
+        return {"error": "No models found"}
+
     comparison = {
-        'models': [],
-        'metrics_comparison': {},
-        'detailed_results': {}
+        "models": list(models["model_name"]),
+        "fair_comparison_possible": True,
+        "track_comparisons": {},
+        "statistical_significance": {},
+        "recommendations": [],
     }
-    
-    # Extract basic info for each model
-    for _, model in models.iterrows():
-        comparison['models'].append({
-            'name': model['model_name'],
-            'author': model['author'],
-            'submission_date': model['submission_date'],
-            'model_type': model['model_type']
-        })
-        
-        # Parse detailed metrics if available
-        try:
-            detailed = json.loads(model['detailed_metrics'])
-            comparison['detailed_results'][model['model_name']] = detailed
-        except:
-            comparison['detailed_results'][model['model_name']] = {}
-    
-    # Compare metrics
-    metrics = ['quality_score', 'bleu', 'chrf', 'rouge1', 'rougeL', 'cer', 'wer']
-    for metric in metrics:
-        if metric in models.columns:
-            comparison['metrics_comparison'][metric] = {
-                model_name: float(score) 
-                for model_name, score in zip(models['model_name'], models[metric])
+
+    # Check if fair comparison is possible
+    categories = models["model_category"].unique()
+    if len(categories) > 1:
+        comparison["recommendations"].append(
+            "⚠️ Comparing models from different categories - interpret results carefully"
+        )
+
+    # For each track, compare models
+    for track_name in EVALUATION_TRACKS.keys():
+        quality_col = f"{track_name}_quality"
+        adequate_col = f"{track_name}_adequate"
+
+        track_models = models[models[adequate_col] & (models[quality_col] > 0)]
+
+        if len(track_models) >= 2:
+            comparison["track_comparisons"][track_name] = {
+                "participating_models": len(track_models),
+                "quality_scores": dict(
+                    zip(track_models["model_name"], track_models[quality_col])
+                ),
+                "confidence_intervals": {},
             }
-    
+
+            # Extract confidence intervals
+            for _, model in track_models.iterrows():
+                ci_lower = model.get(f"{track_name}_ci_lower", 0)
+                ci_upper = model.get(f"{track_name}_ci_upper", 0)
+                comparison["track_comparisons"][track_name]["confidence_intervals"][
+                    model["model_name"]
+                ] = [ci_lower, ci_upper]
+
     return comparison
 
-def export_leaderboard(df: pd.DataFrame, format: str = 'csv', include_detailed: bool = False) -> str:
-    """Export leaderboard in specified format."""
-    
+
+def export_scientific_leaderboard(
+    df: pd.DataFrame,
+    track: str = "all",
+    format: str = "csv",
+    include_detailed: bool = False,
+) -> str:
+    """Export scientific leaderboard in specified format."""
+
     timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-    
-    # Select columns for export
-    if include_detailed:
-        export_df = df.copy()
+
+    if track != "all":
+        # Export specific track
+        export_df = prepare_track_leaderboard_display(df, track)
+        filename_prefix = f"salt_leaderboard_{track}_{timestamp}"
     else:
-        basic_columns = [
-            'model_name', 'author', 'submission_date', 'model_type',
-            'quality_score', 'bleu', 'chrf', 'rouge1', 'rougeL',
-            'total_samples', 'language_pairs_covered', 'coverage_rate'
-        ]
-        export_df = df[basic_columns].copy()
-    
-    if format == 'csv':
-        filename = f"salt_leaderboard_{timestamp}.csv"
+        # Export all tracks
+        if include_detailed:
+            export_df = df.copy()
+        else:
+            # Select essential columns
+            essential_columns = [
+                "model_name",
+                "author",
+                "submission_date",
+                "model_category",
+                "scientific_adequacy_score",
+            ]
+
+            # Add track-specific quality scores
+            for track_name in EVALUATION_TRACKS.keys():
+                essential_columns.extend(
+                    [
+                        f"{track_name}_quality",
+                        f"{track_name}_adequate",
+                    ]
+                )
+
+            available_columns = [col for col in essential_columns if col in df.columns]
+            export_df = df[available_columns].copy()
+
+        filename_prefix = f"salt_leaderboard_scientific_{timestamp}"
+
+    # Export in specified format
+    if format == "csv":
+        filename = f"{filename_prefix}.csv"
         export_df.to_csv(filename, index=False)
-    elif format == 'json':
-        filename = f"salt_leaderboard_{timestamp}.json"
-        export_df.to_json(filename, orient='records', indent=2)
-    elif format == 'xlsx':
-        filename = f"salt_leaderboard_{timestamp}.xlsx"
+    elif format == "json":
+        filename = f"{filename_prefix}.json"
+        export_df.to_json(filename, orient="records", indent=2)
+    elif format == "xlsx":
+        filename = f"{filename_prefix}.xlsx"
         export_df.to_excel(filename, index=False)
     else:
         raise ValueError(f"Unsupported format: {format}")
-    
-    return filename
 
-def get_ranking_history(df: pd.DataFrame, model_name: str) -> Dict:
-    """Get ranking history for a specific model (if multiple submissions)."""
-    
-    model_entries = df[df['model_name'] == model_name].sort_values('submission_date')
-    
-    if len(model_entries) == 0:
-        return {'error': 'Model not found'}
-    
-    history = []
-    for _, entry in model_entries.iterrows():
-        # Calculate rank at time of submission
-        submission_date = entry['submission_date']
-        historical_df = df[df['submission_date'] <= submission_date]
-        rank = (historical_df['quality_score'] > entry['quality_score']).sum() + 1
-        
-        history.append({
-            'submission_date': submission_date,
-            'quality_score': float(entry['quality_score']),
-            'rank': int(rank),
-            'total_models': len(historical_df)
-        })
-    
-    return {
-        'model_name': model_name,
-        'history': history,
-        'current_rank': history[-1]['rank'] if history else None
-    }
+    return filename