app.py

#!/usr/bin/env python3
"""
EuroBench Leaderboard - Hugging Face Space App

Professional leaderboard with improved UI:
- High contrast theme
- Score heatmaps in tables
- Cards and sections
- Better typography
- Export to CSV/JSON
"""

import json
import pathlib
from typing import Any

import gradio as gr
import pandas as pd
from datasets import load_dataset

# Constants
EU_LANGUAGES = {
    "mt": "Maltese",
    "lt": "Lithuanian",
    "sk": "Slovak",
    "sl": "Slovenian",
    "ga": "Irish",
    "lv": "Latvian",
    "et": "Estonian",
    "hr": "Croatian",
}

LANG_ORDER = ["mt", "lt", "sk", "sl", "ga", "lv", "et", "hr"]

RESULTS_DATASET = "EuroBench/results"
RESULTS_FILE = "eurobench_results.json"

# Custom theme with high contrast
theme = gr.themes.Soft(
    primary_hue="blue",
    secondary_hue="indigo",
    neutral_hue="slate",
).set(
    body_background_fill="*neutral_50",
    body_text_color="*neutral_900",
    body_text_size="16px",
    background_fill_primary="*neutral_50",
    background_fill_secondary="*neutral_100",
    border_color_accent="*primary_300",
    border_color_primary="*neutral_200",
    block_label_text_color="*neutral_900",
    block_title_text_color="*neutral_900",
    input_background_fill="*neutral_0",
    input_border_color="*neutral_200",
    input_border_color_focus="*primary_300",
    button_primary_background_fill="*primary_600",
    button_primary_background_fill_hover="*primary_700",
    button_primary_text_color="*neutral_0",
    button_secondary_background_fill="*neutral_100",
    button_secondary_background_fill_hover="*neutral_200",
    button_secondary_text_color="*neutral_900",
    table_row_focus="*primary_100",
    table_even_background_fill="*neutral_0",
    table_odd_background_fill="*neutral_50",
)


def load_results() -> dict[str, Any]:
    """Load results from the HF dataset."""
    try:
        dataset = load_dataset(RESULTS_DATASET, split="train")
        if len(dataset) > 0:
            if "results" in dataset.column_names:
                return json.loads(dataset[0]["results"])
            first_col = dataset.column_names[0]
            data = dataset[0][first_col]
            if isinstance(data, str):
                return json.loads(data)
            return data
    except Exception as e:
        print(f"Warning: Could not load from HF dataset: {e}")
    
    local_path = pathlib.Path(RESULTS_FILE)
    if local_path.exists():
        with open(local_path) as f:
            return json.load(f)
    
    return {}


def get_heatmap_style(val, vmin: float = -0.5, vmax: float = 1.0) -> str:
    """Generate HTML style for score heatmap coloring."""
    if val is None:
        return ""
    try:
        val = float(val)
    except (ValueError, TypeError):
        return ""
    # Normalize to 0-1
    normalized = (val - vmin) / (vmax - vmin)
    normalized = max(0.0, min(1.0, normalized))
    
    # Color gradient: red (low) -> yellow (mid) -> green (high)
    if normalized < 0.5:
        # Red to yellow
        r = 255
        g = int(255 * normalized * 2)
        b = 0
    else:
        # Yellow to green
        r = int(255 * (1 - normalized) * 2)
        g = 255
        b = 0
    
    return f"background-color: rgba({r}, {g}, {b}, 0.25); color: #000; font-weight: 600;"


def results_to_dataframe(results: dict, model_type: str) -> pd.DataFrame:
    """Convert results to a pandas DataFrame."""
    models_data = results.get(model_type, {}).get("models", {})
    
    if not models_data:
        return pd.DataFrame()
    
    rows = []
    for model_name, model_data in models_data.items():
        row = {"Model": model_name}
        
        scores = model_data.get("scores", {})
        for lang in LANG_ORDER:
            lang_name = EU_LANGUAGES[lang]
            row[lang_name] = scores.get(lang, None)
        
        row["Average"] = model_data.get("average", None)
        rows.append(row)
    
    df = pd.DataFrame(rows)
    
    # Round numeric columns
    numeric_cols = [col for col in df.columns if col != "Model"]
    for col in numeric_cols:
        if col in df.columns:
            df[col] = df[col].round(3)
    
    return df


def dataframe_to_styled_html(df: pd.DataFrame, title: str) -> str:
    """Convert DataFrame to styled HTML with heatmap."""
    if df.empty:
        return f"""
        <div style="padding: 2rem; text-align: center; color: #6b7280; font-size: 1.1rem;">
            <p>No data available yet. Run evaluations to populate results.</p>
        </div>
        """
    
    # Get all numeric columns
    numeric_cols = [col for col in df.columns if col != "Model"]
    
    # Build header
    header = ""
    for col in df.columns:
        header += f"""
        <th style="padding: 12px 16px; text-align: left; border-bottom: 2px solid #e5e7eb; 
                   font-weight: 700; color: #1e3a5f; background: #eff6ff;">
            {col}
        </th>
        """
    
    # Build rows
    rows = ""
    for i, (_, row) in enumerate(df.iterrows()):
        bg = "#f8fafc" if i % 2 == 0 else "#ffffff"
        cells = ""
        
        for col in df.columns:
            val = row[col]
            if col == "Model" or col == "Type" or col == "Metric":
                cells += f"""
                <td style="padding: 10px 16px; text-align: left; font-weight: 600; 
                           color: #111827; border-bottom: 1px solid #e5e7eb;">
                    {val}
                </td>
                """
            else:
                style = get_heatmap_style(val)
                cells += f"""
                <td style="padding: 10px 16px; text-align: center; border-bottom: 1px solid #e5e7eb;
                           {style}">
                    {f"{float(val):.3f}" if val is not None else "N/A"}
                </td>
                """
        
        rows += f"""
        <tr style="background: {bg};">
            {cells}
        </tr>
        """
    
    html = f"""
    <div style="margin: 1rem 0; border-radius: 8px; overflow: hidden; border: 1px solid #e5e7eb;">
        <table style="width: 100%; border-collapse: collapse; font-size: 0.95rem;">
            <thead>
                <tr>{header}</tr>
            </thead>
            <tbody>
                {rows}
            </tbody>
        </table>
    </div>
    """
    
    return html


def get_demo_results() -> dict:
    """Generate demo results for development."""
    return {
        "slm": {
            "models": {
                "HuggingFaceTB/SmolLM2-135M": {
                    "scores": {"mt": -0.45, "lt": -0.32, "sk": -0.38, "sl": -0.41, "ga": -0.52, "lv": -0.35, "et": -0.40, "hr": -0.37},
                    "average": -0.40,
                },
                "HuggingFaceTB/SmolLM2-360M": {
                    "scores": {"mt": -0.12, "lt": 0.05, "sk": -0.02, "sl": -0.08, "ga": -0.18, "lv": 0.01, "et": -0.05, "hr": -0.03},
                    "average": -0.05,
                },
                "Qwen/Qwen2.5-0.5B": {
                    "scores": {"mt": 0.15, "lt": 0.28, "sk": 0.22, "sl": 0.18, "ga": 0.08, "lv": 0.25, "et": 0.20, "hr": 0.23},
                    "average": 0.20,
                },
                "Qwen/Qwen2.5-1.5B": {
                    "scores": {"mt": 0.42, "lt": 0.55, "sk": 0.48, "sl": 0.45, "ga": 0.35, "lv": 0.52, "et": 0.47, "hr": 0.50},
                    "average": 0.47,
                },
                "google/gemma-2-2b": {
                    "scores": {"mt": 0.61, "lt": 0.72, "sk": 0.68, "sl": 0.65, "ga": 0.55, "lv": 0.70, "et": 0.66, "hr": 0.69},
                    "average": 0.66,
                },
            }
        },
        "embedding": {
            "models": {
                "intfloat/multilingual-e5-small": {
                    "scores": {"mt": 0.72, "lt": 0.78, "sk": 0.75, "sl": 0.73, "ga": 0.68, "lv": 0.77, "et": 0.74, "hr": 0.76},
                    "average": 0.74,
                },
                "intfloat/multilingual-e5-base": {
                    "scores": {"mt": 0.85, "lt": 0.91, "sk": 0.88, "sl": 0.86, "ga": 0.82, "lv": 0.90, "et": 0.87, "hr": 0.89},
                    "average": 0.87,
                },
                "BAAI/bge-m3": {
                    "scores": {"mt": 0.92, "lt": 0.98, "sk": 0.95, "sl": 0.93, "ga": 0.89, "lv": 0.97, "et": 0.94, "hr": 0.96},
                    "average": 0.94,
                },
            }
        }
    }


def export_to_csv(df: pd.DataFrame) -> str:
    """Export DataFrame to CSV."""
    if df.empty:
        return "No data to export"
    return df.to_csv(index=False)


def export_to_json(results: dict, model_type: str) -> str:
    """Export results to JSON."""
    models_data = results.get(model_type, {}).get("models", {})
    return json.dumps(models_data, indent=2)


def create_app() -> gr.Blocks:
    """Create the Gradio leaderboard app."""
    
    results = load_results()
    if not results:
        results = get_demo_results()
    
    slm_df = results_to_dataframe(results, "slm")
    embedding_df = results_to_dataframe(results, "embedding")
    
    with gr.Blocks(
        title="EuroBench Leaderboard",
        theme=theme,
        css="""
        .header-card {
            background: linear-gradient(135deg, #1e3a5f 0%, #2c5282 100%);
            color: white;
            padding: 2rem;
            border-radius: 12px;
            margin-bottom: 1.5rem;
            box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1);
        }
        .header-card h1 {
            color: white;
            font-size: 2.25rem;
            font-weight: 800;
            margin-bottom: 0.5rem;
        }
        .header-card p {
            color: #e0e7ff;
            font-size: 1.1rem;
            line-height: 1.6;
        }
        .header-card .tag {
            display: inline-block;
            background: rgba(255,255,255,0.15);
            padding: 4px 12px;
            border-radius: 20px;
            font-size: 0.85rem;
            margin: 0.25rem;
            color: #c7d2fe;
        }
        .info-card {
            background: #ffffff;
            border: 1px solid #e5e7eb;
            border-radius: 8px;
            padding: 1.5rem;
            margin: 1rem 0;
            box-shadow: 0 1px 3px rgba(0,0,0,0.08);
        }
        .info-card h3 {
            color: #1e3a5f;
            font-size: 1.2rem;
            font-weight: 700;
            margin-bottom: 0.75rem;
            border-bottom: 2px solid #e0e7ff;
            padding-bottom: 0.5rem;
        }
        .info-card ul {
            color: #374151;
            line-height: 1.8;
            padding-left: 1.2rem;
        }
        .info-card li {
            margin-bottom: 0.25rem;
        }
        .section-title {
            color: #1e3a5f;
            font-size: 1.5rem;
            font-weight: 700;
            margin: 1.5rem 0 1rem 0;
            padding-bottom: 0.5rem;
            border-bottom: 3px solid #e0e7ff;
        }
        .stat-badge {
            display: inline-flex;
            align-items: center;
            gap: 0.5rem;
            background: #eff6ff;
            border: 1px solid #dbeafe;
            padding: 0.5rem 1rem;
            border-radius: 6px;
            font-weight: 600;
            color: #1e3a5f;
            margin: 0.25rem;
        }
        .footer {
            margin-top: 2rem;
            padding: 1.5rem;
            background: #f8fafc;
            border-top: 1px solid #e5e7eb;
            text-align: center;
            color: #6b7280;
            font-size: 0.9rem;
        }
        """
    ) as app:
        
        # Header
        gr.Markdown("""
        <div class="header-card">
            <h1>🏆 EuroBench Leaderboard</h1>
            <p>Benchmarking sub-3B SLMs and compact embedding models on 8 low-resource EU languages</p>
            <div style="margin-top: 1rem;">
                <span class="tag">Maltese</span>
                <span class="tag">Lithuanian</span>
                <span class="tag">Slovak</span>
                <span class="tag">Slovenian</span>
                <span class="tag">Irish</span>
                <span class="tag">Latvian</span>
                <span class="tag">Estonian</span>
                <span class="tag">Croatian</span>
            </div>
            <div style="margin-top: 0.75rem; font-size: 0.9rem; color: #c7d2fe;">
                Scoring: Z-score normalisation per task type, equal weight per task type, one score per model per language
            </div>
        </div>
        """, elem_classes=["header-card"])
        
        with gr.Tabs():
            # Tab 1: SLM Models
            with gr.TabItem("🤖 SLM Models"):
                gr.Markdown("""<div class="section-title">Sub-3B Language Models</div>""")
                
                if not slm_df.empty:
                    slm_html = dataframe_to_styled_html(slm_df, "SLM Results")
                    gr.HTML(slm_html)
                    
                    with gr.Row():
                        slm_csv_btn = gr.Button("📥 Export CSV", variant="primary")
                        slm_json_btn = gr.Button("📥 Export JSON", variant="secondary")
                    
                    slm_csv_output = gr.Textbox(label="CSV Output", visible=False)
                    slm_json_output = gr.Textbox(label="JSON Output", visible=False)
                    
                    slm_csv_btn.click(
                        fn=lambda: export_to_csv(slm_df),
                        outputs=slm_csv_output,
                    )
                    slm_json_btn.click(
                        fn=lambda: export_to_json(results, "slm"),
                        outputs=slm_json_output,
                    )
                else:
                    gr.HTML("""
                    <div style="padding: 3rem; text-align: center; background: #f8fafc; border-radius: 8px;">
                        <p style="color: #6b7280; font-size: 1.1rem;">No SLM results available yet.</p>
                    </div>
                    """)
                
                gr.HTML("""
                <div class="info-card">
                    <h3>Models</h3>
                    <ul>
                        <li><strong>HuggingFaceTB/SmolLM2-135M</strong></li>
                        <li><strong>HuggingFaceTB/SmolLM2-360M</strong></li>
                        <li><strong>HuggingFaceTB/SmolLM2-1.7B</strong></li>
                        <li><strong>Qwen/Qwen2.5-0.5B</strong></li>
                        <li><strong>Qwen/Qwen2.5-1.5B</strong></li>
                        <li><strong>Qwen/Qwen2.5-3B</strong></li>
                        <li><strong>meta-llama/Llama-3.2-1B</strong></li>
                        <li><strong>meta-llama/Llama-3.2-3B</strong></li>
                        <li><strong>google/gemma-2-2b</strong></li>
                    </ul>
                    <h3>Tasks</h3>
                    <ul>
                        <li><strong>Belebele</strong> - Reading comprehension</li>
                        <li><strong>Global-PIQA</strong> - Physical reasoning</li>
                        <li><strong>MultiEURLEX</strong> - Legal text classification</li>
                        <li><strong>Maltese NLI</strong> - Natural language inference</li>
                        <li><strong>Global-MMLU</strong> - Knowledge evaluation</li>
                    </ul>
                </div>
                """)
            
            # Tab 2: Embedding Models
            with gr.TabItem("🔗 Embedding Models"):
                gr.Markdown("""<div class="section-title">Compact Embedding Models</div>""")
                
                if not embedding_df.empty:
                    emb_html = dataframe_to_styled_html(embedding_df, "Embedding Results")
                    gr.HTML(emb_html)
                    
                    with gr.Row():
                        emb_csv_btn = gr.Button("📥 Export CSV", variant="primary")
                        emb_json_btn = gr.Button("📥 Export JSON", variant="secondary")
                    
                    emb_csv_output = gr.Textbox(label="CSV Output", visible=False)
                    emb_json_output = gr.Textbox(label="JSON Output", visible=False)
                    
                    emb_csv_btn.click(
                        fn=lambda: export_to_csv(embedding_df),
                        outputs=emb_csv_output,
                    )
                    emb_json_btn.click(
                        fn=lambda: export_to_json(results, "embedding"),
                        outputs=emb_json_output,
                    )
                else:
                    gr.HTML("""
                    <div style="padding: 3rem; text-align: center; background: #f8fafc; border-radius: 8px;">
                        <p style="color: #6b7280; font-size: 1.1rem;">No embedding results available yet.</p>
                    </div>
                    """)
                
                gr.HTML("""
                <div class="info-card">
                    <h3>Models</h3>
                    <ul>
                        <li><strong>intfloat/multilingual-e5-small</strong> (118M)</li>
                        <li><strong>intfloat/multilingual-e5-base</strong> (278M)</li>
                        <li><strong>BAAI/bge-m3</strong> (568M)</li>
                        <li><strong>Alibaba-NLP/gte-multilingual-base</strong> (305M)</li>
                        <li><strong>sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2</strong> (118M)</li>
                    </ul>
                    <h3>Tasks</h3>
                    <ul>
                        <li><strong>BitextMining</strong> - Cross-lingual sentence similarity (BibleNLP, Flores, NTREX, Tatoeba)</li>
                        <li><strong>Retrieval</strong> - Passage retrieval (BelebeleRetrieval, WebFAQRetrieval)</li>
                        <li><strong>Classification</strong> - Topic categorisation (SIB200, MultiEURLEX)</li>
                    </ul>
                </div>
                """)
            
            # Tab 3: Per-language Breakdown
            with gr.TabItem("🌍 Per-language Breakdown"):
                gr.Markdown("""<div class="section-title">Drill-down by Language</div>""")
                
                lang_dropdown = gr.Dropdown(
                    choices=[(f"{name} ({code})", code) for code, name in EU_LANGUAGES.items()],
                    value="mt",
                    label="Select Language",
                )
                
                lang_html = gr.HTML()
                
                def update_lang_table(lang):
                    """Update table for selected language."""
                    rows = []
                    
                    for model_type in ["slm", "embedding"]:
                        for model_name, model_data in results.get(model_type, {}).get("models", {}).items():
                            scores = model_data.get("scores", {})
                            if lang in scores:
                                rows.append({
                                    "Model": model_name,
                                    "Type": model_type.upper(),
                                    "Score": round(scores[lang], 3),
                                })
                    
                    if not rows:
                        return """
                        <div style="padding: 2rem; text-align: center; color: #6b7280;">
                            No data for this language.
                        </div>
                        """
                    
                    df = pd.DataFrame(rows)
                    df = df.sort_values("Score", ascending=False)
                    
                    return dataframe_to_styled_html(df, f"Results for {EU_LANGUAGES[lang]}")
                
                lang_dropdown.change(
                    fn=update_lang_table,
                    inputs=lang_dropdown,
                    outputs=lang_html,
                )
                
                # Initialize with default
                lang_html.value = update_lang_table("mt")
            
            # Tab 4: Model Comparison
            with gr.TabItem("⚖️ Model Comparison"):
                gr.Markdown("""<div class="section-title">Compare Models Side-by-Side</div>""")
                
                all_models = (
                    list(results.get("slm", {}).get("models", {}).keys()) + 
                    list(results.get("embedding", {}).get("models", {}).keys())
                )
                
                with gr.Row():
                    model1_dropdown = gr.Dropdown(
                        choices=all_models,
                        label="Model 1",
                        value=all_models[0] if all_models else None,
                    )
                    model2_dropdown = gr.Dropdown(
                        choices=all_models,
                        label="Model 2",
                        value=all_models[1] if len(all_models) > 1 else all_models[0] if all_models else None,
                    )
                
                comparison_html = gr.HTML()
                
                def compare_models(model1, model2):
                    """Compare two models."""
                    if not model1 or not model2:
                        return ""
                    
                    rows = []
                    for model_type in ["slm", "embedding"]:
                        models = results.get(model_type, {}).get("models", {})
                        
                        if model1 in models:
                            data1 = models[model1]
                            row1 = {"Metric": model1}
                            for lang in LANG_ORDER:
                                row1[EU_LANGUAGES[lang]] = round(data1.get("scores", {}).get(lang, 0), 3)
                            row1["Average"] = round(data1.get("average", 0), 3)
                            rows.append(row1)
                        
                        if model2 in models:
                            data2 = models[model2]
                            row2 = {"Metric": model2}
                            for lang in LANG_ORDER:
                                row2[EU_LANGUAGES[lang]] = round(data2.get("scores", {}).get(lang, 0), 3)
                            row2["Average"] = round(data2.get("average", 0), 3)
                            rows.append(row2)
                    
                    if not rows:
                        return ""
                    
                    df = pd.DataFrame(rows)
                    return dataframe_to_styled_html(df, "Comparison")
                
                for dropdown in [model1_dropdown, model2_dropdown]:
                    dropdown.change(
                        fn=compare_models,
                        inputs=[model1_dropdown, model2_dropdown],
                        outputs=comparison_html,
                    )
                
                # Initialize
                comparison_html.value = compare_models(
                    all_models[0] if all_models else None,
                    all_models[1] if len(all_models) > 1 else all_models[0] if all_models else None,
                )
            
            # Tab 5: About
            with gr.TabItem("ℹ️ About"):
                gr.HTML("""
                <div class="info-card">
                    <h3>Methodology</h3>
                    <ul>
                        <li><strong>Dataset Curation:</strong> All datasets from MTEB's validated multilingual benchmarks</li>
                        <li><strong>BitextMining:</strong> Cross-lingual sentence similarity (BibleNLP, Flores, NTREX, Tatoeba)</li>
                        <li><strong>Retrieval:</strong> Passage retrieval (BelebeleRetrieval, WebFAQRetrieval)</li>
                        <li><strong>Classification:</strong> Topic categorisation (SIB200, MultiEURLEX)</li>
                        <li><strong>SLM Tasks:</strong> Belebele, Global-PIQA, MultiEURLEX, NLI, Global-MMLU</li>
                    </ul>
                    
                    <h3>Scoring</h3>
                    <ul>
                        <li><strong>Z-score normalisation:</strong> Per task type (mean=0, std=1)</li>
                        <li><strong>Equal weighting:</strong> Each task type contributes equally</li>
                        <li><strong>Aggregation:</strong> Mean of task scores per language, then overall average</li>
                    </ul>
                    
                    <h3>Compute</h3>
                    <ul>
                        <li>Evaluations run on Kaggle free tier (2x T4 GPUs)</li>
                    </ul>
                    
                    <h3>Links</h3>
                    <ul>
                        <li><strong>GitHub:</strong> <a href="https://github.com/andrijdavid/eurobench" target="_blank">andrijdavid/eurobench</a></li>
                        <li><strong>Hugging Face:</strong> <a href="https://huggingface.co/EuroBench" target="_blank">EuroBench</a></li>
                        <li><strong>Results Dataset:</strong> <a href="https://huggingface.co/datasets/EuroBench/results" target="_blank">EuroBench/results</a></li>
                    </ul>
                    
                    <h3>License</h3>
                    <p>Results and code are released under the AGPL-3.0 license.</p>
                </div>
                """)
        
        # Footer
        gr.HTML("""
        <div class="footer">
            <p>EuroBench 2026 | Benchmarking European Language Models</p>
        </div>
        """)
    
    return app


if __name__ == "__main__":
    app = create_app()
    app.launch()