#!/usr/bin/env python3 """ EuroBench Leaderboard - Hugging Face Space App Professional leaderboard with improved UI: - High contrast theme - Score heatmaps in tables - Cards and sections - Better typography - Export to CSV/JSON """ import json import pathlib from typing import Any import gradio as gr import pandas as pd from datasets import load_dataset # Constants EU_LANGUAGES = { "mt": "Maltese", "lt": "Lithuanian", "sk": "Slovak", "sl": "Slovenian", "ga": "Irish", "lv": "Latvian", "et": "Estonian", "hr": "Croatian", } LANG_ORDER = ["mt", "lt", "sk", "sl", "ga", "lv", "et", "hr"] RESULTS_DATASET = "EuroBench/results" RESULTS_FILE = "eurobench_results.json" # Custom theme with high contrast theme = gr.themes.Soft( primary_hue="blue", secondary_hue="indigo", neutral_hue="slate", ).set( body_background_fill="*neutral_50", body_text_color="*neutral_900", body_text_size="16px", background_fill_primary="*neutral_50", background_fill_secondary="*neutral_100", border_color_accent="*primary_300", border_color_primary="*neutral_200", block_label_text_color="*neutral_900", block_title_text_color="*neutral_900", input_background_fill="*neutral_0", input_border_color="*neutral_200", input_border_color_focus="*primary_300", button_primary_background_fill="*primary_600", button_primary_background_fill_hover="*primary_700", button_primary_text_color="*neutral_0", button_secondary_background_fill="*neutral_100", button_secondary_background_fill_hover="*neutral_200", button_secondary_text_color="*neutral_900", table_row_focus="*primary_100", table_even_background_fill="*neutral_0", table_odd_background_fill="*neutral_50", ) def load_results() -> dict[str, Any]: """Load results from the HF dataset.""" try: dataset = load_dataset(RESULTS_DATASET, split="train") if len(dataset) > 0: if "results" in dataset.column_names: return json.loads(dataset[0]["results"]) first_col = dataset.column_names[0] data = dataset[0][first_col] if isinstance(data, str): return json.loads(data) return data except Exception as e: print(f"Warning: Could not load from HF dataset: {e}") local_path = pathlib.Path(RESULTS_FILE) if local_path.exists(): with open(local_path) as f: return json.load(f) return {} def get_heatmap_style(val, vmin: float = -0.5, vmax: float = 1.0) -> str: """Generate HTML style for score heatmap coloring.""" if val is None: return "" try: val = float(val) except (ValueError, TypeError): return "" # Normalize to 0-1 normalized = (val - vmin) / (vmax - vmin) normalized = max(0.0, min(1.0, normalized)) # Color gradient: red (low) -> yellow (mid) -> green (high) if normalized < 0.5: # Red to yellow r = 255 g = int(255 * normalized * 2) b = 0 else: # Yellow to green r = int(255 * (1 - normalized) * 2) g = 255 b = 0 return f"background-color: rgba({r}, {g}, {b}, 0.25); color: #000; font-weight: 600;" def results_to_dataframe(results: dict, model_type: str) -> pd.DataFrame: """Convert results to a pandas DataFrame.""" models_data = results.get(model_type, {}).get("models", {}) if not models_data: return pd.DataFrame() rows = [] for model_name, model_data in models_data.items(): row = {"Model": model_name} scores = model_data.get("scores", {}) for lang in LANG_ORDER: lang_name = EU_LANGUAGES[lang] row[lang_name] = scores.get(lang, None) row["Average"] = model_data.get("average", None) rows.append(row) df = pd.DataFrame(rows) # Round numeric columns numeric_cols = [col for col in df.columns if col != "Model"] for col in numeric_cols: if col in df.columns: df[col] = df[col].round(3) return df def dataframe_to_styled_html(df: pd.DataFrame, title: str) -> str: """Convert DataFrame to styled HTML with heatmap.""" if df.empty: return f"""

No data available yet. Run evaluations to populate results.

""" # Get all numeric columns numeric_cols = [col for col in df.columns if col != "Model"] # Build header header = "" for col in df.columns: header += f""" {col} """ # Build rows rows = "" for i, (_, row) in enumerate(df.iterrows()): bg = "#f8fafc" if i % 2 == 0 else "#ffffff" cells = "" for col in df.columns: val = row[col] if col == "Model" or col == "Type" or col == "Metric": cells += f""" {val} """ else: style = get_heatmap_style(val) cells += f""" {f"{float(val):.3f}" if val is not None else "N/A"} """ rows += f""" {cells} """ html = f"""
{header} {rows}
""" return html def get_demo_results() -> dict: """Generate demo results for development.""" return { "slm": { "models": { "HuggingFaceTB/SmolLM2-135M": { "scores": {"mt": -0.45, "lt": -0.32, "sk": -0.38, "sl": -0.41, "ga": -0.52, "lv": -0.35, "et": -0.40, "hr": -0.37}, "average": -0.40, }, "HuggingFaceTB/SmolLM2-360M": { "scores": {"mt": -0.12, "lt": 0.05, "sk": -0.02, "sl": -0.08, "ga": -0.18, "lv": 0.01, "et": -0.05, "hr": -0.03}, "average": -0.05, }, "Qwen/Qwen2.5-0.5B": { "scores": {"mt": 0.15, "lt": 0.28, "sk": 0.22, "sl": 0.18, "ga": 0.08, "lv": 0.25, "et": 0.20, "hr": 0.23}, "average": 0.20, }, "Qwen/Qwen2.5-1.5B": { "scores": {"mt": 0.42, "lt": 0.55, "sk": 0.48, "sl": 0.45, "ga": 0.35, "lv": 0.52, "et": 0.47, "hr": 0.50}, "average": 0.47, }, "google/gemma-2-2b": { "scores": {"mt": 0.61, "lt": 0.72, "sk": 0.68, "sl": 0.65, "ga": 0.55, "lv": 0.70, "et": 0.66, "hr": 0.69}, "average": 0.66, }, } }, "embedding": { "models": { "intfloat/multilingual-e5-small": { "scores": {"mt": 0.72, "lt": 0.78, "sk": 0.75, "sl": 0.73, "ga": 0.68, "lv": 0.77, "et": 0.74, "hr": 0.76}, "average": 0.74, }, "intfloat/multilingual-e5-base": { "scores": {"mt": 0.85, "lt": 0.91, "sk": 0.88, "sl": 0.86, "ga": 0.82, "lv": 0.90, "et": 0.87, "hr": 0.89}, "average": 0.87, }, "BAAI/bge-m3": { "scores": {"mt": 0.92, "lt": 0.98, "sk": 0.95, "sl": 0.93, "ga": 0.89, "lv": 0.97, "et": 0.94, "hr": 0.96}, "average": 0.94, }, } } } def export_to_csv(df: pd.DataFrame) -> str: """Export DataFrame to CSV.""" if df.empty: return "No data to export" return df.to_csv(index=False) def export_to_json(results: dict, model_type: str) -> str: """Export results to JSON.""" models_data = results.get(model_type, {}).get("models", {}) return json.dumps(models_data, indent=2) def create_app() -> gr.Blocks: """Create the Gradio leaderboard app.""" results = load_results() if not results: results = get_demo_results() slm_df = results_to_dataframe(results, "slm") embedding_df = results_to_dataframe(results, "embedding") with gr.Blocks( title="EuroBench Leaderboard", theme=theme, css=""" .header-card { background: linear-gradient(135deg, #1e3a5f 0%, #2c5282 100%); color: white; padding: 2rem; border-radius: 12px; margin-bottom: 1.5rem; box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1); } .header-card h1 { color: white; font-size: 2.25rem; font-weight: 800; margin-bottom: 0.5rem; } .header-card p { color: #e0e7ff; font-size: 1.1rem; line-height: 1.6; } .header-card .tag { display: inline-block; background: rgba(255,255,255,0.15); padding: 4px 12px; border-radius: 20px; font-size: 0.85rem; margin: 0.25rem; color: #c7d2fe; } .info-card { background: #ffffff; border: 1px solid #e5e7eb; border-radius: 8px; padding: 1.5rem; margin: 1rem 0; box-shadow: 0 1px 3px rgba(0,0,0,0.08); } .info-card h3 { color: #1e3a5f; font-size: 1.2rem; font-weight: 700; margin-bottom: 0.75rem; border-bottom: 2px solid #e0e7ff; padding-bottom: 0.5rem; } .info-card ul { color: #374151; line-height: 1.8; padding-left: 1.2rem; } .info-card li { margin-bottom: 0.25rem; } .section-title { color: #1e3a5f; font-size: 1.5rem; font-weight: 700; margin: 1.5rem 0 1rem 0; padding-bottom: 0.5rem; border-bottom: 3px solid #e0e7ff; } .stat-badge { display: inline-flex; align-items: center; gap: 0.5rem; background: #eff6ff; border: 1px solid #dbeafe; padding: 0.5rem 1rem; border-radius: 6px; font-weight: 600; color: #1e3a5f; margin: 0.25rem; } .footer { margin-top: 2rem; padding: 1.5rem; background: #f8fafc; border-top: 1px solid #e5e7eb; text-align: center; color: #6b7280; font-size: 0.9rem; } """ ) as app: # Header gr.Markdown("""

🏆 EuroBench Leaderboard

Benchmarking sub-3B SLMs and compact embedding models on 8 low-resource EU languages

Maltese Lithuanian Slovak Slovenian Irish Latvian Estonian Croatian
Scoring: Z-score normalisation per task type, equal weight per task type, one score per model per language
""", elem_classes=["header-card"]) with gr.Tabs(): # Tab 1: SLM Models with gr.TabItem("🤖 SLM Models"): gr.Markdown("""
Sub-3B Language Models
""") if not slm_df.empty: slm_html = dataframe_to_styled_html(slm_df, "SLM Results") gr.HTML(slm_html) with gr.Row(): slm_csv_btn = gr.Button("đŸ“Ĩ Export CSV", variant="primary") slm_json_btn = gr.Button("đŸ“Ĩ Export JSON", variant="secondary") slm_csv_output = gr.Textbox(label="CSV Output", visible=False) slm_json_output = gr.Textbox(label="JSON Output", visible=False) slm_csv_btn.click( fn=lambda: export_to_csv(slm_df), outputs=slm_csv_output, ) slm_json_btn.click( fn=lambda: export_to_json(results, "slm"), outputs=slm_json_output, ) else: gr.HTML("""

No SLM results available yet.

""") gr.HTML("""

Models

Tasks

""") # Tab 2: Embedding Models with gr.TabItem("🔗 Embedding Models"): gr.Markdown("""
Compact Embedding Models
""") if not embedding_df.empty: emb_html = dataframe_to_styled_html(embedding_df, "Embedding Results") gr.HTML(emb_html) with gr.Row(): emb_csv_btn = gr.Button("đŸ“Ĩ Export CSV", variant="primary") emb_json_btn = gr.Button("đŸ“Ĩ Export JSON", variant="secondary") emb_csv_output = gr.Textbox(label="CSV Output", visible=False) emb_json_output = gr.Textbox(label="JSON Output", visible=False) emb_csv_btn.click( fn=lambda: export_to_csv(embedding_df), outputs=emb_csv_output, ) emb_json_btn.click( fn=lambda: export_to_json(results, "embedding"), outputs=emb_json_output, ) else: gr.HTML("""

No embedding results available yet.

""") gr.HTML("""

Models

Tasks

""") # Tab 3: Per-language Breakdown with gr.TabItem("🌍 Per-language Breakdown"): gr.Markdown("""
Drill-down by Language
""") lang_dropdown = gr.Dropdown( choices=[(f"{name} ({code})", code) for code, name in EU_LANGUAGES.items()], value="mt", label="Select Language", ) lang_html = gr.HTML() def update_lang_table(lang): """Update table for selected language.""" rows = [] for model_type in ["slm", "embedding"]: for model_name, model_data in results.get(model_type, {}).get("models", {}).items(): scores = model_data.get("scores", {}) if lang in scores: rows.append({ "Model": model_name, "Type": model_type.upper(), "Score": round(scores[lang], 3), }) if not rows: return """
No data for this language.
""" df = pd.DataFrame(rows) df = df.sort_values("Score", ascending=False) return dataframe_to_styled_html(df, f"Results for {EU_LANGUAGES[lang]}") lang_dropdown.change( fn=update_lang_table, inputs=lang_dropdown, outputs=lang_html, ) # Initialize with default lang_html.value = update_lang_table("mt") # Tab 4: Model Comparison with gr.TabItem("⚖ī¸ Model Comparison"): gr.Markdown("""
Compare Models Side-by-Side
""") all_models = ( list(results.get("slm", {}).get("models", {}).keys()) + list(results.get("embedding", {}).get("models", {}).keys()) ) with gr.Row(): model1_dropdown = gr.Dropdown( choices=all_models, label="Model 1", value=all_models[0] if all_models else None, ) model2_dropdown = gr.Dropdown( choices=all_models, label="Model 2", value=all_models[1] if len(all_models) > 1 else all_models[0] if all_models else None, ) comparison_html = gr.HTML() def compare_models(model1, model2): """Compare two models.""" if not model1 or not model2: return "" rows = [] for model_type in ["slm", "embedding"]: models = results.get(model_type, {}).get("models", {}) if model1 in models: data1 = models[model1] row1 = {"Metric": model1} for lang in LANG_ORDER: row1[EU_LANGUAGES[lang]] = round(data1.get("scores", {}).get(lang, 0), 3) row1["Average"] = round(data1.get("average", 0), 3) rows.append(row1) if model2 in models: data2 = models[model2] row2 = {"Metric": model2} for lang in LANG_ORDER: row2[EU_LANGUAGES[lang]] = round(data2.get("scores", {}).get(lang, 0), 3) row2["Average"] = round(data2.get("average", 0), 3) rows.append(row2) if not rows: return "" df = pd.DataFrame(rows) return dataframe_to_styled_html(df, "Comparison") for dropdown in [model1_dropdown, model2_dropdown]: dropdown.change( fn=compare_models, inputs=[model1_dropdown, model2_dropdown], outputs=comparison_html, ) # Initialize comparison_html.value = compare_models( all_models[0] if all_models else None, all_models[1] if len(all_models) > 1 else all_models[0] if all_models else None, ) # Tab 5: About with gr.TabItem("ℹī¸ About"): gr.HTML("""

Methodology

Scoring

Compute

Links

License

Results and code are released under the AGPL-3.0 license.

""") # Footer gr.HTML("""

EuroBench 2026 | Benchmarking European Language Models

""") return app if __name__ == "__main__": app = create_app() app.launch()