Add temporary DABstep leaderboard mirror

README.md

@@ -1,14 +1,29 @@
 ---
-title: DABstep Temp
-emoji: 👁
-colorFrom: yellow
-colorTo: gray
+title: DABstep Leaderboard — Temporary Mirror
+emoji: 🕺
+colorFrom: purple
+colorTo: blue
 sdk: gradio
 sdk_version: 6.1.0
 app_file: app.py
 pinned: false
 license: cc-by-4.0
-short_description: Temporary Leaderboard for DABstep benchmark
+short_description: Temporary mirror of DABstep benchmark leaderboard
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🕺 DABstep Leaderboard — Temporary Mirror
+
+This is a **temporary mirror** of the [DABstep Benchmark Leaderboard](https://huggingface.co/spaces/adyen/DABstep) while the official Adyen leaderboard is experiencing issues.
+
+## ⚠️ Important Notes
+
+- Submissions here **will NOT sync** to the official Adyen leaderboard
+- Once the official leaderboard is restored, please re-submit there
+- For questions: [support@genesiscomputing.ai](mailto:support@genesiscomputing.ai)
+- Hosted by [Genesis Computing](https://genesiscomputing.ai)
+
+## About DABstep
+
+DABstep (Data Agent Benchmark for Multi-Step Reasoning) evaluates AI agents on complex data analysis tasks requiring multi-step reasoning over structured datasets.
+
+See the [HF Discussion](https://huggingface.co/spaces/adyen/DABstep/discussions/17) about the official leaderboard status.

app.py

@@ -0,0 +1,219 @@
+"""
+DABstep Leaderboard - Genesis Edition
+A self-contained leaderboard for the DABstep benchmark.
+
+Original source: https://huggingface.co/spaces/adyen/DABstep
+"""
+import os
+
+import gradio as gr
+
+from dabstep_benchmark.content import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    INTRODUCTION_TEXT,
+    SUBMISSION_TEXT,
+    TEMPORARY_NOTICE,
+    TITLE,
+    VALIDATION_GUIDELINES,
+)
+from dabstep_benchmark.leaderboard import (
+    generate_leaderboard_df,
+    process_submission,
+    refresh,
+)
+
+
+def download_leaderboard(lb_type: str) -> str:
+    """Download the leaderboard as CSV."""
+    validated_lb, unvalidated_lb = generate_leaderboard_df()
+    
+    if lb_type == "validated":
+        df_to_download = validated_lb
+    else:
+        df_to_download = unvalidated_lb
+    
+    os.makedirs("data", exist_ok=True)
+    path = f"data/{lb_type}_leaderboard.csv"
+    
+    if os.path.exists(path):
+        os.remove(path)
+    
+    df_to_download.to_csv(path, index=False)
+    return path
+
+
+# Custom CSS for better styling
+CUSTOM_CSS = """
+.markdown-text {
+    font-size: 16px;
+}
+.gradio-container {
+    max-width: 1200px !important;
+}
+#citation-button {
+    font-family: monospace;
+    font-size: 12px;
+}
+.notice-box {
+    background: transparent !important;
+    border: none !important;
+    padding: 0 !important;
+    margin: 8px 0 16px 0 !important;
+}
+.notice-box p {
+    margin: 0 !important;
+    padding: 12px 16px !important;
+    background: #2a2a2a !important;
+    border: 1px solid #444 !important;
+    border-radius: 6px !important;
+    color: #ccc !important;
+    font-size: 14px !important;
+    line-height: 1.5 !important;
+}
+.notice-box a {
+    color: #6cb6ff !important;
+}
+"""
+
+if __name__ == "__main__":
+    # Ensure data directories exist
+    os.makedirs("data/task_scores", exist_ok=True)
+    os.makedirs("data/submissions", exist_ok=True)
+    
+    # Load data once at startup (cached for subsequent calls)
+    validated_lb, unvalidated_lb = generate_leaderboard_df()
+    
+    # Build the Gradio app
+    demo = gr.Blocks(title="DABstep Leaderboard - Temporary Mirror")
+    
+    with demo:
+        gr.HTML(f"<style>{CUSTOM_CSS}</style>")
+        gr.Markdown(TITLE)
+        gr.Markdown(TEMPORARY_NOTICE, elem_classes="notice-box")
+        gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+        
+        with gr.Tab("📊 Leaderboard"):
+            with gr.Tab("Validated"):
+                verified_table = gr.Dataframe(
+                    value=validated_lb,
+                    datatype=["markdown", "str", "str", "str", "markdown", "str", "str"],
+                    interactive=False,
+                    column_widths=["20%"],
+                    wrap=True,
+                )
+                verified_download = gr.DownloadButton(
+                    label="📥 Download Leaderboard CSV",
+                    elem_id="download-verified-lb",
+                )
+            
+            with gr.Tab("Unvalidated"):
+                unverified_table = gr.Dataframe(
+                    value=unvalidated_lb,
+                    datatype=["markdown", "str", "str", "str", "markdown", "str", "str"],
+                    interactive=False,
+                    column_widths=["20%"],
+                    wrap=True,
+                )
+                unverified_download = gr.DownloadButton(
+                    label="📥 Download Full Leaderboard CSV",
+                    elem_id="download-unverified-lb",
+                )
+            
+            # Refresh button
+            refresh_button = gr.Button("🔄 Refresh Leaderboard", variant="secondary")
+            
+            def do_refresh():
+                """Clear cache and reload leaderboard data."""
+                return refresh(only_leaderboard=True)
+            
+            refresh_button.click(
+                fn=do_refresh,
+                inputs=None,
+                outputs=[verified_table, unverified_table],
+            )
+        
+        # Download handlers
+        verified_download.click(
+            download_leaderboard,
+            inputs=[gr.Textbox(value="validated", visible=False)],
+            outputs=[verified_download]
+        )
+        unverified_download.click(
+            download_leaderboard,
+            inputs=[gr.Textbox(value="unvalidated", visible=False)],
+            outputs=[unverified_download]
+        )
+        
+        
+        gr.Markdown(VALIDATION_GUIDELINES, elem_classes="markdown-text")
+        
+        with gr.Tab("📤 Submit"):
+            gr.Markdown(SUBMISSION_TEXT, elem_classes="markdown-text")
+            
+            with gr.Row():
+                with gr.Column():
+                    split = gr.Radio(["all"], value="all", label="Split", visible=False)
+                    agent_name_textbox = gr.Textbox(
+                        label="Agent Name",
+                        placeholder="e.g., MyDataAgent-v1"
+                    )
+                    model_family_textbox = gr.Textbox(
+                        label="Model Family",
+                        placeholder="e.g., GPT-4, Claude, Llama"
+                    )
+                    repo_url_textbox = gr.Textbox(
+                        label="Repository URL (optional)",
+                        placeholder="https://github.com/..."
+                    )
+                
+                with gr.Column():
+                    organisation = gr.Textbox(
+                        label="Organization",
+                        placeholder="e.g., Genesis Computing"
+                    )
+                    mail = gr.Textbox(
+                        label="Contact Email",
+                        placeholder="your@email.com"
+                    )
+                    file_output = gr.File(
+                        label="Upload Submission (.jsonl)",
+                        file_types=[".jsonl", ".json"]
+                    )
+            
+            with gr.Row():
+                submit_button = gr.Button("🚀 Submit Answers", variant="primary")
+            
+            submission_result = gr.Markdown()
+            
+            submit_button.click(
+                process_submission,
+                inputs=[
+                    split,
+                    agent_name_textbox,
+                    model_family_textbox,
+                    repo_url_textbox,
+                    file_output,
+                    organisation,
+                    mail
+                ],
+                outputs=submission_result,
+            )
+        
+        with gr.Tab("📚 Citation"):
+            with gr.Accordion("📙 How to Cite", open=True):
+                citation_button = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    lines=len(CITATION_BUTTON_TEXT.split("\n")),
+                    elem_id="citation-button",
+                )
+    
+    # Launch the app
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+        debug=True
+    )
+

dabstep_benchmark/__init__.py

@@ -0,0 +1,2 @@
+# DABstep Benchmark Package
+

dabstep_benchmark/content.py

@@ -0,0 +1,72 @@
+"""
+DABstep Benchmark Content
+Text content for the leaderboard UI.
+"""
+
+TITLE = """# 🕺 DABstep Leaderboard — Temporary Mirror"""
+
+TEMPORARY_NOTICE = """
+⚠️ **Temporary mirror** — The [official Adyen leaderboard](https://huggingface.co/spaces/adyen/DABstep) is currently down. Submissions are accepted here for testing, but will likely need to be re-submitted to Adyen once it's back online. Hosted by [Genesis Computing](https://genesiscomputing.ai) • [support@genesiscomputing.ai](mailto:support@genesiscomputing.ai)
+"""
+
+INTRODUCTION_TEXT = """
+The [Data Agent Benchmark for Multi-step Reasoning (DABstep)](https://huggingface.co/blog/dabstep) measures and pushes the state-of-the-art in Data Analysis by LLMs.
+
+The benchmark is composed of ~450 data analysis questions centered around documents that agents must understand and cross-reference to answer correctly.
+
+### Resources
+- 📊 [Original Dataset](https://huggingface.co/datasets/adyen/DABstep)
+- 📄 [Adyen Technical Report](https://www.adyen.com/knowledge-hub/data-agent-benchmark-for-multi-step-reasoning-dabstep)
+- 📝 [Hugging Face Blog Post](https://huggingface.co/blog/dabstep)
+- 🔗 [Colab Notebook Baseline](https://colab.research.google.com/drive/1pXi5ffBFNJQ5nn1111SnIfjfKCOlunxu)
+- 💬 [HF Discussion: Leaderboard is down](https://huggingface.co/spaces/adyen/DABstep/discussions/17)
+- 🌐 [Genesis Computing](https://genesiscomputing.ai) — Mirror host
+"""
+
+SUBMISSION_TEXT = """
+## Submission Format
+
+Submit a JSON Lines (.jsonl) file with the following format:
+
+```json
+{"task_id": "1", "agent_answer": "Your answer", "reasoning_trace": "Optional: how your model reached this answer"}
+{"task_id": "2", "agent_answer": "Another answer", "reasoning_trace": "Optional trace"}
+```
+
+**Required fields:**
+- `task_id`: The task identifier (string)
+- `agent_answer`: Your agent's answer (string)
+
+**Optional fields:**
+- `reasoning_trace`: The reasoning steps (string)
+
+Scores are expressed as the percentage of correct answers. Evaluation uses quasi-exact match 
+between your answer and the ground truth (with normalization for numbers, lists, etc.).
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+
+CITATION_BUTTON_TEXT = r"""@misc{DABstep,
+    title={DABstep: Data Agent Benchmark for Multi-step Reasoning},
+    author={Alex Egg and Martin Iglesias Goyanes and Friso Kingma and Andreu Mora and Leandro von Werra and Thomas Wolf},
+    year={2025},
+    eprint={2506.23719},
+    archivePrefix={arXiv},
+    primaryClass={cs.LG},
+    url={https://arxiv.org/abs/2506.23719}
+}"""
+
+VALIDATION_GUIDELINES = """
+## About This Leaderboard
+
+This is an independent instance of the DABstep leaderboard. Submissions are scored automatically 
+against the ground truth answers using the official DABstep scorer.
+
+**Scoring:**
+- Easy Level: Tasks 1-72 (basic data analysis)
+- Hard Level: Tasks 73-450 (complex multi-step reasoning)
+
+**Note:** This leaderboard stores submissions locally. For official benchmark results, 
+please submit to the [official DABstep leaderboard](https://huggingface.co/spaces/adyen/DABstep).
+"""
+

dabstep_benchmark/evaluation/__init__.py

@@ -0,0 +1,2 @@
+# DABstep Evaluation Package
+

dabstep_benchmark/evaluation/scorer.py

@@ -0,0 +1,149 @@
+"""
+DABstep Benchmark Scorer
+Original source: https://huggingface.co/spaces/adyen/DABstep/blob/main/dabstep_benchmark/evaluation/scorer.py
+"""
+from __future__ import annotations
+
+import math
+import re
+from difflib import SequenceMatcher
+
+
+def is_numeric_with_commas(value: str) -> bool:
+    """
+    True for strings that are either:
+    - numbers using comma thousands-separators (at least one comma),
+      with optional dot-decimal, e.g. "1,000" or "12,345.67"
+    OR
+    - pure decimals (no separators) with a decimal point or comma,
+      e.g. "0.99" or "0,99"
+    Plain ints without commas (e.g. "64") are rejected.
+    """
+    v = value.strip()
+    pattern = r'''
+    ^\$?                              # optional dollar sign
+    (?:                               # two alternate groups:
+        \d{1,3}(?:,\d{3})+(?:\.\d+)?  # 1) at least one comma-group + optional .decimal
+      | \d+[.,]\d+                    # 2) or plain decimal with . or ,
+    )
+    $                                 # end of string
+    '''
+    return bool(re.match(pattern, v, re.VERBOSE))
+
+
+def question_scorer(input1: str, input2: str) -> bool:
+    """Score a single question answer against the ground truth."""
+    # Remove leading/trailing whitespace and convert to lowercase
+    input1 = input1.strip().lower()
+    input2 = input2.strip().lower()
+
+    # Check if inputs are numeric with commas
+    if is_numeric_with_commas(input1) or is_numeric_with_commas(input2):
+        num1 = extract_numeric(input1)
+        num2 = extract_numeric(input2)
+        return compare_numeric(num1, num2) if num1 is not None and num2 is not None else False
+
+    # Check for list match
+    if ';' in input1 or ';' in input2 or ',' in input1 or ',' in input2:
+        return compare_lists(input1, input2)
+
+    # Extract numeric values if present
+    num1 = extract_numeric(input1)
+    num2 = extract_numeric(input2)
+
+    # If both inputs have numeric values, compare them
+    if num1 is not None and num2 is not None:
+        return compare_numeric(num1, num2)
+
+    # Check for string match or subset
+    return compare_strings(input1, input2)
+
+
+def extract_numeric(value: str) -> float | None:
+    """Extract numeric value from a string."""
+    # Remove commas and currency symbols from the value string
+    value = value.replace(',', '').replace('$', '')
+
+    # Extract the first occurrence of a numeric value
+    match = re.search(r'(\d*\.\d+|\d+\.?\d*)%?', value)
+    if match:
+        num_str = match.group(1)
+        try:
+            return float(num_str)
+        except ValueError:
+            return None
+    return None
+
+
+def compare_numeric(num1: float, num2: float) -> bool:
+    """Compare two numeric values with tolerance."""
+    # Check for exact equality first
+    if num1 == num2:
+        return True
+
+    # For percentages and small numbers, use a more lenient comparison
+    if num1 < 1 and num2 < 1:
+        return math.isclose(num1, num2, rel_tol=1e-4, abs_tol=1e-4)
+
+    # For larger numbers, use the original comparison method
+    dec_places1 = len(str(num1).split('.')[-1]) if '.' in str(num1) else 0
+    dec_places2 = len(str(num2).split('.')[-1]) if '.' in str(num2) else 0
+    round_to = min(dec_places1, dec_places2)
+    rounded1 = round(num1, round_to)
+    rounded2 = round(num2, round_to)
+
+    if rounded1 == rounded2:
+        return True
+
+    return math.isclose(num1, num2, rel_tol=1e-4, abs_tol=1e-4)
+
+
+def compare_strings(str1: str, str2: str) -> bool:
+    """Compare two strings for similarity."""
+    # Remove all whitespace and punctuation
+    clean1 = re.sub(r'[^\w]', '', str1)
+    clean2 = re.sub(r'[^\w]', '', str2)
+
+    if clean1 == clean2:
+        return True
+
+    words1 = re.findall(r'\b\w+\b', str1.lower())
+    words2 = re.findall(r'\b\w+\b', str2.lower())
+
+    # Only do subset comparison if neither list is empty
+    if (len(words1) == 1 or len(words2) == 1) and words1 and words2:
+        return set(words1).issubset(set(words2)) or set(words2).issubset(set(words1))
+
+    # Use similarity score
+    similarity = SequenceMatcher(None, str1, str2).ratio()
+    return similarity > 0.95
+
+
+def compare_lists(list1: str, list2: str) -> bool:
+    """Compare two list-formatted strings."""
+    # Normalize list representations by removing brackets
+    list1 = re.sub(r'^\[|\]$', '', list1.strip())
+    list2 = re.sub(r'^\[|\]$', '', list2.strip())
+
+    # Split the lists and remove whitespace
+    items1 = [item.strip() for item in re.split(r'[,;]', list1) if item.strip()]
+    items2 = [item.strip() for item in re.split(r'[,;]', list2) if item.strip()]
+
+    # Sort the items to handle different order
+    items1.sort()
+    items2.sort()
+
+    # Check if the lists are identical
+    if items1 == items2:
+        return True
+
+    # If lists are not identical, compare each item
+    if len(items1) != len(items2):
+        return False
+
+    for item1, item2 in zip(items1, items2):
+        if not question_scorer(item1, item2):
+            return False
+
+    return True
+

dabstep_benchmark/leaderboard.py

@@ -0,0 +1,506 @@
+"""
+DABstep Leaderboard Logic
+Handles submission processing, scoring, and leaderboard generation.
+"""
+from __future__ import annotations
+
+import datetime
+import json
+import os
+import re
+from email.utils import parseaddr
+from pathlib import Path
+
+import gradio as gr
+import pandas as pd
+
+from dabstep_benchmark.utils import (
+    evaluate,
+    format_error,
+    format_log,
+    format_warning,
+    is_valid_https_url,
+)
+
+# Paths
+DATA_DIR = Path("data")
+SUBMISSIONS_DIR = DATA_DIR / "submissions"
+TASK_SCORES_DIR = DATA_DIR / "task_scores"
+METADATA_FILE = DATA_DIR / "metadata.jsonl"  # Small file with just submission metadata
+SCORES_SUMMARY_FILE = DATA_DIR / "scores_summary.jsonl"  # Pre-aggregated scores (128 KB)
+GROUND_TRUTH_FILE = Path("ground_truth.jsonl")
+
+# In-memory cache
+GROUND_TRUTH_DF = None
+SUBMISSIONS_DF = None
+TASK_SCORES_DF = None
+METADATA_DF = None
+SCORES_SUMMARY_DF = None
+LEADERBOARD_CACHE = None  # Cached (validated_df, unvalidated_df)
+
+
+def ensure_directories():
+    """Ensure data directories exist."""
+    SUBMISSIONS_DIR.mkdir(parents=True, exist_ok=True)
+    TASK_SCORES_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def load_ground_truth() -> pd.DataFrame:
+    """Load ground truth answers from JSONL file or HF Secret.
+    
+    For HuggingFace Spaces deployment:
+    1. Set the GROUND_TRUTH_DATA secret in Space Settings
+    2. The secret should contain the full JSONL content (one JSON object per line)
+    3. Delete the ground_truth.jsonl file from the repo to hide answers
+    
+    For local development:
+    - Just use the ground_truth.jsonl file
+    """
+    global GROUND_TRUTH_DF
+    
+    if GROUND_TRUTH_DF is not None:
+        return GROUND_TRUTH_DF
+    
+    records = []
+    
+    # Try loading from HF Secret first (for production)
+    ground_truth_data = os.environ.get("GROUND_TRUTH_DATA")
+    if ground_truth_data:
+        print("Loading ground truth from HF Secret...")
+        for line in ground_truth_data.strip().split("\n"):
+            if line.strip():
+                record = json.loads(line)
+                task_id = str(record["task_id"])
+                level = "easy" if int(task_id) <= 72 else "hard"
+                records.append({
+                    "task_id": task_id,
+                    "answer": str(record["agent_answer"]),
+                    "level": level
+                })
+    else:
+        # Fall back to file for local development
+        if not GROUND_TRUTH_FILE.exists():
+            raise FileNotFoundError(
+                f"Ground truth not found. Either set GROUND_TRUTH_DATA env var "
+                f"or provide {GROUND_TRUTH_FILE}"
+            )
+        
+        print("Loading ground truth from file...")
+        with open(GROUND_TRUTH_FILE) as f:
+            for line in f:
+                record = json.loads(line)
+                task_id = str(record["task_id"])
+                level = "easy" if int(task_id) <= 72 else "hard"
+                records.append({
+                    "task_id": task_id,
+                    "answer": str(record["agent_answer"]),
+                    "level": level
+                })
+    
+    GROUND_TRUTH_DF = pd.DataFrame(records)
+    print(f"Loaded {len(GROUND_TRUTH_DF)} ground truth answers")
+    return GROUND_TRUTH_DF
+
+
+def load_submissions() -> pd.DataFrame:
+    """Load all submissions from the submissions directory."""
+    global SUBMISSIONS_DF
+    
+    submissions = []
+    for file_path in SUBMISSIONS_DIR.glob("*.jsonl"):
+        try:
+            df = pd.read_json(file_path, lines=True, dtype=str)
+            submissions.append(df)
+        except Exception as e:
+            print(f"Error loading {file_path}: {e}")
+    
+    if submissions:
+        SUBMISSIONS_DF = pd.concat(submissions, ignore_index=True)
+    else:
+        SUBMISSIONS_DF = pd.DataFrame(columns=[
+            "submission_id", "task_id", "agent_answer", "agent_name",
+            "model_family", "organisation", "repo_url", "date"
+        ])
+    
+    return SUBMISSIONS_DF
+
+
+def load_task_scores() -> pd.DataFrame:
+    """Load all task scores from the task_scores directory."""
+    global TASK_SCORES_DF
+    
+    scores = []
+    for file_path in TASK_SCORES_DIR.glob("*.jsonl"):
+        try:
+            with open(file_path) as f:
+                for line in f:
+                    scores.append(json.loads(line))
+        except Exception as e:
+            print(f"Error loading {file_path}: {e}")
+    
+    if scores:
+        TASK_SCORES_DF = pd.DataFrame(scores)
+    else:
+        TASK_SCORES_DF = pd.DataFrame(columns=[
+            "submission_id", "task_id", "score", "level", "agent_answer"
+        ])
+    
+    return TASK_SCORES_DF
+
+
+def load_metadata() -> pd.DataFrame:
+    """Load submission metadata from the small metadata file."""
+    global METADATA_DF
+    
+    # Return cached data if available
+    if METADATA_DF is not None:
+        return METADATA_DF
+    
+    if not METADATA_FILE.exists():
+        print(f"No metadata file found at {METADATA_FILE}")
+        METADATA_DF = pd.DataFrame(columns=[
+            "submission_id", "agent_name", "model_family", "organisation", 
+            "repo_url", "date", "validated"
+        ])
+        return METADATA_DF
+    
+    try:
+        METADATA_DF = pd.read_json(METADATA_FILE, lines=True, dtype=str)
+        # Convert validated to boolean
+        if "validated" in METADATA_DF.columns:
+            METADATA_DF["validated"] = METADATA_DF["validated"].apply(
+                lambda x: str(x).lower() == "true" if pd.notna(x) else False
+            )
+        print(f"Loaded metadata for {len(METADATA_DF)} submissions")
+    except Exception as e:
+        print(f"Error loading metadata: {e}")
+        METADATA_DF = pd.DataFrame(columns=[
+            "submission_id", "agent_name", "model_family", "organisation", 
+            "repo_url", "date", "validated"
+        ])
+    
+    return METADATA_DF
+
+
+def save_metadata(submission_id: str, agent_name: str, organisation: str, 
+                  model_family: str, repo_url: str, date: str, validated: bool = False):
+    """Append a new submission's metadata to the metadata file."""
+    metadata = {
+        "submission_id": submission_id,
+        "agent_name": agent_name,
+        "organisation": organisation,
+        "model_family": model_family,
+        "repo_url": repo_url,
+        "date": date,
+        "validated": validated
+    }
+    
+    with open(METADATA_FILE, "a") as f:
+        f.write(json.dumps(metadata) + "\n")
+    print(f"Saved metadata for {submission_id}")
+
+
+def load_scores_summary() -> pd.DataFrame:
+    """Load pre-aggregated scores summary (128 KB instead of 514 MB)."""
+    global SCORES_SUMMARY_DF
+    
+    # Return cached data if available
+    if SCORES_SUMMARY_DF is not None:
+        return SCORES_SUMMARY_DF
+    
+    if not SCORES_SUMMARY_FILE.exists():
+        print(f"No scores summary file at {SCORES_SUMMARY_FILE}")
+        SCORES_SUMMARY_DF = pd.DataFrame(columns=[
+            "submission_id", "easy_accuracy", "hard_accuracy"
+        ])
+        return SCORES_SUMMARY_DF
+    
+    try:
+        SCORES_SUMMARY_DF = pd.read_json(SCORES_SUMMARY_FILE, lines=True)
+        print(f"Loaded scores summary for {len(SCORES_SUMMARY_DF)} submissions")
+    except Exception as e:
+        print(f"Error loading scores summary: {e}")
+        SCORES_SUMMARY_DF = pd.DataFrame(columns=[
+            "submission_id", "easy_accuracy", "hard_accuracy"
+        ])
+    
+    return SCORES_SUMMARY_DF
+
+
+def save_scores_summary(submission_id: str, easy_accuracy: float, hard_accuracy: float):
+    """Append a new submission's aggregated scores to the summary file."""
+    entry = {
+        "submission_id": submission_id,
+        "easy_accuracy": round(easy_accuracy, 2),
+        "hard_accuracy": round(hard_accuracy, 2)
+    }
+    with open(SCORES_SUMMARY_FILE, "a") as f:
+        f.write(json.dumps(entry) + "\n")
+    print(f"Saved scores summary for {submission_id}")
+
+
+def refresh(only_leaderboard: bool = False) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Refresh data and regenerate leaderboard."""
+    global GROUND_TRUTH_DF, SUBMISSIONS_DF, TASK_SCORES_DF, METADATA_DF, SCORES_SUMMARY_DF, LEADERBOARD_CACHE
+    
+    ensure_directories()
+    
+    if not only_leaderboard:
+        GROUND_TRUTH_DF = None
+        load_ground_truth()
+    
+    SUBMISSIONS_DF = None
+    TASK_SCORES_DF = None
+    METADATA_DF = None
+    SCORES_SUMMARY_DF = None
+    LEADERBOARD_CACHE = None
+    
+    return generate_leaderboard_df()
+
+
+def validate_submission(submission_df: pd.DataFrame) -> str | None:
+    """Validate a submission DataFrame."""
+    mandatory_columns = ["agent_answer", "task_id"]
+    expected_columns = [*mandatory_columns, "reasoning_trace"]
+    
+    # Check for missing mandatory columns
+    missing_columns = [col for col in mandatory_columns if col not in submission_df.columns]
+    if missing_columns:
+        return format_error(f"Missing mandatory columns: {', '.join(missing_columns)}")
+    
+    # Check for unexpected columns
+    unexpected_columns = [col for col in submission_df.columns if col not in expected_columns]
+    if unexpected_columns:
+        return format_error(f"Unexpected columns: {', '.join(unexpected_columns)}")
+    
+    # Check for NaN values in any column
+    if submission_df.isnull().values.any():
+        return format_error("Submission contains NaN values. Please ensure no missing data.")
+    
+    # Check if all columns are of string type
+    non_string_columns = [col for col in submission_df.columns if submission_df[col].dtype != 'object']
+    if non_string_columns:
+        return format_error(f"Columns with non-string data type: {', '.join(non_string_columns)}")
+    
+    return None
+
+
+def process_submission(
+    split: str,
+    agent_name: str,
+    model_family: str,
+    repo_url: str,
+    path_to_file: str,
+    organisation: str,
+    mail: str,
+) -> str:
+    """Process a new submission."""
+    # Validate inputs
+    if agent_name == "":
+        return format_warning("Please provide an agent name")
+    if organisation == "":
+        return format_warning("Please provide an organisation")
+    if mail == "":
+        return format_warning("Please provide an email")
+    if model_family == "":
+        return format_warning("Please provide a model family")
+    
+    allowed_pattern = re.compile(r'^[a-zA-Z0-9 _.-]+$')
+    if not allowed_pattern.match(agent_name):
+        return format_warning(
+            f"Agent name can only contain alphanumeric characters, spaces, dashes (-), and underscores (_)"
+        )
+    if not allowed_pattern.match(organisation):
+        return format_warning(
+            f"Organisation can only contain alphanumeric characters, spaces, dashes (-), and underscores (_)"
+        )
+    
+    # Basic email validation
+    _, parsed_mail = parseaddr(mail)
+    if "@" not in parsed_mail:
+        return format_warning("Please provide a valid email address.")
+    
+    if repo_url != "" and not is_valid_https_url(repo_url):
+        return format_warning("If you provide a URL it must be a valid one. You can also leave it empty")
+    
+    # Validate submission file
+    if path_to_file is None:
+        return format_warning("Please attach a file.")
+    
+    submission_path = path_to_file.name
+    try:
+        submission_df = pd.read_json(submission_path, lines=True, dtype=str)
+        validation_error = validate_submission(submission_df)
+        if validation_error:
+            return validation_error
+    except Exception as exc:
+        return format_error(f"Submission file is incorrectly formatted. Please fix it and resubmit your file. {exc!s}")
+    
+    submission_id = f"{organisation}-{agent_name}"
+    print(f"Processing submission_id={submission_id}...")
+    gr.Info(f"Processing submission of {agent_name}...")
+    
+    # Reload data
+    ensure_directories()
+    load_ground_truth()
+    submissions_df = load_submissions()
+    
+    # Check if already submitted
+    if len(submissions_df) > 0 and submission_id in submissions_df['submission_id'].values:
+        return format_warning(f"This {submission_id} pair has already been submitted.")
+    
+    # Add metadata to submission
+    submission_df["submission_id"] = submission_id
+    submission_df["agent_name"] = agent_name
+    submission_df["model_family"] = model_family
+    submission_df["organisation"] = organisation
+    submission_df["repo_url"] = repo_url
+    submission_df["date"] = datetime.date.today().strftime("%d-%m-%Y")
+    
+    if "reasoning_trace" not in submission_df.columns:
+        submission_df["reasoning_trace"] = ""
+    
+    # Evaluate submission
+    try:
+        task_scores = evaluate(
+            agent_answers=submission_df,
+            tasks_with_gt=GROUND_TRUTH_DF,
+            submission_id=submission_id
+        )
+    except KeyError as exc:
+        return format_error(str(exc))
+    
+    # Save submission
+    filename_id = f"v1__{organisation}-{agent_name}__{datetime.datetime.today().strftime('%d-%m-%Y')}"
+    submission_file = SUBMISSIONS_DIR / f"{filename_id}.jsonl"
+    submission_df.to_json(submission_file, orient="records", lines=True)
+    print(f"Saved submission to {submission_file}")
+    
+    # Save scores
+    scores_file = TASK_SCORES_DIR / f"{filename_id}.jsonl"
+    with open(scores_file, "w") as f:
+        for score in task_scores:
+            f.write(json.dumps(score) + "\n")
+    print(f"Saved task scores to {scores_file}")
+    
+    # Calculate aggregated scores for the summary
+    easy_scores = [s["score"] for s in task_scores if s["level"] == "easy"]
+    hard_scores = [s["score"] for s in task_scores if s["level"] == "hard"]
+    easy_accuracy = (sum(easy_scores) / len(easy_scores) * 100) if easy_scores else 0
+    hard_accuracy = (sum(hard_scores) / len(hard_scores) * 100) if hard_scores else 0
+    
+    # Save scores summary (for fast leaderboard rendering)
+    save_scores_summary(
+        submission_id=submission_id,
+        easy_accuracy=easy_accuracy,
+        hard_accuracy=hard_accuracy
+    )
+    
+    # Save metadata (small file for fast leaderboard rendering)
+    date_str = datetime.datetime.today().strftime('%d-%m-%Y')
+    save_metadata(
+        submission_id=submission_id,
+        agent_name=agent_name,
+        organisation=organisation,
+        model_family=model_family,
+        repo_url=repo_url,
+        date=date_str,
+        validated=False
+    )
+    
+    return format_log(f"""
+        Agent {agent_name} submitted by {organisation} successfully!
+        Please refresh the leaderboard to see your score.
+    """)
+
+
+def generate_leaderboard_df() -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Generate the leaderboard DataFrames.
+    
+    Uses pre-aggregated files for fast loading:
+    - metadata.jsonl (284 KB) - submission metadata
+    - scores_summary.jsonl (128 KB) - pre-aggregated scores
+    
+    Total: ~400 KB instead of 3.1 GB
+    """
+    global LEADERBOARD_CACHE
+    
+    # Return cached result if available
+    if LEADERBOARD_CACHE is not None:
+        return LEADERBOARD_CACHE
+    
+    # Load pre-aggregated scores (128 KB instead of 514 MB)
+    scores_summary_df = load_scores_summary()
+    
+    # Load metadata (284 KB instead of 2.6 GB)
+    metadata_df = load_metadata()
+    
+    if len(metadata_df) == 0 or len(scores_summary_df) == 0:
+        empty_df = pd.DataFrame(columns=[
+            "Agent", "Easy Level Accuracy (%)", "Hard Level Accuracy (%)",
+            "Organization", "Repo URL", "Model Family", "Date"
+        ])
+        return empty_df, empty_df
+    
+    # Join metadata with pre-aggregated scores
+    leaderboard_df = pd.merge(metadata_df, scores_summary_df, on="submission_id", how="inner")
+    
+    # Rename columns (scores_summary already has percentage values)
+    col_map = {
+        "agent_name": "Agent",
+        "easy_accuracy": "Easy Level Accuracy (%)",
+        "hard_accuracy": "Hard Level Accuracy (%)",
+        "organisation": "Organization",
+        "repo_url": "Repo URL",
+        "model_family": "Model Family",
+        "date": "Date",
+    }
+    
+    leaderboard_df.rename(columns=col_map, inplace=True)
+    
+    # Format columns (keep 'validated' for later splitting)
+    available_cols = [col for col in col_map.values() if col in leaderboard_df.columns]
+    keep_cols = available_cols.copy()
+    if "validated" in leaderboard_df.columns:
+        keep_cols.append("validated")
+    leaderboard_df = leaderboard_df[keep_cols]
+    
+    # Make repo URL clickable
+    if "Repo URL" in leaderboard_df.columns:
+        leaderboard_df["Repo URL"] = leaderboard_df["Repo URL"].apply(
+            lambda x: f"[Link]({x})" if x != "" else ""
+        )
+    
+    # Make agent name bold
+    if "Agent" in leaderboard_df.columns:
+        leaderboard_df["Agent"] = leaderboard_df["Agent"].apply(lambda x: f"**{x}**")
+    
+    # Sort by best score
+    sort_cols = []
+    if "Hard Level Accuracy (%)" in leaderboard_df.columns:
+        sort_cols.append("Hard Level Accuracy (%)")
+    if "Easy Level Accuracy (%)" in leaderboard_df.columns:
+        sort_cols.append("Easy Level Accuracy (%)")
+    
+    if sort_cols:
+        leaderboard_df.sort_values(by=sort_cols, ascending=[False] * len(sort_cols), inplace=True)
+    
+    # Split into validated and unvalidated based on the 'validated' field
+    display_cols = [c for c in leaderboard_df.columns if c != "validated"]
+    
+    if "validated" in leaderboard_df.columns:
+        # Convert validated field to boolean (handles string "true"/"false" and bool)
+        leaderboard_df["_is_validated"] = leaderboard_df["validated"].apply(
+            lambda x: str(x).lower() in ("true", "1", "yes") if pd.notna(x) else False
+        )
+        validated_lb = leaderboard_df[leaderboard_df["_is_validated"]][display_cols].copy()
+        unvalidated_lb = leaderboard_df[~leaderboard_df["_is_validated"]][display_cols].copy()
+    else:
+        # No validated field - all go to unvalidated
+        validated_lb = pd.DataFrame(columns=display_cols)
+        unvalidated_lb = leaderboard_df[display_cols].copy()
+    
+    # Cache the result
+    LEADERBOARD_CACHE = (validated_lb, unvalidated_lb)
+    return validated_lb, unvalidated_lb
+

dabstep_benchmark/utils.py

@@ -0,0 +1,89 @@
+"""
+DABstep Benchmark Utilities
+Adapted from: https://huggingface.co/spaces/adyen/DABstep/blob/main/dabstep_benchmark/utils.py
+"""
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from dabstep_benchmark.evaluation.scorer import question_scorer
+
+
+def format_error(msg: str) -> str:
+    """Format an error message in red."""
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{msg}</p>"
+
+
+def format_warning(msg: str) -> str:
+    """Format a warning message in orange."""
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>"
+
+
+def format_log(msg: str) -> str:
+    """Format a log message in green."""
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"
+
+
+def model_hyperlink(link: str, model_name: str) -> str:
+    """Create a hyperlink for a model."""
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def is_valid_https_url(url: str) -> bool:
+    """Validate that a URL is a valid HTTPS URL."""
+    pattern = re.compile(
+        r'^https://'  # URL must start with 'https://'
+        r'(?!10(?:\.\d{1,3}){3})'  # Exclude private IP 10.x.x.x
+        r'(?!127(?:\.\d{1,3}){3})'  # Exclude loopback IP 127.x.x.x
+        r'(?!169\.254(?:\.\d{1,3}){2})'  # Exclude link-local IP 169.254.x.x
+        r'(?!192\.168(?:\.\d{1,3}){2})'  # Exclude private IP 192.168.x.x
+        r'(?!172\.(?:1[6-9]|2[0-9]|3[0-1])(?:\.\d{1,3}){2})'  # Exclude private IP 172.16.x.x - 172.31.x.x
+        r'(?:(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,})'  # Match domain name
+        r'(?::\d{2,5})?'  # Optional port
+        r'(?:/[^\s]*)?$',  # Optional path
+        re.IGNORECASE
+    )
+    return re.match(pattern, url) is not None
+
+
+def evaluate(
+    agent_answers: pd.DataFrame,
+    tasks_with_gt: pd.DataFrame,
+    submission_id: str = ""
+) -> list[dict]:
+    """
+    Evaluate agent answers against ground truth.
+    
+    Args:
+        agent_answers: DataFrame with columns 'task_id' and 'agent_answer'
+        tasks_with_gt: DataFrame with columns 'task_id', 'answer', and 'level'
+        submission_id: Identifier for the submission
+    
+    Returns:
+        List of score dictionaries for each task
+    """
+    task_scores = []
+    
+    for _, row in tasks_with_gt.iterrows():
+        correct_answer = row["answer"]
+        level = str(row["level"])
+        task_id = str(row["task_id"])
+        
+        if task_id not in agent_answers["task_id"].values:
+            raise KeyError(f"Task ID: {task_id} not found. Are you sure you submitted the correct file?")
+        
+        agent_answer = agent_answers.loc[agent_answers.task_id == task_id, "agent_answer"].values[0]
+        score = question_scorer(agent_answer, correct_answer)
+        
+        task_scores.append({
+            "submission_id": submission_id,
+            "task_id": task_id,
+            "score": score,
+            "level": level,
+            "agent_answer": agent_answer,
+        })
+    
+    return task_scores
+

requirements.txt

@@ -0,0 +1,4 @@
+gradio>=4.0.0
+pandas>=2.0.0
+numpy>=1.24.0
+