update through web

.gitignore

@@ -0,0 +1,14 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/
+.history/

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README copy.md

@@ -0,0 +1,48 @@
+---
+title: Vla4ad
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+app_file: app.py
+pinned: true
+license: apache-2.0
+short_description: Duplicate this leaderboard to initialize your own!
+sdk_version: 5.43.1
+tags:
+- leaderboard
+---
+
+# Start the configuration
+
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
+
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find 
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -0,0 +1,559 @@
+import os
+import json
+import pandas as pd
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import gradio as gr
+
+# 加载数据
+def load_json(filename):
+    possible_paths = [
+        f'results/{filename}',
+        filename,
+        os.path.join(os.path.dirname(__file__), 'results', filename),
+        os.path.join(os.path.dirname(__file__), filename)
+    ]
+    for path in possible_paths:
+        if os.path.exists(path):
+            with open(path, 'r', encoding='utf-8') as f:
+                return json.load(f)
+    raise FileNotFoundError(f"Cannot find {filename}")
+
+# 加载所有数据
+va_data = load_json('va_models.json')
+vla_data = load_json('vla_models.json')
+datasets_data = load_json('datasets.json')
+nuscenes_data = load_json('nuscenes_results.json')
+wod_data = load_json('wod_results.json')
+navsim_data = load_json('navsim_results.json')
+bench2drive_data = load_json('bench2drive_results.json')
+
+df_va = pd.DataFrame(va_data)
+df_vla = pd.DataFrame(vla_data)
+df_datasets = pd.DataFrame(datasets_data)
+df_nuscenes = pd.DataFrame(nuscenes_data)
+df_wod = pd.DataFrame(wod_data)
+df_navsim = pd.DataFrame(navsim_data)
+df_bench2drive = pd.DataFrame(bench2drive_data)
+
+INPUT_ICONS = {'camera': '📷', 'lidar': '🔍', 'status': '⚙️', 'prompt': '💭', 
+               'instruction': '📝', 'scene': '🌆', 'traffic': '🚦', 'context': '📚'}
+
+ACTION_LABELS = {'RL': 'Policy w/ RL', 'REG': 'Decoder+MLP', 'SEL': 'Traj. Selection',
+                'GEN': 'Generative Model', 'LH': 'Language Head'}
+
+# 格式化函数
+def format_inputs(inputs):
+    if isinstance(inputs, list):
+        return ' '.join([INPUT_ICONS.get(inp, inp) for inp in inputs])
+    return str(inputs)
+
+def format_datasets(datasets):
+    if isinstance(datasets, list):
+        return ', '.join(datasets[:3])
+    return str(datasets)
+
+# VA Models
+def update_va_table(category, venue, action, search):
+    df = df_va.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    if venue:
+        df = df[df['venue'].str.contains(venue, case=False, na=False)]
+    if action != "All":
+        df = df[df['action'] == action]
+    if search:
+        mask = df['model'].str.contains(search, case=False, na=False) | df['vision'].str.contains(search, case=False, na=False)
+        df = df[mask]
+    display_df = df[['id', 'model', 'venue', 'input', 'dataset', 'vision', 'action', 'output', 'category']].copy()
+    display_df['input'] = display_df['input'].apply(format_inputs)
+    display_df['dataset'] = display_df['dataset'].apply(format_datasets)
+    display_df['action'] = display_df['action'].map(ACTION_LABELS)
+    return display_df, f"**📊 Statistics:** Total {len(df)} models | Categories: {df['category'].nunique()}"
+
+def create_va_plot(category):
+    df = df_va.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    bg_color, panel_color = "#0e1117", "#161b22"
+    bar_colors = ['#4cc9f0', '#f72585', '#7209b7', '#3a0ca3', '#4361ee', '#4895ef', '#560bad']
+    text_color, grid_color = "#c9d1d9", "#30363d"
+    
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
+    fig.patch.set_facecolor(bg_color)
+    
+    category_counts = df['category'].value_counts()
+    ax1.barh(category_counts.index, category_counts.values, color=bar_colors[:len(category_counts)])
+    ax1.set_facecolor(panel_color)
+    ax1.set_xlabel('Count', color=text_color, fontsize=11)
+    ax1.set_title('Models by Category', color=text_color, fontsize=13, fontweight='bold')
+    ax1.tick_params(colors=text_color)
+    ax1.grid(axis='x', linestyle='--', alpha=0.3, color=grid_color)
+    ax1.xaxis.set_major_locator(plt.MaxNLocator(integer=True))
+    for spine in ['top', 'right', 'left']:
+        ax1.spines[spine].set_visible(False)
+    ax1.spines['bottom'].set_color(grid_color)
+    
+    action_counts = df['action'].value_counts()
+    ax2.bar(range(len(action_counts)), action_counts.values, color=bar_colors[:len(action_counts)])
+    ax2.set_facecolor(panel_color)
+    ax2.set_xticks(range(len(action_counts)))
+    ax2.set_xticklabels([ACTION_LABELS.get(a, a) for a in action_counts.index], rotation=45, ha='right')
+    ax2.set_ylabel('Count', color=text_color, fontsize=11)
+    ax2.set_title('Models by Action Type', color=text_color, fontsize=13, fontweight='bold')
+    ax2.tick_params(colors=text_color)
+    ax2.grid(axis='y', linestyle='--', alpha=0.3, color=grid_color)
+    ax2.yaxis.set_major_locator(plt.MaxNLocator(integer=True))
+    for spine in ['top', 'right']:
+        ax2.spines[spine].set_visible(False)
+    ax2.spines['left'].set_color(grid_color)
+    ax2.spines['bottom'].set_color(grid_color)
+    
+    plt.tight_layout()
+    result = fig
+    plt.close('all')
+    return result
+
+# VLA Models
+def update_vla_table(category, venue, language, search):
+    df = df_vla.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    if venue:
+        df = df[df['venue'].str.contains(venue, case=False, na=False)]
+    if language:
+        df = df[df['language'].str.contains(language, case=False, na=False)]
+    if search:
+        mask = df['model'].str.contains(search, case=False, na=False) | df['vision'].str.contains(search, case=False, na=False) | df['language'].str.contains(search, case=False, na=False)
+        df = df[mask]
+    display_df = df[['id', 'model', 'venue', 'input', 'dataset', 'vision', 'language', 'action', 'output', 'category']].copy()
+    display_df['input'] = display_df['input'].apply(format_inputs)
+    display_df['dataset'] = display_df['dataset'].apply(format_datasets)
+    display_df['action'] = display_df['action'].map(ACTION_LABELS)
+    return display_df, f"**📊 Statistics:** Total {len(df)} models | Categories: {df['category'].nunique()}"
+
+def create_vla_plot(category):
+    df = df_vla.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    bg_color, panel_color = "#0e1117", "#161b22"
+    bar_colors = ['#4cc9f0', '#f72585', '#7209b7', '#3a0ca3']
+    text_color, grid_color = "#c9d1d9", "#30363d"
+    
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
+    fig.patch.set_facecolor(bg_color)
+    
+    category_counts = df['category'].value_counts()
+    ax1.barh(category_counts.index, category_counts.values, color=bar_colors[:len(category_counts)])
+    ax1.set_facecolor(panel_color)
+    ax1.set_xlabel('Count', color=text_color, fontsize=11)
+    ax1.set_title('VLA Models by Category', color=text_color, fontsize=13, fontweight='bold')
+    ax1.tick_params(colors=text_color)
+    ax1.grid(axis='x', linestyle='--', alpha=0.3, color=grid_color)
+    ax1.xaxis.set_major_locator(plt.MaxNLocator(integer=True))
+    for spine in ['top', 'right', 'left']:
+        ax1.spines[spine].set_visible(False)
+    ax1.spines['bottom'].set_color(grid_color)
+    
+    language_counts = df['language'].value_counts().head(10)
+    ax2.bar(range(len(language_counts)), language_counts.values, color=bar_colors[:len(language_counts)])
+    ax2.set_facecolor(panel_color)
+    ax2.set_xticks(range(len(language_counts)))
+    ax2.set_xticklabels(language_counts.index, rotation=45, ha='right')
+    ax2.set_ylabel('Count', color=text_color, fontsize=11)
+    ax2.set_title('Top Language Models', color=text_color, fontsize=13, fontweight='bold')
+    ax2.tick_params(colors=text_color)
+    ax2.grid(axis='y', linestyle='--', alpha=0.3, color=grid_color)
+    ax2.yaxis.set_major_locator(plt.MaxNLocator(integer=True))
+    for spine in ['top', 'right']:
+        ax2.spines[spine].set_visible(False)
+    ax2.spines['left'].set_color(grid_color)
+    ax2.spines['bottom'].set_color(grid_color)
+    
+    plt.tight_layout()
+    result = fig
+    plt.close('all')
+    return result
+
+# Datasets
+def update_datasets_table(category, year, search):
+    df = df_datasets.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    if year:
+        try:
+            df = df[df['year'] == int(year)]
+        except:
+            pass
+    if search:
+        mask = df['dataset'].str.contains(search, case=False, na=False) | df['sensor'].str.contains(search, case=False, na=False)
+        df = df[mask]
+    return df, f"**📊 Statistics:** Total {len(df)} datasets | Years: {df['year'].nunique()}"
+
+# Benchmark tables update functions
+def update_nuscenes_table(category, search):
+    df = df_nuscenes.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    if search:
+        mask = df['model'].str.contains(search, case=False, na=False)
+        df = df[mask]
+    return df, f"**📊 Statistics:** Total {len(df)} models"
+
+def create_nuscenes_plot(category):
+    """创建nuScenes统计图表"""
+    df = df_nuscenes.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    
+    bg_color, panel_color = "#0e1117", "#161b22"
+    bar_colors = ['#4cc9f0', '#f72585', '#7209b7', '#3a0ca3']
+    text_color, grid_color = "#c9d1d9", "#30363d"
+    
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
+    fig.patch.set_facecolor(bg_color)
+    
+    # 1. 按类别统计
+    category_counts = df['category'].value_counts()
+    ax1.bar(range(len(category_counts)), category_counts.values, color=bar_colors[:len(category_counts)])
+    ax1.set_facecolor(panel_color)
+    ax1.set_xticks(range(len(category_counts)))
+    ax1.set_xticklabels(category_counts.index, rotation=15, ha='right')
+    ax1.set_ylabel('Count', color=text_color, fontsize=11)
+    ax1.set_title('Models by Category', color=text_color, fontsize=13, fontweight='bold')
+    ax1.tick_params(colors=text_color)
+    ax1.grid(axis='y', linestyle='--', alpha=0.3, color=grid_color)
+    ax1.yaxis.set_major_locator(plt.MaxNLocator(integer=True))
+    for spine in ['top', 'right']:
+        ax1.spines[spine].set_visible(False)
+    ax1.spines['left'].set_color(grid_color)
+    ax1.spines['bottom'].set_color(grid_color)
+    
+    # 2. Top 10 模型按L2 Avg排序
+    top_models = df.nsmallest(10, 'l2_avg')[['model', 'l2_avg']].dropna()
+    ax2.barh(range(len(top_models)), top_models['l2_avg'].values, color=bar_colors[2])
+    ax2.set_facecolor(panel_color)
+    ax2.set_yticks(range(len(top_models)))
+    ax2.set_yticklabels(top_models['model'].values, fontsize=9)
+    ax2.set_xlabel('L2 Avg (m)', color=text_color, fontsize=11)
+    ax2.set_title('Top 10 Models by L2 Error', color=text_color, fontsize=13, fontweight='bold')
+    ax2.tick_params(colors=text_color)
+    ax2.grid(axis='x', linestyle='--', alpha=0.3, color=grid_color)
+    ax2.invert_yaxis()
+    for spine in ['top', 'right', 'left']:
+        ax2.spines[spine].set_visible(False)
+    ax2.spines['bottom'].set_color(grid_color)
+    
+    plt.tight_layout()
+    result = fig
+    plt.close('all')
+    return result
+
+def update_wod_table(category, search):
+    df = df_wod.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    if search:
+        mask = df['model'].str.contains(search, case=False, na=False)
+        df = df[mask]
+    return df, f"**📊 Statistics:** Total {len(df)} models"
+
+def create_wod_plot(category):
+    """创建WOD统计图表"""
+    df = df_wod.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    
+    bg_color, panel_color = "#0e1117", "#161b22"
+    bar_colors = ['#4cc9f0', '#f72585', '#7209b7', '#3a0ca3']
+    text_color, grid_color = "#c9d1d9", "#30363d"
+    
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
+    fig.patch.set_facecolor(bg_color)
+    
+    # 1. 按类别统计
+    category_counts = df['category'].value_counts()
+    ax1.bar(range(len(category_counts)), category_counts.values, color=bar_colors[:len(category_counts)])
+    ax1.set_facecolor(panel_color)
+    ax1.set_xticks(range(len(category_counts)))
+    ax1.set_xticklabels(category_counts.index, rotation=15, ha='right')
+    ax1.set_ylabel('Count', color=text_color, fontsize=11)
+    ax1.set_title('Models by Category', color=text_color, fontsize=13, fontweight='bold')
+    ax1.tick_params(colors=text_color)
+    ax1.grid(axis='y', linestyle='--', alpha=0.3, color=grid_color)
+    ax1.yaxis.set_major_locator(plt.MaxNLocator(integer=True))
+    for spine in ['top', 'right']:
+        ax1.spines[spine].set_visible(False)
+    ax1.spines['left'].set_color(grid_color)
+    ax1.spines['bottom'].set_color(grid_color)
+    
+    # 2. Top模型按RFS Overall排序
+    top_models = df.nlargest(10, 'rfs_overall')[['model', 'rfs_overall']].dropna()
+    if len(top_models) > 0:
+        ax2.barh(range(len(top_models)), top_models['rfs_overall'].values, color=bar_colors[1])
+        ax2.set_facecolor(panel_color)
+        ax2.set_yticks(range(len(top_models)))
+        ax2.set_yticklabels(top_models['model'].values, fontsize=9)
+        ax2.set_xlabel('RFS Overall', color=text_color, fontsize=11)
+        ax2.set_title('Top Models by RFS Overall', color=text_color, fontsize=13, fontweight='bold')
+        ax2.tick_params(colors=text_color)
+        ax2.grid(axis='x', linestyle='--', alpha=0.3, color=grid_color)
+        ax2.invert_yaxis()
+    else:
+        ax2.text(0.5, 0.5, 'No data available', ha='center', va='center', 
+                color=text_color, fontsize=12, transform=ax2.transAxes)
+        ax2.set_facecolor(panel_color)
+    
+    for spine in ['top', 'right', 'left']:
+        ax2.spines[spine].set_visible(False)
+    ax2.spines['bottom'].set_color(grid_color)
+    
+    plt.tight_layout()
+    result = fig
+    plt.close('all')
+    return result
+
+def update_navsim_table(category, search):
+    df = df_navsim.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    if search:
+        mask = df['model'].str.contains(search, case=False, na=False)
+        df = df[mask]
+    return df, f"**📊 Statistics:** Total {len(df)} models"
+
+def create_navsim_plot(category):
+    """创建NAVSIM统计图表"""
+    df = df_navsim.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    
+    bg_color, panel_color = "#0e1117", "#161b22"
+    bar_colors = ['#4cc9f0', '#f72585', '#7209b7', '#3a0ca3']
+    text_color, grid_color = "#c9d1d9", "#30363d"
+    
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
+    fig.patch.set_facecolor(bg_color)
+    
+    # 1. 按类别统计
+    category_counts = df['category'].value_counts()
+    ax1.bar(range(len(category_counts)), category_counts.values, color=bar_colors[:len(category_counts)])
+    ax1.set_facecolor(panel_color)
+    ax1.set_xticks(range(len(category_counts)))
+    ax1.set_xticklabels(category_counts.index, rotation=15, ha='right')
+    ax1.set_ylabel('Count', color=text_color, fontsize=11)
+    ax1.set_title('Models by Category', color=text_color, fontsize=13, fontweight='bold')
+    ax1.tick_params(colors=text_color)
+    ax1.grid(axis='y', linestyle='--', alpha=0.3, color=grid_color)
+    ax1.yaxis.set_major_locator(plt.MaxNLocator(integer=True))
+    for spine in ['top', 'right']:
+        ax1.spines[spine].set_visible(False)
+    ax1.spines['left'].set_color(grid_color)
+    ax1.spines['bottom'].set_color(grid_color)
+    
+    # 2. Top 10 模��按PDMS排序
+    top_models = df.nlargest(10, 'pdms')[['model', 'pdms']]
+    ax2.barh(range(len(top_models)), top_models['pdms'].values, color=bar_colors[0])
+    ax2.set_facecolor(panel_color)
+    ax2.set_yticks(range(len(top_models)))
+    ax2.set_yticklabels(top_models['model'].values, fontsize=9)
+    ax2.set_xlabel('PDMS Score', color=text_color, fontsize=11)
+    ax2.set_title('Top 10 Models by PDMS', color=text_color, fontsize=13, fontweight='bold')
+    ax2.tick_params(colors=text_color)
+    ax2.grid(axis='x', linestyle='--', alpha=0.3, color=grid_color)
+    ax2.invert_yaxis()
+    for spine in ['top', 'right', 'left']:
+        ax2.spines[spine].set_visible(False)
+    ax2.spines['bottom'].set_color(grid_color)
+    
+    plt.tight_layout()
+    result = fig
+    plt.close('all')
+    return result
+
+def update_bench2drive_table(category, search):
+    df = df_bench2drive.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    if search:
+        mask = df['model'].str.contains(search, case=False, na=False)
+        df = df[mask]
+    return df, f"**📊 Statistics:** Total {len(df)} models"
+
+def create_bench2drive_plot(category):
+    """创建Bench2Drive统计图表"""
+    df = df_bench2drive.copy()
+    if category != "All":
+        df = df[df['category'] == category]
+    
+    bg_color, panel_color = "#0e1117", "#161b22"
+    bar_colors = ['#4cc9f0', '#f72585', '#7209b7', '#3a0ca3']
+    text_color, grid_color = "#c9d1d9", "#30363d"
+    
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
+    fig.patch.set_facecolor(bg_color)
+    
+    # 1. 按类别统计
+    category_counts = df['category'].value_counts()
+    ax1.bar(range(len(category_counts)), category_counts.values, color=bar_colors[:len(category_counts)])
+    ax1.set_facecolor(panel_color)
+    ax1.set_xticks(range(len(category_counts)))
+    ax1.set_xticklabels(category_counts.index, rotation=15, ha='right')
+    ax1.set_ylabel('Count', color=text_color, fontsize=11)
+    ax1.set_title('Models by Category', color=text_color, fontsize=13, fontweight='bold')
+    ax1.tick_params(colors=text_color)
+    ax1.grid(axis='y', linestyle='--', alpha=0.3, color=grid_color)
+    ax1.yaxis.set_major_locator(plt.MaxNLocator(integer=True))
+    for spine in ['top', 'right']:
+        ax1.spines[spine].set_visible(False)
+    ax1.spines['left'].set_color(grid_color)
+    ax1.spines['bottom'].set_color(grid_color)
+    
+    # 2. Top 10 模型按DS排序
+    top_models = df.nlargest(10, 'ds')[['model', 'ds']]
+    ax2.barh(range(len(top_models)), top_models['ds'].values, color=bar_colors[3])
+    ax2.set_facecolor(panel_color)
+    ax2.set_yticks(range(len(top_models)))
+    ax2.set_yticklabels(top_models['model'].values, fontsize=9)
+    ax2.set_xlabel('Driving Score', color=text_color, fontsize=11)
+    ax2.set_title('Top 10 Models by DS', color=text_color, fontsize=13, fontweight='bold')
+    ax2.tick_params(colors=text_color)
+    ax2.grid(axis='x', linestyle='--', alpha=0.3, color=grid_color)
+    ax2.invert_yaxis()
+    for spine in ['top', 'right', 'left']:
+        ax2.spines[spine].set_visible(False)
+    ax2.spines['bottom'].set_color(grid_color)
+    
+    plt.tight_layout()
+    result = fig
+    plt.close('all')
+    return result
+
+# 预生成初始图表
+initial_va_plot = create_va_plot("All")
+initial_vla_plot = create_vla_plot("All")
+initial_nuscenes_plot = create_nuscenes_plot("All")
+initial_wod_plot = create_wod_plot("All")
+initial_navsim_plot = create_navsim_plot("All")
+initial_bench2drive_plot = create_bench2drive_plot("All")
+
+# Gradio UI
+with gr.Blocks(css="#title {text-align: center;} .gradio-container {max-width: 100% !important;}") as demo:
+    gr.Markdown("# 🚗 VLA for Autonomous Driving: Model Leaderboard", elem_id="title")
+    gr.Markdown("### 📄 Survey: *Vision-Language-Action Models for Autonomous Driving*")
+    
+    with gr.Tabs():
+        # Tab 1: VA Models
+        with gr.Tab("Vision-Action Models (VA)"):
+            gr.Markdown("### Table 1: Vision-Action Models in Autonomous Driving")
+            va_stats = gr.Markdown(f"**📊 Statistics:** Total {len(df_va)} models")
+            with gr.Row():
+                va_category = gr.Dropdown(label="Category", choices=["All"] + sorted(df_va['category'].unique().tolist()), value="All")
+                va_venue = gr.Textbox(label="Filter by Venue", placeholder="e.g., CVPR, ICCV...")
+                va_action = gr.Dropdown(label="Action Type", choices=["All"] + sorted(df_va['action'].unique().tolist()), value="All")
+                va_search = gr.Textbox(label="Search", placeholder="e.g., TransFuser...")
+            va_update_btn = gr.Button("🔍 Update", variant="primary")
+            va_table = gr.Dataframe(value=df_va[['id', 'model', 'venue', 'input', 'dataset', 'vision', 'action', 'output', 'category']], label="VA Models", interactive=False, wrap=True)
+            va_plot = gr.Plot(label="Statistics", value=initial_va_plot, format="png")
+            va_update_btn.click(fn=lambda cat, ven, act, search: (*update_va_table(cat, ven, act, search), create_va_plot(cat)),
+                               inputs=[va_category, va_venue, va_action, va_search], outputs=[va_table, va_stats, va_plot])
+        
+        # Tab 2: VLA Models
+        with gr.Tab("Vision-Language-Action Models (VLA)"):
+            gr.Markdown("### Table 3: Vision-Language-Action Models in Autonomous Driving")
+            vla_stats = gr.Markdown(f"**📊 Statistics:** Total {len(df_vla)} models")
+            with gr.Row():
+                vla_category = gr.Dropdown(label="Category", choices=["All"] + sorted(df_vla['category'].unique().tolist()), value="All")
+                vla_venue = gr.Textbox(label="Filter by Venue", placeholder="e.g., CVPR...")
+                vla_language = gr.Textbox(label="Filter by Language Model", placeholder="e.g., Qwen...")
+                vla_search = gr.Textbox(label="Search", placeholder="e.g., AutoVLA...")
+            vla_update_btn = gr.Button("🔍 Update", variant="primary")
+            vla_table = gr.Dataframe(value=df_vla[['id', 'model', 'venue', 'input', 'dataset', 'vision', 'language', 'action', 'output', 'category']], label="VLA Models", interactive=False, wrap=True)
+            vla_plot = gr.Plot(label="Statistics", value=initial_vla_plot, format="png")
+            vla_update_btn.click(fn=lambda cat, ven, lang, search: (*update_vla_table(cat, ven, lang, search), create_vla_plot(cat)),
+                                inputs=[vla_category, vla_venue, vla_language, vla_search], outputs=[vla_table, vla_stats, vla_plot])
+        
+        # Tab 3: Datasets
+        with gr.Tab("Datasets & Benchmarks"):
+            gr.Markdown("### Table 4: Summary of Datasets & Benchmarks")
+            datasets_stats = gr.Markdown(f"**📊 Statistics:** Total {len(df_datasets)} datasets")
+            with gr.Row():
+                datasets_category = gr.Dropdown(label="Category", choices=["All"] + sorted(df_datasets['category'].unique().tolist()), value="All")
+                datasets_year = gr.Textbox(label="Filter by Year", placeholder="e.g., 2024...")
+                datasets_search = gr.Textbox(label="Search", placeholder="e.g., nuScenes...")
+            datasets_update_btn = gr.Button("🔍 Update", variant="primary")
+            datasets_table = gr.Dataframe(value=df_datasets, label="Datasets", interactive=False, wrap=True)
+            datasets_update_btn.click(fn=update_datasets_table, inputs=[datasets_category, datasets_year, datasets_search], outputs=[datasets_table, datasets_stats])
+        
+        # Tab 4: nuScenes
+        with gr.Tab("nuScenes Open-Loop"):
+            gr.Markdown("### Table 5: Open-Loop Planning Results on nuScenes")
+            gr.Markdown("**Metrics:** L2 Error (m) ↓ | Collision Rate ↓")
+            nuscenes_stats = gr.Markdown(f"**📊 Statistics:** Total {len(df_nuscenes)} models")
+            with gr.Row():
+                nuscenes_category = gr.Dropdown(label="Category", choices=["All", "Vision-Action", "Vision-Language-Action"], value="All")
+                nuscenes_search = gr.Textbox(label="Search Model", placeholder="e.g., UniAD, EMMA...")
+            nuscenes_update_btn = gr.Button("🔍 Update", variant="primary")
+            nuscenes_table = gr.Dataframe(value=df_nuscenes, label="nuScenes Results", interactive=False, wrap=True)
+            nuscenes_plot = gr.Plot(label="Statistics", value=initial_nuscenes_plot, format="png")
+            nuscenes_update_btn.click(fn=lambda cat, search: (*update_nuscenes_table(cat, search), create_nuscenes_plot(cat)),
+                                     inputs=[nuscenes_category, nuscenes_search], 
+                                     outputs=[nuscenes_table, nuscenes_stats, nuscenes_plot])
+        
+        # Tab 5: WOD-E2E
+        with gr.Tab("WOD-E2E"):
+            gr.Markdown("### Table 6: Results on WOD-E2E Test Split")
+            gr.Markdown("**Metrics:** RFS (Overall/Spotlight) ↑ | ADE (5s/3s) ↓")
+            wod_stats = gr.Markdown(f"**📊 Statistics:** Total {len(df_wod)} models")
+            with gr.Row():
+                wod_category = gr.Dropdown(label="Category", choices=["All", "Vision-Action", "Vision-Language-Action"], value="All")
+                wod_search = gr.Textbox(label="Search Model", placeholder="e.g., AutoVLA...")
+            wod_update_btn = gr.Button("🔍 Update", variant="primary")
+            wod_table = gr.Dataframe(value=df_wod, label="WOD-E2E Results", interactive=False, wrap=True)
+            wod_plot = gr.Plot(label="Statistics", value=initial_wod_plot, format="png")
+            wod_update_btn.click(fn=lambda cat, search: (*update_wod_table(cat, search), create_wod_plot(cat)),
+                                inputs=[wod_category, wod_search], 
+                                outputs=[wod_table, wod_stats, wod_plot])
+        
+        # Tab 6: NAVSIM
+        with gr.Tab("NAVSIM Closed-Loop"):
+            gr.Markdown("### Table 7: Closed-Loop Results on NAVSIM")
+            gr.Markdown("**Metrics:** NC/DAC/TTC/Comf/EP/PDMS ↑")
+            navsim_stats = gr.Markdown(f"**📊 Statistics:** Total {len(df_navsim)} models")
+            with gr.Row():
+                navsim_category = gr.Dropdown(label="Category", choices=["All", "Vision-Action", "Vision-Language-Action"], value="All")
+                navsim_search = gr.Textbox(label="Search Model", placeholder="e.g., ReflectDrive...")
+            navsim_update_btn = gr.Button("🔍 Update", variant="primary")
+            navsim_table = gr.Dataframe(value=df_navsim, label="NAVSIM Results", interactive=False, wrap=True)
+            navsim_plot = gr.Plot(label="Statistics", value=initial_navsim_plot, format="png")
+            navsim_update_btn.click(fn=lambda cat, search: (*update_navsim_table(cat, search), create_navsim_plot(cat)),
+                                   inputs=[navsim_category, navsim_search], 
+                                   outputs=[navsim_table, navsim_stats, navsim_plot])
+        
+        # Tab 7: Bench2Drive
+        with gr.Tab("Bench2Drive"):
+            gr.Markdown("### Table 8: Closed-Loop & Open-Loop Results on Bench2Drive")
+            gr.Markdown("**Metrics:** DS/SR ↑ | L2 Avg ↓")
+            bench2drive_stats = gr.Markdown(f"**📊 Statistics:** Total {len(df_bench2drive)} models")
+            with gr.Row():
+                bench2drive_category = gr.Dropdown(label="Category", choices=["All", "Vision-Action", "Vision-Language-Action"], value="All")
+                bench2drive_search = gr.Textbox(label="Search Model", placeholder="e.g., SimLingo...")
+            bench2drive_update_btn = gr.Button("🔍 Update", variant="primary")
+            bench2drive_table = gr.Dataframe(value=df_bench2drive, label="Bench2Drive Results", interactive=False, wrap=True)
+            bench2drive_plot = gr.Plot(label="Statistics", value=initial_bench2drive_plot, format="png")
+            bench2drive_update_btn.click(fn=lambda cat, search: (*update_bench2drive_table(cat, search), create_bench2drive_plot(cat)),
+                                        inputs=[bench2drive_category, bench2drive_search], 
+                                        outputs=[bench2drive_table, bench2drive_stats, bench2drive_plot])
+    
+    gr.Markdown("""
+    ---
+    **Legend:** 📷 Camera | 🔍 LiDAR | ⚙️ Status | 💭 Prompt | 📝 Instruction | 🌆 Scene | 🚦 Traffic | 📚 Context
+    
+    **Action Types:** RL=Reinforcement Learning | REG=Decoder+MLP | SEL=Selection | GEN=Generative | LH=Language Head
+    
+    📖 **Project:** [https://worldbench.github.io/vla4ad](https://worldbench.github.io/vla4ad) | 🔗 **GitHub:** [https://github.com/worldbench/awesome-vla-for-ad](https://github.com/worldbench/awesome-vla-for-ad)
+    """)
+
+if __name__ == "__main__":
+    demo.launch()

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,16 @@
+APScheduler
+black
+datasets
+gradio
+gradio[oauth]
+gradio_leaderboard==0.0.13
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib
+numpy
+pandas
+python-dateutil
+tqdm
+transformers
+tokenizers>=0.15.0
+sentencepiece

results/bench2drive_results.json

@@ -0,0 +1,22 @@
+[
+    {"id": 1, "model": "TCP", "year": 2022, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "ds": 40.70, "sr": 15.00, "efficiency": 54.26, "comfort": 47.80, "l2_avg": 1.70, "category": "Vision-Action"},
+    {"id": 2, "model": "ThinkTwice", "year": 2023, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "ds": 62.44, "sr": 31.23, "efficiency": 69.33, "comfort": 16.22, "l2_avg": 0.95, "category": "Vision-Action"},
+    {"id": 3, "model": "DriveAdapter", "year": 2023, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "REG", "ds": 64.22, "sr": 33.08, "efficiency": 70.22, "comfort": 16.01, "l2_avg": 1.01, "category": "Vision-Action"},
+    {"id": 4, "model": "UniAD-Base", "year": 2023, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "ds": 45.81, "sr": 16.36, "efficiency": 129.21, "comfort": 43.58, "l2_avg": 0.73, "category": "Vision-Action"},
+    {"id": 5, "model": "VAD", "year": 2023, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "ds": 42.35, "sr": 15.00, "efficiency": 157.94, "comfort": 46.01, "l2_avg": 0.91, "category": "Vision-Action"},
+    {"id": 6, "model": "GenAD", "year": 2024, "input": "camera", "vision": "ResNet", "language": "-", "action": "GEN", "ds": 44.81, "sr": 15.90, "efficiency": null, "comfort": null, "l2_avg": null, "category": "Vision-Action"},
+    {"id": 7, "model": "DriveTransformer", "year": 2025, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "ds": 63.46, "sr": 35.01, "efficiency": 100.64, "comfort": 20.78, "l2_avg": 0.62, "category": "Vision-Action"},
+    {"id": 8, "model": "ETA", "year": 2025, "input": "camera", "vision": "CLIP", "language": "-", "action": "REG", "ds": 69.53, "sr": 38.64, "efficiency": 184.51, "comfort": 28.43, "l2_avg": null, "category": "Vision-Action"},
+    {"id": 9, "model": "WoTE", "year": 2025, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "SEL", "ds": 61.71, "sr": 31.36, "efficiency": null, "comfort": null, "l2_avg": null, "category": "Vision-Action"},
+    {"id": 10, "model": "GuideFlow", "year": 2025, "input": "camera", "vision": "ResNet", "language": "-", "action": "GEN", "ds": 75.21, "sr": 51.36, "efficiency": null, "comfort": null, "l2_avg": null, "category": "Vision-Action"},
+    {"id": 11, "model": "Raw2Drive", "year": 2025, "input": "camera", "vision": "ResNet", "language": "-", "action": "RL", "ds": 71.36, "sr": 50.24, "efficiency": 214.17, "comfort": 22.42, "l2_avg": null, "category": "Vision-Action"},
+    {"id": 12, "model": "ORION", "year": 2025, "input": "camera, prompt, command", "vision": "EVA-02", "language": "Vicuna-1.5", "action": "GEN", "ds": 77.74, "sr": 54.62, "efficiency": 151.48, "comfort": 17.38, "l2_avg": 0.68, "category": "Vision-Language-Action"},
+    {"id": 13, "model": "AutoVLA", "year": 2025, "input": "camera, prompt, command, status", "vision": "SigLIP", "language": "Qwen2.5-VL", "action": "LH", "ds": 78.84, "sr": 57.73, "efficiency": 146.93, "comfort": 39.33, "l2_avg": null, "category": "Vision-Language-Action"},
+    {"id": 14, "model": "SimLingo-Base", "year": 2025, "input": "camera, prompt, command", "vision": "InternViT", "language": "Qwen2", "action": "REG", "ds": 85.94, "sr": 66.82, "efficiency": 244.18, "comfort": 25.49, "l2_avg": null, "category": "Vision-Language-Action"},
+    {"id": 15, "model": "SimLingo", "year": 2025, "input": "camera, prompt, command", "vision": "InternViT", "language": "Qwen2", "action": "REG", "ds": 85.07, "sr": 67.27, "efficiency": 259.23, "comfort": 33.67, "l2_avg": null, "category": "Vision-Language-Action"},
+    {"id": 16, "model": "ReAL-AD (QwenVL)", "year": 2025, "input": "camera, prompt, command", "vision": "ResNet", "language": "QwenVL", "action": "REG", "ds": 40.76, "sr": 10.93, "efficiency": null, "comfort": null, "l2_avg": 0.87, "category": "Vision-Language-Action"},
+    {"id": 17, "model": "ReasonPlan", "year": 2025, "input": "camera, prompt, command, status, context", "vision": "SigLIP", "language": "Qwen", "action": "LH", "ds": 64.01, "sr": 34.55, "efficiency": 180.64, "comfort": 25.63, "l2_avg": 0.61, "category": "Vision-Language-Action"},
+    {"id": 18, "model": "DriveMoE", "year": 2025, "input": "camera, prompt, status", "vision": "BEV Encoder", "language": "LLaMA", "action": "REG", "ds": 74.22, "sr": 48.64, "efficiency": 175.96, "comfort": 15.31, "l2_avg": 0.31, "category": "Vision-Language-Action"},
+    {"id": 19, "model": "VDRive", "year": 2025, "input": "camera, prompt, command, status", "vision": "Qwen2.5-VL, CVQ", "language": "InternVL3", "action": "GEN", "ds": 66.15, "sr": 50.51, "efficiency": 110.23, "comfort": 22.90, "l2_avg": 0.55, "category": "Vision-Language-Action"},
+    {"id": 20, "model": "CoReVLA", "year": 2025, "input": "camera, prompt, command, context", "vision": "Qwen2.5-VL", "language": "Qwen2.5-VL", "action": "LH", "ds": 72.18, "sr": 50.00, "efficiency": 145.41, "comfort": 34.35, "l2_avg": null, "category": "Vision-Language-Action"}
+  ]

results/datasets.json

@@ -0,0 +1,33 @@
+[
+  {"id": 1, "dataset": "BDD100K", "year": 2020, "sensor": "camera", "type": "Real", "scale": "120M", "language": "-", "action_type": "Traj.", "category": "Vision-Action"},
+  {"id": 2, "dataset": "nuScenes", "year": 2020, "sensor": "camera, lidar, radar", "type": "Real", "scale": "1.4M", "language": "-", "action_type": "Traj.", "category": "Vision-Action"},
+  {"id": 3, "dataset": "Waymo", "year": 2020, "sensor": "camera, lidar, radar", "type": "Real", "scale": "200M", "language": "-", "action_type": "Traj.", "category": "Vision-Action"},
+  {"id": 4, "dataset": "nuPlan", "year": 2021, "sensor": "camera, lidar, map", "type": "Real", "scale": "4.6M", "language": "-", "action_type": "Ctrl, Traj.", "category": "Vision-Action"},
+  {"id": 5, "dataset": "Argoverse 2", "year": 2021, "sensor": "camera, lidar, map", "type": "Real", "scale": "300K", "language": "-", "action_type": "Traj.", "category": "Vision-Action"},
+  {"id": 6, "dataset": "Bench2Drive", "year": 2024, "sensor": "camera, lidar, map", "type": "Sim", "scale": "2M", "language": "-", "action_type": "Traj.", "category": "Vision-Action"},
+  {"id": 7, "dataset": "RoboBEV", "year": 2025, "sensor": "camera, lidar, radar", "type": "Real", "scale": "866K", "language": "-", "action_type": "Traj.", "category": "Vision-Action"},
+  {"id": 8, "dataset": "WOD-E2E", "year": 2025, "sensor": "camera", "type": "Real", "scale": "800K", "language": "-", "action_type": "Traj.", "category": "Vision-Action"},
+  {"id": 9, "dataset": "BDD-X", "year": 2018, "sensor": "camera", "type": "Real", "scale": "8.4M", "language": "Caption (26K)", "action_type": "Ctrl", "category": "Vision-Language-Action"},
+  {"id": 10, "dataset": "Talk2Car", "year": 2022, "sensor": "camera, lidar, radar", "type": "Real", "scale": "400K", "language": "Caption (12K)", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 11, "dataset": "SDN", "year": 2022, "sensor": "camera", "type": "Sim", "scale": "-", "language": "Instruction, QA (8.4K)", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 12, "dataset": "DriveMLM", "year": 2023, "sensor": "camera, lidar", "type": "Sim", "scale": "-", "language": "Reason., Deci.", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 13, "dataset": "LMDrive", "year": 2024, "sensor": "camera, lidar", "type": "Sim", "scale": "3M", "language": "Instruction (64K)", "action_type": "Traj.", "category": "Vision-Language-Action"},
+  {"id": 14, "dataset": "DriveLM-N", "year": 2024, "sensor": "camera, lidar, radar", "type": "Real", "scale": "4.8K", "language": "QA (445K)", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 15, "dataset": "DriveLM-C", "year": 2024, "sensor": "camera, lidar, radar", "type": "Sim", "scale": "64K", "language": "QA (3.76M)", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 16, "dataset": "HBD", "year": 2024, "sensor": "camera", "type": "Real, Sim", "scale": "-", "language": "Deci., Descrip., QA", "action_type": "Traj.", "category": "Vision-Language-Action"},
+  {"id": 17, "dataset": "VLAAD", "year": 2024, "sensor": "camera", "type": "Real", "scale": "-", "language": "Reason., QA (64K)", "action_type": "Ctrl.", "category": "Vision-Language-Action"},
+  {"id": 18, "dataset": "SUP-AD", "year": 2024, "sensor": "camera", "type": "Real", "scale": "-", "language": "Action, Reason., QA", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 19, "dataset": "NuInstruct", "year": 2024, "sensor": "camera, lidar, radar", "type": "Real", "scale": "11.8K", "language": "Instruction (91K)", "action_type": "Ctrl", "category": "Vision-Language-Action"},
+  {"id": 20, "dataset": "WOMD-Reason", "year": 2024, "sensor": "camera", "type": "Real", "scale": "63K", "language": "QA (2940K)", "action_type": "Plan.", "category": "Vision-Language-Action"},
+  {"id": 21, "dataset": "DriveCoT", "year": 2024, "sensor": "camera, lidar", "type": "Sim", "scale": "-", "language": "CoT, Deci. (36K)", "action_type": "Ctrl", "category": "Vision-Language-Action"},
+  {"id": 22, "dataset": "Reason2Drive", "year": 2024, "sensor": "camera, lidar, radar", "type": "Real", "scale": "-", "language": "Reason., QA (632K)", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 23, "dataset": "DriveBench", "year": 2025, "sensor": "camera, lidar, radar", "type": "Real", "scale": "19.2K", "language": "QA (20.5K)", "action_type": "Ctrl", "category": "Vision-Language-Action"},
+  {"id": 24, "dataset": "MetaAD", "year": 2025, "sensor": "camera", "type": "Real", "scale": "120K", "language": "Reason., Plan, QA (30K)", "action_type": "Ctrl", "category": "Vision-Language-Action"},
+  {"id": 25, "dataset": "OmniDrive", "year": 2025, "sensor": "camera, lidar, radar", "type": "Real", "scale": "-", "language": "Reason., QA", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 26, "dataset": "NuInteract", "year": 2025, "sensor": "camera, lidar, radar", "type": "Real", "scale": "34K", "language": "Caption, QA (1.5M)", "action_type": "Ctrl", "category": "Vision-Language-Action"},
+  {"id": 27, "dataset": "DriveAction", "year": 2025, "sensor": "camera", "type": "Real", "scale": "2.6K", "language": "QA (16.18K)", "action_type": "Ctrl", "category": "Vision-Language-Action"},
+  {"id": 28, "dataset": "ImpromptuVLA", "year": 2025, "sensor": "camera", "type": "Real, Sim", "scale": "2M", "language": "Instruction, QA (80K)", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 29, "dataset": "CoVLA", "year": 2025, "sensor": "camera", "type": "Real", "scale": "6M", "language": "Caption (6M)", "action_type": "Traj.", "category": "Vision-Language-Action"},
+  {"id": 30, "dataset": "OmniReason-N", "year": 2025, "sensor": "camera, lidar, radar", "type": "Real", "scale": "-", "language": "QA", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"},
+  {"id": 31, "dataset": "OmniReason-B2D", "year": 2025, "sensor": "camera", "type": "Sim", "scale": "-", "language": "QA", "action_type": "Ctrl, Traj.", "category": "Vision-Language-Action"}
+]

results/navsim_results.json

@@ -0,0 +1,26 @@
+[
+    {"id": 1, "model": "TransFuser", "year": 2022, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "REG", "nc": 97.7, "dac": 92.8, "ttc": 92.8, "comf": 100.0, "ep": 79.2, "pdms": 84.0, "category": "Vision-Action"},
+    {"id": 2, "model": "UniAD", "year": 2023, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "nc": 97.8, "dac": 91.9, "ttc": 92.9, "comf": 100.0, "ep": 78.8, "pdms": 83.4, "category": "Vision-Action"},
+    {"id": 3, "model": "VADv2", "year": 2024, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "nc": 97.2, "dac": 89.1, "ttc": 91.6, "comf": 100.0, "ep": 76.0, "pdms": 80.9, "category": "Vision-Action"},
+    {"id": 4, "model": "PARA-Drive", "year": 2024, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "nc": 97.9, "dac": 92.4, "ttc": 93.0, "comf": 99.8, "ep": 79.3, "pdms": 84.0, "category": "Vision-Action"},
+    {"id": 5, "model": "LAW", "year": 2024, "input": "camera", "vision": "Swin-T", "language": "-", "action": "REG", "nc": 96.4, "dac": 95.4, "ttc": 88.7, "comf": 99.9, "ep": 81.7, "pdms": 84.6, "category": "Vision-Action"},
+    {"id": 6, "model": "DRAMA", "year": 2024, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "REG", "nc": 98.0, "dac": 93.1, "ttc": 94.8, "comf": 100.0, "ep": 80.1, "pdms": 85.5, "category": "Vision-Action"},
+    {"id": 7, "model": "DiffusionDrive", "year": 2024, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "GEN", "nc": 98.2, "dac": 96.2, "ttc": 94.7, "comf": 100.0, "ep": 82.2, "pdms": 88.1, "category": "Vision-Action"},
+    {"id": 8, "model": "WoTE", "year": 2025, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "SEL", "nc": 98.5, "dac": 96.8, "ttc": 94.9, "comf": 99.9, "ep": 81.9, "pdms": 88.3, "category": "Vision-Action"},
+    {"id": 9, "model": "World4Drive", "year": 2025, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "nc": 97.4, "dac": 94.3, "ttc": 92.8, "comf": 100.0, "ep": 79.9, "pdms": 85.1, "category": "Vision-Action"},
+    {"id": 10, "model": "DrivingGPT", "year": 2025, "input": "camera", "vision": "VQ-VAE", "language": "-", "action": "REG", "nc": 98.9, "dac": 90.7, "ttc": 94.9, "comf": 95.6, "ep": 79.7, "pdms": 82.4, "category": "Vision-Action"},
+    {"id": 11, "model": "AD-R1", "year": 2025, "input": "camera, lidar, status", "vision": "-", "language": "-", "action": "RL", "nc": 98.7, "dac": 97.8, "ttc": 94.8, "comf": 100.0, "ep": 87.5, "pdms": 91.9, "category": "Vision-Action"},
+    {"id": 12, "model": "SeerDrive", "year": 2025, "input": "camera, lidar", "vision": "VoVNet", "language": "-", "action": "SEL", "nc": 98.8, "dac": 98.6, "ttc": 95.8, "comf": 100.0, "ep": 84.2, "pdms": 90.7, "category": "Vision-Action"},
+    {"id": 13, "model": "Epona", "year": 2025, "input": "camera, status", "vision": "DC-AE", "language": "-", "action": "REG", "nc": 97.9, "dac": 95.1, "ttc": 93.8, "comf": 99.9, "ep": 80.4, "pdms": 86.2, "category": "Vision-Action"},
+    {"id": 14, "model": "GoalFlow", "year": 2025, "input": "camera, lidar, status", "vision": "VoVNet", "language": "-", "action": "GEN", "nc": 98.4, "dac": 98.3, "ttc": 94.6, "comf": 100.0, "ep": 85.0, "pdms": 90.3, "category": "Vision-Action"},
+    {"id": 15, "model": "TrajDiff", "year": 2025, "input": "camera, lidar, status", "vision": "ResNet", "language": "-", "action": "GEN", "nc": 98.1, "dac": 97.0, "ttc": 94.3, "comf": 100.0, "ep": 82.7, "pdms": 88.5, "category": "Vision-Action"},
+    {"id": 16, "model": "DiffusionDriveV2", "year": 2025, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "GEN", "nc": 98.3, "dac": 97.9, "ttc": 94.8, "comf": 99.9, "ep": 87.5, "pdms": 91.2, "category": "Vision-Action"},
+    {"id": 17, "model": "NaviHydra", "year": 2025, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "SEL", "nc": 98.7, "dac": 98.6, "ttc": 88.7, "comf": 96.2, "ep": 100.0, "pdms": 92.7, "category": "Vision-Action"},
+    {"id": 18, "model": "Mimir", "year": 2025, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "GEN", "nc": 98.2, "dac": 97.5, "ttc": 94.6, "comf": 100.0, "ep": 83.6, "pdms": 89.3, "category": "Vision-Action"},
+    {"id": 19, "model": "ReCogDrive", "year": 2025, "input": "camera, prompt, command", "vision": "InternViT", "language": "Qwen2.5-VL", "action": "GEN", "nc": 98.2, "dac": 97.8, "ttc": 95.2, "comf": 99.8, "ep": 83.5, "pdms": 89.6, "category": "Vision-Language-Action"},
+    {"id": 20, "model": "AutoVLA", "year": 2025, "input": "camera, prompt, command, status", "vision": "Qwen2.5-VL", "language": "Qwen2.5-VL", "action": "LH", "nc": 99.1, "dac": 97.1, "ttc": 97.1, "comf": 99.9, "ep": 87.6, "pdms": 92.1, "category": "Vision-Language-Action"},
+    {"id": 21, "model": "ReflectDrive", "year": 2025, "input": "camera, prompt, scene, status", "vision": "LLaDA-V", "language": "LLaDA-V", "action": "GEN", "nc": 99.7, "dac": 99.5, "ttc": 99.1, "comf": 99.9, "ep": 88.9, "pdms": 94.7, "category": "Vision-Language-Action"},
+    {"id": 22, "model": "AdaThinkDrive", "year": 2025, "input": "camera, prompt, command, status", "vision": "InternVL3", "language": "InternVL3", "action": "REG", "nc": 99.1, "dac": 98.8, "ttc": 97.2, "comf": 100.0, "ep": 87.9, "pdms": 93.0, "category": "Vision-Language-Action"},
+    {"id": 23, "model": "Percept-WAM", "year": 2025, "input": "camera, lidar, prompt, command", "vision": "InternViT", "language": "InternVL2", "action": "REG", "nc": 98.8, "dac": 98.6, "ttc": 94.4, "comf": 99.5, "ep": 84.8, "pdms": 90.2, "category": "Vision-Language-Action"},
+    {"id": 24, "model": "Reasoning-VLA", "year": 2025, "input": "camera, prompt, status", "vision": "Qwen2.5-VL", "language": "Qwen2.5-VL", "action": "REG", "nc": 97.8, "dac": 93.2, "ttc": 98.1, "comf": 99.8, "ep": 80.7, "pdms": 91.7, "category": "Vision-Language-Action"}
+  ]

results/nuscenes_results.json

@@ -0,0 +1,60 @@
+[
+    {"id": 1, "model": "ST-P3", "year": 2022, "input": "camera", "vision": "EfficientNet", "language": "-", "action": "REG", "l2_1s": 1.33, "l2_2s": 2.11, "l2_3s": 2.90, "l2_avg": 2.11, "cr_1s": 0.23, "cr_2s": 0.62, "cr_3s": 1.27, "cr_avg": 0.71, "category": "Vision-Action"},
+    {"id": 2, "model": "UniAD", "year": 2022, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.44, "l2_2s": 0.67, "l2_3s": 0.96, "l2_avg": 0.69, "cr_1s": 0.04, "cr_2s": 0.08, "cr_3s": 0.23, "cr_avg": 0.12, "category": "Vision-Action"},
+    {"id": 3, "model": "VAD", "year": 2023, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.17, "l2_2s": 0.34, "l2_3s": 0.60, "l2_avg": 0.37, "cr_1s": 0.07, "cr_2s": 0.10, "cr_3s": 0.24, "cr_avg": 0.14, "category": "Vision-Action"},
+    {"id": 4, "model": "OccNet", "year": 2023, "input": "camera", "vision": "ResNet", "language": "-", "action": "SEL", "l2_1s": 1.29, "l2_2s": 2.13, "l2_3s": 2.99, "l2_avg": 2.14, "cr_1s": 0.21, "cr_2s": 0.59, "cr_3s": 1.37, "cr_avg": 0.72, "category": "Vision-Action"},
+    {"id": 5, "model": "BEV-Planner", "year": 2024, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.30, "l2_2s": 0.52, "l2_3s": 0.83, "l2_avg": 0.55, "cr_1s": 0.10, "cr_2s": 0.37, "cr_3s": 1.30, "cr_avg": 0.59, "category": "Vision-Action"},
+    {"id": 6, "model": "Drive-WM", "year": 2024, "input": "camera, status", "vision": "ConvNeXt", "language": "-", "action": "SEL", "l2_1s": 0.43, "l2_2s": 0.77, "l2_3s": 1.20, "l2_avg": 0.80, "cr_1s": 0.10, "cr_2s": 0.21, "cr_3s": 0.48, "cr_avg": 0.26, "category": "Vision-Action"},
+    {"id": 7, "model": "GenAD", "year": 2024, "input": "camera", "vision": "ResNet", "language": "-", "action": "GEN", "l2_1s": 0.36, "l2_2s": 0.83, "l2_3s": 1.55, "l2_avg": 0.91, "cr_1s": 0.06, "cr_2s": 0.23, "cr_3s": 1.00, "cr_avg": 0.43, "category": "Vision-Action"},
+    {"id": 8, "model": "OccWorld", "year": 2024, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.43, "l2_2s": 1.08, "l2_3s": 1.99, "l2_avg": 1.17, "cr_1s": 0.07, "cr_2s": 0.38, "cr_3s": 1.35, "cr_avg": 0.60, "category": "Vision-Action"},
+    {"id": 9, "model": "DriveDreamer", "year": 2024, "input": "camera, status", "vision": "SD", "language": "-", "action": "REG", "l2_1s": null, "l2_2s": null, "l2_3s": null, "l2_avg": 0.29, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": 0.15, "category": "Vision-Action"},
+    {"id": 10, "model": "SparseAD", "year": 2024, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.15, "l2_2s": 0.31, "l2_3s": 0.56, "l2_avg": 0.34, "cr_1s": 0.00, "cr_2s": 0.04, "cr_3s": 0.15, "cr_avg": 0.06, "category": "Vision-Action"},
+    {"id": 11, "model": "GaussianAD", "year": 2024, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.40, "l2_2s": 0.64, "l2_3s": 0.88, "l2_avg": 0.64, "cr_1s": 0.09, "cr_2s": 0.38, "cr_3s": 0.81, "cr_avg": 0.42, "category": "Vision-Action"},
+    {"id": 12, "model": "LAW", "year": 2024, "input": "camera", "vision": "Swin-T", "language": "-", "action": "REG", "l2_1s": 0.24, "l2_2s": 0.46, "l2_3s": 0.76, "l2_avg": 0.49, "cr_1s": 0.08, "cr_2s": 0.10, "cr_3s": 0.39, "cr_avg": 0.19, "category": "Vision-Action"},
+    {"id": 13, "model": "SSR", "year": 2024, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.18, "l2_2s": 0.36, "l2_3s": 0.63, "l2_avg": 0.39, "cr_1s": 0.01, "cr_2s": 0.04, "cr_3s": 0.12, "cr_avg": 0.06, "category": "Vision-Action"},
+    {"id": 14, "model": "Drive-OccWorld", "year": 2025, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.25, "l2_2s": 0.44, "l2_3s": 0.72, "l2_avg": 0.47, "cr_1s": 0.03, "cr_2s": 0.08, "cr_3s": 0.22, "cr_avg": 0.11, "category": "Vision-Action"},
+    {"id": 15, "model": "DriveTransformer", "year": 2025, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.19, "l2_2s": 0.34, "l2_3s": 0.66, "l2_avg": 0.40, "cr_1s": 0.03, "cr_2s": 0.10, "cr_3s": 0.21, "cr_avg": 0.11, "category": "Vision-Action"},
+    {"id": 16, "model": "DiffusionDrive", "year": 2025, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "GEN", "l2_1s": 0.27, "l2_2s": 0.54, "l2_3s": 0.90, "l2_avg": 0.57, "cr_1s": 0.03, "cr_2s": 0.05, "cr_3s": 0.16, "cr_avg": 0.08, "category": "Vision-Action"},
+    {"id": 17, "model": "World4Drive", "year": 2025, "input": "camera", "vision": "ResNet", "language": "-", "action": "REG", "l2_1s": 0.23, "l2_2s": 0.47, "l2_3s": 0.81, "l2_avg": 0.50, "cr_1s": 0.02, "cr_2s": 0.12, "cr_3s": 0.33, "cr_avg": 0.16, "category": "Vision-Action"},
+    {"id": 18, "model": "Epona", "year": 2025, "input": "camera, status", "vision": "DC-AE", "language": "-", "action": "REG", "l2_1s": 0.61, "l2_2s": 1.17, "l2_3s": 1.98, "l2_avg": 1.25, "cr_1s": 0.01, "cr_2s": 0.22, "cr_3s": 0.85, "cr_avg": 0.36, "category": "Vision-Action"},
+    {"id": 19, "model": "SeerDrive", "year": 2025, "input": "camera, lidar", "vision": "ResNet", "language": "-", "action": "SEL", "l2_1s": 0.20, "l2_2s": 0.39, "l2_3s": 0.69, "l2_avg": 0.43, "cr_1s": 0.00, "cr_2s": 0.05, "cr_3s": 0.14, "cr_avg": 0.06, "category": "Vision-Action"},
+    {"id": 20, "model": "GuideFlow", "year": 2025, "input": "camera", "vision": "ResNet", "language": "-", "action": "GEN", "l2_1s": null, "l2_2s": null, "l2_3s": null, "l2_avg": null, "cr_1s": 0.00, "cr_2s": 0.02, "cr_3s": 0.18, "cr_avg": 0.07, "category": "Vision-Action"},
+    {"id": 21, "model": "Agent-Driver", "year": 2023, "input": "camera, prompt", "vision": "-", "language": "GPT-3.5", "action": "LH", "l2_1s": 0.16, "l2_2s": 0.34, "l2_3s": 0.61, "l2_avg": 0.37, "cr_1s": 0.02, "cr_2s": 0.07, "cr_3s": 0.18, "cr_avg": 0.09, "category": "Vision-Language-Action"},
+    {"id": 22, "model": "DriveVLM", "year": 2024, "input": "camera, prompt, scene, status", "vision": "ViT", "language": "QwenVL", "action": "GEN", "l2_1s": 0.18, "l2_2s": 0.34, "l2_3s": 0.68, "l2_avg": 0.40, "cr_1s": 0.10, "cr_2s": 0.22, "cr_3s": 0.45, "cr_avg": 0.27, "category": "Vision-Language-Action"},
+    {"id": 23, "model": "DriveVLM-Dual", "year": 2024, "input": "camera, prompt", "vision": "ViT", "language": "QwenVL", "action": "REG", "l2_1s": 0.15, "l2_2s": 0.29, "l2_3s": 0.48, "l2_avg": 0.31, "cr_1s": 0.05, "cr_2s": 0.08, "cr_3s": 0.17, "cr_avg": 0.10, "category": "Vision-Language-Action"},
+    {"id": 24, "model": "RAG-Driver", "year": 2024, "input": "camera, prompt, command, context, status", "vision": "CLIP", "language": "Vicuna-1.5", "action": "LH", "l2_1s": 0.34, "l2_2s": 0.37, "l2_3s": 0.69, "l2_avg": 0.40, "cr_1s": 0.01, "cr_2s": 0.05, "cr_3s": 0.26, "cr_avg": 0.10, "category": "Vision-Language-Action"},
+    {"id": 25, "model": "Senna", "year": 2024, "input": "camera, prompt, command, status", "vision": "ViT", "language": "Vicuna-1.5", "action": "REG", "l2_1s": 0.37, "l2_2s": 0.54, "l2_3s": 0.86, "l2_avg": 0.59, "cr_1s": 0.09, "cr_2s": 0.12, "cr_3s": 0.33, "cr_avg": 0.18, "category": "Vision-Language-Action"},
+    {"id": 26, "model": "Doe-1", "year": 2024, "input": "camera, prompt, status, scene", "vision": "Lumina-mGPT", "language": "BPE tokenizer", "action": "LH", "l2_1s": 0.37, "l2_2s": 0.67, "l2_3s": 1.07, "l2_avg": 0.70, "cr_1s": 0.02, "cr_2s": 0.14, "cr_3s": 0.47, "cr_avg": 0.21, "category": "Vision-Language-Action"},
+    {"id": 27, "model": "VLP", "year": 2024, "input": "camera, prompt, scene, command, status", "vision": "UniAD, VAD", "language": "CLIP", "action": "REG", "l2_1s": 0.30, "l2_2s": 0.53, "l2_3s": 0.84, "l2_avg": 0.55, "cr_1s": 0.01, "cr_2s": 0.07, "cr_3s": 0.38, "cr_avg": 0.15, "category": "Vision-Language-Action"},
+    {"id": 28, "model": "VLM-AD", "year": 2024, "input": "camera, prompt, scene, status", "vision": "UniAD, VAD", "language": "CLIP-ViT", "action": "REG", "l2_1s": 0.24, "l2_2s": 0.46, "l2_3s": 0.75, "l2_avg": 0.48, "cr_1s": 0.12, "cr_2s": 0.17, "cr_3s": 0.41, "cr_avg": 0.23, "category": "Vision-Language-Action"},
+    {"id": 29, "model": "OpenDriveVLA", "year": 2025, "input": "camera, prompt, scene, command, status", "vision": "ResNet", "language": "Qwen2.5-VL", "action": "LH", "l2_1s": 0.14, "l2_2s": 0.30, "l2_3s": 0.55, "l2_avg": 0.33, "cr_1s": 0.02, "cr_2s": 0.07, "cr_3s": 0.22, "cr_avg": 0.10, "category": "Vision-Language-Action"},
+    {"id": 30, "model": "OmniDrive", "year": 2025, "input": "camera, prompt, command, scene, status, traffic", "vision": "EVA-02", "language": "LLaMA2", "action": "LH", "l2_1s": 0.40, "l2_2s": 0.80, "l2_3s": 1.32, "l2_avg": 0.84, "cr_1s": 0.04, "cr_2s": 0.46, "cr_3s": 2.32, "cr_avg": 0.94, "category": "Vision-Language-Action"},
+    {"id": 31, "model": "ORION", "year": 2025, "input": "camera, prompt, command", "vision": "EVA-02", "language": "Vicuna-1.5", "action": "GEN", "l2_1s": 0.17, "l2_2s": 0.31, "l2_3s": 0.55, "l2_avg": 0.34, "cr_1s": 0.05, "cr_2s": 0.25, "cr_3s": 0.80, "cr_avg": 0.37, "category": "Vision-Language-Action"},
+    {"id": 32, "model": "EMMA", "year": 2025, "input": "camera, prompt, command, status", "vision": "Gemini-VLM", "language": "Gemini", "action": "LH", "l2_1s": 0.14, "l2_2s": 0.29, "l2_3s": 0.54, "l2_avg": 0.32, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": null, "category": "Vision-Language-Action"},
+    {"id": 33, "model": "WKER", "year": 2025, "input": "camera, prompt, command, status", "vision": "EVA-02", "language": "LLaMA3", "action": "LH", "l2_1s": 0.14, "l2_2s": 0.30, "l2_3s": 0.55, "l2_avg": 0.33, "cr_1s": 0.07, "cr_2s": 0.14, "cr_3s": 0.32, "cr_avg": 0.18, "category": "Vision-Language-Action"},
+    {"id": 34, "model": "Drive-R1", "year": 2025, "input": "camera, prompt, command, status", "vision": "InternVL2", "language": "InternVL2", "action": "LH", "l2_1s": 0.14, "l2_2s": 0.28, "l2_3s": 0.50, "l2_avg": 0.31, "cr_1s": 0.02, "cr_2s": 0.06, "cr_3s": 0.19, "cr_avg": 0.09, "category": "Vision-Language-Action"},
+    {"id": 35, "model": "ReAL-AD (MiniCPM)", "year": 2025, "input": "camera, prompt, command", "vision": "ResNet", "language": "MiniCPM", "action": "REG", "l2_1s": 0.30, "l2_2s": 0.48, "l2_3s": 0.67, "l2_avg": 0.48, "cr_1s": 0.07, "cr_2s": 0.10, "cr_3s": 0.28, "cr_avg": 0.15, "category": "Vision-Language-Action"},
+    {"id": 36, "model": "ReAL-AD (QwenVL)", "year": 2025, "input": "camera, prompt, command", "vision": "ResNet", "language": "QwenVL", "action": "REG", "l2_1s": 0.35, "l2_2s": 0.53, "l2_3s": 0.71, "l2_avg": 0.53, "cr_1s": 0.09, "cr_2s": 0.12, "cr_3s": 0.31, "cr_avg": 0.17, "category": "Vision-Language-Action"},
+    {"id": 37, "model": "DiMA", "year": 2025, "input": "camera, prompt, status", "vision": "VAD", "language": "LLaVA-1.5", "action": "REG", "l2_1s": 0.18, "l2_2s": 0.50, "l2_3s": 1.03, "l2_avg": 0.57, "cr_1s": 0.00, "cr_2s": 0.05, "cr_3s": 0.16, "cr_avg": 0.08, "category": "Vision-Language-Action"},
+    {"id": 38, "model": "FasionAD", "year": 2025, "input": "camera, prompt, scene, command, status", "vision": "BEVFormer", "language": "VLM+Thinking", "action": "GEN", "l2_1s": 0.19, "l2_2s": 0.62, "l2_3s": 1.25, "l2_avg": 0.69, "cr_1s": 0.02, "cr_2s": 0.09, "cr_3s": 0.44, "cr_avg": 0.18, "category": "Vision-Language-Action"},
+    {"id": 39, "model": "InsightDrive", "year": 2025, "input": "camera, prompt, scene, status", "vision": "ResNet", "language": "VLMs", "action": "REG", "l2_1s": 0.23, "l2_2s": 0.41, "l2_3s": 0.68, "l2_avg": 0.44, "cr_1s": 0.09, "cr_2s": 0.10, "cr_3s": 0.27, "cr_avg": 0.15, "category": "Vision-Language-Action"},
+    {"id": 40, "model": "S4-Driver", "year": 2025, "input": "camera, prompt, command, status", "vision": "ViT-G", "language": "PaLI3", "action": "LH", "l2_1s": 0.13, "l2_2s": 0.28, "l2_3s": 0.51, "l2_avg": 0.31, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": null, "category": "Vision-Language-Action"},
+    {"id": 41, "model": "SOLVE", "year": 2025, "input": "camera, prompt, status", "vision": "EVA-02", "language": "LLaVA-1.5", "action": "REG", "l2_1s": 0.13, "l2_2s": 0.25, "l2_3s": 0.47, "l2_avg": 0.28, "cr_1s": 0.00, "cr_2s": 0.16, "cr_3s": 0.43, "cr_avg": 0.20, "category": "Vision-Language-Action"},
+    {"id": 42, "model": "VERDI", "year": 2025, "input": "camera, prompt, scene, status", "vision": "VAD", "language": "Qwen2.5-VL", "action": "REG", "l2_1s": 0.36, "l2_2s": 0.62, "l2_3s": 0.96, "l2_avg": 0.65, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": null, "category": "Vision-Language-Action"},
+    {"id": 43, "model": "OmniReason", "year": 2025, "input": "camera, prompt, command, scene, status, traffic, context", "vision": "EVA-02", "language": "LLaVA-1.5", "action": "LH", "l2_1s": 0.15, "l2_2s": 0.31, "l2_3s": 0.57, "l2_avg": 0.34, "cr_1s": 0.04, "cr_2s": 0.18, "cr_3s": 0.98, "cr_avg": 0.40, "category": "Vision-Language-Action"},
+    {"id": 44, "model": "FutureSightDrive", "year": 2025, "input": "camera, prompt, command, status", "vision": "ViT", "language": "Qwen2-VL", "action": "LH", "l2_1s": 0.14, "l2_2s": 0.25, "l2_3s": 0.46, "l2_avg": 0.28, "cr_1s": 0.03, "cr_2s": 0.06, "cr_3s": 0.21, "cr_avg": 0.10, "category": "Vision-Language-Action"},
+    {"id": 45, "model": "Occ-LLM", "year": 2025, "input": "camera, prompt, status", "vision": "-", "language": "LLaMA2", "action": "LH", "l2_1s": 0.12, "l2_2s": 0.24, "l2_3s": 0.49, "l2_avg": 0.28, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": null, "category": "Vision-Language-Action"},
+    {"id": 46, "model": "FastDriveVLA", "year": 2025, "input": "camera, prompt", "vision": "Qwen2.5-VL", "language": "Qwen2.5-VL", "action": "LH", "l2_1s": 0.14, "l2_2s": 0.29, "l2_3s": 0.54, "l2_avg": 0.33, "cr_1s": 0.00, "cr_2s": 0.18, "cr_3s": 0.70, "cr_avg": 0.29, "category": "Vision-Language-Action"},
+    {"id": 47, "model": "AutoDrive-R²", "year": 2025, "input": "camera, prompt, status", "vision": "Qwen2.5-VL", "language": "Qwen2.5-VL", "action": "LH", "l2_1s": 0.13, "l2_2s": 0.19, "l2_3s": 0.25, "l2_avg": 0.19, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": null, "category": "Vision-Language-Action"},
+    {"id": 48, "model": "VDrive", "year": 2025, "input": "camera, prompt, command, status", "vision": "Qwen2.5-VL, CVQ", "language": "InternVL3", "action": "GEN", "l2_1s": 0.12, "l2_2s": 0.26, "l2_3s": 0.50, "l2_avg": 0.29, "cr_1s": 0.03, "cr_2s": 0.16, "cr_3s": 0.36, "cr_avg": 0.18, "category": "Vision-Language-Action"},
+    {"id": 49, "model": "OccVLA", "year": 2025, "input": "camera, prompt, status", "vision": "VQ-VAE", "language": "PaliGemma-2", "action": "REG", "l2_1s": 0.18, "l2_2s": 0.26, "l2_3s": 0.40, "l2_avg": 0.28, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": null, "category": "Vision-Language-Action"},
+    {"id": 50, "model": "FasionAD++", "year": 2024, "input": "camera, prompt, command, status", "vision": "GenAD", "language": "CLIP, QwenVL", "action": "GEN", "l2_1s": 0.13, "l2_2s": 0.26, "l2_3s": 0.45, "l2_avg": 0.28, "cr_1s": 0.05, "cr_2s": 0.08, "cr_3s": 0.15, "cr_avg": 0.09, "category": "Vision-Language-Action"},
+    {"id": 51, "model": "ALN-P3", "year": 2025, "input": "camera, prompt, scene", "vision": "VAD", "language": "CLIP, LLaMA-2", "action": "REG", "l2_1s": null, "l2_2s": null, "l2_3s": null, "l2_avg": null, "cr_1s": 0.05, "cr_2s": 0.09, "cr_3s": 0.35, "cr_avg": 0.16, "category": "Vision-Language-Action"},
+    {"id": 52, "model": "VLM-E2E", "year": 2025, "input": "camera, prompt, scene", "vision": "EfficientNet", "language": "CLIP", "action": "REG", "l2_1s": 0.28, "l2_2s": 0.50, "l2_3s": 0.80, "l2_avg": 0.53, "cr_1s": 0.01, "cr_2s": 0.06, "cr_3s": 0.20, "cr_avg": 0.09, "category": "Vision-Language-Action"},
+    {"id": 53, "model": "NetRoller", "year": 2025, "input": "camera, prompt", "vision": "CLIP", "language": "LLaMA-2", "action": "REG", "l2_1s": 0.38, "l2_2s": 0.66, "l2_3s": 1.01, "l2_avg": 0.68, "cr_1s": 0.06, "cr_2s": 0.13, "cr_3s": 0.30, "cr_avg": 0.16, "category": "Vision-Language-Action"},
+    {"id": 54, "model": "OmniScene", "year": 2025, "input": "camera, prompt, scene, status", "vision": "ResNet", "language": "Qwen2.5-VL", "action": "REG", "l2_1s": 0.28, "l2_2s": 0.53, "l2_3s": 0.91, "l2_avg": 0.57, "cr_1s": 0.00, "cr_2s": 0.04, "cr_3s": 0.19, "cr_avg": 0.08, "category": "Vision-Language-Action"},
+    {"id": 55, "model": "Sce2DriveX", "year": 2025, "input": "camera, prompt, command, status", "vision": "OpenCLIP", "language": "Vicuna-1.5", "action": "LH", "l2_1s": 0.15, "l2_2s": 0.33, "l2_3s": 0.59, "l2_avg": 0.36, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": null, "category": "Vision-Language-Action"},
+    {"id": 56, "model": "dVLM-AD", "year": 2025, "input": "camera, prompt, status", "vision": "SigLIP2", "language": "LLaDA-V", "action": "LH", "l2_1s": 0.15, "l2_2s": 0.40, "l2_3s": 0.68, "l2_avg": 0.41, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": null, "category": "Vision-Language-Action"},
+    {"id": 57, "model": "Percept-WAM", "year": 2025, "input": "camera, lidar, prompt, command", "vision": "InternViT", "language": "InternVL2", "action": "REG", "l2_1s": 0.16, "l2_2s": 0.33, "l2_3s": 0.60, "l2_avg": 0.36, "cr_1s": null, "cr_2s": null, "cr_3s": null, "cr_avg": null, "category": "Vision-Language-Action"},
+    {"id": 58, "model": "Reasoning-VLA", "year": 2025, "input": "camera, prompt, status", "vision": "Qwen2.5-VL", "language": "Qwen2.5-VL", "action": "REG", "l2_1s": 0.05, "l2_2s": 0.19, "l2_3s": 0.41, "l2_avg": 0.22, "cr_1s": 0.02, "cr_2s": 0.06, "cr_3s": 0.13, "cr_avg": 0.07, "category": "Vision-Language-Action"}
+  ]

results/va_models.json

@@ -0,0 +1,911 @@
+[
+  {
+    "id": 1,
+    "model": "LBC",
+    "venue": "CoRL'20",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "CARLA",
+      "NoCrash"
+    ],
+    "vision": "ResNet",
+    "action": "RL",
+    "output": "Ctrl.+Traj.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 2,
+    "model": "Latent-DRL",
+    "venue": "CVPR'20",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "ResNet",
+    "action": "RL",
+    "output": "Ctrl.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 3,
+    "model": "NEAT",
+    "venue": "ICCV'21",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 4,
+    "model": "Roach",
+    "venue": "ICCV'21",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "CARLA",
+      "NoCrash"
+    ],
+    "vision": "ResNet",
+    "action": "RL",
+    "output": "Ctrl.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 5,
+    "model": "WoR",
+    "venue": "ICCV'21",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "CARLA",
+      "NoCrash",
+      "ProcGen"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Ctrl.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 6,
+    "model": "TCP",
+    "venue": "NeurIPS'22",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Ctrl.+Traj.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 7,
+    "model": "Urban-Driver",
+    "venue": "CoRL'22",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "Lyft"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 8,
+    "model": "LAV",
+    "venue": "CVPR'22",
+    "input": [
+      "camera",
+      "lidar"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Ctrl.+Traj.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 9,
+    "model": "TransFuser",
+    "venue": "TPAMI'23",
+    "input": [
+      "camera",
+      "lidar"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 10,
+    "model": "GRI",
+    "venue": "Robotics'23",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "EfficientNet",
+    "action": "RL",
+    "output": "Ctrl.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 11,
+    "model": "BEVPlanner",
+    "venue": "CVPR'24",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 12,
+    "model": "Raw2Drive",
+    "venue": "NeurIPS'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "CARLA",
+      "Bench2Drive"
+    ],
+    "vision": "ResNet",
+    "action": "RL",
+    "output": "Ctrl.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 13,
+    "model": "RAD",
+    "venue": "NeurIPS'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "Private"
+    ],
+    "vision": "ResNet",
+    "action": "RL",
+    "output": "Traj.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 14,
+    "model": "TrajDiff",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "lidar",
+      "status"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "ResNet",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Action-Only Models"
+  },
+  {
+    "id": 15,
+    "model": "ST-P3",
+    "venue": "ECCV'22",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes",
+      "CARLA"
+    ],
+    "vision": "EfficientNet",
+    "action": "SEL",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 16,
+    "model": "UniAD",
+    "venue": "CVPR'23",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 17,
+    "model": "VAD",
+    "venue": "ICCV'23",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 18,
+    "model": "OccNet",
+    "venue": "ICCV'23",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes",
+      "OpenOcc"
+    ],
+    "vision": "ResNet",
+    "action": "SEL",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 19,
+    "model": "GenAD",
+    "venue": "ECCV'24",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 20,
+    "model": "PARA-Drive",
+    "venue": "CVPR'24",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 21,
+    "model": "Hydra-MDP",
+    "venue": "CVPRW'24",
+    "input": [
+      "camera",
+      "lidar"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "ResNet",
+    "action": "SEL",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 22,
+    "model": "SparseAD",
+    "venue": "arXiv'24",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 23,
+    "model": "GaussianAD",
+    "venue": "arXiv'24",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 24,
+    "model": "DiFSD",
+    "venue": "arXiv'24",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 25,
+    "model": "DriveTransformer",
+    "venue": "ICLR'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Bench2Drive"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 26,
+    "model": "SparseDrive",
+    "venue": "ICRA'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 27,
+    "model": "DiffusionDrive",
+    "venue": "CVPR'25",
+    "input": [
+      "camera",
+      "lidar"
+    ],
+    "dataset": [
+      "nuScenes",
+      "NAVSIM"
+    ],
+    "vision": "ResNet",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 28,
+    "model": "GoalFlow",
+    "venue": "CVPR'25",
+    "input": [
+      "camera",
+      "lidar",
+      "status"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "VoVNet",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 29,
+    "model": "GuideFlow",
+    "venue": "arXiv'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes",
+      "NAVSIM",
+      "Bench2Drive"
+    ],
+    "vision": "ResNet",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 30,
+    "model": "ETA",
+    "venue": "arXiv'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "Bench2Drive"
+    ],
+    "vision": "CLIP-ViT",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 31,
+    "model": "Geo",
+    "venue": "arXiv'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 32,
+    "model": "DiffusionDriveV2",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "lidar"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "ResNet",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 33,
+    "model": "NaviHydra",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "lidar"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "ResNet",
+    "action": "SEL",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 34,
+    "model": "Mimir",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "lidar"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "ResNet",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Perception-Action Models"
+  },
+  {
+    "id": 35,
+    "model": "DriveDreamer",
+    "venue": "ECCV'24",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "SD",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Image-Based World Models"
+  },
+  {
+    "id": 36,
+    "model": "GenAD",
+    "venue": "CVPR'24",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "OpenDV"
+    ],
+    "vision": "SDXL",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Image-Based World Models"
+  },
+  {
+    "id": 37,
+    "model": "Drive-WM",
+    "venue": "CVPR'24",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ConvNeXt",
+    "action": "SEL",
+    "output": "Traj.",
+    "category": "Image-Based World Models"
+  },
+  {
+    "id": 38,
+    "model": "DrivingWorld",
+    "venue": "arXiv'24",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "nuPlan"
+    ],
+    "vision": "VQ-VAE",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Image-Based World Models"
+  },
+  {
+    "id": 39,
+    "model": "Imagine-2-Drive",
+    "venue": "IROS'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "SVD",
+    "action": "SEL",
+    "output": "Traj.",
+    "category": "Image-Based World Models"
+  },
+  {
+    "id": 40,
+    "model": "DrivingGPT",
+    "venue": "ICCV'25",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "nuPlan",
+      "NAVSIM"
+    ],
+    "vision": "VQ-VAE",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Image-Based World Models"
+  },
+  {
+    "id": 41,
+    "model": "Epona",
+    "venue": "ICCV'25",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "NAVSIM",
+      "nuPlan"
+    ],
+    "vision": "DC-AE",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Image-Based World Models"
+  },
+  {
+    "id": 42,
+    "model": "VaViM",
+    "venue": "arXiv'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "OpenDV",
+      "nuScenes",
+      "nuPlan"
+    ],
+    "vision": "LLaMAGen",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Image-Based World Models"
+  },
+  {
+    "id": 43,
+    "model": "OccWorld",
+    "venue": "ECCV'24",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Occ3D"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Occupancy-Based World Models"
+  },
+  {
+    "id": 44,
+    "model": "NeMo",
+    "venue": "ECCV'24",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Occupancy-Based World Models"
+  },
+  {
+    "id": 45,
+    "model": "OccVAR",
+    "venue": "-",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Occ3D"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Occupancy-Based World Models"
+  },
+  {
+    "id": 46,
+    "model": "RenderWorld",
+    "venue": "arXiv'24",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Occ3D"
+    ],
+    "vision": "Swin-T",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Occupancy-Based World Models"
+  },
+  {
+    "id": 47,
+    "model": "DFIT-OccWorld",
+    "venue": "arXiv'24",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Occ3D"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Occupancy-Based World Models"
+  },
+  {
+    "id": 48,
+    "model": "Drive-OccWorld",
+    "venue": "AAAI'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Cam4DOcc"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Occupancy-Based World Models"
+  },
+  {
+    "id": 49,
+    "model": "T³Former",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Occ3D"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Occupancy-Based World Models"
+  },
+  {
+    "id": 50,
+    "model": "AD-R1",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "lidar",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "NAVSIM"
+    ],
+    "vision": "-",
+    "action": "RL",
+    "output": "Traj.",
+    "category": "Occupancy-Based World Models"
+  },
+  {
+    "id": 51,
+    "model": "Covariate-Shift",
+    "venue": "arXiv'24",
+    "input": [
+      "camera",
+      "status"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "DINOv2",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Latent-Based World Models"
+  },
+  {
+    "id": 52,
+    "model": "World4Drive",
+    "venue": "ICCV'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes",
+      "NAVSIM"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Latent-Based World Models"
+  },
+  {
+    "id": 53,
+    "model": "WoTE",
+    "venue": "ICCV'25",
+    "input": [
+      "camera",
+      "lidar"
+    ],
+    "dataset": [
+      "NAVSIM",
+      "Bench2Drive"
+    ],
+    "vision": "ResNet",
+    "action": "SEL",
+    "output": "Traj.",
+    "category": "Latent-Based World Models"
+  },
+  {
+    "id": 54,
+    "model": "LAW",
+    "venue": "ICLR'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes",
+      "NAVSIM",
+      "CARLA"
+    ],
+    "vision": "Swin-T",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Latent-Based World Models"
+  },
+  {
+    "id": 55,
+    "model": "SSR",
+    "venue": "ICLR'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes",
+      "CARLA"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Latent-Based World Models"
+  },
+  {
+    "id": 56,
+    "model": "Echo-Planning",
+    "venue": "arXiv'25",
+    "input": [
+      "camera"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Latent-Based World Models"
+  },
+  {
+    "id": 57,
+    "model": "SeerDrive",
+    "venue": "NeurIPS'25",
+    "input": [
+      "camera",
+      "lidar"
+    ],
+    "dataset": [
+      "nuScenes",
+      "NAVSIM"
+    ],
+    "vision": "VoVNet",
+    "action": "SEL",
+    "output": "Traj.",
+    "category": "Latent-Based World Models"
+  }
+]

results/vla_models.json

@@ -0,0 +1,1499 @@
+[
+  {
+    "id": 1,
+    "model": "DriveMLM",
+    "venue": "arXiv'23",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "traffic"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "EVA-CLIP",
+    "language": "LLaMA",
+    "action": "LH",
+    "output": "Desc.+Meta.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 2,
+    "model": "RAG-Driver",
+    "venue": "RSS'24",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "context",
+      "status"
+    ],
+    "dataset": [
+      "BDD-X"
+    ],
+    "vision": "CLIP",
+    "language": "Vicuna-1.5",
+    "action": "LH",
+    "output": "Desc.+Ctrl.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 3,
+    "model": "RDA-Driver",
+    "venue": "ECCV'24",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "DriveLM",
+      "nuScenes"
+    ],
+    "vision": "BEVFusion",
+    "language": "LLaMA",
+    "action": "LH",
+    "output": "Desc.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 4,
+    "model": "DriveLM",
+    "venue": "ECCV'24",
+    "input": [
+      "camera",
+      "prompt"
+    ],
+    "dataset": [
+      "DriveLM"
+    ],
+    "vision": "BLIP-2",
+    "language": "LLaMA",
+    "action": "LH",
+    "output": "Meta.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 5,
+    "model": "DriveGPT4",
+    "venue": "RA-L'24",
+    "input": [
+      "camera",
+      "prompt"
+    ],
+    "dataset": [
+      "BDD-X"
+    ],
+    "vision": "CLIP",
+    "language": "LLaMA-2",
+    "action": "LH",
+    "output": "Desc.+Ctrl.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 6,
+    "model": "DriVLMe",
+    "venue": "IROS'24",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "BDD-X",
+      "SDN",
+      "CARLA"
+    ],
+    "vision": "CLIP",
+    "language": "Vicuna",
+    "action": "LH",
+    "output": "Desc.+Ctrl.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 7,
+    "model": "LLaDA",
+    "venue": "CVPR'24",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "scene",
+      "traffic"
+    ],
+    "dataset": [
+      "nuScenes",
+      "nuPlan"
+    ],
+    "vision": "-",
+    "language": "GPT-4",
+    "action": "LH",
+    "output": "Ctrl.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 8,
+    "model": "VLAAD",
+    "venue": "WACVW'24",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "VLAAD"
+    ],
+    "vision": "BLIP-2",
+    "language": "LLaMA-2",
+    "action": "LH",
+    "output": "Ctrl.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 9,
+    "model": "OccLLaMA",
+    "venue": "arXiv'24",
+    "input": [
+      "camera",
+      "prompt"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "Swin-T, PointPillar",
+    "language": "LLaMA-3",
+    "action": "LH",
+    "output": "Ctrl.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 10,
+    "model": "Doe-1",
+    "venue": "arXiv'24",
+    "input": [
+      "camera",
+      "prompt",
+      "status",
+      "scene"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "Lumina-mGPT",
+    "language": "BPE tokenizer",
+    "action": "LH",
+    "output": "Ctrl.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 11,
+    "model": "LINGO-2",
+    "venue": "-",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "Private"
+    ],
+    "vision": "Wayve Vision",
+    "language": "Wayve VLA",
+    "action": "LH",
+    "output": "Desc.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 12,
+    "model": "SafeAuto",
+    "venue": "ICML'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status",
+      "traffic",
+      "context"
+    ],
+    "dataset": [
+      "BDD-X",
+      "DriveLM"
+    ],
+    "vision": "LanguageBind",
+    "language": "Video-LLaVA",
+    "action": "LH",
+    "output": "Desc.+Ctrl.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 13,
+    "model": "OpenEMMA",
+    "venue": "WACV'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "Qwen2-VL",
+    "language": "Qwen2-VL",
+    "action": "LH",
+    "output": "Desc.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 14,
+    "model": "ReasonPlan",
+    "venue": "CoRL'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "Bench2Drive"
+    ],
+    "vision": "SigLIP",
+    "language": "Qwen",
+    "action": "LH",
+    "output": "Desc.+Traj.+Meta.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 15,
+    "model": "FutureSightDrive",
+    "venue": "NeurIPS'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "DriveLM"
+    ],
+    "vision": "ViT",
+    "language": "Qwen2-VL",
+    "action": "LH",
+    "output": "Desc.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 16,
+    "model": "ImpromptuVLA",
+    "venue": "NeurIPS'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "status",
+      "traffic"
+    ],
+    "dataset": [
+      "ImpromptuVLA"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "LH",
+    "output": "Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 17,
+    "model": "WKER",
+    "venue": "AAAI'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "EVA-02",
+    "language": "LLaMA3",
+    "action": "LH",
+    "output": "Desc.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 18,
+    "model": "OmniDrive",
+    "venue": "CVPR'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "scene",
+      "status",
+      "traffic"
+    ],
+    "dataset": [
+      "nuScenes",
+      "DriveLM"
+    ],
+    "vision": "EVA-02",
+    "language": "LLaMA2",
+    "action": "LH",
+    "output": "Desc.+Traj",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 19,
+    "model": "S4-Driver",
+    "venue": "CVPR'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "Waymo",
+      "nuScenes"
+    ],
+    "vision": "PaLI3",
+    "language": "PaLI3",
+    "action": "LH",
+    "output": "Meta.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 20,
+    "model": "EMMA",
+    "venue": "TMLR'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "Waymo",
+      "nuScenes"
+    ],
+    "vision": "Gemini-VLM",
+    "language": "Gemini",
+    "action": "LH",
+    "output": "Desc.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 21,
+    "model": "Occ-LLM",
+    "venue": "ICRA'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "-",
+    "language": "LLaMA2",
+    "action": "LH",
+    "output": "Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 22,
+    "model": "Sce2DriveX",
+    "venue": "RA-L'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Bench2Drive"
+    ],
+    "vision": "OpenCLIP",
+    "language": "Vicuna-1.5",
+    "action": "LH",
+    "output": "Ctrl.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 23,
+    "model": "DriveAgent-R1",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Private"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "LH",
+    "output": "Desc.+Meta.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 24,
+    "model": "Drive-R1",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "DriveLM"
+    ],
+    "vision": "InternVL2",
+    "language": "InternVL2",
+    "action": "LH",
+    "output": "Desc.+Traj.+Meta.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 25,
+    "model": "FastDriveVLA",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "LH",
+    "output": "Desc.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 26,
+    "model": "WiseAD",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt"
+    ],
+    "dataset": [
+      "BDD-X",
+      "CARLA",
+      "DriveLM"
+    ],
+    "vision": "CLIP",
+    "language": "MobileVLM",
+    "action": "LH",
+    "output": "Traj.+Ctrl.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 27,
+    "model": "AutoDrive-R²",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Waymo"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "LH",
+    "output": "Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 28,
+    "model": "OmniReason",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "scene",
+      "status",
+      "traffic",
+      "context"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Bench2Drive"
+    ],
+    "vision": "EVA-02",
+    "language": "LLaVA-1.5",
+    "action": "LH",
+    "output": "Meta.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 29,
+    "model": "OpenREAD",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "scene",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "Qwen3",
+    "language": "Qwen3",
+    "action": "LH",
+    "output": "Desc.+Meta.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 30,
+    "model": "dVLM-AD",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "Waymo",
+      "nuScenes"
+    ],
+    "vision": "SigLIP2",
+    "language": "LLaDA-V",
+    "action": "LH",
+    "output": "Desc.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 31,
+    "model": "PLA",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "scene",
+      "status",
+      "context"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "Sensor Encoder",
+    "language": "GPT-4.1",
+    "action": "LH",
+    "output": "Desc.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 32,
+    "model": "AlphaDrive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "MetaAD"
+    ],
+    "vision": "Qwen2-VL",
+    "language": "Qwen2-VL",
+    "action": "LH",
+    "output": "Desc.+Meta.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 33,
+    "model": "CoReVLA",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "context"
+    ],
+    "dataset": [
+      "Bench2Drive"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "LH",
+    "output": "Ctrl.+Traj.",
+    "category": "Textural Action Generator"
+  },
+  {
+    "id": 34,
+    "model": "LMDrive",
+    "venue": "CVPR'24",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "instruction"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "ResNet",
+    "language": "LLaMA, Vicuna",
+    "action": "REG",
+    "output": "Ctrl.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 35,
+    "model": "BEVDriver",
+    "venue": "IROS'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "InterFuser",
+    "language": "LLaMA-3.1",
+    "action": "REG",
+    "output": "Ctrl.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 36,
+    "model": "CoVLA-Agent",
+    "venue": "WACV'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "CoVLA"
+    ],
+    "vision": "CLIP",
+    "language": "LLaMA2",
+    "action": "REG",
+    "output": "Desc.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 37,
+    "model": "ORION",
+    "venue": "ICCV'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Bench2Drive"
+    ],
+    "vision": "EVA-02",
+    "language": "Vicuna-1.5",
+    "action": "GEN",
+    "output": "Desc.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 38,
+    "model": "SimLingo",
+    "venue": "CVPR'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "CARLA",
+      "Bench2Drive"
+    ],
+    "vision": "InternViT",
+    "language": "Qwen2",
+    "action": "REG",
+    "output": "Ctrl.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 39,
+    "model": "DriveGPT4-V2",
+    "venue": "CVPR'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "scene",
+      "status"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "CLIP, SigLIP",
+    "language": "Qwen, Tinyllama",
+    "action": "REG",
+    "output": "Ctrl.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 40,
+    "model": "AutoVLA",
+    "venue": "NeurIPS'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Bench2Drive",
+      "NAVSIM",
+      "Waymo"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "LH",
+    "output": "Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 41,
+    "model": "DriveMoE",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "Bench2Drive"
+    ],
+    "vision": "PaliGemma",
+    "language": "PaliGemma",
+    "action": "GEN",
+    "output": "Ctrl.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 42,
+    "model": "DSDrive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "ResNet",
+    "language": "LLaMA",
+    "action": "REG",
+    "output": "Desc.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 43,
+    "model": "OccVLA",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "VQ-VAE",
+    "language": "PaliGemma-2",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 44,
+    "model": "VDRive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Bench2Drive"
+    ],
+    "vision": "Qwen2.5-VL, CVQ",
+    "language": "InternVL3",
+    "action": "GEN",
+    "output": "Desc.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 45,
+    "model": "ReflectDrive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "LLaDA-V",
+    "language": "LLaDA-V",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 46,
+    "model": "E3AD",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status",
+      "context"
+    ],
+    "dataset": [
+      "Talk2Car"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 47,
+    "model": "LCDrive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "PhysicalAI-AV"
+    ],
+    "vision": "DINOv2",
+    "language": "Qwen3",
+    "action": "LH",
+    "output": "Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 48,
+    "model": "Alpamayo-R1",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status",
+      "context"
+    ],
+    "dataset": [
+      "Private"
+    ],
+    "vision": "Cosmos-Reason1",
+    "language": "Cosmos-Reason1",
+    "action": "REG",
+    "output": "Desc.+Ctrl.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 49,
+    "model": "UniUGP",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "Waymo",
+      "nuScenes",
+      "nuPlan"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "GEN",
+    "output": "Desc.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 50,
+    "model": "MindDrive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "ResNet-34",
+    "language": "LLaVA-1B",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 51,
+    "model": "AdaThinkDrive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "InternVL3",
+    "language": "InternVL3",
+    "action": "REG",
+    "output": "Desc.+Traj.+Meta.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 52,
+    "model": "Percept-WAM",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "lidar",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "nuScenes",
+      "NAVSIM"
+    ],
+    "vision": "InternViT",
+    "language": "InternVL2",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 53,
+    "model": "Reasoning-VLA",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "NAVSIM",
+      "Waymo",
+      "Argoverse"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 54,
+    "model": "SpaceDrive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status",
+      "scene"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Bench2Drive"
+    ],
+    "vision": "Qwen2.5-VL",
+    "language": "Qwen2.5-VL",
+    "action": "REG",
+    "output": "Desc.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 55,
+    "model": "OpenDriveVLA",
+    "venue": "AAAI'26",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "BEVFormer",
+    "language": "Qwen2.5",
+    "action": "LH",
+    "output": "Ctrl.+Traj.",
+    "category": "Numerical Action Generator"
+  },
+  {
+    "id": 56,
+    "model": "DriveVLM",
+    "venue": "CoRL'24",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "SUPAD"
+    ],
+    "vision": "-",
+    "language": "QwenVL",
+    "action": "REG",
+    "output": "Desc.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 57,
+    "model": "LeapAD",
+    "venue": "NeurIPS'24",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "traffic"
+    ],
+    "dataset": [
+      "CARLA"
+    ],
+    "vision": "QwenVL",
+    "language": "GPT-4, Qwen1.5",
+    "action": "LH",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 58,
+    "model": "FasionAD",
+    "venue": "arXiv'24",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "CARLA"
+    ],
+    "vision": "GenAD",
+    "language": "CLIP, QwenVL",
+    "action": "GEN",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 59,
+    "model": "Senna",
+    "venue": "arXiv'24",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "VADv2, ViT",
+    "language": "Vicuna",
+    "action": "REG",
+    "output": "Desc.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 60,
+    "model": "DualAD",
+    "venue": "IROS'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "status"
+    ],
+    "dataset": [
+      "nuPlan"
+    ],
+    "vision": "-",
+    "language": "GPT-4o, GLM-4",
+    "action": "SEL",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 61,
+    "model": "DME-Driver",
+    "venue": "AAAI'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "HBD"
+    ],
+    "vision": "UniAD",
+    "language": "LLaVA",
+    "action": "REG",
+    "output": "Desc.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 62,
+    "model": "SOLVE",
+    "venue": "CVPR'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "EVA-02",
+    "language": "LLaVA-1.5",
+    "action": "REG",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 63,
+    "model": "ReAL-AD",
+    "venue": "ICCV'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction"
+    ],
+    "dataset": [
+      "nuScenes",
+      "Bench2Drive"
+    ],
+    "vision": "UniAD, VAD",
+    "language": "MiniCPM-2.5",
+    "action": "REG",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 64,
+    "model": "LeapVAD",
+    "venue": "TNNLS'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "status"
+    ],
+    "dataset": [
+      "DriveLM",
+      "CARLA"
+    ],
+    "vision": "QwenVL, InternVL2",
+    "language": "GPT-4o",
+    "action": "LH",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 65,
+    "model": "DiffVLA",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "CLIP",
+    "language": "Vicuna-1.5",
+    "action": "GEN",
+    "output": "Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 66,
+    "model": "FasionAD++",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes",
+      "CARLA"
+    ],
+    "vision": "BEVFormer",
+    "language": "Vicuna-1.5, QwenVL",
+    "action": "GEN",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Explicit Action Guidance"
+  },
+  {
+    "id": 67,
+    "model": "VLP",
+    "venue": "CVPR'24",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "instruction",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "UniAD, VAD",
+    "language": "CLIP",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 68,
+    "model": "VLM-AD",
+    "venue": "CoRL'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "UniAD, VAD",
+    "language": "CLIP",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 69,
+    "model": "DiMA",
+    "venue": "CVPR'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "UniAD, VAD",
+    "language": "LLaVA-1.5",
+    "action": "REG",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 70,
+    "model": "ALN-P3",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "VAD",
+    "language": "LLaMA-2",
+    "action": "REG",
+    "output": "Desc.+Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 71,
+    "model": "VERDI",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "VAD",
+    "language": "Qwen2.5-VL",
+    "action": "REG",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 72,
+    "model": "VLM-E2E",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "EfficientNet",
+    "language": "CLIP",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 73,
+    "model": "ReCogDrive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "status"
+    ],
+    "dataset": [
+      "NAVSIM"
+    ],
+    "vision": "InternViT",
+    "language": "Qwen2.5",
+    "action": "GEN",
+    "output": "Desc.+Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 74,
+    "model": "InsightDrive",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "language": "BERT",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 75,
+    "model": "NetRoller",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt"
+    ],
+    "dataset": [
+      "nuScenes",
+      "DriveLM"
+    ],
+    "vision": "CLIP",
+    "language": "LLaMA-2",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 76,
+    "model": "ViLaD",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "context"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "SigLIP-2",
+    "language": "LLaDA-V",
+    "action": "GEN",
+    "output": "Ctrl.+Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 77,
+    "model": "OmniScene",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt",
+      "scene",
+      "status"
+    ],
+    "dataset": [
+      "nuScenes"
+    ],
+    "vision": "ResNet",
+    "language": "Qwen2.5-VL",
+    "action": "REG",
+    "output": "Traj.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  },
+  {
+    "id": 78,
+    "model": "LMAD",
+    "venue": "arXiv'25",
+    "input": [
+      "camera",
+      "prompt"
+    ],
+    "dataset": [
+      "nuScenes",
+      "DriveLM"
+    ],
+    "vision": "VAD",
+    "language": "LLaVA-1.5",
+    "action": "LH",
+    "output": "Desc.+Meta.",
+    "category": "Dual-System: Implicit Representations Transfer"
+  }
+]

results/wod_results.json

@@ -0,0 +1,12 @@
+[
+    {"id": 1, "model": "Waymo Baseline", "year": 2025, "input": "camera", "vision": "-", "language": "-", "action": "-", "rfs_overall": 7.53, "rfs_spotlight": 6.60, "ade_5s": 3.02, "ade_3s": 1.32, "category": "Vision-Action"},
+    {"id": 2, "model": "Swin-Trajectory", "year": 2025, "input": "camera", "vision": "SwinT", "language": "-", "action": "REG", "rfs_overall": 7.54, "rfs_spotlight": 6.68, "ade_5s": 2.81, "ade_3s": 1.21, "category": "Vision-Action"},
+    {"id": 3, "model": "DiffusionDrive", "year": 2025, "input": "camera", "vision": "ResNet", "language": "-", "action": "GEN", "rfs_overall": 7.69, "rfs_spotlight": 6.65, "ade_5s": 2.99, "ade_3s": 1.31, "category": "Vision-Action"},
+    {"id": 4, "model": "RAP-DINO", "year": 2025, "input": "camera", "vision": "DINO", "language": "-", "action": "REG", "rfs_overall": 8.04, "rfs_spotlight": 7.20, "ade_5s": 2.65, "ade_3s": 1.17, "category": "Vision-Action"},
+    {"id": 5, "model": "OpenEMMA", "year": 2025, "input": "camera, prompt, status", "vision": "Qwen2-VL", "language": "Qwen2-VL", "action": "LH", "rfs_overall": 5.16, "rfs_spotlight": 4.71, "ade_5s": 12.74, "ade_3s": 6.68, "category": "Vision-Language-Action"},
+    {"id": 6, "model": "HMVLM", "year": 2025, "input": "camera, prompt, command, status", "vision": "ViT", "language": "Qwen2.5-VL", "action": "LH", "rfs_overall": 7.74, "rfs_spotlight": 6.73, "ade_5s": 3.07, "ade_3s": 1.33, "category": "Vision-Language-Action"},
+    {"id": 7, "model": "AutoVLA", "year": 2025, "input": "camera, prompt, command, status", "vision": "Qwen2.5-VL", "language": "Qwen2.5-VL", "action": "LH", "rfs_overall": 7.56, "rfs_spotlight": 6.94, "ade_5s": 2.96, "ade_3s": 1.35, "category": "Vision-Language-Action"},
+    {"id": 8, "model": "Poutine", "year": 2025, "input": "camera, prompt, status, command", "vision": "ViT", "language": "Qwen2.5-VL", "action": "LH", "rfs_overall": 7.99, "rfs_spotlight": 6.89, "ade_5s": 2.74, "ade_3s": 1.21, "category": "Vision-Language-Action"},
+    {"id": 9, "model": "LightEMMA", "year": 2025, "input": "camera, prompt", "vision": "Qwen2.5-VL", "language": "Qwen2.5-VL", "action": "LH", "rfs_overall": 6.52, "rfs_spotlight": 5.71, "ade_5s": 3.73, "ade_3s": 1.71, "category": "Vision-Language-Action"},
+    {"id": 10, "model": "dVLM-AD", "year": 2025, "input": "camera, prompt, status", "vision": "SigLIP2", "language": "LLaDA-V", "action": "LH", "rfs_overall": 7.63, "rfs_spotlight": null, "ade_5s": 3.02, "ade_3s": 1.29, "category": "Vision-Language-Action"}
+  ]

src/UNKNOWN.egg-info/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.4
+Name: UNKNOWN
+Version: 0.0.0

src/UNKNOWN.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+README.md
+pyproject.toml
+src/about.py
+src/envs.py
+src/populate.py
+src/UNKNOWN.egg-info/PKG-INFO
+src/UNKNOWN.egg-info/SOURCES.txt
+src/UNKNOWN.egg-info/dependency_links.txt
+src/UNKNOWN.egg-info/top_level.txt
+src/display/css_html_js.py
+src/display/formatting.py
+src/display/utils.py
+src/leaderboard/read_evals.py
+src/submission/check_validity.py
+src/submission/submit.py

src/UNKNOWN.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

src/UNKNOWN.egg-info/top_level.txt

@@ -0,0 +1,6 @@
+about
+display
+envs
+leaderboard
+populate
+submission

src/about.py

@@ -0,0 +1,72 @@
+from dataclasses import dataclass
+from enum import Enum
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+
+# Select your tasks here
+# ---------------------------------------------------
+class Tasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    task0 = Task("anli_r1", "acc", "ANLI")
+    task1 = Task("logiqa", "acc_norm", "LogiQA")
+
+NUM_FEWSHOT = 0 # Change with your few shot
+# ---------------------------------------------------
+
+
+
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+
+# What does your leaderboard evaluate?
+INTRODUCTION_TEXT = """
+Intro text
+"""
+
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = f"""
+## How it works
+
+## Reproducibility
+To reproduce our results, here is the commands you can run:
+
+"""
+
+EVALUATION_QUEUE_TEXT = """
+## Some good practices before submitting a model
+
+### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+
+### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+
+### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+
+### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+
+## In case of model failure
+If your model is displayed in the `FAILED` category, its execution stopped.
+Make sure you have followed the above steps first.
+If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""
+"""

src/display/css_html_js.py

@@ -0,0 +1,105 @@
+custom_css = """
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+#leaderboard-table td:nth-child(2),
+#leaderboard-table th:nth-child(2) {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -0,0 +1,27 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,110 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+
+import pandas as pd
+
+from src.about import Tasks
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+## All the model information that we might need
+@dataclass
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = "" # emoji
+
+
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
+
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    Unknown = ModelDetails("?")
+
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        return Precision.Unknown
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+

src/envs.py

@@ -0,0 +1,25 @@
+import os
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+
+OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+REPO_ID = f"{OWNER}/leaderboard"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+
+API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -0,0 +1,196 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str # org_model_precision (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    revision: str # commit hash, "" if main
+    results: dict
+    precision: Precision = Precision.Unknown
+    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original # Original or Adapter
+    architecture: str = "Unknown" 
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = "" # submission date of request file
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Precision
+        precision = Precision.from_str(config.get("model_dtype"))
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}_{precision.value.name}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}_{precision.value.name}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, _, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,  
+            revision= config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.precision.name: self.precision.value.name,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED"]
+                and req_content["precision"] == precision.split(".")[-1]
+            ):
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results

src/populate.py

@@ -0,0 +1,58 @@
+import json
+import os
+
+import pandas as pd
+
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.leaderboard.read_evals import get_raw_eval_results
+
+
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_eval_results(results_path, requests_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return df
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -0,0 +1,99 @@
+import json
+import os
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+
+import huggingface_hub
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig
+from transformers.models.auto.tokenization_auto import AutoTokenizer
+
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    """Checks if the model card and license exist and have been filled"""
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        if test_tokenizer:
+            try:
+                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+            except ValueError as e:
+                return (
+                    False,
+                    f"uses a tokenizer which is not in a transformers release: {e}",
+                    None
+                )
+            except Exception as e:
+                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None
+        )
+
+    except Exception as e:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
+    return model_info.config.get("architectures", "Unknown")
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    """Gather a list of already submitted models to avoid duplicates"""
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -0,0 +1,119 @@
+import json
+import os
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.submission.check_validity import (
+    already_submitted_models,
+    check_model_card,
+    get_model_size,
+    is_model_on_hub,
+)
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    weight_type: str,
+    model_type: str,
+):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # Does the model actually exist?
+    if revision == "":
+        revision = "main"
+
+    # Is the model on the hub?
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    if not weight_type == "Adapter":
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+
+    # Were the model card and license filled?
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+        "private": False,
+    }
+
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )