Init Commit

app.py

@@ -1,8 +1,6 @@
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
+from collections import OrderedDict
 
 from src.about import (
     CITATION_BUTTON_LABEL,
@@ -13,181 +11,198 @@ from src.about import (
     TITLE,
 )
 from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
 
 
+# ============================================================
+# Static Leaderboard Data for VBVR-Bench
+# ============================================================
+
+# Column group definitions (ordered for display)
+COLUMN_GROUPS = OrderedDict([
+    ("Overall", ["Overall"]),
+    ("Overall by Category", [
+        "Abst.(All)", "Know.(All)", "Perc.(All)", "Spat.(All)", "Trans.(All)",
+    ]),
+    ("In-Domain (ID)", ["ID"]),
+    ("In-Domain by Category", [
+        "Abst.(ID)", "Know.(ID)", "Perc.(ID)", "Spat.(ID)", "Trans.(ID)",
+    ]),
+    ("Out-of-Domain (OOD)", ["OOD"]),
+    ("Out-of-Domain by Category", [
+        "Abst.(OOD)", "Know.(OOD)", "Perc.(OOD)", "Spat.(OOD)", "Trans.(OOD)",
+    ]),
+])
+
+# Default column groups to show (matching LaTeX table layout)
+DEFAULT_GROUPS = [
+    "Overall",
+    "In-Domain (ID)",
+    "In-Domain by Category",
+    "Out-of-Domain (OOD)",
+    "Out-of-Domain by Category",
+]
+
+# Columns always shown regardless of group selection
+ALWAYS_VISIBLE_COLS = ["Model", "Type"]
+
+# ============================================================
+# Static model scores data
+# ============================================================
+MODELS_DATA = [
+    {
+        "Model": "Human",
+        "Type": "👤 Reference",
+        "Overall": 0.974, "ID": 0.960, "OOD": 0.988,
+        "Abst.(All)": 0.947, "Know.(All)": 0.972, "Perc.(All)": 0.994, "Spat.(All)": 0.969, "Trans.(All)": 0.981,
+        "Abst.(ID)": 0.919, "Know.(ID)": 0.956, "Perc.(ID)": 1.000, "Spat.(ID)": 0.950, "Trans.(ID)": 1.000,
+        "Abst.(OOD)": 1.000, "Know.(OOD)": 1.000, "Perc.(OOD)": 0.990, "Spat.(OOD)": 1.000, "Trans.(OOD)": 0.970,
+    },
+    # ---- Open-source Models ----
+    {
+        "Model": "CogVideoX1.5-5B-I2V",
+        "Type": "🟢 Open-source",
+        "Overall": 0.2727, "ID": 0.2831, "OOD": 0.2623,
+        "Abst.(All)": 0.2548, "Know.(All)": 0.2952, "Perc.(All)": 0.2525, "Spat.(All)": 0.2996, "Trans.(All)": 0.2903,
+        "Abst.(ID)": 0.2408, "Know.(ID)": 0.3285, "Perc.(ID)": 0.2567, "Spat.(ID)": 0.3281, "Trans.(ID)": 0.3051,
+        "Abst.(OOD)": 0.2809, "Know.(OOD)": 0.2352, "Perc.(OOD)": 0.2501, "Spat.(OOD)": 0.2539, "Trans.(OOD)": 0.2824,
+    },
+    {
+        "Model": "HunyuanVideo-I2V",
+        "Type": "🟢 Open-source",
+        "Overall": 0.2726, "ID": 0.2799, "OOD": 0.2653,
+        "Abst.(All)": 0.1956, "Know.(All)": 0.3614, "Perc.(All)": 0.2910, "Spat.(All)": 0.2698, "Trans.(All)": 0.2733,
+        "Abst.(ID)": 0.2068, "Know.(ID)": 0.3573, "Perc.(ID)": 0.2933, "Spat.(ID)": 0.2802, "Trans.(ID)": 0.3160,
+        "Abst.(OOD)": 0.1747, "Know.(OOD)": 0.3688, "Perc.(OOD)": 0.2897, "Spat.(OOD)": 0.2530, "Trans.(OOD)": 0.2502,
+    },
+    {
+        "Model": "Wan2.2-I2V-A14B",
+        "Type": "🟢 Open-source",
+        "Overall": 0.3714, "ID": 0.4125, "OOD": 0.3287,
+        "Abst.(All)": 0.4212, "Know.(All)": 0.3556, "Perc.(All)": 0.3710, "Spat.(All)": 0.3397, "Trans.(All)": 0.3465,
+        "Abst.(ID)": 0.4301, "Know.(ID)": 0.3823, "Perc.(ID)": 0.4147, "Spat.(ID)": 0.4043, "Trans.(ID)": 0.4192,
+        "Abst.(OOD)": 0.4046, "Know.(OOD)": 0.3077, "Perc.(OOD)": 0.3427, "Spat.(OOD)": 0.2364, "Trans.(OOD)": 0.3073,
+    },
+    {
+        "Model": "LTX-2",
+        "Type": "🟢 Open-source",
+        "Overall": 0.3129, "ID": 0.3287, "OOD": 0.2971,
+        "Abst.(All)": 0.2908, "Know.(All)": 0.3531, "Perc.(All)": 0.3200, "Spat.(All)": 0.2980, "Trans.(All)": 0.3093,
+        "Abst.(ID)": 0.3156, "Know.(ID)": 0.3621, "Perc.(ID)": 0.3257, "Spat.(ID)": 0.3399, "Trans.(ID)": 0.3060,
+        "Abst.(OOD)": 0.2444, "Know.(OOD)": 0.3369, "Perc.(OOD)": 0.3167, "Spat.(OOD)": 0.2308, "Trans.(OOD)": 0.3110,
+    },
+    # ---- Proprietary Models ----
+    {
+        "Model": "Runway Gen-4 Turbo",
+        "Type": "🔵 Proprietary",
+        "Overall": 0.4031, "ID": 0.3920, "OOD": 0.4141,
+        "Abst.(All)": 0.4370, "Know.(All)": 0.4165, "Perc.(All)": 0.4223, "Spat.(All)": 0.3357, "Trans.(All)": 0.3696,
+        "Abst.(ID)": 0.3956, "Know.(ID)": 0.4094, "Perc.(ID)": 0.4288, "Spat.(ID)": 0.3409, "Trans.(ID)": 0.3629,
+        "Abst.(OOD)": 0.5147, "Know.(OOD)": 0.4294, "Perc.(OOD)": 0.4185, "Spat.(OOD)": 0.3274, "Trans.(OOD)": 0.3733,
+    },
+    {
+        "Model": "Sora 2",
+        "Type": "🔵 Proprietary",
+        "Overall": 0.5457, "ID": 0.5691, "OOD": 0.5225,
+        "Abst.(All)": 0.5824, "Know.(All)": 0.4749, "Perc.(All)": 0.5458, "Spat.(All)": 0.5298, "Trans.(All)": 0.5640,
+        "Abst.(ID)": 0.6023, "Know.(ID)": 0.4767, "Perc.(ID)": 0.5810, "Spat.(ID)": 0.5720, "Trans.(ID)": 0.5967,
+        "Abst.(OOD)": 0.5462, "Know.(OOD)": 0.4715, "Perc.(OOD)": 0.5254, "Spat.(OOD)": 0.4623, "Trans.(OOD)": 0.5465,
+    },
+    {
+        "Model": "Kling 2.6",
+        "Type": "🔵 Proprietary",
+        "Overall": 0.3691, "ID": 0.4082, "OOD": 0.3300,
+        "Abst.(All)": 0.4866, "Know.(All)": 0.2556, "Perc.(All)": 0.3095, "Spat.(All)": 0.3504, "Trans.(All)": 0.4149,
+        "Abst.(ID)": 0.4647, "Know.(ID)": 0.3225, "Perc.(ID)": 0.3749, "Spat.(ID)": 0.3471, "Trans.(ID)": 0.5193,
+        "Abst.(OOD)": 0.5277, "Know.(OOD)": 0.1350, "Perc.(OOD)": 0.2717, "Spat.(OOD)": 0.3556, "Trans.(OOD)": 0.3588,
+    },
+    {
+        "Model": "Veo 3.1",
+        "Type": "🔵 Proprietary",
+        "Overall": 0.4800, "ID": 0.5307, "OOD": 0.4288,
+        "Abst.(All)": 0.5991, "Know.(All)": 0.4225, "Perc.(All)": 0.4568, "Spat.(All)": 0.4430, "Trans.(All)": 0.4413,
+        "Abst.(ID)": 0.6109, "Know.(ID)": 0.5032, "Perc.(ID)": 0.5196, "Spat.(ID)": 0.4443, "Trans.(ID)": 0.5103,
+        "Abst.(OOD)": 0.5770, "Know.(OOD)": 0.2772, "Perc.(OOD)": 0.4204, "Spat.(OOD)": 0.4406, "Trans.(OOD)": 0.4041,
+    },
+    # ---- Data Scaling Strong Baseline ----
+    {
+        "Model": "VBVR-Wan2.2",
+        "Type": "⭐ Strong Baseline",
+        "Overall": 0.6848, "ID": 0.7599, "OOD": 0.6097,
+        "Abst.(All)": 0.7394, "Know.(All)": 0.6864, "Perc.(All)": 0.6333, "Spat.(All)": 0.6960, "Trans.(All)": 0.6909,
+        "Abst.(ID)": 0.7240, "Know.(ID)": 0.7500, "Perc.(ID)": 0.7817, "Spat.(ID)": 0.7446, "Trans.(ID)": 0.8327,
+        "Abst.(OOD)": 0.7682, "Know.(OOD)": 0.5720, "Perc.(OOD)": 0.5474, "Spat.(OOD)": 0.6182, "Trans.(OOD)": 0.6145,
+    },
+]
+
+
+def build_full_dataframe():
+    """Build the complete DataFrame with all columns, sorted by Overall descending."""
+    df = pd.DataFrame(MODELS_DATA)
+    # Ensure column order: always-visible cols first, then groups in defined order
+    all_cols = list(ALWAYS_VISIBLE_COLS)
+    for group_cols in COLUMN_GROUPS.values():
+        all_cols.extend(group_cols)
+    df = df[all_cols]
+    # Sort by Overall descending
+    df = df.sort_values("Overall", ascending=False).reset_index(drop=True)
+    # Round numeric columns to 3 decimal places for clean display
+    numeric_cols = df.select_dtypes(include="number").columns
+    df[numeric_cols] = df[numeric_cols].round(3)
+    return df
+
+
+FULL_DF = build_full_dataframe()
+
+
+def get_filtered_df(selected_groups):
+    """Filter DataFrame columns based on selected column groups."""
+    if not selected_groups:
+        selected_groups = ["Overall"]  # Always show at least Overall
+
+    cols = list(ALWAYS_VISIBLE_COLS)
+    for group_name, group_cols in COLUMN_GROUPS.items():
+        if group_name in selected_groups:
+            cols.extend(group_cols)
+
+    return FULL_DF[cols]
+
+
+# ============================================================
+# Gradio Interface
+# ============================================================
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+        with gr.TabItem("🏅 VBVR-Bench Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                column_selector = gr.CheckboxGroup(
+                    choices=list(COLUMN_GROUPS.keys()),
+                    value=DEFAULT_GROUPS,
+                    label="Select Column Groups to Display:",
+                    interactive=True,
+                )
+
+            leaderboard_table = gr.Dataframe(
+                value=get_filtered_df(DEFAULT_GROUPS),
+                interactive=False,
+                elem_id="leaderboard-table",
+            )
 
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
+            column_selector.change(
+                fn=get_filtered_df,
+                inputs=[column_selector],
+                outputs=[leaderboard_table],
             )
 
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=1):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("🚀 Submit", elem_id="llm-benchmark-tab-submit", id=2):
+            gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(
@@ -198,7 +213,4 @@ with demo:
                 show_copy_button=True,
             )
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch()

src/about.py

@@ -11,7 +11,7 @@ class Task:
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
     task0 = Task("anli_r1", "acc", "ANLI")
     task1 = Task("logiqa", "acc_norm", "LogiQA")
 
@@ -21,52 +21,102 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">VBVR-Bench Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+**VBVR-Bench** is a comprehensive benchmark for evaluating **video reasoning capabilities**.
+
+To systematically assess model reasoning capabilities, VBVR-Bench employs a **dual-split evaluation strategy** across **100 diverse tasks**:
+- **In-Domain (ID)**: 50 tasks that overlap with training categories but differ in unseen parameter configurations and sample instances, testing *in-domain generalization*.
+- **Out-of-Domain (OOD)**: 50 entirely novel tasks designed to measure *out-of-domain generalization*, testing whether models acquire transferable reasoning primitives rather than relying on task-specific memorization.
+
+Each task consists of **5 test samples**, enabling statistically robust evaluation across diverse reasoning scenarios.
+
+Use the column group selector below to customize which score groups are displayed.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## How it works
+## About VBVR-Bench
+
+### Rule-Based Evaluation Framework
+
+A key feature of VBVR-Bench is its fully **rule-based evaluation framework**. Most test tasks have a unique, verifiable correct answer, allowing interpretable evaluation based on spatial position, color, object identity, path, or logical outcome. Geometric, physical, and deductive constraints are also considered in the scoring rubrics.
+
+Each of the 100 test tasks is paired with a dedicated evaluation rule, with scores on multiple aspects to compute a weighted, comprehensive score. Sub-criteria include:
+- **Spatial Accuracy**: Correctness of object positions and arrangements
+- **Trajectory Correctness**: Validity of movement paths
+- **Temporal Consistency**: Smooth frame-by-frame progression
+- **Logical Validity**: Adherence to task-specific reasoning constraints
+
+### Example: Task G-45 (Key Door Matching)
+
+A green dot agent must first locate a color-specified key and then navigate to the matching door within a grid maze. Performance is scored across four weighted dimensions:
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+| Dimension | Weight | Description |
+|-----------|--------|-------------|
+| Target Identification | 30% | Correct key and door selection without color confusion |
+| Path Validity | 30% | Following allowed paths without wall collisions |
+| Path Efficiency | 20% | Comparison to optimal BFS path |
+| Animation Quality | 20% | Smooth movement and precise object alignment |
 
+A perfect score requires all four dimensions to be satisfied.
+
+### Key Benefits
+
+- **Reproducibility and Determinism**: Fully deterministic evaluation avoiding stochastic variability or hallucinations associated with LLM-based judgments.
+- **Granular Verifiability**: Each task is decomposed into interpretable vectors, allowing precise measurement of spatial, temporal, and logical correctness at the pixel or object-property level.
+- **Transparent Diagnosis**: By explicitly encoding reasoning constraints, the benchmark not only ranks models but also reveals systematic capability gaps and cross-domain performance trends.
+
+### Model Categories
+- 👤 **Reference**: Human performance baseline
+- 🟢 **Open-source**: Publicly available models  
+- 🔵 **Proprietary**: Commercial/closed-source models
+- ⭐ **Strong Baseline**: Data scaling strong baseline (VBVR-Wan2.2)
 """
 
 EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
+## How to Submit Your Results
+
+We welcome submissions from the research community! To submit your model's evaluation results to the VBVR-Bench leaderboard:
+
+### Submission Process
+
+📧 **Email your submission to: [C200210@e.ntu.edu.sg](mailto:C200210@e.ntu.edu.sg)**
+
+Please include the following in your submission:
 
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+### Required Materials
 
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+1. **Model Information**
+   - Model name and version
+   - Model type (Open-source / Proprietary)
+   - Link to model (if publicly available)
+   - Brief model description
 
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+2. **Evaluation Results**
+   - Complete evaluation scores in JSON format
+   - Scores for all 100 tasks (50 ID + 50 OOD)
+   - Category-wise breakdown (Abstraction, Knowledge, Perception, Spatiality, Transformation)
 
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+3. **Evaluation Logs**
+   - Full evaluation logs for verification
+   - Generated videos for a subset of tasks (optional but recommended)
 
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+4. **Technical Details**
+   - Inference configuration (resolution, frame rate, etc.)
+   - Hardware used for generation
+   - Any preprocessing or postprocessing applied
 
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+We will review your submission and add it to the leaderboard within 1-2 weeks.
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
+@article{vbvr2026,
+  title={A Very Big Video Reasoning Suite},
+  author={Wang, Maijunxian and Wang, Ruisi and Lin, Juyi and Ji, Ran and Wiedemer, Thaddäus and Gao, Qingying and Luo, Dezhi and Qian, Yaoyao and Huang, Lianyu and Hong, Zelong and Ge, Jiahui and Ma, Qianli and He, Hang and Zhou, Yifan and Guo, Lingzi and Mei, Lantao and Li, Jiachen and Xing, Hanwen and Zhao, Tianqi and Yu, Fengyuan and Xiao, Weihang and Jiao, Yizheng and Hou, Jianheng and Zhang, Danyang and Xu, Pengcheng and Zhong, Boyang and Zhao, Zehong and Fang, Gaoyun and Kitaoka, John and Xu, Yile and Xu, Hua and Blacutt, Kenton and Nguyen, Tin and Song, Siyuan and Sun, Haoran and Wen, Shaoyue and He, Linyang and Wang, Runming and Wang, Yanzhi and Yang, Mengyue and Ma, Ziqiao and Millière, Raphaël and Shi, Freda and Vasconcelos, Nuno and Khashabi, Daniel and Yuille, Alan and Du, Yilun and Liu, Ziming and Lin, Dahua and Liu, Ziwei and Kumar, Vikash and Li, Yijiang and Yang, Lei and Cai, Zhongang and Deng, Hokin},
+  year={2026}
+}
 """