Build MBench leaderboard Space

.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.pyc
+.env
+.venv/
+mbench_leaderboard_submission/
+membench_leaderboard_submission/

README.md

@@ -1,13 +1,33 @@
 ---
 title: MBench Leaderboard
-emoji: 🏃
-colorFrom: green
-colorTo: red
+emoji: 🏆
+colorFrom: indigo
+colorTo: purple
 sdk: gradio
-sdk_version: 6.15.1
-python_version: '3.13'
+sdk_version: 4.36.1
+python_version: 3.10
 app_file: app.py
 pinned: false
+license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# MBench Leaderboard
+
+MBench is a benchmark for evaluating the memory capability of video world models. It focuses on whether a model can preserve a coherent world state across long-horizon video continuation and interaction.
+
+The benchmark is organized around three core memory dimensions:
+
+- **Entity Consistency:** persistent object and human identity, geometry, texture, and appearance.
+- **Environment Consistency:** stable spatial layout, reprojection behavior, lighting, and style.
+- **Causal Consistency:** reliable state evolution and interaction consequences over time.
+
+MBench uses trigger-conditioned scoring: Trigger Coverage measures whether the model actually enters the intended memory challenge, Memory Reliability measures consistency after the challenge is triggered, and M-Score balances both with a harmonic mean.
+
+The bundled seed leaderboard is transcribed from Table 2 of the MBench paper. Aggregate leaderboard columns are derived as unweighted averages over the reported sub-dimensions until official leaderboard totals are released.
+
+## Links
+
+- **Dataset:** `studyOverflow/TempMemoryData`
+- **Leaderboard data repo:** `PeanutUp/membench_leaderboard_submission`
+- **GitHub repo:** https://github.com/study-overflow/MBench
+- **Project page:** https://peanutup.github.io/MBench-project/

app.py

@@ -0,0 +1,450 @@
+import json
+import os
+import re
+import shutil
+import tempfile
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+from huggingface_hub import HfApi, snapshot_download
+
+from constants import (
+    ALL_COLUMNS,
+    LEADERBOARD_INTRO,
+    LEADERBOARD_REPO,
+    LOCAL_LEADERBOARD_DIR,
+    METRIC_COLUMNS,
+    MODEL_INFO_COLUMNS,
+    MODEL_TYPE_CHOICES,
+    RESULTS_CSV,
+    SUBMIT_INTRO,
+)
+from scripts.validate_submission import validate_submission_json
+
+
+SPACE_ROOT = Path(__file__).resolve().parent
+LOCAL_LEADERBOARD_PATH = Path(LOCAL_LEADERBOARD_DIR).resolve()
+RESULTS_PATH = Path(RESULTS_CSV).resolve()
+SEED_RESULTS_PATH = SPACE_ROOT / "seed" / "results.csv"
+PENDING_DIR = LOCAL_LEADERBOARD_PATH / "submissions" / "pending"
+VERIFIED_DIR = LOCAL_LEADERBOARD_PATH / "submissions" / "verified"
+NUMERIC_COLUMNS = [
+    "Total M-Score",
+    "Entity Score",
+    "Environment Score",
+    "Causal Score",
+    *METRIC_COLUMNS,
+]
+UPLOAD_ALLOW_PATTERNS = ["results.csv", "submissions/**"]
+
+
+def empty_results() -> pd.DataFrame:
+    return pd.DataFrame(columns=ALL_COLUMNS)
+
+
+def ensure_columns(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+    for column in ALL_COLUMNS:
+        if column not in df.columns:
+            df[column] = 0 if column in NUMERIC_COLUMNS or column == "Rank" else ""
+    return df[ALL_COLUMNS]
+
+
+def clean_numeric_columns(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+    for column in NUMERIC_COLUMNS:
+        df[column] = pd.to_numeric(df[column], errors="coerce").fillna(0.0)
+    return df
+
+
+def read_results_csv(path: Path) -> pd.DataFrame:
+    df = pd.read_csv(path)
+    return clean_numeric_columns(ensure_columns(df))
+
+
+def load_seed_results(reason: str) -> tuple[pd.DataFrame, str]:
+    if SEED_RESULTS_PATH.exists():
+        try:
+            df = read_results_csv(SEED_RESULTS_PATH)
+            return (
+                df,
+                f"{reason}\n\nUsing bundled seed results from Table 2 of the MBench paper.",
+            )
+        except Exception as exc:
+            return empty_results(), f"{reason}\n\nCould not read bundled seed results: {exc}"
+    return empty_results(), f"{reason}\n\nBundled seed results are missing."
+
+
+def load_remote_results() -> tuple[pd.DataFrame, str]:
+    try:
+        snapshot_download(
+            repo_id=LEADERBOARD_REPO,
+            repo_type="dataset",
+            local_dir=str(LOCAL_LEADERBOARD_PATH),
+            local_dir_use_symlinks=False,
+        )
+    except Exception as exc:
+        message = (
+            "Leaderboard data is not available yet. Please run "
+            "`python scripts/upload_seed_results.py` after setting `HF_TOKEN`."
+            f"\n\nDetails: {exc}"
+        )
+        return load_seed_results(message)
+
+    if not RESULTS_PATH.exists():
+        message = (
+            "`results.csv` was not found in the leaderboard data repo. Please run "
+            "`python scripts/upload_seed_results.py` to initialize it."
+        )
+        return load_seed_results(message)
+
+    try:
+        df = read_results_csv(RESULTS_PATH)
+    except Exception as exc:
+        return load_seed_results(f"Could not read `results.csv`: {exc}")
+
+    return df, f"Loaded results from `{LEADERBOARD_REPO}`."
+
+
+def prepare_leaderboard(
+    model_type: str,
+    selected_metrics: list[str] | None,
+) -> tuple[pd.DataFrame, str]:
+    df, status = load_remote_results()
+
+    if model_type and model_type != "All" and not df.empty:
+        df = df[df["Model Type"] == model_type].copy()
+
+    if not df.empty:
+        df = df.sort_values(
+            by="Total M-Score",
+            ascending=False,
+            kind="mergesort",
+        ).reset_index(drop=True)
+        df["Rank"] = np.arange(1, len(df) + 1)
+
+    metrics = [metric for metric in (selected_metrics or []) if metric in METRIC_COLUMNS]
+    columns = MODEL_INFO_COLUMNS + metrics
+    return df[columns], status
+
+
+def sanitize_filename(value: str) -> str:
+    cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "_", value.strip())
+    cleaned = cleaned.strip("._-")
+    return cleaned or "model"
+
+
+def get_uploaded_path(file_obj: Any) -> Path | None:
+    if file_obj is None:
+        return None
+    if isinstance(file_obj, (str, os.PathLike)):
+        return Path(file_obj)
+    if isinstance(file_obj, dict):
+        path = file_obj.get("path") or file_obj.get("name")
+        return Path(path) if path else None
+    name = getattr(file_obj, "name", None)
+    return Path(name) if name else None
+
+
+def safe_extract_zip(zip_path: Path, target_dir: Path) -> None:
+    target_root = target_dir.resolve()
+    with zipfile.ZipFile(zip_path, "r") as zip_ref:
+        for member in zip_ref.infolist():
+            member_path = (target_root / member.filename).resolve()
+            try:
+                member_path.relative_to(target_root)
+            except ValueError:
+                raise ValueError("ZIP contains an unsafe path.")
+        zip_ref.extractall(target_root)
+
+
+def read_first_json_from_zip(zip_path: Path) -> dict:
+    if not zipfile.is_zipfile(zip_path):
+        raise ValueError("Uploaded file must be a valid ZIP archive.")
+
+    with tempfile.TemporaryDirectory(prefix="mbench_submission_") as tmp_dir:
+        extract_dir = Path(tmp_dir)
+        safe_extract_zip(zip_path, extract_dir)
+        json_files = sorted(
+            path for path in extract_dir.rglob("*.json") if path.is_file()
+        )
+        if not json_files:
+            raise ValueError("No JSON file found inside the ZIP archive.")
+
+        with json_files[0].open("r", encoding="utf-8") as handle:
+            data = json.load(handle)
+
+    if not isinstance(data, dict):
+        raise ValueError("The first JSON file must contain a JSON object.")
+    return data
+
+
+def require_text(value: str, label: str) -> str:
+    if value is None or not str(value).strip():
+        raise ValueError(f"{label} is required.")
+    return str(value).strip()
+
+
+def ensure_submission_dirs() -> None:
+    PENDING_DIR.mkdir(parents=True, exist_ok=True)
+    VERIFIED_DIR.mkdir(parents=True, exist_ok=True)
+    (PENDING_DIR / ".gitkeep").touch(exist_ok=True)
+    (VERIFIED_DIR / ".gitkeep").touch(exist_ok=True)
+
+
+def ensure_local_results_file() -> None:
+    if RESULTS_PATH.exists() or not SEED_RESULTS_PATH.exists():
+        return
+    RESULTS_PATH.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(SEED_RESULTS_PATH, RESULTS_PATH)
+
+
+def save_pending_submission(
+    zip_path: Path,
+    result_json: dict,
+    model_name: str,
+    model_link: str,
+    team_name: str,
+    contact_email: str,
+    model_type: str,
+    accessibility: str,
+) -> tuple[Path, Path]:
+    ensure_submission_dirs()
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+    safe_model_name = sanitize_filename(model_name)
+    stem = f"{timestamp}_{safe_model_name}"
+
+    payload = {
+        "submitted_at_utc": datetime.now(timezone.utc).isoformat(),
+        "status": "pending",
+        "model_name": model_name,
+        "model_link": model_link,
+        "team_name": team_name,
+        "contact_email": contact_email,
+        "model_type": model_type,
+        "accessibility": accessibility,
+        "result_json": result_json,
+    }
+
+    json_path = PENDING_DIR / f"{stem}.json"
+    raw_zip_path = PENDING_DIR / f"{stem}.zip"
+    json_path.write_text(
+        json.dumps(payload, indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    shutil.copyfile(zip_path, raw_zip_path)
+    return json_path, raw_zip_path
+
+
+def upload_local_leaderboard(token: str, model_name: str) -> None:
+    api = HfApi(token=token)
+    api.create_repo(repo_id=LEADERBOARD_REPO, repo_type="dataset", exist_ok=True)
+    ensure_local_results_file()
+    api.upload_folder(
+        folder_path=str(LOCAL_LEADERBOARD_PATH),
+        repo_id=LEADERBOARD_REPO,
+        repo_type="dataset",
+        allow_patterns=UPLOAD_ALLOW_PATTERNS,
+        commit_message=f"Add pending MBench submission for {model_name}",
+    )
+
+
+def submit_result(
+    zip_file: Any,
+    model_name: str,
+    model_link: str,
+    team_name: str,
+    contact_email: str,
+    model_type: str,
+    accessibility: str,
+) -> str:
+    token = os.environ.get("HF_TOKEN")
+    if not token:
+        return "HF_TOKEN is not set. Please add it in Space Settings -> Secrets."
+
+    try:
+        model_name = require_text(model_name, "Model name")
+        model_link = require_text(model_link, "Model link")
+        contact_email = require_text(contact_email, "Contact email")
+        team_name = str(team_name or "").strip()
+        accessibility = str(accessibility or "Unknown").strip()
+
+        if model_type not in MODEL_TYPE_CHOICES[1:]:
+            raise ValueError("Model type must be text-conditioned or action-conditioned.")
+
+        zip_path = get_uploaded_path(zip_file)
+        if zip_path is None or not zip_path.exists():
+            raise ValueError("Please upload a ZIP file.")
+
+        result_json = read_first_json_from_zip(zip_path)
+        ok, message = validate_submission_json(result_json)
+        if not ok:
+            raise ValueError(message)
+
+        # Refresh the local dataset checkout before adding the pending submission.
+        try:
+            snapshot_download(
+                repo_id=LEADERBOARD_REPO,
+                repo_type="dataset",
+                local_dir=str(LOCAL_LEADERBOARD_PATH),
+                token=token,
+                local_dir_use_symlinks=False,
+            )
+        except Exception:
+            ensure_submission_dirs()
+
+        ensure_local_results_file()
+        save_pending_submission(
+            zip_path=zip_path,
+            result_json=result_json,
+            model_name=model_name,
+            model_link=model_link,
+            team_name=team_name,
+            contact_email=contact_email,
+            model_type=model_type,
+            accessibility=accessibility,
+        )
+        upload_local_leaderboard(token, model_name)
+    except Exception as exc:
+        return f"Submission failed: {exc}"
+
+    return "Submission received. It is pending official verification."
+
+
+def about_markdown() -> str:
+    return """
+# About MBench
+
+MBench evaluates whether video world models preserve memory over long-horizon generation and interaction.
+
+**Entity Consistency** measures whether objects and humans keep stable identity, geometry, texture, and appearance after occlusion, exit/re-entry, or viewpoint changes.
+
+**Environment Consistency** measures whether the world layout remains stable, including epipolar geometry, reprojection consistency, lighting, and style.
+
+**Causal Consistency** measures whether earlier events continue to affect later frames through state progress, physical plausibility, and text/action interaction.
+
+**Trigger Coverage** checks whether a generated video actually enters the intended memory challenge. **Memory Reliability** evaluates consistency only after that challenge is triggered. **M-Score** is the harmonic mean of Trigger Coverage and Memory Reliability.
+
+Submitted results are not automatically verified. Official results will be added after MBench team review.
+"""
+
+
+def build_demo() -> gr.Blocks:
+    initial_df, initial_status = prepare_leaderboard("All", METRIC_COLUMNS)
+
+    css = """
+    .metric-panel textarea { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
+    footer { visibility: hidden; }
+    """
+
+    with gr.Blocks(
+        title="MBench Leaderboard",
+        theme=gr.themes.Soft(primary_hue="indigo", secondary_hue="purple"),
+        css=css,
+    ) as demo:
+        with gr.Tab("Leaderboard"):
+            gr.Markdown(LEADERBOARD_INTRO)
+            status = gr.Markdown(initial_status)
+
+            with gr.Row():
+                model_type_filter = gr.Radio(
+                    choices=MODEL_TYPE_CHOICES,
+                    value="All",
+                    label="Model Type",
+                )
+                refresh_button = gr.Button("Refresh", variant="primary")
+
+            metric_selector = gr.CheckboxGroup(
+                choices=METRIC_COLUMNS,
+                value=METRIC_COLUMNS,
+                label="Metric Columns",
+            )
+
+            leaderboard_table = gr.Dataframe(
+                value=initial_df,
+                label="MBench Results",
+                interactive=False,
+                wrap=True,
+                height=520,
+            )
+
+            refresh_button.click(
+                fn=prepare_leaderboard,
+                inputs=[model_type_filter, metric_selector],
+                outputs=[leaderboard_table, status],
+            )
+            model_type_filter.change(
+                fn=prepare_leaderboard,
+                inputs=[model_type_filter, metric_selector],
+                outputs=[leaderboard_table, status],
+            )
+            metric_selector.change(
+                fn=prepare_leaderboard,
+                inputs=[model_type_filter, metric_selector],
+                outputs=[leaderboard_table, status],
+            )
+
+        with gr.Tab("Submit"):
+            gr.Markdown(SUBMIT_INTRO)
+            with gr.Row():
+                with gr.Column():
+                    zip_input = gr.File(
+                        label="Submission ZIP",
+                        file_types=[".zip"],
+                        type="filepath",
+                    )
+                    model_name_input = gr.Textbox(label="Model Name")
+                    model_link_input = gr.Textbox(label="Model Link")
+                    team_name_input = gr.Textbox(label="Team Name")
+                    contact_email_input = gr.Textbox(label="Contact Email")
+                    model_type_input = gr.Dropdown(
+                        choices=MODEL_TYPE_CHOICES[1:],
+                        value="text-conditioned",
+                        label="Model Type",
+                    )
+                    accessibility_input = gr.Dropdown(
+                        choices=[
+                            "Open weights",
+                            "API only",
+                            "Closed",
+                            "Research preview",
+                            "Unknown",
+                        ],
+                        value="Unknown",
+                        label="Accessibility",
+                    )
+                    submit_button = gr.Button("Submit", variant="primary")
+
+                with gr.Column():
+                    submit_status = gr.Markdown()
+
+            submit_button.click(
+                fn=submit_result,
+                inputs=[
+                    zip_input,
+                    model_name_input,
+                    model_link_input,
+                    team_name_input,
+                    contact_email_input,
+                    model_type_input,
+                    accessibility_input,
+                ],
+                outputs=submit_status,
+            )
+
+        with gr.Tab("About"):
+            gr.Markdown(about_markdown())
+
+    return demo
+
+
+demo = build_demo()
+
+
+if __name__ == "__main__":
+    demo.launch()

constants.py

@@ -0,0 +1,71 @@
+BENCHMARK_DATASET_REPO = "studyOverflow/TempMemoryData"
+LEADERBOARD_REPO = "PeanutUp/membench_leaderboard_submission"
+LOCAL_LEADERBOARD_DIR = "./membench_leaderboard_submission"
+RESULTS_CSV = "./membench_leaderboard_submission/results.csv"
+
+MODEL_INFO_COLUMNS = [
+    "Rank",
+    "Model Name",
+    "Model Link",
+    "Model Type",
+    "Certification",
+    "Accessibility",
+    "Sampled by",
+    "Evaluated by",
+    "Date",
+    "Total M-Score",
+    "Entity Score",
+    "Environment Score",
+    "Causal Score",
+]
+
+METRIC_COLUMNS = [
+    "Object Geometry",
+    "Object Texture",
+    "Human Identity",
+    "Human Appearance",
+    "Epipolar Geometry",
+    "Reprojection Consistency",
+    "Lighting Consistency",
+    "Style Consistency",
+    "State Progress",
+    "Physical Plausibility",
+    "Text Interaction",
+    "Action Interaction",
+    "Trigger Coverage",
+    "Memory Reliability",
+]
+
+ALL_COLUMNS = MODEL_INFO_COLUMNS + METRIC_COLUMNS
+
+MODEL_TYPE_CHOICES = [
+    "All",
+    "text-conditioned",
+    "action-conditioned",
+]
+
+LEADERBOARD_INTRO = """
+# MBench Leaderboard
+
+**MBench** evaluates memory capability in video world models: whether generated worlds keep entities, environments, and causal state coherent across long horizons and interaction.
+
+- **Entity Consistency:** object geometry/texture and human identity/appearance.
+- **Environment Consistency:** spatial consistency, reprojection, lighting, and style stability.
+- **Causal Consistency:** state progress, physical plausibility, and text/action interaction consistency.
+- **Trigger-Conditioned Scoring:** M-Score is the harmonic mean of Trigger Coverage and Memory Reliability, so models are rewarded only when they both trigger the memory challenge and remain consistent after it.
+
+Seed leaderboard values are transcribed from Table 2 of the MBench paper. Aggregate columns are derived as unweighted averages over the reported sub-dimensions until official leaderboard totals are released.
+
+Dataset: `studyOverflow/TempMemoryData`  
+Leaderboard data repo: `PeanutUp/membench_leaderboard_submission`  
+Project page: https://peanutup.github.io/MBench-project/  
+GitHub: https://github.com/study-overflow/MBench
+"""
+
+SUBMIT_INTRO = """
+# Submit to MBench
+
+Upload a ZIP file containing one JSON result file. Submissions are saved to `submissions/pending/` in the leaderboard data repo and are **not** added to the public leaderboard automatically.
+
+Required JSON keys: `model_name`, `model_link`, `model_type`, and `total_m_score`.
+"""

requirements.txt

@@ -0,0 +1,7 @@
+gradio==4.36.1
+pandas
+numpy
+huggingface_hub==0.23.4
+fastapi==0.111.0
+starlette==0.37.2
+socksio

scripts/upload_seed_results.py

@@ -0,0 +1,60 @@
+import os
+import shutil
+import sys
+import tempfile
+from pathlib import Path
+
+from huggingface_hub import HfApi
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from constants import LEADERBOARD_REPO
+
+
+REPO_TYPE = "dataset"
+
+
+def main() -> None:
+    token = os.environ.get("HF_TOKEN")
+
+    seed_results = ROOT / "seed" / "results.csv"
+    if not seed_results.exists():
+        raise FileNotFoundError(f"Seed file not found: {seed_results}")
+
+    api = HfApi(token=token)
+    try:
+        api.whoami(token=token)
+    except Exception as exc:
+        raise RuntimeError(
+            "HF_TOKEN is not set and no cached Hugging Face login was found. "
+            "Please run `export HF_TOKEN=...` or `hf auth login` first."
+        ) from exc
+
+    api.create_repo(repo_id=LEADERBOARD_REPO, repo_type=REPO_TYPE, exist_ok=True)
+
+    with tempfile.TemporaryDirectory(prefix="mbench_seed_") as tmp_dir:
+        tmp_root = Path(tmp_dir)
+        shutil.copy2(seed_results, tmp_root / "results.csv")
+
+        pending_dir = tmp_root / "submissions" / "pending"
+        verified_dir = tmp_root / "submissions" / "verified"
+        pending_dir.mkdir(parents=True, exist_ok=True)
+        verified_dir.mkdir(parents=True, exist_ok=True)
+        (pending_dir / ".gitkeep").write_text("", encoding="utf-8")
+        (verified_dir / ".gitkeep").write_text("", encoding="utf-8")
+
+        api.upload_folder(
+            folder_path=str(tmp_root),
+            repo_id=LEADERBOARD_REPO,
+            repo_type=REPO_TYPE,
+            commit_message="Initialize MBench leaderboard seed results",
+        )
+
+    print(f"Uploaded seed results to {LEADERBOARD_REPO}: results.csv")
+    print("Created submissions/pending/.gitkeep and submissions/verified/.gitkeep")
+
+
+if __name__ == "__main__":
+    main()

scripts/validate_submission.py

@@ -0,0 +1,58 @@
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from constants import METRIC_COLUMNS
+
+
+ALLOWED_MODEL_TYPES = {"text-conditioned", "action-conditioned"}
+REQUIRED_KEYS = ["model_name", "model_link", "model_type", "total_m_score"]
+SUMMARY_SCORE_FIELDS = [
+    "total_m_score",
+    "entity_score",
+    "environment_score",
+    "causal_score",
+]
+
+
+def _normalize_key(key: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "_", key.lower()).strip("_")
+
+
+def _is_present(value: Any) -> bool:
+    return value is not None and str(value).strip() != ""
+
+
+def validate_submission_json(data: dict) -> tuple[bool, str]:
+    if not isinstance(data, dict):
+        return False, "Submission JSON must be an object."
+
+    normalized = {_normalize_key(str(key)): value for key, value in data.items()}
+
+    for key in REQUIRED_KEYS:
+        if key not in normalized or not _is_present(normalized[key]):
+            return False, f"Missing required field: {key}"
+
+    model_type = str(normalized["model_type"]).strip()
+    if model_type not in ALLOWED_MODEL_TYPES:
+        return (
+            False,
+            "model_type must be either text-conditioned or action-conditioned.",
+        )
+
+    score_fields = set(SUMMARY_SCORE_FIELDS)
+    score_fields.update(_normalize_key(column) for column in METRIC_COLUMNS)
+
+    for key, value in normalized.items():
+        if key in score_fields and _is_present(value):
+            try:
+                float(value)
+            except (TypeError, ValueError):
+                return False, f"Score field {key} must be convertible to float."
+
+    return True, "Submission JSON is valid."

seed/results.csv

@@ -0,0 +1,15 @@
+Rank,Model Name,Model Link,Model Type,Certification,Accessibility,Sampled by,Evaluated by,Date,Total M-Score,Entity Score,Environment Score,Causal Score,Object Geometry,Object Texture,Human Identity,Human Appearance,Epipolar Geometry,Reprojection Consistency,Lighting Consistency,Style Consistency,State Progress,Physical Plausibility,Text Interaction,Action Interaction,Trigger Coverage,Memory Reliability
+1,HY-WorldPlay,,action-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,67.25,58.67,81.69,55.53,47.12,68.54,52.46,66.58,83.86,68.17,82.67,92.04,0,0,25.38,85.69,0,0
+2,Infinite-World,,action-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,56.59,41.88,71.13,56.95,35.70,61.88,23.08,46.85,74.04,61.51,62.63,86.34,0,0,27.53,86.37,0,0
+3,Matrix-Game 3.0,,action-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,53.79,48.16,60.20,52.21,44.15,58.22,42.38,47.91,61.99,32.86,62.06,83.90,0,0,22.48,81.93,0,0
+4,Self Forcing,,text-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,49.44,61.42,46.77,30.81,85.23,66.16,37.67,56.64,52.76,8.70,52.01,73.61,33.50,0,28.13,0,0,0
+5,Yume-1.5,,action-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,46.67,42.23,50.57,47.76,60.96,49.99,17.41,40.57,51.86,24.55,51.21,74.65,0,0,33.32,62.20,0,0
+6,LongLive,,text-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,45.28,55.88,36.10,42.45,75.29,55.83,34.89,57.49,31.11,2.38,43.58,67.32,54.22,0,30.69,0,0,0
+7,MemFlow,,text-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,45.13,56.51,37.36,37.92,80.60,63.78,31.44,50.22,42.00,1.29,39.03,67.12,45.72,0,30.13,0,0,0
+8,Skyreels V2,,text-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,38.16,40.10,35.13,40.34,63.54,40.94,14.62,41.31,33.43,2.49,31.08,73.54,51.92,0,28.77,0,0,0
+9,LongCat-Video,,text-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,36.83,35.90,30.48,51.38,45.97,39.15,16.32,42.16,16.66,0.74,40.08,64.46,72.67,0,30.08,0,0,0
+10,Helios,,text-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,34.73,42.86,26.80,34.32,71.72,49.60,20.28,29.85,14.37,2.47,27.20,63.16,41.11,0,27.53,0,0,0
+11,Causal Forcing,,text-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,34.26,42.10,24.34,38.44,49.77,39.32,28.16,51.14,10.02,0.00,41.46,45.86,47.92,0,28.95,0,0,0
+12,Lingbot-World,,action-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,34.12,30.71,30.81,47.54,33.20,44.54,11.57,33.53,22.12,7.57,40.06,53.50,0,0,31.76,63.32,0,0
+13,Cosmos-Predict 2.5,,text-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,30.93,27.81,24.25,50.51,36.98,32.49,9.53,32.26,5.14,0.17,40.56,51.14,71.92,0,29.10,0,0,0
+14,Matrix-Game 2.0,,action-conditioned,MBench Paper,TBD,MBench Team,MBench Team,2026-05-28,20.34,11.44,23.44,31.93,14.62,28.99,1.22,0.94,14.78,3.08,38.79,37.10,0,0,16.00,47.86,0,0