| BENCHMARK_DATASET_REPO = "studyOverflow/TempMemoryData" |
| LEADERBOARD_REPO = "PeanutUp/membench_leaderboard_submission" |
| LOCAL_LEADERBOARD_DIR = "./membench_leaderboard_submission" |
| RESULTS_CSV = "./membench_leaderboard_submission/results.csv" |
|
|
| MODEL_INFO_COLUMNS = [ |
| "Rank", |
| "Model Name", |
| "Model Link", |
| "Model Type", |
| "Certification", |
| "Accessibility", |
| "Sampled by", |
| "Evaluated by", |
| "Date", |
| "Total M-Score", |
| "Entity Score", |
| "Environment Score", |
| "Causal Score", |
| ] |
|
|
| METRIC_COLUMNS = [ |
| "Object Geometry", |
| "Object Texture", |
| "Human Identity", |
| "Human Appearance", |
| "Epipolar Geometry", |
| "Reprojection Consistency", |
| "Lighting Consistency", |
| "Style Consistency", |
| "State Progress", |
| "Physical Plausibility", |
| "Text Interaction", |
| "Action Interaction", |
| ] |
|
|
| ALL_COLUMNS = MODEL_INFO_COLUMNS + METRIC_COLUMNS |
|
|
| MODEL_TYPE_CHOICES = [ |
| "All", |
| "text-conditioned", |
| "action-conditioned", |
| ] |
|
|
| LEADERBOARD_INTRO = """ |
| # MBench Leaderboard |
| |
| **MBench** evaluates memory capability in video world models: whether generated worlds keep entities, environments, and causal state coherent across long horizons and interaction. |
| |
| - **Entity Consistency:** object geometry/texture and human identity/appearance. |
| - **Environment Consistency:** spatial consistency, reprojection, lighting, and style stability. |
| - **Causal Consistency:** state progress, physical plausibility, and text/action interaction consistency. |
| |
| Seed leaderboard values are transcribed from Table 2 of the MBench paper. Aggregate columns are derived as unweighted averages over the reported sub-dimensions until official leaderboard totals are released. |
| |
| Dataset: `studyOverflow/TempMemoryData` |
| Leaderboard data repo: `PeanutUp/membench_leaderboard_submission` |
| Project page: https://peanutup.github.io/MBench-project/ |
| GitHub: https://github.com/study-overflow/MBench |
| """ |
|
|
| SUBMIT_INTRO = """ |
| # Submit to MBench |
| |
| Upload a ZIP file containing one JSON result file. Submissions are saved to `submissions/pending/` in the leaderboard data repo and are **not** added to the public leaderboard automatically. |
| |
| Required JSON keys: `model_name`, `model_link`, `model_type`, and `total_m_score`. |
| """ |
|
|
