| { |
| "schema_version": "1.0", |
| "benchmark_version": "1.0", |
| "updated": "2026-05-29", |
| "models": [ |
| { |
| "id": "google-gemini-3-1-pro", |
| "name": "Gemini 3.1 Pro", |
| "organization": "Google", |
| "params": "—", |
| "type": "closed", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://deepmind.google/technologies/gemini/https://deepmind.google/technologies/gemini/", |
| "overall": 61.915, |
| "spatial": 71.72, |
| "spatio_temporal": 64.35, |
| "temporal": 40.74, |
| "semantic": 70.86, |
| "2d_localization": 76.15, |
| "2d_referring_expressions": 67.17, |
| "2d_spatial_pointing": 71.84, |
| "single_object_tracking": 64.35, |
| "temporal_localization": 45.69, |
| "dense_video_captioning": 35.78, |
| "event_verification": 69.83, |
| "video_qa": 71.88 |
| }, |
| { |
| "id": "alibaba-qwen3-vl-32b", |
| "name": "Qwen3-VL-32B-Instruct", |
| "organization": "Alibaba", |
| "params": "32B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct", |
| "overall": 55.04875, |
| "spatial": 70.78, |
| "spatio_temporal": 44.19, |
| "temporal": 39.56, |
| "semantic": 65.67, |
| "2d_localization": 64.33, |
| "2d_referring_expressions": 72.39, |
| "2d_spatial_pointing": 75.62, |
| "single_object_tracking": 44.19, |
| "temporal_localization": 49.71, |
| "dense_video_captioning": 29.4, |
| "event_verification": 60.04, |
| "video_qa": 71.3 |
| }, |
| { |
| "id": "nvidia-cosmos-reason2-8b", |
| "name": "Cosmos-Reason2-8B", |
| "organization": "NVIDIA", |
| "params": "8B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-8B", |
| "overall": 54.1825, |
| "spatial": 73.12, |
| "spatio_temporal": 37.69, |
| "temporal": 39.9, |
| "semantic": 66.02, |
| "2d_localization": 83.88, |
| "2d_referring_expressions": 66.88, |
| "2d_spatial_pointing": 68.6, |
| "single_object_tracking": 37.69, |
| "temporal_localization": 47.3, |
| "dense_video_captioning": 32.5, |
| "event_verification": 64.09, |
| "video_qa": 67.95 |
| }, |
| { |
| "id": "alibaba-qwen3-vl-8b", |
| "name": "Qwen3-VL-8B-Instruct", |
| "organization": "Alibaba", |
| "params": "8B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct", |
| "overall": 50.06875, |
| "spatial": 67.26, |
| "spatio_temporal": 33.12, |
| "temporal": 36.98, |
| "semantic": 62.92, |
| "2d_localization": 59.93, |
| "2d_referring_expressions": 73.29, |
| "2d_spatial_pointing": 68.56, |
| "single_object_tracking": 33.12, |
| "temporal_localization": 44.32, |
| "dense_video_captioning": 29.64, |
| "event_verification": 59.39, |
| "video_qa": 66.44 |
| }, |
| { |
| "id": "alibaba-qwen3-5-vl-27b", |
| "name": "Qwen3.5-27B", |
| "organization": "Alibaba", |
| "params": "27B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-27B-Instruct", |
| "overall": 49.25333333, |
| "spatial": 79.04, |
| "spatio_temporal": 24.4, |
| "temporal": 31.93, |
| "semantic": 61.65, |
| "2d_localization": 85.46, |
| "2d_referring_expressions": 76.35, |
| "2d_spatial_pointing": 75.32, |
| "single_object_tracking": 24.4, |
| "temporal_localization": 36.88, |
| "dense_video_captioning": 26.97, |
| "event_verification": 55.34, |
| "video_qa": 67.95 |
| }, |
| { |
| "id": "alibaba-qwen3-5-vl-9b", |
| "name": "Qwen3.5-9B", |
| "organization": "Alibaba", |
| "params": "9B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-9B-Instruct", |
| "overall": 48.48166667, |
| "spatial": 75.26, |
| "spatio_temporal": 23.52, |
| "temporal": 35.95, |
| "semantic": 59.21, |
| "2d_localization": 76.87, |
| "2d_referring_expressions": 77.16, |
| "2d_spatial_pointing": 71.74, |
| "single_object_tracking": 23.52, |
| "temporal_localization": 42.53, |
| "dense_video_captioning": 29.36, |
| "event_verification": 46.78, |
| "video_qa": 71.63 |
| }, |
| { |
| "id": "google-gemini-3-2-flash-lite", |
| "name": "Gemini 3.1 Flash Lite", |
| "organization": "Google", |
| "params": "—", |
| "type": "closed", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://deepmind.google/technologies/gemini/", |
| "overall": 51.335, |
| "spatial": 62.06, |
| "spatio_temporal": 46.68, |
| "temporal": 35.17, |
| "semantic": 61.43, |
| "2d_localization": 69.88, |
| "2d_referring_expressions": 52.62, |
| "2d_spatial_pointing": 63.68, |
| "single_object_tracking": 46.68, |
| "temporal_localization": 37.72, |
| "dense_video_captioning": 32.62, |
| "event_verification": 54.83, |
| "video_qa": 68.03 |
| }, |
| { |
| "id": "nvidia-cosmos-reason2-32b", |
| "name": "Cosmos-Reason2-32B", |
| "organization": "NVIDIA", |
| "params": "32B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-32B", |
| "overall": 46.66833333, |
| "spatial": 41.36, |
| "spatio_temporal": 33.12, |
| "temporal": 40.26, |
| "semantic": 71.94, |
| "2d_localization": 4.35, |
| "2d_referring_expressions": 45.71, |
| "2d_spatial_pointing": 74.03, |
| "single_object_tracking": 33.12, |
| "temporal_localization": 50.47, |
| "dense_video_captioning": 30.04, |
| "event_verification": 73.58, |
| "video_qa": 70.29 |
| }, |
| { |
| "id": "nvidia-cosmos3-super", |
| "name": "Cosmos3-Super", |
| "organization": "NVIDIA", |
| "params": "—", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-05-29", |
| "model_url": "https://huggingface.co/nvidia/Cosmos3-Super", |
| "overall": 63.01333333, |
| "spatial": 78.71, |
| "spatio_temporal": 62.25, |
| "temporal": 40.72, |
| "semantic": 70.37, |
| "2d_localization": 86.97, |
| "2d_referring_expressions": 76.23, |
| "2d_spatial_pointing": 72.94, |
| "single_object_tracking": 62.25, |
| "temporal_localization": 51.9, |
| "dense_video_captioning": 29.54, |
| "event_verification": 71.28, |
| "video_qa": 69.46 |
| }, |
| { |
| "id": "nvidia-cosmos3-nano", |
| "name": "Cosmos3-Nano", |
| "organization": "NVIDIA", |
| "params": "—", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-05-29", |
| "model_url": "https://huggingface.co/nvidia/Cosmos3-Nano", |
| "overall": 60.67291667, |
| "spatial": 74.84, |
| "spatio_temporal": 59.21, |
| "temporal": 39.73, |
| "semantic": 68.92, |
| "2d_localization": 74.11, |
| "2d_referring_expressions": 75.57, |
| "2d_spatial_pointing": 74.83, |
| "single_object_tracking": 59.21, |
| "temporal_localization": 48.04, |
| "dense_video_captioning": 31.42, |
| "event_verification": 68.88, |
| "video_qa": 68.95 |
| } |
| ] |
| } |
|
|
