{
  "schema_version": "1.0",
  "benchmark_version": "1.0",
  "updated": "2026-05-29",
  "models": [
    {
      "id": "google-gemini-3-1-pro",
      "name": "Gemini 3.1 Pro",
      "organization": "Google",
      "params": "—",
      "type": "closed",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://deepmind.google/technologies/gemini/https://deepmind.google/technologies/gemini/",
      "overall": 61.915,
      "spatial": 71.72,
      "spatio_temporal": 64.35,
      "temporal": 40.74,
      "semantic": 70.86,
      "2d_localization": 76.15,
      "2d_referring_expressions": 67.17,
      "2d_spatial_pointing": 71.84,
      "single_object_tracking": 64.35,
      "temporal_localization": 45.69,
      "dense_video_captioning": 35.78,
      "event_verification": 69.83,
      "video_qa": 71.88
    },
    {
      "id": "alibaba-qwen3-vl-32b",
      "name": "Qwen3-VL-32B-Instruct",
      "organization": "Alibaba",
      "params": "32B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct",
      "overall": 55.04875,
      "spatial": 70.78,
      "spatio_temporal": 44.19,
      "temporal": 39.56,
      "semantic": 65.67,
      "2d_localization": 64.33,
      "2d_referring_expressions": 72.39,
      "2d_spatial_pointing": 75.62,
      "single_object_tracking": 44.19,
      "temporal_localization": 49.71,
      "dense_video_captioning": 29.4,
      "event_verification": 60.04,
      "video_qa": 71.3
    },
    {
      "id": "nvidia-cosmos-reason2-8b",
      "name": "Cosmos-Reason2-8B",
      "organization": "NVIDIA",
      "params": "8B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-8B",
      "overall": 54.1825,
      "spatial": 73.12,
      "spatio_temporal": 37.69,
      "temporal": 39.9,
      "semantic": 66.02,
      "2d_localization": 83.88,
      "2d_referring_expressions": 66.88,
      "2d_spatial_pointing": 68.6,
      "single_object_tracking": 37.69,
      "temporal_localization": 47.3,
      "dense_video_captioning": 32.5,
      "event_verification": 64.09,
      "video_qa": 67.95
    },
    {
      "id": "alibaba-qwen3-vl-8b",
      "name": "Qwen3-VL-8B-Instruct",
      "organization": "Alibaba",
      "params": "8B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct",
      "overall": 50.06875,
      "spatial": 67.26,
      "spatio_temporal": 33.12,
      "temporal": 36.98,
      "semantic": 62.92,
      "2d_localization": 59.93,
      "2d_referring_expressions": 73.29,
      "2d_spatial_pointing": 68.56,
      "single_object_tracking": 33.12,
      "temporal_localization": 44.32,
      "dense_video_captioning": 29.64,
      "event_verification": 59.39,
      "video_qa": 66.44
    },
    {
      "id": "alibaba-qwen3-5-vl-27b",
      "name": "Qwen3.5-27B",
      "organization": "Alibaba",
      "params": "27B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-27B-Instruct",
      "overall": 49.25333333,
      "spatial": 79.04,
      "spatio_temporal": 24.4,
      "temporal": 31.93,
      "semantic": 61.65,
      "2d_localization": 85.46,
      "2d_referring_expressions": 76.35,
      "2d_spatial_pointing": 75.32,
      "single_object_tracking": 24.4,
      "temporal_localization": 36.88,
      "dense_video_captioning": 26.97,
      "event_verification": 55.34,
      "video_qa": 67.95
    },
    {
      "id": "alibaba-qwen3-5-vl-9b",
      "name": "Qwen3.5-9B",
      "organization": "Alibaba",
      "params": "9B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-9B-Instruct",
      "overall": 48.48166667,
      "spatial": 75.26,
      "spatio_temporal": 23.52,
      "temporal": 35.95,
      "semantic": 59.21,
      "2d_localization": 76.87,
      "2d_referring_expressions": 77.16,
      "2d_spatial_pointing": 71.74,
      "single_object_tracking": 23.52,
      "temporal_localization": 42.53,
      "dense_video_captioning": 29.36,
      "event_verification": 46.78,
      "video_qa": 71.63
    },
    {
      "id": "google-gemini-3-2-flash-lite",
      "name": "Gemini 3.1 Flash Lite",
      "organization": "Google",
      "params": "—",
      "type": "closed",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://deepmind.google/technologies/gemini/",
      "overall": 51.335,
      "spatial": 62.06,
      "spatio_temporal": 46.68,
      "temporal": 35.17,
      "semantic": 61.43,
      "2d_localization": 69.88,
      "2d_referring_expressions": 52.62,
      "2d_spatial_pointing": 63.68,
      "single_object_tracking": 46.68,
      "temporal_localization": 37.72,
      "dense_video_captioning": 32.62,
      "event_verification": 54.83,
      "video_qa": 68.03
    },
    {
      "id": "nvidia-cosmos-reason2-32b",
      "name": "Cosmos-Reason2-32B",
      "organization": "NVIDIA",
      "params": "32B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-32B",
      "overall": 46.66833333,
      "spatial": 41.36,
      "spatio_temporal": 33.12,
      "temporal": 40.26,
      "semantic": 71.94,
      "2d_localization": 4.35,
      "2d_referring_expressions": 45.71,
      "2d_spatial_pointing": 74.03,
      "single_object_tracking": 33.12,
      "temporal_localization": 50.47,
      "dense_video_captioning": 30.04,
      "event_verification": 73.58,
      "video_qa": 70.29
    },
    {
      "id": "nvidia-cosmos3-super",
      "name": "Cosmos3-Super",
      "organization": "NVIDIA",
      "params": "—",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-05-29",
      "model_url": "https://huggingface.co/nvidia/Cosmos3-Super",
      "overall": 63.01333333,
      "spatial": 78.71,
      "spatio_temporal": 62.25,
      "temporal": 40.72,
      "semantic": 70.37,
      "2d_localization": 86.97,
      "2d_referring_expressions": 76.23,
      "2d_spatial_pointing": 72.94,
      "single_object_tracking": 62.25,
      "temporal_localization": 51.9,
      "dense_video_captioning": 29.54,
      "event_verification": 71.28,
      "video_qa": 69.46
    },
    {
      "id": "nvidia-cosmos3-nano",
      "name": "Cosmos3-Nano",
      "organization": "NVIDIA",
      "params": "—",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-05-29",
      "model_url": "https://huggingface.co/nvidia/Cosmos3-Nano",
      "overall": 60.67291667,
      "spatial": 74.84,
      "spatio_temporal": 59.21,
      "temporal": 39.73,
      "semantic": 68.92,
      "2d_localization": 74.11,
      "2d_referring_expressions": 75.57,
      "2d_spatial_pointing": 74.83,
      "single_object_tracking": 59.21,
      "temporal_localization": 48.04,
      "dense_video_captioning": 31.42,
      "event_verification": 68.88,
      "video_qa": 68.95
    }
  ]
}