{
 "benchmarks": {
  "semantic/cityscapes": {
   "capability": "2d",
   "dataset_key": "semantic",
   "root": "data/raw/cityscapes",
   "split": "val",
   "native_resolution": true,
   "dataset_kwargs": {},
   "samples": 500,
   "source_samples": 500,
   "dataset_fingerprint": "4800afab564d03887787292933196637cb183d48e4460eae3e71e2d3056649e3",
   "paper_reference": {
    "benchmark": "semantic",
    "dataset": "cityscapes",
    "split": "val",
    "metric": "mIoU",
    "paper": 0.699,
    "note": "Cityscapes val, 19-class legend including absent classes"
   },
   "decoded_metrics": {
    "mIoU": 0.21488772332668304,
    "pixel_accuracy": 0.6165922528089834,
    "per_class_iou": [
     0.9111851453781128,
     0.5919085144996643,
     0.0002911773626692593,
     0.0245357658714056,
     0.14803653955459595,
     0.030759338289499283,
     0.0013343992177397013,
     0.32152560353279114,
     0.8069646954536438,
     0.18592989444732666,
     0.8710127472877502,
     0.03289300203323364,
     0.15516267716884613,
     2.678571945580188e-06,
     0.0,
     0.0,
     9.742333531903569e-06,
     0.0,
     0.0013146874262019992
    ],
    "valid_pixels": 917018489
   },
   "merged_shards": {
    "count": 2,
    "blocks": [
     [
      0,
      250
     ],
     [
      250,
      500
     ]
    ]
   }
  },
  "depth/diode": {
   "capability": "3d",
   "dataset_key": "diode",
   "root": "data/raw/diode",
   "split": "val",
   "native_resolution": true,
   "dataset_kwargs": {
    "scene_type": "indoors"
   },
   "samples": 325,
   "source_samples": 325,
   "dataset_fingerprint": "232d580aad6e70e832582574952aae2881d5446df76700a65263699e67b4f7b1",
   "paper_reference": {
    "benchmark": "depth",
    "dataset": "diode",
    "metric": "delta_1.25",
    "paper": 0.917,
    "paper_absrel": 0.108,
    "note": "DIODE-Indoor"
   },
   "decoded_metrics": {
    "AbsRel": 2369.7988923507623,
    "SqRel": 2298440481.7564754,
    "RMSE": 86640.62652330476,
    "RMSE_log": 2.0387466503864773,
    "delta_1.25": 0.07701537999848698,
    "delta_1.25^2": 0.1616714156651281,
    "delta_1.25^3": 0.26458824917186297,
    "valid_pixels": 251430909,
    "AbsRel_per_image": 2510.145138869721,
    "RMSE_per_image": 24996.572537966968,
    "delta_1.25_per_image": 0.07618638903987175,
    "images": 325,
    "AbsRel_aligned": 884.0034807925355,
    "delta_1.25_aligned": 0.5702212775305245,
    "RMSE_aligned": 121488.23251882072,
    "aligned_valid_pixels": 250644970,
    "skipped_samples": 0,
    "depth_curve": "metric_power",
    "depth_units": "metres"
   }
  },
  "depth/eth3d": {
   "capability": "3d",
   "dataset_key": "eth3d",
   "root": "data/raw/eth3d",
   "split": "val",
   "native_resolution": true,
   "dataset_kwargs": {},
   "samples": 454,
   "source_samples": 454,
   "dataset_fingerprint": "fe5358029536a5dcf622d0263432d53bb5ec737d4805b5078c06760748a840e2",
   "paper_reference": {
    "benchmark": "depth",
    "dataset": "eth3d",
    "metric": "delta_1.25",
    "paper": 0.935,
    "paper_absrel": 0.103
   },
   "decoded_metrics": {
    "AbsRel": 2.5183310746559884,
    "SqRel": 79.38748302880606,
    "RMSE": 17.986439332820574,
    "RMSE_log": 1.9709849963989834,
    "delta_1.25": 0.11925263062829346,
    "delta_1.25^2": 0.23443484210639565,
    "delta_1.25^3": 0.33277493130946484,
    "valid_pixels": 127712646,
    "AbsRel_per_image": 2.5737888975651515,
    "RMSE_per_image": 12.462342106002806,
    "delta_1.25_per_image": 0.1285937691920154,
    "images": 454,
    "AbsRel_aligned": 1.9480595149436388,
    "delta_1.25_aligned": 0.7040436249285035,
    "RMSE_aligned": 134.9028541449569,
    "aligned_valid_pixels": 124336433,
    "skipped_samples": 0,
    "depth_curve": "metric_power",
    "depth_units": "metres"
   },
   "merged_shards": {
    "count": 2,
    "blocks": [
     [
      0,
      227
     ],
     [
      227,
      454
     ]
    ]
   }
  },
  "depth/nyu": {
   "capability": "3d",
   "dataset_key": "nyu_depth",
   "root": "data/raw/nyu_depth_v2",
   "split": "val",
   "native_resolution": true,
   "dataset_kwargs": {},
   "samples": 654,
   "source_samples": 654,
   "dataset_fingerprint": "7b97e1125ad9f4322e0bec7a2c6a354dc809ab2f3ed2451b2c68d9c8f7ecb953",
   "paper_reference": {
    "benchmark": "depth",
    "dataset": "nyu",
    "metric": "delta_1.25",
    "paper": 0.948,
    "paper_absrel": 0.081
   },
   "decoded_metrics": {
    "AbsRel": 1.722833826843339,
    "SqRel": 9.855502423959873,
    "RMSE": 4.574520489150392,
    "RMSE_log": 1.6363381378304396,
    "delta_1.25": 0.0723108977514424,
    "delta_1.25^2": 0.16879996851650167,
    "delta_1.25^3": 0.30323655105967096,
    "valid_pixels": 156018240,
    "AbsRel_per_image": 1.7228337658928075,
    "RMSE_per_image": 4.180940210636967,
    "delta_1.25_per_image": 0.07231089552970879,
    "images": 654,
    "AbsRel_aligned": 2.283583821095263,
    "delta_1.25_aligned": 0.6566354722876154,
    "RMSE_aligned": 24.507983961218898,
    "aligned_valid_pixels": 154348320,
    "skipped_samples": 0,
    "depth_curve": "metric_power",
    "depth_units": "metres"
   },
   "merged_shards": {
    "count": 2,
    "blocks": [
     [
      0,
      327
     ],
     [
      327,
      654
     ]
    ]
   }
  },
  "referring/reasonseg": {
   "capability": "2d",
   "dataset_key": "reasonseg",
   "root": "data/raw/reasonseg",
   "split": "val",
   "native_resolution": true,
   "dataset_kwargs": {},
   "samples": 200,
   "source_samples": 200,
   "dataset_fingerprint": "0506037a629f6ca966100ce63f2aa1df9d56e192228e9741cbb0c9d5402fac84",
   "paper_reference": {
    "benchmark": "referring",
    "dataset": "reasonseg",
    "split": "val",
    "metric": "gIoU",
    "paper": 0.793,
    "note": "ReasonSeg val; paper rewrites queries with Gemini 2.5 Pro"
   },
   "decoded_metrics": {
    "IoU": 0.2543628777739729,
    "cIoU": 0.1731907588495319,
    "gIoU": 0.2543628777739729,
    "Precision@0.5": 0.18,
    "Precision": 0.17935487757824095,
    "Recall": 0.8344168456960918,
    "F-score": 0.29524739696955726,
    "samples": 200
   }
  },
  "referring/refcocog": {
   "capability": "2d",
   "dataset_key": "referring",
   "root": "data/raw/refer/refcocog",
   "split": "val",
   "native_resolution": true,
   "dataset_kwargs": {
    "protocol": "umd",
    "strict": true
   },
   "samples": 4896,
   "source_samples": 4896,
   "dataset_fingerprint": "e570d18c8f6517ec0a39616a19a91c568ca7f8b655055180a462c6e28fe5b298",
   "paper_reference": {
    "benchmark": "referring",
    "dataset": "refcocog",
    "split": "val",
    "metric": "cIoU",
    "paper": 0.738,
    "note": "RefCOCOg UMD val"
   },
   "decoded_metrics": {
    "IoU": 0.5253263977825223,
    "cIoU": 0.37619617476788464,
    "gIoU": 0.5253263977825223,
    "Precision@0.5": 0.5526960784313726,
    "Precision": 0.39575273335307415,
    "Recall": 0.8838940809294463,
    "F-score": 0.5467188205654409,
    "samples": 4896
   },
   "merged_shards": {
    "count": 10,
    "blocks": [
     [
      0,
      490
     ],
     [
      490,
      980
     ],
     [
      980,
      1470
     ],
     [
      1470,
      1960
     ],
     [
      1960,
      2450
     ],
     [
      2450,
      2940
     ],
     [
      2940,
      3430
     ],
     [
      3430,
      3920
     ],
     [
      3920,
      4410
     ],
     [
      4410,
      4896
     ]
    ]
   }
  }
 }
}