| { |
| "benchmarks": { |
| "semantic/cityscapes": { |
| "capability": "2d", |
| "dataset_key": "semantic", |
| "root": "data/raw/cityscapes", |
| "split": "val", |
| "native_resolution": true, |
| "dataset_kwargs": {}, |
| "samples": 500, |
| "source_samples": 500, |
| "dataset_fingerprint": "4800afab564d03887787292933196637cb183d48e4460eae3e71e2d3056649e3", |
| "paper_reference": { |
| "benchmark": "semantic", |
| "dataset": "cityscapes", |
| "split": "val", |
| "metric": "mIoU", |
| "paper": 0.699, |
| "note": "Cityscapes val, 19-class legend including absent classes" |
| }, |
| "decoded_metrics": { |
| "mIoU": 0.21488772332668304, |
| "pixel_accuracy": 0.6165922528089834, |
| "per_class_iou": [ |
| 0.9111851453781128, |
| 0.5919085144996643, |
| 0.0002911773626692593, |
| 0.0245357658714056, |
| 0.14803653955459595, |
| 0.030759338289499283, |
| 0.0013343992177397013, |
| 0.32152560353279114, |
| 0.8069646954536438, |
| 0.18592989444732666, |
| 0.8710127472877502, |
| 0.03289300203323364, |
| 0.15516267716884613, |
| 2.678571945580188e-06, |
| 0.0, |
| 0.0, |
| 9.742333531903569e-06, |
| 0.0, |
| 0.0013146874262019992 |
| ], |
| "valid_pixels": 917018489 |
| }, |
| "merged_shards": { |
| "count": 2, |
| "blocks": [ |
| [ |
| 0, |
| 250 |
| ], |
| [ |
| 250, |
| 500 |
| ] |
| ] |
| } |
| }, |
| "depth/diode": { |
| "capability": "3d", |
| "dataset_key": "diode", |
| "root": "data/raw/diode", |
| "split": "val", |
| "native_resolution": true, |
| "dataset_kwargs": { |
| "scene_type": "indoors" |
| }, |
| "samples": 325, |
| "source_samples": 325, |
| "dataset_fingerprint": "232d580aad6e70e832582574952aae2881d5446df76700a65263699e67b4f7b1", |
| "paper_reference": { |
| "benchmark": "depth", |
| "dataset": "diode", |
| "metric": "delta_1.25", |
| "paper": 0.917, |
| "paper_absrel": 0.108, |
| "note": "DIODE-Indoor" |
| }, |
| "decoded_metrics": { |
| "AbsRel": 2369.7988923507623, |
| "SqRel": 2298440481.7564754, |
| "RMSE": 86640.62652330476, |
| "RMSE_log": 2.0387466503864773, |
| "delta_1.25": 0.07701537999848698, |
| "delta_1.25^2": 0.1616714156651281, |
| "delta_1.25^3": 0.26458824917186297, |
| "valid_pixels": 251430909, |
| "AbsRel_per_image": 2510.145138869721, |
| "RMSE_per_image": 24996.572537966968, |
| "delta_1.25_per_image": 0.07618638903987175, |
| "images": 325, |
| "AbsRel_aligned": 884.0034807925355, |
| "delta_1.25_aligned": 0.5702212775305245, |
| "RMSE_aligned": 121488.23251882072, |
| "aligned_valid_pixels": 250644970, |
| "skipped_samples": 0, |
| "depth_curve": "metric_power", |
| "depth_units": "metres" |
| } |
| }, |
| "depth/eth3d": { |
| "capability": "3d", |
| "dataset_key": "eth3d", |
| "root": "data/raw/eth3d", |
| "split": "val", |
| "native_resolution": true, |
| "dataset_kwargs": {}, |
| "samples": 454, |
| "source_samples": 454, |
| "dataset_fingerprint": "fe5358029536a5dcf622d0263432d53bb5ec737d4805b5078c06760748a840e2", |
| "paper_reference": { |
| "benchmark": "depth", |
| "dataset": "eth3d", |
| "metric": "delta_1.25", |
| "paper": 0.935, |
| "paper_absrel": 0.103 |
| }, |
| "decoded_metrics": { |
| "AbsRel": 2.5183310746559884, |
| "SqRel": 79.38748302880606, |
| "RMSE": 17.986439332820574, |
| "RMSE_log": 1.9709849963989834, |
| "delta_1.25": 0.11925263062829346, |
| "delta_1.25^2": 0.23443484210639565, |
| "delta_1.25^3": 0.33277493130946484, |
| "valid_pixels": 127712646, |
| "AbsRel_per_image": 2.5737888975651515, |
| "RMSE_per_image": 12.462342106002806, |
| "delta_1.25_per_image": 0.1285937691920154, |
| "images": 454, |
| "AbsRel_aligned": 1.9480595149436388, |
| "delta_1.25_aligned": 0.7040436249285035, |
| "RMSE_aligned": 134.9028541449569, |
| "aligned_valid_pixels": 124336433, |
| "skipped_samples": 0, |
| "depth_curve": "metric_power", |
| "depth_units": "metres" |
| }, |
| "merged_shards": { |
| "count": 2, |
| "blocks": [ |
| [ |
| 0, |
| 227 |
| ], |
| [ |
| 227, |
| 454 |
| ] |
| ] |
| } |
| }, |
| "depth/nyu": { |
| "capability": "3d", |
| "dataset_key": "nyu_depth", |
| "root": "data/raw/nyu_depth_v2", |
| "split": "val", |
| "native_resolution": true, |
| "dataset_kwargs": {}, |
| "samples": 654, |
| "source_samples": 654, |
| "dataset_fingerprint": "7b97e1125ad9f4322e0bec7a2c6a354dc809ab2f3ed2451b2c68d9c8f7ecb953", |
| "paper_reference": { |
| "benchmark": "depth", |
| "dataset": "nyu", |
| "metric": "delta_1.25", |
| "paper": 0.948, |
| "paper_absrel": 0.081 |
| }, |
| "decoded_metrics": { |
| "AbsRel": 1.722833826843339, |
| "SqRel": 9.855502423959873, |
| "RMSE": 4.574520489150392, |
| "RMSE_log": 1.6363381378304396, |
| "delta_1.25": 0.0723108977514424, |
| "delta_1.25^2": 0.16879996851650167, |
| "delta_1.25^3": 0.30323655105967096, |
| "valid_pixels": 156018240, |
| "AbsRel_per_image": 1.7228337658928075, |
| "RMSE_per_image": 4.180940210636967, |
| "delta_1.25_per_image": 0.07231089552970879, |
| "images": 654, |
| "AbsRel_aligned": 2.283583821095263, |
| "delta_1.25_aligned": 0.6566354722876154, |
| "RMSE_aligned": 24.507983961218898, |
| "aligned_valid_pixels": 154348320, |
| "skipped_samples": 0, |
| "depth_curve": "metric_power", |
| "depth_units": "metres" |
| }, |
| "merged_shards": { |
| "count": 2, |
| "blocks": [ |
| [ |
| 0, |
| 327 |
| ], |
| [ |
| 327, |
| 654 |
| ] |
| ] |
| } |
| }, |
| "referring/reasonseg": { |
| "capability": "2d", |
| "dataset_key": "reasonseg", |
| "root": "data/raw/reasonseg", |
| "split": "val", |
| "native_resolution": true, |
| "dataset_kwargs": {}, |
| "samples": 200, |
| "source_samples": 200, |
| "dataset_fingerprint": "0506037a629f6ca966100ce63f2aa1df9d56e192228e9741cbb0c9d5402fac84", |
| "paper_reference": { |
| "benchmark": "referring", |
| "dataset": "reasonseg", |
| "split": "val", |
| "metric": "gIoU", |
| "paper": 0.793, |
| "note": "ReasonSeg val; paper rewrites queries with Gemini 2.5 Pro" |
| }, |
| "decoded_metrics": { |
| "IoU": 0.2543628777739729, |
| "cIoU": 0.1731907588495319, |
| "gIoU": 0.2543628777739729, |
| "Precision@0.5": 0.18, |
| "Precision": 0.17935487757824095, |
| "Recall": 0.8344168456960918, |
| "F-score": 0.29524739696955726, |
| "samples": 200 |
| } |
| }, |
| "referring/refcocog": { |
| "capability": "2d", |
| "dataset_key": "referring", |
| "root": "data/raw/refer/refcocog", |
| "split": "val", |
| "native_resolution": true, |
| "dataset_kwargs": { |
| "protocol": "umd", |
| "strict": true |
| }, |
| "samples": 4896, |
| "source_samples": 4896, |
| "dataset_fingerprint": "e570d18c8f6517ec0a39616a19a91c568ca7f8b655055180a462c6e28fe5b298", |
| "paper_reference": { |
| "benchmark": "referring", |
| "dataset": "refcocog", |
| "split": "val", |
| "metric": "cIoU", |
| "paper": 0.738, |
| "note": "RefCOCOg UMD val" |
| }, |
| "decoded_metrics": { |
| "IoU": 0.5253263977825223, |
| "cIoU": 0.37619617476788464, |
| "gIoU": 0.5253263977825223, |
| "Precision@0.5": 0.5526960784313726, |
| "Precision": 0.39575273335307415, |
| "Recall": 0.8838940809294463, |
| "F-score": 0.5467188205654409, |
| "samples": 4896 |
| }, |
| "merged_shards": { |
| "count": 10, |
| "blocks": [ |
| [ |
| 0, |
| 490 |
| ], |
| [ |
| 490, |
| 980 |
| ], |
| [ |
| 980, |
| 1470 |
| ], |
| [ |
| 1470, |
| 1960 |
| ], |
| [ |
| 1960, |
| 2450 |
| ], |
| [ |
| 2450, |
| 2940 |
| ], |
| [ |
| 2940, |
| 3430 |
| ], |
| [ |
| 3430, |
| 3920 |
| ], |
| [ |
| 3920, |
| 4410 |
| ], |
| [ |
| 4410, |
| 4896 |
| ] |
| ] |
| } |
| } |
| } |
| } |
