| { |
| "metadata": { |
| "evaluation_mode": "decoded_task_metrics", |
| "model_name": "qwen-image-2.1", |
| "lora_path": null, |
| "cpu_offload": false, |
| "quantize": null, |
| "device": "cuda", |
| "dtype": "bfloat16", |
| "resolution": 512, |
| "batch_size": 1, |
| "num_inference_steps": 50, |
| "strength": 0.85, |
| "seed": 42, |
| "skipped_benchmarks": [], |
| "python": "3.11.13", |
| "torch": "2.7.1+cu126", |
| "capability": "depth", |
| "paper_capabilities": { |
| "2d": { |
| "cityscapes": { |
| "benchmark": "semantic", |
| "dataset": "cityscapes", |
| "split": "val", |
| "metric": "mIoU", |
| "paper": 0.699, |
| "note": "Cityscapes val, 19-class legend including absent classes" |
| }, |
| "refcocog": { |
| "benchmark": "referring", |
| "dataset": "refcocog", |
| "split": "val", |
| "metric": "cIoU", |
| "paper": 0.738, |
| "note": "RefCOCOg UMD val" |
| }, |
| "reasonseg": { |
| "benchmark": "referring", |
| "dataset": "reasonseg", |
| "split": "val", |
| "metric": "gIoU", |
| "paper": 0.793, |
| "note": "ReasonSeg val; paper rewrites queries with Gemini 2.5 Pro" |
| }, |
| "saco": { |
| "benchmark": "instance", |
| "dataset": "saco", |
| "split": "val", |
| "metric": "cgF1", |
| "paper": 47.5, |
| "note": "SA-Co/Gold; paper filters NPs with Gemini 3.1 Flash-Lite" |
| } |
| }, |
| "3d": { |
| "nyu": { |
| "benchmark": "depth", |
| "dataset": "nyu", |
| "metric": "delta_1.25", |
| "paper": 0.948, |
| "paper_absrel": 0.081 |
| }, |
| "ibims": { |
| "benchmark": "depth", |
| "dataset": "ibims", |
| "metric": "delta_1.25", |
| "paper": 0.934, |
| "paper_absrel": 0.078 |
| }, |
| "eth3d": { |
| "benchmark": "depth", |
| "dataset": "eth3d", |
| "metric": "delta_1.25", |
| "paper": 0.935, |
| "paper_absrel": 0.103 |
| }, |
| "diode": { |
| "benchmark": "depth", |
| "dataset": "diode", |
| "metric": "delta_1.25", |
| "paper": 0.917, |
| "paper_absrel": 0.108, |
| "note": "DIODE-Indoor" |
| }, |
| "kitti": { |
| "benchmark": "depth", |
| "dataset": "kitti", |
| "metric": "delta_1.25", |
| "paper": 0.915, |
| "paper_absrel": 0.107 |
| }, |
| "nuscenes": { |
| "benchmark": "depth", |
| "dataset": "nuscenes", |
| "metric": "delta_1.25", |
| "paper": 0.643, |
| "paper_absrel": 0.219, |
| "note": "80 m cap; paper does not disclose a crop" |
| } |
| }, |
| "generation": { |
| "genai_bench": { |
| "benchmark": "generation", |
| "dataset": "genai_bench", |
| "metric": "human_win_rate", |
| "paper": 0.535, |
| "note": "GenAI-Bench T2I human pairwise; CLIP/FID are proxies only" |
| }, |
| "imgedit": { |
| "benchmark": "generation", |
| "dataset": "imgedit", |
| "metric": "human_win_rate", |
| "paper": 0.478, |
| "note": "ImgEdit human pairwise; not a reconstruction score" |
| } |
| } |
| }, |
| "depth_colormap": "cube_edge", |
| "depth_curve": "metric_power", |
| "depth_log_alpha": 0.4, |
| "invert_depth": false, |
| "instance_decoding": { |
| "color_tolerance": 14.0, |
| "min_area_fraction": 0.0002, |
| "erosion_threshold": 0.1, |
| "bbox_expansion_factor": 5.0, |
| "grouping_mode": "seed", |
| "adaptive_color_tolerance": false, |
| "seed_merge_rule": "pixel_only", |
| "spatial_merge_area_basis": "bbox" |
| }, |
| "depth_prompt": null, |
| "relative_luminance": false, |
| "inference_protocol": { |
| "num_inference_steps": 50, |
| "guidance_scale": 4.0, |
| "strength": 0.85, |
| "seed": 42, |
| "canvas_resolution": 512, |
| "native_resolution_scoring": true, |
| "note": "The paper does not state inference settings; these are the repository values for this run (exp_setting.md E1.3). num_inference_steps=1 is a smoke setting and must not be reported as a result." |
| }, |
| "protocol": { |
| "depth": "decoded RGB to metric depth with curve=metric_power (paper Eq.1); log_relative is an opt-in relative ablation and must not be cited as paper AbsRel; valid GT pixels only; NYU Eigen crop + 10m cap / KITTI Garg crop + 80m cap / ETH3D 80m cap / nuScenes 80m cap (no paper crop) when those datasets are named; AbsRel_aligned/delta_1.25_aligned are per-image median-scale diagnostics, not paper protocol; *_per_image keys are means of per-image metrics (headline keys are pixel-weighted); crop/cap/aggregation are conventions, not paper-disclosed", |
| "normal": "decoded RGB to unit camera-space normals; valid pixels are finite, non-zero GT (plus DIODE mask when present); mean_angle_deg_per_image is the mean of per-image means; NYU/ScanNet require community normal maps; VKITTI2 ships no normals, so VirtualKITTINormalDataset derives them from its centimetre depth + intrinsic.txt in the paper frame (used only to build GT, never as model input); DIODE is indoors", |
| "semantic": "decoded RGB to class labels; Cityscapes labelIds mapped to trainIds; global confusion matrix; official gtFine only, never cityscapes8; with --native-resolution the canvas is decoded first and the label map nearest-upsampled to the 2048x1024 GT (interpolating RGB would blend boundary colours into phantom classes)", |
| "referring": "RefCOCOg requires refs(umd).p with no protocol fallback and validates images, expressions, and nonempty native COCO polygon/RLE masks using pycocotools; ReasonSeg val uses LISA JSON-next-to-image; every sentence/query is a sample; cIoU/gIoU from masks, never from bounding boxes; paper ReasonSeg gIoU includes a Gemini 2.5 Pro rewrite that this command does not run unless rewritten_text is already in the JSON", |
| "instance": "SA-Co/Gold query JSONL when present, else color/instance pairs; cgF1 = 100 * pmF1 * IL_MCC; paper IL_MCC uses Gemini 3.1 Flash-Lite presence filtering, which this command does not invent", |
| "generation": "paper GenAI-Bench / ImgEdit numbers are human pairwise win rates; this command only scores an optional CLIP/FID proxy when images exist and never writes those proxies into the paper win-rate column", |
| "aggregation": "pixel-weighted across the complete selected dataset, not batch mean" |
| } |
| }, |
| "preflight": { |
| "ok": true, |
| "jobs": [ |
| { |
| "benchmark": "depth", |
| "dataset": "eth3d", |
| "dataset_key": "eth3d", |
| "root": "data/raw/eth3d", |
| "split": "val", |
| "ok": true, |
| "samples": 454, |
| "dataset_fingerprint": "fe5358029536a5dcf622d0263432d53bb5ec737d4805b5078c06760748a840e2" |
| } |
| ], |
| "skipped": [] |
| }, |
| "benchmarks": { |
| "depth/eth3d": { |
| "capability": "3d", |
| "dataset_key": "eth3d", |
| "root": "data/raw/eth3d", |
| "split": "val", |
| "native_resolution": true, |
| "dataset_kwargs": {}, |
| "samples": 454, |
| "source_samples": 454, |
| "dataset_fingerprint": "fe5358029536a5dcf622d0263432d53bb5ec737d4805b5078c06760748a840e2", |
| "paper_reference": { |
| "benchmark": "depth", |
| "dataset": "eth3d", |
| "metric": "delta_1.25", |
| "paper": 0.935, |
| "paper_absrel": 0.103 |
| }, |
| "decoded_metrics": { |
| "AbsRel": 2.5183310746559884, |
| "SqRel": 79.38748302880606, |
| "RMSE": 17.986439332820574, |
| "RMSE_log": 1.9709849963989834, |
| "delta_1.25": 0.11925263062829346, |
| "delta_1.25^2": 0.23443484210639565, |
| "delta_1.25^3": 0.33277493130946484, |
| "valid_pixels": 127712646, |
| "AbsRel_per_image": 2.5737888975651515, |
| "RMSE_per_image": 12.462342106002806, |
| "delta_1.25_per_image": 0.1285937691920154, |
| "images": 454, |
| "AbsRel_aligned": 1.9480595149436388, |
| "delta_1.25_aligned": 0.7040436249285035, |
| "RMSE_aligned": 134.9028541449569, |
| "aligned_valid_pixels": 124336433, |
| "skipped_samples": 0, |
| "depth_curve": "metric_power", |
| "depth_units": "metres" |
| }, |
| "merged_shards": { |
| "count": 2, |
| "blocks": [ |
| [ |
| 0, |
| 227 |
| ], |
| [ |
| 227, |
| 454 |
| ] |
| ] |
| } |
| } |
| } |
| } |
