File size: 2,646 Bytes
56baf43 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | {
"schema_version": "medmlx.reference.v1",
"model": "nv-reason-ct",
"date": "2026-10-12",
"case": {
"dataset": "LUNA16 subset0",
"case_id": "1.3.6.1.4.1.14519.5.2.1.6279.6001.105756658031515062000744821260",
"input_sha256": "5a7c3a4e681dcba5585bb49e2f355cbc08311b2944614003966409f9b6a71dfe"
},
"original": {
"code": "nvidia/NV-Reason-CT (Hugging Face remote model.py)",
"version_or_commit": "386b93e034983f6c1fc841a43833a1b6a0cd9c13",
"device": "cpu",
"precision": "FP32",
"checkpoint_sha256": "655c15e59dfa1957f7d5f475b1702d689fb7107160f21256028391bb5f6e6ba5",
"command": "PYTHONDONTWRITEBYTECODE=1 work/reference/nv-reason-ct/.venv/bin/python benchmarks/reference/nv-reason-ct.py",
"mode": "greedy_generation",
"duration_s": 1244.5534047500114,
"environment": {
"torch": "2.13.0",
"transformers": "5.6.2",
"dynamic-network-architectures": "0.4.3",
"timm": "1.0.22",
"monai": "1.6.0rc1",
"numpy": "2.3.5",
"nibabel": "5.4.2"
}
},
"mlx": {
"run_record": "results/runs/nv-reason-ct.json",
"output_path": "<paper-repo>/work/runs/nv-reason-ct/out/run.json",
"output_sha256": "a9af3ecaf7d50ee2669a10699a773bc96274cef4aa073885eb6fcf2ad6b20fb9",
"report": {
"path": "<paper-repo>/work/runs/nv-reason-ct/out/report.txt",
"sha256": "06d5943d8465cbda9743273566eb8ac802ebf44e0621f08a83f7eb78977bc282"
},
"preprocessing_revision": "eb2c10baf688b32eee082dbb8973e88b4555db7a"
},
"comparison": {
"metric": "token_identical",
"value": true,
"threshold": true,
"result": "pass",
"first_divergence": null,
"original_generated_tokens": 208,
"mlx_generated_tokens": 208,
"original_terminated_by_eos": true,
"original_truncated_by_max_new_tokens": false
},
"preprocessing": {
"max_abs_diff": 9.375810623168945e-05,
"prompt_tokens_identical": true,
"pixel_input": "recorded MLX revision's deterministic host preprocessing"
},
"notes": "Same SHA-256 verified LUNA16 NIfTI and original model.safetensors. Source snapshot relocated from radnn to research/radnn; source/model/template/tokenizer files match the recorded pins. Same prompt: write a structured chest CT report. Thinking off, greedy, max_new_tokens 512, FP32 CPU, SDPA, cache enabled. Original processor/template and prompt tokens are checked against recorded native host preprocessing; the identical recorded host-preprocessed voxels are explicitly supplied to the original network. Saved MLX output tokens come directly from its hash-verified run.json. No MLX generation rerun.",
"failure": null
}
|