| { |
| "protocol": { |
| "benchmark": "https://github.com/opper-ai/jevman-benchmark", |
| "revision": "412ed72484ffeb3bd205a64be2d2ea5e233aac88", |
| "map": "Original fixed map", |
| "max_gameplay_seconds": 300, |
| "parallel_games": 1, |
| "added_latency_ms": 0, |
| "local_hardware": "NVIDIA DGX Spark / GB10", |
| "local_loops": 2, |
| "local_probability_calibration": false, |
| "local_server": "Fresh server per game; startup excluded from gameplay", |
| "hosted_route": "OpenRouter; HTTP latency includes network and provider overhead", |
| "sampling": "Repeated real-time games; not independent maps or paired random seeds", |
| "jev_sampling": "All 12 completed games from two six-game runs using the same protocol", |
| "replay_verification": "All completed replays passed the upstream verifier" |
| }, |
| "conditions": [ |
| { |
| "model": "Decisive Dory", |
| "route": "NVFP4 \u00b7 Eider", |
| "games": [ |
| { |
| "score": 2300, |
| "survived_seconds": 92.66160000000217, |
| "calls": 191, |
| "latency_ms_sum": 30752, |
| "replay_sha256": "ab452665d336a32a32a02f92b1fbe2412afb0107f7d6a664d98a8a6ba8d5afaf" |
| }, |
| { |
| "score": 2040, |
| "survived_seconds": 97.0343000000044, |
| "calls": 212, |
| "latency_ms_sum": 34994, |
| "replay_sha256": "962baf1ab3627b119e0884158bbb7eb9f47285ea3f40c49179f7cdf376239a22" |
| }, |
| { |
| "score": 2830, |
| "survived_seconds": 96.17950000000396, |
| "calls": 195, |
| "latency_ms_sum": 30941, |
| "replay_sha256": "77f0471014575ad5c2f27d9878d71297300a9d5038562951f5f7047526fd43e7" |
| }, |
| { |
| "score": 2260, |
| "survived_seconds": 76.8986000000027, |
| "calls": 150, |
| "latency_ms_sum": 24841, |
| "replay_sha256": "226832364e02185418f46e0b2ae7051fe692f2eb51283c72e2322f42d9a09c15" |
| }, |
| { |
| "score": 2690, |
| "survived_seconds": 81.84360000000173, |
| "calls": 159, |
| "latency_ms_sum": 24517, |
| "replay_sha256": "6f58daea7fa64b0eb14520b820a30ff6fa1ab21b89add0a31649f3e36899b332" |
| }, |
| { |
| "score": 1910, |
| "survived_seconds": 67.23850000000128, |
| "calls": 143, |
| "latency_ms_sum": 25136, |
| "replay_sha256": "fd5b20647dd61a04aa71565e5540e3ac0b8d2f25a8878a742790f3af33271982" |
| } |
| ], |
| "game_count": 6, |
| "mean_score": 2338.3333333333335, |
| "median_score": 2280.0, |
| "mean_decision_ms": 163.02952380952382, |
| "scheduler_errors": 0, |
| "resolved_models": [ |
| "local-dory" |
| ] |
| }, |
| { |
| "model": "Decisive Dory", |
| "route": "Q8_0 \u00b7 llama.cpp", |
| "games": [ |
| { |
| "score": 2220, |
| "survived_seconds": 90.00120000000175, |
| "calls": 151, |
| "latency_ms_sum": 48905, |
| "replay_sha256": "97f4bcc396b43e9e8848bd0093977b531aa8193c1871d3f26917c0ea290acb39" |
| }, |
| { |
| "score": 2310, |
| "survived_seconds": 65.34389999999954, |
| "calls": 114, |
| "latency_ms_sum": 40117, |
| "replay_sha256": "1e9b0d7114f5fa33d5719e8d20bfb89ffdc96d2753e4a128ee31e94797f6ca8f" |
| }, |
| { |
| "score": 2310, |
| "survived_seconds": 64.91600000000018, |
| "calls": 114, |
| "latency_ms_sum": 40300, |
| "replay_sha256": "efc1ee7b3c96469a866e5aadd94708104a76dbc6520f69dc3247a32a03af412f" |
| }, |
| { |
| "score": 2130, |
| "survived_seconds": 58.775600000001006, |
| "calls": 102, |
| "latency_ms_sum": 35172, |
| "replay_sha256": "c2ddfdb40daf0a7ca943d6c012690bef8b5ac49f9589681c7e09e9483ad0d002" |
| }, |
| { |
| "score": 2220, |
| "survived_seconds": 90.04169999999984, |
| "calls": 153, |
| "latency_ms_sum": 51135, |
| "replay_sha256": "c0b76b482c1aabec20cd11b314051537517fedc3012f998995d9a3f0bf0fcd6d" |
| }, |
| { |
| "score": 2980, |
| "survived_seconds": 87.8558000000031, |
| "calls": 152, |
| "latency_ms_sum": 52981, |
| "replay_sha256": "287970b9664eea214c9c28dd092d1a6dc942430eabc8394a2970360bb357fce5" |
| } |
| ], |
| "game_count": 6, |
| "mean_score": 2361.6666666666665, |
| "median_score": 2265.0, |
| "mean_decision_ms": 341.74300254452925, |
| "scheduler_errors": 0, |
| "resolved_models": [ |
| "dory-jevman-q8" |
| ] |
| }, |
| { |
| "model": "Jev 1.14", |
| "route": "OpenRouter", |
| "games": [ |
| { |
| "score": 4770, |
| "survived_seconds": 66.7314000000021, |
| "calls": 135, |
| "latency_ms_sum": 33957, |
| "replay_sha256": "f33346b8d20393805575649c416a2c4343e9d7c4e6011009188dc5c3b055e662" |
| }, |
| { |
| "score": 2460, |
| "survived_seconds": 46.36650000000062, |
| "calls": 95, |
| "latency_ms_sum": 16876, |
| "replay_sha256": "2a295eb5464d7ce30be8323147cd936ce40a85473f3a110e7ff5c5c2960827ce" |
| }, |
| { |
| "score": 2540, |
| "survived_seconds": 45.76800000000065, |
| "calls": 93, |
| "latency_ms_sum": 17012, |
| "replay_sha256": "f8d9967634a0b6a067e158213785c57e51af9ad2763b457e925a28d360110ea5" |
| }, |
| { |
| "score": 6050, |
| "survived_seconds": 105.78640000000006, |
| "calls": 204, |
| "latency_ms_sum": 35635, |
| "replay_sha256": "b299862de3623ebb332c36646d3d202cda701b0d02495dba118141eb050c12d4" |
| }, |
| { |
| "score": 3020, |
| "survived_seconds": 65.42200000000238, |
| "calls": 124, |
| "latency_ms_sum": 22516, |
| "replay_sha256": "817ef7cf8f250fcde6d8ef31ac5c511977cc7822e4c80938fbc5a7f9a607e367" |
| }, |
| { |
| "score": 5170, |
| "survived_seconds": 113.07670000000334, |
| "calls": 211, |
| "latency_ms_sum": 41874, |
| "replay_sha256": "dca22fa9a3b7e7e7adf20fb7350ed04f5b75869e0c7eebecfb9cdef00dc8471e" |
| }, |
| { |
| "score": 3360, |
| "survived_seconds": 65.91620000000175, |
| "calls": 115, |
| "latency_ms_sum": 27609, |
| "replay_sha256": "f3f2a114c9c196317d76daad98ffeb7760f6ebfeedaec2910369274670d9fcb0" |
| }, |
| { |
| "score": 2510, |
| "survived_seconds": 56.38020000000047, |
| "calls": 110, |
| "latency_ms_sum": 20643, |
| "replay_sha256": "405afa6aa1e283518d90e4b4e2105be5e79e9c726d2c9d758431e4d0226c231c" |
| }, |
| { |
| "score": 6610, |
| "survived_seconds": 100.2825000000033, |
| "calls": 196, |
| "latency_ms_sum": 33076, |
| "replay_sha256": "ee10c68d2ef0f7081379508d658bc45ab926cb86a36d5be53105d249ac8f9026" |
| }, |
| { |
| "score": 2400, |
| "survived_seconds": 50.01130000000068, |
| "calls": 95, |
| "latency_ms_sum": 23380, |
| "replay_sha256": "425ff1f1258f15bad585284621e1ae135d6ab50092b0fedace4915bc9a945365" |
| }, |
| { |
| "score": 2190, |
| "survived_seconds": 50.25400000000027, |
| "calls": 88, |
| "latency_ms_sum": 22038, |
| "replay_sha256": "49f635dabfa938af30377099bdaa9e43666402997c8a08153b34ce31d570d325" |
| }, |
| { |
| "score": 3370, |
| "survived_seconds": 78.94970000000323, |
| "calls": 138, |
| "latency_ms_sum": 29063, |
| "replay_sha256": "3cb6f1436d33144467147d9e4ec45fc8d4ce23eb43d571ba3e3cbfc8762ace4a" |
| } |
| ], |
| "game_count": 12, |
| "mean_score": 3704.1666666666665, |
| "median_score": 3190.0, |
| "mean_decision_ms": 201.79488778054863, |
| "scheduler_errors": 4, |
| "resolved_models": [ |
| "typesafe/jev-1.14-20261014" |
| ] |
| } |
| ], |
| "eider_weights_sha256": "62bc20a6fbafaf157154650dfb4e9ec32a19d67e8e1f60ddcc3318fd37d6ccb4", |
| "llama_q8_weights_sha256": "bb36b8ed263ce633649ccfad9c6e1dc065e9c3d333624fc27028f496089a38b8", |
| "base_reference": { |
| "model": "NVIDIA Dory 1.0 4B (unadapted)", |
| "route": "NVFP4 \u00b7 Eider", |
| "games": [ |
| { |
| "score": 1490, |
| "survived_seconds": 48.98480000000033, |
| "calls": 115, |
| "latency_ms_sum": 22525, |
| "replay_sha256": "f06d720ed6fa6881a82cbb58ab0da4446e9553ee620b3d82047872e52577a27c" |
| }, |
| { |
| "score": 680, |
| "survived_seconds": 35.804200000000534, |
| "calls": 82, |
| "latency_ms_sum": 14881, |
| "replay_sha256": "6eae5fd6214d64446a285712a0e4b76a736edf90b9ea78ebedce31fdb3b1b8c1" |
| }, |
| { |
| "score": 1210, |
| "survived_seconds": 36.868400000000015, |
| "calls": 83, |
| "latency_ms_sum": 13773, |
| "replay_sha256": "64a36d3e4c9d8c4be8e20caac33c39478f84ca81190151d9538022ef3e744935" |
| }, |
| { |
| "score": 1080, |
| "survived_seconds": 32.14370000000079, |
| "calls": 62, |
| "latency_ms_sum": 10051, |
| "replay_sha256": "4090ddef29ed6b35294af7153ae6eb46de68a7c2ec60f381c1b5b1ec279c97e7" |
| }, |
| { |
| "score": 1490, |
| "survived_seconds": 42.16530000000059, |
| "calls": 90, |
| "latency_ms_sum": 14641, |
| "replay_sha256": "155a735437f3788abbddee63382697b0b7b0d47ba4ea0aa77e5f07a2f13254d0" |
| }, |
| { |
| "score": 630, |
| "survived_seconds": 33.85650000000001, |
| "calls": 75, |
| "latency_ms_sum": 13615, |
| "replay_sha256": "32499a938cc6265eae1f45f313d5962bdb79d8a07e14ee5bcec4d45766e045ea" |
| } |
| ], |
| "game_count": 6, |
| "mean_score": 1096.6666666666667, |
| "median_score": 1145.0, |
| "mean_decision_ms": 176.5009861932939, |
| "scheduler_errors": 0, |
| "weights_sha256": "2dc8446f2c52d3249eac5bd2154f1f4a8f206152e3b66d7077965b9ad991bbf7", |
| "quantization": "Same ModelOpt recipe and calibration examples as Decisive Dory" |
| } |
| } |
|
|
