| { |
| "schema": "dory-llama-q8-comparison/v1", |
| "test_data_sha256": "e2ca7413a306fc2b0e9e044aa685c94d5aa8e0def7cfcec6448941b7369ae625", |
| "server_sha256": "577c2b294fbf44483b3999032f6db11743c42dc0d5420f2b07f8e6cf76665ccb", |
| "llama_revision": "3834b4ae1ee6f63b8bf314a07dfb9c45407975b9", |
| "local_llama_changes": "", |
| "quantization": "Q8_0", |
| "device": "NVIDIA DGX Spark / GB10", |
| "loops": 2, |
| "calibrated": false, |
| "same_prompt_and_labels_as_eider": true, |
| "base": { |
| "questions": 512, |
| "modal_accuracy": 0.384765625, |
| "brier": 0.6735945644865479, |
| "median_request_ms": 108.9351789560169, |
| "mean_request_ms": 133.98210436844238, |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.4921875, |
| "brier": 0.3753410996249456, |
| "median_request_ms": 210.46100556850433, |
| "mean_request_ms": 206.47532288785442 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.3802083333333333, |
| "brier": 0.650590381539687, |
| "median_request_ms": 111.42371664755046, |
| "mean_request_ms": 118.42679404556596 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.3177083333333333, |
| "brier": 0.8954343906744768, |
| "median_request_ms": 102.34942147508264, |
| "mean_request_ms": 101.2086023450441 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.2863070539419087, |
| "brier": 0.7614253393748526, |
| "median_request_ms": 109.7046872600913, |
| "mean_request_ms": 119.73183251206311 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.5106382978723404, |
| "brier": 0.6795592626548467, |
| "median_request_ms": 101.0505030862987, |
| "mean_request_ms": 96.79506232149582 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.4519774011299435, |
| "brier": 0.5508380764871043, |
| "median_request_ms": 132.75975128635764, |
| "mean_request_ms": 173.13406747465922 |
| } |
| }, |
| "model_sha256": "b04185f003dff2c18097b565427b8c0e7be3e203e56cbad084069aa887042bad", |
| "loops": 2, |
| "prompt_parity_questions": 512, |
| "command": [ |
| "llama-server", |
| "--model", |
| "dory-base-Q8_0.gguf", |
| "--alias", |
| "dory-gguf-eval", |
| "--host", |
| "127.0.0.1", |
| "--port", |
| "18091", |
| "--ctx-size", |
| "4096", |
| "--parallel", |
| "1", |
| "--threads", |
| "4", |
| "--threads-batch", |
| "4", |
| "--batch-size", |
| "512", |
| "--ubatch-size", |
| "512", |
| "--fit", |
| "off", |
| "--gpu-layers", |
| "all", |
| "--no-warmup" |
| ] |
| }, |
| "fine_tuned": { |
| "questions": 512, |
| "modal_accuracy": 0.77734375, |
| "brier": 0.2599162990600157, |
| "median_request_ms": 109.77665497921407, |
| "mean_request_ms": 135.0687600670426, |
| "by_source": { |
| "helpsteer2": { |
| "questions": 128, |
| "modal_accuracy": 0.515625, |
| "brier": 0.23377195573417558, |
| "median_request_ms": 211.45723038353026, |
| "mean_request_ms": 208.28389131929725 |
| }, |
| "planning": { |
| "questions": 192, |
| "modal_accuracy": 0.96875, |
| "brier": 0.14051244957828152, |
| "median_request_ms": 112.26724414154887, |
| "mean_request_ms": 119.26124551003643 |
| }, |
| "scone": { |
| "questions": 192, |
| "modal_accuracy": 0.7604166666666666, |
| "brier": 0.3967497107589765, |
| "median_request_ms": 103.08765154331923, |
| "mean_request_ms": 102.06618712254567 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "questions": 241, |
| "modal_accuracy": 0.8257261410788381, |
| "brier": 0.3344495604077365, |
| "median_request_ms": 110.2855778299272, |
| "mean_request_ms": 120.80057657659981 |
| }, |
| "noul": { |
| "questions": 94, |
| "modal_accuracy": 0.9680851063829787, |
| "brier": 0.07220308279158415, |
| "median_request_ms": 101.73197486437857, |
| "mean_request_ms": 97.29551964815944 |
| }, |
| "score": { |
| "questions": 177, |
| "modal_accuracy": 0.6101694915254238, |
| "brier": 0.2581226625878792, |
| "median_request_ms": 134.17465519160032, |
| "mean_request_ms": 174.55642571999024 |
| } |
| }, |
| "model_sha256": "bb36b8ed263ce633649ccfad9c6e1dc065e9c3d333624fc27028f496089a38b8", |
| "loops": 2, |
| "prompt_parity_questions": 512, |
| "command": [ |
| "llama-server", |
| "--model", |
| "dory-decisions-Q8_0.gguf", |
| "--alias", |
| "dory-gguf-eval", |
| "--host", |
| "127.0.0.1", |
| "--port", |
| "18091", |
| "--ctx-size", |
| "4096", |
| "--parallel", |
| "1", |
| "--threads", |
| "4", |
| "--threads-batch", |
| "4", |
| "--batch-size", |
| "512", |
| "--ubatch-size", |
| "512", |
| "--fit", |
| "off", |
| "--gpu-layers", |
| "all", |
| "--no-warmup" |
| ] |
| }, |
| "cross_entropy_omitted": "The native API returns binary probability rather than raw logits; subtraction can round the complement to zero.", |
| "base_model": "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16", |
| "base_model_revision": "b366b87f762ea334ba3eb55912031baba1515810", |
| "description": "Same 512 questions as the NVFP4 comparison; native /v1/systemone endpoint, no source patches, no calibration, no generated reasoning." |
| } |
|
|
