{
  "schema": "dory-llama-q8-comparison/v1",
  "test_data_sha256": "e2ca7413a306fc2b0e9e044aa685c94d5aa8e0def7cfcec6448941b7369ae625",
  "server_sha256": "577c2b294fbf44483b3999032f6db11743c42dc0d5420f2b07f8e6cf76665ccb",
  "llama_revision": "3834b4ae1ee6f63b8bf314a07dfb9c45407975b9",
  "local_llama_changes": "",
  "quantization": "Q8_0",
  "device": "NVIDIA DGX Spark / GB10",
  "loops": 2,
  "calibrated": false,
  "same_prompt_and_labels_as_eider": true,
  "base": {
    "questions": 512,
    "modal_accuracy": 0.384765625,
    "brier": 0.6735945644865479,
    "median_request_ms": 108.9351789560169,
    "mean_request_ms": 133.98210436844238,
    "by_source": {
      "helpsteer2": {
        "questions": 128,
        "modal_accuracy": 0.4921875,
        "brier": 0.3753410996249456,
        "median_request_ms": 210.46100556850433,
        "mean_request_ms": 206.47532288785442
      },
      "planning": {
        "questions": 192,
        "modal_accuracy": 0.3802083333333333,
        "brier": 0.650590381539687,
        "median_request_ms": 111.42371664755046,
        "mean_request_ms": 118.42679404556596
      },
      "scone": {
        "questions": 192,
        "modal_accuracy": 0.3177083333333333,
        "brier": 0.8954343906744768,
        "median_request_ms": 102.34942147508264,
        "mean_request_ms": 101.2086023450441
      }
    },
    "by_kind": {
      "choice": {
        "questions": 241,
        "modal_accuracy": 0.2863070539419087,
        "brier": 0.7614253393748526,
        "median_request_ms": 109.7046872600913,
        "mean_request_ms": 119.73183251206311
      },
      "noul": {
        "questions": 94,
        "modal_accuracy": 0.5106382978723404,
        "brier": 0.6795592626548467,
        "median_request_ms": 101.0505030862987,
        "mean_request_ms": 96.79506232149582
      },
      "score": {
        "questions": 177,
        "modal_accuracy": 0.4519774011299435,
        "brier": 0.5508380764871043,
        "median_request_ms": 132.75975128635764,
        "mean_request_ms": 173.13406747465922
      }
    },
    "model_sha256": "b04185f003dff2c18097b565427b8c0e7be3e203e56cbad084069aa887042bad",
    "loops": 2,
    "prompt_parity_questions": 512,
    "command": [
      "llama-server",
      "--model",
      "dory-base-Q8_0.gguf",
      "--alias",
      "dory-gguf-eval",
      "--host",
      "127.0.0.1",
      "--port",
      "18091",
      "--ctx-size",
      "4096",
      "--parallel",
      "1",
      "--threads",
      "4",
      "--threads-batch",
      "4",
      "--batch-size",
      "512",
      "--ubatch-size",
      "512",
      "--fit",
      "off",
      "--gpu-layers",
      "all",
      "--no-warmup"
    ]
  },
  "fine_tuned": {
    "questions": 512,
    "modal_accuracy": 0.77734375,
    "brier": 0.2599162990600157,
    "median_request_ms": 109.77665497921407,
    "mean_request_ms": 135.0687600670426,
    "by_source": {
      "helpsteer2": {
        "questions": 128,
        "modal_accuracy": 0.515625,
        "brier": 0.23377195573417558,
        "median_request_ms": 211.45723038353026,
        "mean_request_ms": 208.28389131929725
      },
      "planning": {
        "questions": 192,
        "modal_accuracy": 0.96875,
        "brier": 0.14051244957828152,
        "median_request_ms": 112.26724414154887,
        "mean_request_ms": 119.26124551003643
      },
      "scone": {
        "questions": 192,
        "modal_accuracy": 0.7604166666666666,
        "brier": 0.3967497107589765,
        "median_request_ms": 103.08765154331923,
        "mean_request_ms": 102.06618712254567
      }
    },
    "by_kind": {
      "choice": {
        "questions": 241,
        "modal_accuracy": 0.8257261410788381,
        "brier": 0.3344495604077365,
        "median_request_ms": 110.2855778299272,
        "mean_request_ms": 120.80057657659981
      },
      "noul": {
        "questions": 94,
        "modal_accuracy": 0.9680851063829787,
        "brier": 0.07220308279158415,
        "median_request_ms": 101.73197486437857,
        "mean_request_ms": 97.29551964815944
      },
      "score": {
        "questions": 177,
        "modal_accuracy": 0.6101694915254238,
        "brier": 0.2581226625878792,
        "median_request_ms": 134.17465519160032,
        "mean_request_ms": 174.55642571999024
      }
    },
    "model_sha256": "bb36b8ed263ce633649ccfad9c6e1dc065e9c3d333624fc27028f496089a38b8",
    "loops": 2,
    "prompt_parity_questions": 512,
    "command": [
      "llama-server",
      "--model",
      "dory-decisions-Q8_0.gguf",
      "--alias",
      "dory-gguf-eval",
      "--host",
      "127.0.0.1",
      "--port",
      "18091",
      "--ctx-size",
      "4096",
      "--parallel",
      "1",
      "--threads",
      "4",
      "--threads-batch",
      "4",
      "--batch-size",
      "512",
      "--ubatch-size",
      "512",
      "--fit",
      "off",
      "--gpu-layers",
      "all",
      "--no-warmup"
    ]
  },
  "cross_entropy_omitted": "The native API returns binary probability rather than raw logits; subtraction can round the complement to zero.",
  "base_model": "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16",
  "base_model_revision": "b366b87f762ea334ba3eb55912031baba1515810",
  "description": "Same 512 questions as the NVFP4 comparison; native /v1/systemone endpoint, no source patches, no calibration, no generated reasoning."
}