evaluation/merge-verification.json

{
  "version": 1,
  "reference_report": "reports/decision-0.8b-v2/decision-development.json",
  "merged_report": "reports/decision-0.8b-v2/merged-development.json",
  "reference_sha256": "701bf9841acbb867d113dff015a52323832deeaeff5b5174be63f1d05f9cf321",
  "merged_sha256": "eeedeb35361995f7186d1ec3b2eafffd584eb71e559f2c930edbbe576831c0d8",
  "reference_model": {
    "provider": "local",
    "name": "models/Qwen--Qwen3.5-0.8B",
    "adapter": "runs/decision-0.8b-v2/adapter",
    "identity_version": 2,
    "prompt_sha256": "b2c4baf03d0224bf37ea1b2e11c2fa2c98c26d09a7a881a4565150829d1e2af0",
    "model_weights_sha256": "79353a8fb811beddb052c75f92debca361226dcb0700d8f46004a77bae214f7c",
    "model_config_sha256": "779572e46ae6552ba1826a162ab198560403ab1adeed8654e24e45c14e8ba978",
    "tokenizer_sha256": "4773ccc9cb7087ffa57977144fc5786e4c1b5ef4d2b25a850bd25d3318581142",
    "snapshot_sha256": "6aa297f2cadde0920e13ed5e2e0b78a33a36046d0d5fcb8efd8b03b4cd7e5ab3",
    "weight_identity_status": "verified_local_content",
    "encoding_sha256": "1e1255124fb4f698cabd531cbc251e2b0b91b8d119a53cf88974792f196a2c30",
    "inference_dtype": "bfloat16",
    "adapter_artifacts": {
      "config_sha256": "9fee8484ccc91ee15604c0264461eecfa46b2f5605362c662d31191c08ed900e",
      "weights_sha256": "cb80a8e8a5ffd1232abaecd37a3f91a58c0f268d218069ce4cb779f70158aff8"
    }
  },
  "merged_model": {
    "provider": "local",
    "name": "exports/decision-0.8b-hf-release-v1",
    "adapter": null,
    "identity_version": 2,
    "prompt_sha256": "b2c4baf03d0224bf37ea1b2e11c2fa2c98c26d09a7a881a4565150829d1e2af0",
    "model_weights_sha256": "847a5e57da6235e29efe54367b1160d208b3cc3d9e7eac4ed2a7869dcf66aba0",
    "model_config_sha256": "3df71add00d308cead66382bfea530aa71b0c584dddda6e58e8a0a1d6b0df329",
    "tokenizer_sha256": "09f1f26effaa7c51589555eacfbcece6043764d227c11be74d2b05c90584a548",
    "snapshot_sha256": "71a36baa11a7946ce7f84fe3edcad65305e03a5d0c0cd3cd7650e144ab98640c",
    "weight_identity_status": "verified_local_content",
    "encoding_sha256": "1e1255124fb4f698cabd531cbc251e2b0b91b8d119a53cf88974792f196a2c30",
    "inference_dtype": "bfloat16",
    "adapter_artifacts": null
  },
  "dataset_sha256": "21977c3441de6aff6f320084c0531e4131996b62b22c80b7d4c9cc158a7b6fa2",
  "tokenizer_equivalence": {
    "count": 892,
    "all_rendered_prompts_equal": true,
    "all_prompt_label_and_eos_token_ids_equal": true,
    "max_completion_tokens": 1511,
    "digests": {
      "rendered_prompts": "97f395740b34c18dbb84d6760700131f7064cd9cee1ab2da099c1bf05df0ba18",
      "prompt_and_option_ids": "3e6a8ab7b7b10ef88a50e79b5d92aa008e159b09bcaa9f25ddf81eb5397ee2c2"
    }
  },
  "comparison": {
    "records": 892,
    "prediction_change_count": 10,
    "label_agreement": 0.9887892376681614,
    "maximum_absolute_probability_difference": 0.08797600865364075,
    "mean_absolute_probability_difference": 0.0031891730525986763,
    "reference_accuracy": 0.7589686098654709,
    "accuracy_delta": 0.0033632286995515237,
    "predeclared_gate": {
      "maximum_accuracy_drop": 0.005,
      "passed": true,
      "scope": "Development diagnostic only; not evidence of iPhone performance or final-test accuracy."
    },
    "paired_group_bootstrap": {
      "accuracy_delta": 0.0033632286995515697,
      "ci95": [
        -0.0033632286995515697,
        0.009917212832011038
      ],
      "samples": 2000,
      "seed": 42,
      "by_source": {
        "ag_news": {
          "count": 50,
          "groups": 50,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "banking77": {
          "count": 50,
          "groups": 50,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "boolq": {
          "count": 50,
          "groups": 50,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": -0.02,
          "ci95": [
            -0.06,
            0.0
          ]
        },
        "clinc150": {
          "count": 50,
          "groups": 50,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "go_emotions": {
          "count": 50,
          "groups": 50,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "helpsteer2": {
          "count": 50,
          "groups": 37,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": 0.02,
          "ci95": [
            -0.0425531914893617,
            0.08515263644773333
          ]
        },
        "mnli": {
          "count": 50,
          "groups": 50,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "paws": {
          "count": 50,
          "groups": 40,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "policy": {
          "count": 48,
          "groups": 16,
          "fixed_weight": 0.053811659192825115,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "policy_v2": {
          "count": 48,
          "groups": 8,
          "fixed_weight": 0.053811659192825115,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "research_taxonomy_v21": {
          "count": 48,
          "groups": 6,
          "fixed_weight": 0.053811659192825115,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "routing_v2": {
          "count": 48,
          "groups": 8,
          "fixed_weight": 0.053811659192825115,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "sst5": {
          "count": 50,
          "groups": 50,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": 0.02,
          "ci95": [
            0.0,
            0.06
          ]
        },
        "vitaminc": {
          "count": 50,
          "groups": 49,
          "fixed_weight": 0.05605381165919283,
          "accuracy_delta": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        "workflow.simple_quorum": {
          "count": 100,
          "groups": 20,
          "fixed_weight": 0.11210762331838565,
          "accuracy_delta": 0.01,
          "ci95": [
            0.0,
            0.03
          ]
        },
        "workflow.veto_alternative": {
          "count": 100,
          "groups": 20,
          "fixed_weight": 0.11210762331838565,
          "accuracy_delta": 0.01,
          "ci95": [
            -0.02,
            0.040249999999998634
          ]
        }
      },
      "method": "paired percentile bootstrap; whole groups within source; fixed observed source weights"
    },
    "candidate_accuracy": 0.7623318385650224,
    "nll_delta": 0.0009865970977848137,
    "ece_delta": -0.0018650550111259517,
    "family_macro_accuracy_delta": 0.0025000000000000577
  },
  "verification_script_sha256": "96621f2caa4c0aae82a0ae99580de631b0594337c5ce940979188d5c0a85d469",
  "export_manifest_sha256": "755f610140d17483feb5b540dab7162badb2b00ca0f6b686cc389268d7e9a6ba",
  "method": "Full matched development panel; optional strict 32-record exporter diagnostic omitted.",
  "source_report_sha256": "99a4c9b7372d45c40e27c27f5ca2fe8bf18f8cb5ae1f1c44658149d8f929080a"
}