{
  "format": "vimeml_public_evaluation_v2",
  "model_version": "2.1-extend5-step40000-int8-b32-v1",
  "checkpoint_sha256": "fa202f18d00cf5e77469bd3be5b8b264117fb73233ec9966fa3213257a470104",
  "coreml_manifest_sha256": "1dda48c76eacfa3ba99764098fab721eed4ada413130b99c0324620365aca0b7",
  "training": {
    "full_validation_bpc_fp32": 3.0802686334,
    "test_used_for_selection": false,
    "v20_epochs": 4,
    "restart1_epochs": 2,
    "extend5_stopped_step": 161095,
    "selected_extend5_step": 40000
  },
  "ranking": {
    "ajimee": {
      "original_order": {
        "cases": 200,
        "top1_correct": 87,
        "top1_accuracy": 0.435,
        "top5_correct": 143,
        "top5_accuracy": 0.715,
        "candidate_pool_covered": 162,
        "candidate_pool_recall": 0.81,
        "covered_top1_accuracy": 0.5370370370370371,
        "mean_min_cer": 0.0821630611981915,
        "mrr": 0.554232017982018,
        "corrected_vs_azookey": 0,
        "regressed_vs_azookey": 0,
        "fallback_cases": 0
      },
      "fp32_lm": {
        "cases": 200,
        "top1_correct": 144,
        "top1_accuracy": 0.72,
        "top5_correct": 160,
        "top5_accuracy": 0.8,
        "candidate_pool_covered": 162,
        "candidate_pool_recall": 0.81,
        "covered_top1_accuracy": 0.8888888888888888,
        "mean_min_cer": 0.030729146273316313,
        "mrr": 0.7542142857142857,
        "corrected_vs_azookey": 61,
        "regressed_vs_azookey": 4,
        "fallback_cases": 0
      },
      "int8_lm": {
        "cases": 200,
        "top1_correct": 144,
        "top1_accuracy": 0.72,
        "top5_correct": 160,
        "top5_accuracy": 0.8,
        "candidate_pool_covered": 162,
        "candidate_pool_recall": 0.81,
        "covered_top1_accuracy": 0.8888888888888888,
        "mean_min_cer": 0.03116485517226051,
        "mrr": 0.7533809523809524,
        "corrected_vs_azookey": 62,
        "regressed_vs_azookey": 5,
        "fallback_cases": 0
      },
      "fp32_comparison": {
        "order_changed_cases": 163,
        "top1_changed_cases": 7,
        "max_score_sum_abs": 1.8200132250785828,
        "mean_score_sum_abs": 0.2882139610526506
      },
      "labels_formal_gold": null
    },
    "development": {
      "original_order": {
        "cases": 137,
        "top1_correct": 111,
        "top1_accuracy": 0.8102189781021898,
        "top5_correct": 130,
        "top5_accuracy": 0.948905109489051,
        "candidate_pool_covered": 136,
        "candidate_pool_recall": 0.9927007299270073,
        "covered_top1_accuracy": 0.8161764705882353,
        "mean_min_cer": 0.04746998743349109,
        "mrr": 0.8745925911071898,
        "corrected_vs_azookey": 0,
        "regressed_vs_azookey": 0,
        "fallback_cases": 0
      },
      "fp32_lm": {
        "cases": 137,
        "top1_correct": 122,
        "top1_accuracy": 0.8905109489051095,
        "top5_correct": 136,
        "top5_accuracy": 0.9927007299270073,
        "candidate_pool_covered": 136,
        "candidate_pool_recall": 0.9927007299270073,
        "covered_top1_accuracy": 0.8970588235294118,
        "mean_min_cer": 0.029006070976873896,
        "mrr": 0.936374695863747,
        "corrected_vs_azookey": 14,
        "regressed_vs_azookey": 3,
        "fallback_cases": 0
      },
      "int8_lm": {
        "cases": 137,
        "top1_correct": 122,
        "top1_accuracy": 0.8905109489051095,
        "top5_correct": 136,
        "top5_accuracy": 0.9927007299270073,
        "candidate_pool_covered": 136,
        "candidate_pool_recall": 0.9927007299270073,
        "covered_top1_accuracy": 0.8970588235294118,
        "mean_min_cer": 0.029006070976873896,
        "mrr": 0.9351581508515816,
        "corrected_vs_azookey": 14,
        "regressed_vs_azookey": 3,
        "fallback_cases": 0
      },
      "fp32_comparison": {
        "order_changed_cases": 121,
        "top1_changed_cases": 1,
        "max_score_sum_abs": 1.3734395503997803,
        "mean_score_sum_abs": 0.19201541958957397
      },
      "labels_formal_gold": null
    },
    "expanded-development": {
      "original_order": {
        "cases": 2000,
        "top1_correct": 1224,
        "top1_accuracy": 0.612,
        "top5_correct": 1711,
        "top5_accuracy": 0.8555,
        "candidate_pool_covered": 1756,
        "candidate_pool_recall": 0.878,
        "covered_top1_accuracy": 0.6970387243735763,
        "mean_min_cer": 0.049437279587549655,
        "mrr": 0.714617032967033,
        "corrected_vs_azookey": 0,
        "regressed_vs_azookey": 0,
        "fallback_cases": 0
      },
      "fp32_lm": {
        "cases": 2000,
        "top1_correct": 1487,
        "top1_accuracy": 0.7435,
        "top5_correct": 1748,
        "top5_accuracy": 0.874,
        "candidate_pool_covered": 1756,
        "candidate_pool_recall": 0.878,
        "covered_top1_accuracy": 0.8468109339407744,
        "mean_min_cer": 0.029395604106192606,
        "mrr": 0.8023303571428563,
        "corrected_vs_azookey": 322,
        "regressed_vs_azookey": 59,
        "fallback_cases": 0
      },
      "int8_lm": {
        "cases": 2000,
        "top1_correct": 1481,
        "top1_accuracy": 0.7405,
        "top5_correct": 1748,
        "top5_accuracy": 0.874,
        "candidate_pool_covered": 1756,
        "candidate_pool_recall": 0.878,
        "covered_top1_accuracy": 0.8433940774487472,
        "mean_min_cer": 0.02966218634430054,
        "mrr": 0.800903769841269,
        "corrected_vs_azookey": 319,
        "regressed_vs_azookey": 62,
        "fallback_cases": 0
      },
      "fp32_comparison": {
        "order_changed_cases": 404,
        "top1_changed_cases": 30,
        "max_score_sum_abs": 1.790113139897585,
        "mean_score_sum_abs": 0.22886438840285703
      },
      "labels_formal_gold": false
    }
  },
  "alignment": {
    "fp32": {
      "strict_passed": true,
      "tolerances": {
        "atol": 0.0003,
        "rtol": 0.0003
      },
      "logits": [
        {
          "max_abs": 9.5367431640625e-06,
          "mean_abs": 1.2576065273606218e-06,
          "rmse": 1.6289324748934612e-06,
          "outside_tolerance": 0,
          "values": 16384,
          "id": "bos",
          "length": 1,
          "valid_prefix": 1,
          "last_top1_equal": true
        },
        {
          "max_abs": 3.910064697265625e-05,
          "mean_abs": 3.185306411523925e-06,
          "rmse": 4.141091950346693e-06,
          "outside_tolerance": 0,
          "values": 262144,
          "id": "short",
          "length": 16,
          "valid_prefix": 16,
          "last_top1_equal": true
        },
        {
          "max_abs": 4.57763671875e-05,
          "mean_abs": 4.321694975506236e-06,
          "rmse": 5.538972619049361e-06,
          "outside_tolerance": 0,
          "values": 2097152,
          "id": "long",
          "length": 128,
          "valid_prefix": 128,
          "last_top1_equal": true
        },
        {
          "max_abs": 4.482269287109375e-05,
          "mean_abs": 3.3942287953614425e-06,
          "rmse": 4.3666045771710725e-06,
          "outside_tolerance": 0,
          "values": 262144,
          "id": "right_pad",
          "length": 128,
          "valid_prefix": 16,
          "last_top1_equal": true
        },
        {
          "max_abs": 4.482269287109375e-05,
          "mean_abs": 3.3942287953614425e-06,
          "rmse": 4.3666045771710725e-06,
          "outside_tolerance": 0,
          "values": 262144,
          "id": "changed_future",
          "length": 128,
          "valid_prefix": 16,
          "last_top1_equal": true
        }
      ],
      "invariants": {
        "right_pad": {
          "max_abs": 0.0,
          "mean_abs": 0.0,
          "rmse": 0.0,
          "outside_tolerance": 0,
          "values": 262144
        },
        "changed_future": {
          "max_abs": 0.0,
          "mean_abs": 0.0,
          "rmse": 0.0,
          "outside_tolerance": 0,
          "values": 262144
        }
      }
    },
    "int8": {
      "strict_passed": false,
      "tolerances": {
        "atol": 0.0003,
        "rtol": 0.0003
      },
      "logits": [
        {
          "max_abs": 0.1717967987060547,
          "mean_abs": 0.03132702315087954,
          "rmse": 0.03977237435193969,
          "outside_tolerance": 15692,
          "values": 16384,
          "id": "bos",
          "length": 1,
          "valid_prefix": 1,
          "last_top1_equal": true
        },
        {
          "max_abs": 0.8956146240234375,
          "mean_abs": 0.10476142495688912,
          "rmse": 0.13775674684387576,
          "outside_tolerance": 257685,
          "values": 262144,
          "id": "short",
          "length": 16,
          "valid_prefix": 16,
          "last_top1_equal": true
        },
        {
          "max_abs": 1.0094914436340332,
          "mean_abs": 0.10851713189305306,
          "rmse": 0.1394378905311297,
          "outside_tolerance": 2067332,
          "values": 2097152,
          "id": "long",
          "length": 128,
          "valid_prefix": 128,
          "last_top1_equal": true
        },
        {
          "max_abs": 0.8956174850463867,
          "mean_abs": 0.10476151736959416,
          "rmse": 0.13775683790182477,
          "outside_tolerance": 257684,
          "values": 262144,
          "id": "right_pad",
          "length": 128,
          "valid_prefix": 16,
          "last_top1_equal": true
        },
        {
          "max_abs": 0.8956174850463867,
          "mean_abs": 0.10476151736959416,
          "rmse": 0.13775683790182477,
          "outside_tolerance": 257684,
          "values": 262144,
          "id": "changed_future",
          "length": 128,
          "valid_prefix": 16,
          "last_top1_equal": true
        }
      ],
      "invariants": {
        "right_pad": {
          "max_abs": 0.0,
          "mean_abs": 0.0,
          "rmse": 0.0,
          "outside_tolerance": 0,
          "values": 262144
        },
        "changed_future": {
          "max_abs": 0.0,
          "mean_abs": 0.0,
          "rmse": 0.0,
          "outside_tolerance": 0,
          "values": 262144
        }
      }
    }
  },
  "fixed_generation": {
    "next_token_top1_equal": 12,
    "greedy_32_token_equal": 8,
    "total": 12
  },
  "quantization_diagnostics": {
    "format": "vimeml_quantization_analysis_v1",
    "date": "2026-10-08",
    "source_paths": [
      "outputs/ime-eval/expanded-v21-dev-draft-extend5-best/scores.jsonl",
      "outputs/deployment/v21-int8-expanded-development-v1/scores.jsonl"
    ],
    "scope": "Existing frozen draft development scores; no model execution, blind data or label edits",
    "cases": 2000,
    "top1_changes": 30,
    "wins": 8,
    "losses": 14,
    "paired_exact_two_sided_p": 0.28627872467041016,
    "fp32_top1_margin_by_change": {
      "changed": {
        "cases": 30,
        "median_logp_sum": 0.07003979850560427,
        "maximum_logp_sum": 0.36277501191943884,
        "count_margin_le_0_25": 27
      },
      "unchanged": {
        "cases": 1910,
        "median_logp_sum": 3.0718008121475577,
        "maximum_logp_sum": 26.55984030291438,
        "count_margin_le_0_25": 85
      }
    },
    "interpretation": "No statistically significant accuracy decrease detected; this does not establish equivalence or formal gold quality."
  },
  "physical_keyboard_extension": {
    "device": "iPhone 16 Pro Max",
    "os": "iOS 27.2 (24B5099f)",
    "pid": 17014,
    "model_version": "2.1-extend5-step40000-int8-b32-v1",
    "instruments_duration_seconds": 201.29728,
    "instruments_samples": 164,
    "in_process_duration_seconds": 191.90217375,
    "in_process_samples": 1920,
    "kernel_lifetime_footprint_peak_mib": 34.720596,
    "in_process_sampled_footprint_peak_mib": 28.298721,
    "instruments_sampled_footprint_peak_mib": 28.001846,
    "instruments_private_resident_peak_mib": 73.78125,
    "lm_scoring_p95_ms": 53.915625,
    "lm_next_words_p95_ms": 38.114333,
    "measurement_enabled_after_completion": false,
    "input_text_recorded": false,
    "scope": "Bounded real extension workload; separate from test host, sampled resident values and lifetime physical footprint"
  },
  "limitations": [
    {
      "item": "INT8 strict frozen logits alignment",
      "result": "failed",
      "max_absolute_error": 1.0094914436340332,
      "atol": 0.0003,
      "rtol": 0.0003
    },
    {
      "item": "Expanded development draft Top-1",
      "fp32": 1487,
      "int8": 1481,
      "total": 2000,
      "delta_percentage_points": -0.3
    },
    {
      "item": "Actual extension result-to-UI long tail",
      "maximum_ms": 1926.0947,
      "cause": "unlocated; no per-event correlation timestamps"
    },
    {
      "item": "Late-window extension footprint growth and second model load",
      "cause": "unlocated; insufficient evidence for leak/cache/host attribution"
    },
    {
      "item": "Fully cold startup",
      "status": "not certified: measured first process load may use warm OS/Core ML caches"
    },
    {
      "item": "Long-term footprint plateau/system termination audit",
      "status": "not certified by this bounded workload"
    }
  ],
  "experiment_decision": "V2 series complete; current INT8 task-quality loss and observed UI tail accepted.",
  "source_reports_sha256": {
    "outputs/deployment/v21-int8-ajimee-v1/metrics.json": "7b136dbed85b2f354eeae1838eb7f32e34c8eb3e909fe4fcce1ac449f1480c63",
    "outputs/deployment/v21-int8-development-v1/metrics.json": "f1384fa5d5b7c4da2503a9118300a0857bf2ef5c4ee2d7213817851839f2394c",
    "outputs/deployment/v21-int8-expanded-development-v1/metrics.json": "0a07dbe706add83b3c3e917dce1334d0227f105d29347671bca539442f887226",
    "outputs/deployment/v21-fp32-alignment-v1/alignment.json": "3c0a368f4836388733fcb21882e431b8b7548b1e9ea61fefeeb479ce51954b0b",
    "outputs/deployment/v21-int8-alignment-v1/alignment.json": "791d047cf14d15025a33bc51c0e3e1ebaa986247832bdd42fcda027f8901db7f",
    "outputs/deployment/v21-closeout-v1/acceptance.json": "1992372e8867f29c0d2add7d9d15ab0ace302555e42741c821bad8d6c0e20e3d",
    "outputs/deployment/v21-quantization-analysis-v1/analysis.json": "0fd04daf4e8b86a6cb028d89f4c598312bdafcbca271f2b27d445b6db581a132"
  },
  "scope": "Saved fixed-pool metrics and bounded device workload; draft expanded labels, no blind scoring. No benchmark texts, per-case scores, user inputs or raw device traces distributed."
}