{
  "format": "vimeml_public_evaluation_v4",
  "model_version": "4.2-epoch20-step1081942",
  "checkpoint_sha256": "0e83ea7417b9816b37183044468343b151f4e610da0764c31f1f780c5fa5796d",
  "coreml_manifest_sha256": {
    "stateless_int8": "0c3b6f3b809aaf3dd2f9ae18626c363f72dd34c59953dea48fe865ebbde9d6da",
    "kvchunk_int8": "fd4bbb6c31fa2f71b2c9be6ab5feb0bbb03b7f51216907c7ff271171d26d1b75"
  },
  "training": {
    "epochs": 20,
    "updates": 1081942,
    "seed": 1219827974,
    "supervised_targets": 12104389994,
    "full_validation_lm_bpc_fp32": 3.0451515462768692,
    "full_validation_chat_bpc_fp32": 3.122059834972166,
    "selection": "best complete LM validation BPC (epoch 20), fixed before any IME scoring",
    "test_or_blind_used": false
  },
  "fixed_pool_fp32_cuda": {
    "standard-development": {
      "azookey": {
        "cases": 728,
        "top1_correct": 453,
        "top1_accuracy": 0.6222527472527473,
        "top5_correct": 608,
        "top5_accuracy": 0.8351648351648352,
        "candidate_pool_covered": 630,
        "candidate_pool_recall": 0.8653846153846154,
        "covered_top1_accuracy": 0.719047619047619,
        "mean_min_cer": 0.05160664332397995,
        "mrr": 0.7097974489912511,
        "corrected_vs_azookey": 0,
        "regressed_vs_azookey": 0,
        "fallback_cases": 0
      },
      "lm_context_sum": {
        "cases": 728,
        "top1_correct": 523,
        "top1_accuracy": 0.7184065934065934,
        "top5_correct": 624,
        "top5_accuracy": 0.8571428571428571,
        "candidate_pool_covered": 630,
        "candidate_pool_recall": 0.8653846153846154,
        "covered_top1_accuracy": 0.8301587301587302,
        "mean_min_cer": 0.034705386175760286,
        "mrr": 0.7749688955732909,
        "corrected_vs_azookey": 110,
        "regressed_vs_azookey": 40,
        "fallback_cases": 0
      },
      "lm_no_context_sum": {
        "cases": 728,
        "top1_correct": 510,
        "top1_accuracy": 0.7005494505494505,
        "top5_correct": 621,
        "top5_accuracy": 0.853021978021978,
        "candidate_pool_covered": 630,
        "candidate_pool_recall": 0.8653846153846154,
        "covered_top1_accuracy": 0.8095238095238095,
        "mean_min_cer": 0.03638251311053943,
        "mrr": 0.7641914137449851,
        "corrected_vs_azookey": 108,
        "regressed_vs_azookey": 51,
        "fallback_cases": 0
      }
    },
    "legacy-regression": {
      "azookey": {
        "cases": 500,
        "top1_correct": 317,
        "top1_accuracy": 0.634,
        "top5_correct": 437,
        "top5_accuracy": 0.874,
        "candidate_pool_covered": 443,
        "candidate_pool_recall": 0.886,
        "covered_top1_accuracy": 0.7155756207674944,
        "mean_min_cer": 0.04623841144643269,
        "mrr": 0.7311617826617824,
        "corrected_vs_azookey": 0,
        "regressed_vs_azookey": 0,
        "fallback_cases": 0
      },
      "lm_context_sum": {
        "cases": 500,
        "top1_correct": 370,
        "top1_accuracy": 0.74,
        "top5_correct": 442,
        "top5_accuracy": 0.884,
        "candidate_pool_covered": 443,
        "candidate_pool_recall": 0.886,
        "covered_top1_accuracy": 0.835214446952596,
        "mean_min_cer": 0.028879998926345758,
        "mrr": 0.8080666666666665,
        "corrected_vs_azookey": 76,
        "regressed_vs_azookey": 23,
        "fallback_cases": 0
      },
      "lm_no_context_sum": {
        "cases": 500,
        "top1_correct": 369,
        "top1_accuracy": 0.738,
        "top5_correct": 442,
        "top5_accuracy": 0.884,
        "candidate_pool_covered": 443,
        "candidate_pool_recall": 0.886,
        "covered_top1_accuracy": 0.8329571106094809,
        "mean_min_cer": 0.028316756130444452,
        "mrr": 0.8037999999999997,
        "corrected_vs_azookey": 77,
        "regressed_vs_azookey": 25,
        "fallback_cases": 0
      }
    },
    "ajimee": {
      "azookey": {
        "cases": 200,
        "top1_correct": 87,
        "top1_accuracy": 0.435,
        "top5_correct": 143,
        "top5_accuracy": 0.715,
        "candidate_pool_covered": 162,
        "candidate_pool_recall": 0.81,
        "covered_top1_accuracy": 0.5370370370370371,
        "mean_min_cer": 0.0821630611981915,
        "mrr": 0.554232017982018,
        "corrected_vs_azookey": 0,
        "regressed_vs_azookey": 0,
        "fallback_cases": 0
      },
      "lm_context_sum": {
        "cases": 200,
        "top1_correct": 139,
        "top1_accuracy": 0.695,
        "top5_correct": 161,
        "top5_accuracy": 0.805,
        "candidate_pool_covered": 162,
        "candidate_pool_recall": 0.81,
        "covered_top1_accuracy": 0.8580246913580247,
        "mean_min_cer": 0.03520357489337187,
        "mrr": 0.7448809523809524,
        "corrected_vs_azookey": 58,
        "regressed_vs_azookey": 6,
        "fallback_cases": 0
      },
      "lm_no_context_sum": {
        "cases": 200,
        "top1_correct": 130,
        "top1_accuracy": 0.65,
        "top5_correct": 161,
        "top5_accuracy": 0.805,
        "candidate_pool_covered": 162,
        "candidate_pool_recall": 0.81,
        "covered_top1_accuracy": 0.8024691358024691,
        "mean_min_cer": 0.051505004672742855,
        "mrr": 0.7155833333333333,
        "corrected_vs_azookey": 52,
        "regressed_vs_azookey": 9,
        "fallback_cases": 0
      }
    },
    "historical-development": {
      "azookey": {
        "cases": 137,
        "top1_correct": 111,
        "top1_accuracy": 0.8102189781021898,
        "top5_correct": 130,
        "top5_accuracy": 0.948905109489051,
        "candidate_pool_covered": 136,
        "candidate_pool_recall": 0.9927007299270073,
        "covered_top1_accuracy": 0.8161764705882353,
        "mean_min_cer": 0.04746998743349109,
        "mrr": 0.8745925911071898,
        "corrected_vs_azookey": 0,
        "regressed_vs_azookey": 0,
        "fallback_cases": 0
      },
      "lm_context_sum": {
        "cases": 137,
        "top1_correct": 123,
        "top1_accuracy": 0.8978102189781022,
        "top5_correct": 136,
        "top5_accuracy": 0.9927007299270073,
        "candidate_pool_covered": 136,
        "candidate_pool_recall": 0.9927007299270073,
        "covered_top1_accuracy": 0.9044117647058824,
        "mean_min_cer": 0.02333226384321275,
        "mrr": 0.9388077858880779,
        "corrected_vs_azookey": 15,
        "regressed_vs_azookey": 3,
        "fallback_cases": 0
      },
      "lm_no_context_sum": {
        "cases": 137,
        "top1_correct": 124,
        "top1_accuracy": 0.9051094890510949,
        "top5_correct": 136,
        "top5_accuracy": 0.9927007299270073,
        "candidate_pool_covered": 136,
        "candidate_pool_recall": 0.9927007299270073,
        "covered_top1_accuracy": 0.9117647058823529,
        "mean_min_cer": 0.019706327735524817,
        "mrr": 0.9454987834549878,
        "corrected_vs_azookey": 16,
        "regressed_vs_azookey": 3,
        "fallback_cases": 0
      }
    }
  },
  "fixed_reference_725": {
    "plan_id": "81512f147d3749ef9a36e61470fce173",
    "role": "reference_test_after_consumption",
    "source_metrics": {
      "by_source": {
        "jmultiwoz": {
          "cases": 250,
          "top1_correct": 226,
          "top1_accuracy": 0.904,
          "top5_correct": 245,
          "top5_accuracy": 0.98,
          "candidate_pool_covered": 245,
          "candidate_pool_recall": 0.98,
          "covered_top1_accuracy": 0.9224489795918367,
          "mean_min_cer": 0.009039811881542249,
          "mrr": 0.9376666666666666,
          "corrected_vs_azookey": 35,
          "regressed_vs_azookey": 11,
          "fallback_cases": 0
        },
        "wrime": {
          "cases": 475,
          "top1_correct": 315,
          "top1_accuracy": 0.6631578947368421,
          "top5_correct": 398,
          "top5_accuracy": 0.8378947368421052,
          "candidate_pool_covered": 405,
          "candidate_pool_recall": 0.8526315789473684,
          "covered_top1_accuracy": 0.7777777777777778,
          "mean_min_cer": 0.05086094251484006,
          "mrr": 0.7354361661730081,
          "corrected_vs_azookey": 82,
          "regressed_vs_azookey": 32,
          "fallback_cases": 0
        }
      },
      "independent_micro": {
        "cases": 725,
        "top1_correct": 541,
        "top1_accuracy": 0.7462068965517241,
        "top5_correct": 643,
        "top5_accuracy": 0.886896551724138,
        "candidate_pool_covered": 650,
        "candidate_pool_recall": 0.896551724137931,
        "covered_top1_accuracy": 0.8323076923076923,
        "mean_min_cer": 0.03643986298611669,
        "mrr": 0.805170821515649,
        "corrected_vs_azookey": 117,
        "regressed_vs_azookey": 43,
        "fallback_cases": 0
      },
      "independent_equal_source_macro_top1": 0.783578947368421
    },
    "paired_vs_v21_fp32": {
      "v1_wrong_v2_correct": 40,
      "v1_correct_v2_wrong": 34,
      "both_correct": 501,
      "both_wrong": 150,
      "net_correct": 6,
      "exact_mcnemar_p_two_sided": 0.5613807442989819
    },
    "by_source_vs_v21": {
      "wrime": {
        "v1_wrong_v2_correct": 32,
        "v1_correct_v2_wrong": 26,
        "both_correct": 283,
        "both_wrong": 134,
        "net_correct": 6,
        "exact_mcnemar_p_two_sided": 0.5118423084381689
      },
      "jmultiwoz": {
        "v1_wrong_v2_correct": 8,
        "v1_correct_v2_wrong": 8,
        "both_correct": 218,
        "both_wrong": 16,
        "net_correct": 0,
        "exact_mcnemar_p_two_sided": 1
      }
    }
  },
  "coreml": {
    "top1": {
      "cuda": {
        "standard-development": 523,
        "legacy-regression": 370,
        "ajimee": 139,
        "historical-development": 123
      },
      "mac_pytorch": {
        "standard-development": 523,
        "legacy-regression": 370,
        "ajimee": 139,
        "historical-development": 123
      },
      "coreml_fp32": {
        "standard-development": 523,
        "legacy-regression": 370,
        "ajimee": 139,
        "historical-development": 123
      },
      "coreml_int8": {
        "standard-development": 522,
        "legacy-regression": 371,
        "ajimee": 139,
        "historical-development": 123
      },
      "kv_fp32": {
        "standard-development": 523,
        "legacy-regression": 370,
        "ajimee": 139,
        "historical-development": 123
      },
      "kv_int8": {
        "standard-development": 522,
        "legacy-regression": 371,
        "ajimee": 139,
        "historical-development": 123
      },
      "kvchunk_fp32": {
        "standard-development": 523,
        "legacy-regression": 370,
        "ajimee": 139,
        "historical-development": 123
      },
      "kvchunk_int8": {
        "standard-development": 522,
        "legacy-regression": 371,
        "ajimee": 139,
        "historical-development": 123
      },
      "v21_coreml_fp32": {
        "standard-development": 505,
        "legacy-regression": 374,
        "ajimee": 144,
        "historical-development": 122
      },
      "v21_coreml_int8": {
        "standard-development": 506,
        "legacy-regression": 375,
        "ajimee": 144,
        "historical-development": 122
      }
    },
    "checks": {
      "1_mac_pytorch_fixtures_within_3e-4": true,
      "1_mac_pytorch_top1_equals_recorded_cuda": true,
      "1_mac_pytorch_orders_equal_cuda_all_methods": true,
      "2_fp32_stateless_and_kv_converted": true,
      "3_fp32_stateless_alignment_passed": true,
      "3_fp32_kv_alignment_passed": true,
      "3_coreml_fp32_top1_equals_mac_pytorch": true,
      "3_coreml_fp32_orders_equal_mac_pytorch_all_methods": true,
      "3_kv_fp32_top1_equals_mac_pytorch": true,
      "3_kv_fp32_orders_equal_mac_pytorch_all_methods": true,
      "4_int8_strict_logits_alignment_passed (expected false)": false,
      "4_int8_kv_vs_same_weight_stateless_int8_passed": true,
      "4_kv_int8_top1_equals_stateless_int8": true,
      "4_kv_int8_orders_equal_stateless_int8_all_methods": false,
      "7_kvchunk_fp32_alignment_passed": true,
      "7_kvchunk_int8_vs_same_weight_stateless_int8_passed": true,
      "7_kvchunk_int8_strict_vs_fp32_fixtures_passed (expected false)": false,
      "7_kvchunk_fp32_top1_equals_stateless_fp32": true,
      "7_kvchunk_fp32_orders_equal_stateless_fp32_all_methods": true,
      "7_kvchunk_int8_top1_equals_stateless_int8": true,
      "7_kvchunk_int8_orders_equal_stateless_int8_all_methods": false,
      "8_v21_coreml_fp32_top1_equals_recorded": true,
      "5_measurements_recorded": true,
      "6_ios_resources_prepared": true
    },
    "int8_vs_coreml_fp32": {
      "standard-development": {
        "fp32": 523,
        "int8": 522,
        "v1_wrong_v2_correct": 2,
        "v1_correct_v2_wrong": 3,
        "both_correct": 520,
        "both_wrong": 203,
        "net_correct": -1,
        "exact_mcnemar_p_two_sided": 1,
        "significant_at_0_05": false,
        "order_changed_cases": 254,
        "top1_changed_cases": 11,
        "max_score_sum_abs": 0.7504582202527672,
        "mean_score_sum_abs": 0.12163806840569316
      },
      "legacy-regression": {
        "fp32": 370,
        "int8": 371,
        "v1_wrong_v2_correct": 1,
        "v1_correct_v2_wrong": 0,
        "both_correct": 370,
        "both_wrong": 129,
        "net_correct": 1,
        "exact_mcnemar_p_two_sided": 1,
        "significant_at_0_05": false,
        "order_changed_cases": 46,
        "top1_changed_cases": 2,
        "max_score_sum_abs": 0.8114978885278106,
        "mean_score_sum_abs": 0.11693049636118634
      },
      "ajimee": {
        "fp32": 139,
        "int8": 139,
        "v1_wrong_v2_correct": 0,
        "v1_correct_v2_wrong": 0,
        "both_correct": 139,
        "both_wrong": 61,
        "net_correct": 0,
        "exact_mcnemar_p_two_sided": 1.0,
        "significant_at_0_05": false,
        "order_changed_cases": 106,
        "top1_changed_cases": 0,
        "max_score_sum_abs": 0.9850609144195914,
        "mean_score_sum_abs": 0.1471606100397155
      },
      "historical-development": {
        "fp32": 123,
        "int8": 123,
        "v1_wrong_v2_correct": 0,
        "v1_correct_v2_wrong": 0,
        "both_correct": 123,
        "both_wrong": 14,
        "net_correct": 0,
        "exact_mcnemar_p_two_sided": 1.0,
        "significant_at_0_05": false,
        "order_changed_cases": 88,
        "top1_changed_cases": 0,
        "max_score_sum_abs": 0.7235050201416016,
        "mean_score_sum_abs": 0.09826715727665117
      }
    },
    "v21_int8_vs_v42_int8": {
      "standard-development": {
        "v1_wrong_v2_correct": 44,
        "v1_correct_v2_wrong": 28,
        "both_correct": 478,
        "both_wrong": 178,
        "net_correct": 16,
        "exact_mcnemar_p_two_sided": 0.07636915467183139
      },
      "legacy-regression": {
        "v1_wrong_v2_correct": 13,
        "v1_correct_v2_wrong": 17,
        "both_correct": 358,
        "both_wrong": 112,
        "net_correct": -4,
        "exact_mcnemar_p_two_sided": 0.584664711728692
      },
      "ajimee": {
        "v1_wrong_v2_correct": 9,
        "v1_correct_v2_wrong": 14,
        "both_correct": 130,
        "both_wrong": 47,
        "net_correct": -5,
        "exact_mcnemar_p_two_sided": 0.4048728942871094
      },
      "historical-development": {
        "v1_wrong_v2_correct": 4,
        "v1_correct_v2_wrong": 3,
        "both_correct": 119,
        "both_wrong": 11,
        "net_correct": 1,
        "exact_mcnemar_p_two_sided": 1
      }
    },
    "alignment": {
      "v42-fp32-alignment-v1": {
        "passed": true,
        "checks": 5,
        "max_abs": 5.245208740234375e-05,
        "failed_checks": 0
      },
      "v42-kv-fp32-alignment-v1": {
        "passed": true,
        "checks": 17,
        "max_abs": 4.9591064453125e-05,
        "failed_checks": 0
      },
      "v42-int8-alignment-v1": {
        "passed": false,
        "checks": 5,
        "max_abs": 0.4774494171142578,
        "failed_checks": 5
      },
      "v42-kv-int8-vs-stateless-int8-v1": {
        "passed": true,
        "checks": 16,
        "max_abs": 4.00543212890625e-05,
        "failed_checks": 0
      },
      "v42-kv-int8-vs-fp32-fixtures-v1": {
        "passed": false,
        "checks": 17,
        "max_abs": 0.707888126373291,
        "failed_checks": 17
      },
      "v42-kvchunk-fp32-alignment-v1": {
        "passed": true,
        "checks": 27,
        "max_abs": 5.245208740234375e-05,
        "failed_checks": 0
      },
      "v42-kvchunk-int8-vs-stateless-int8-v1": {
        "passed": true,
        "checks": 26,
        "max_abs": 4.00543212890625e-05,
        "failed_checks": 0
      },
      "v42-kvchunk-int8-vs-fp32-fixtures-v1": {
        "passed": false,
        "checks": 22,
        "max_abs": 0.7078914642333984,
        "failed_checks": 22
      }
    }
  },
  "notation_flip_analysis_vs_v21": {
    "standard-development": {
      "A": {
        "improved": 28,
        "regressed": 23,
        "net": 5
      },
      "S": {
        "improved": 15,
        "regressed": 5,
        "net": 10
      },
      "P": {
        "improved": 3,
        "regressed": 0,
        "net": 3
      },
      "meaning_plus_proper_exact_p": 0.010622024536132812,
      "flips_margin_lt_1nat": {
        "all": 19,
        "of": 74,
        "A": 16
      }
    },
    "legacy-regression": {
      "A": {
        "improved": 12,
        "regressed": 17,
        "net": -5
      },
      "S": {
        "improved": 3,
        "regressed": 2,
        "net": 1
      },
      "P": {
        "improved": 0,
        "regressed": 0,
        "net": 0
      },
      "meaning_plus_proper_exact_p": 1.0,
      "flips_margin_lt_1nat": {
        "all": 12,
        "of": 34,
        "A": 11
      }
    },
    "ajimee": {
      "A": {
        "improved": 1,
        "regressed": 2,
        "net": -1
      },
      "S": {
        "improved": 7,
        "regressed": 11,
        "net": -4
      },
      "P": {
        "improved": 0,
        "regressed": 0,
        "net": 0
      },
      "meaning_plus_proper_exact_p": 0.480682373046875,
      "flips_margin_lt_1nat": {
        "all": 6,
        "of": 21,
        "A": 2
      }
    },
    "historical-development": {
      "A": {
        "improved": 3,
        "regressed": 1,
        "net": 2
      },
      "S": {
        "improved": 1,
        "regressed": 2,
        "net": -1
      },
      "P": {
        "improved": 0,
        "regressed": 0,
        "net": 0
      },
      "meaning_plus_proper_exact_p": 1.0,
      "flips_margin_lt_1nat": {
        "all": 3,
        "of": 7,
        "A": 2
      }
    },
    "reference725": {
      "A": {
        "improved": 27,
        "regressed": 24,
        "net": 3
      },
      "S": {
        "improved": 11,
        "regressed": 7,
        "net": 4
      },
      "P": {
        "improved": 2,
        "regressed": 3,
        "net": -1
      },
      "meaning_plus_proper_exact_p": 0.6776394844055176,
      "flips_margin_lt_1nat": {
        "all": 23,
        "of": 74,
        "A": 19
      }
    }
  },
  "measurements": {
    "mac": {
      "source_sha256": "66eeb948028365445bb1a8dad6e84e9f8d69806ac9702dd8293821b1a47e2a53",
      "hardware": "Apple M3, macOS 27.2, CPU_ONLY",
      "stateless_int8_warm_p50_ms": {
        "T1": 1.3,
        "T128": 6.07
      },
      "v21_int8_warm_p50_ms": {
        "T1": 0.99,
        "T128": 5.68
      },
      "kvchunk_int8_prefill_p50_ms": {
        "P7": 2.49,
        "P31": 3.52,
        "P63": 4.55,
        "P119": 6.56
      },
      "v21_kv_int8_prefill_p50_ms": {
        "P7": 2.19,
        "P31": 3.03,
        "P63": 4.13,
        "P119": 6.11
      }
    },
    "iphone16promax_xctest_host": {
      "source_sha256": "3c8566cef8ad78b11d5a9d1bf3eb55d6eadef8aaf3c1d0f5aa7680615a562cbb",
      "conditions": "iOS 27.2, wired and charging, thermal state fair",
      "v42_int8": {
        "predict_T128_p50_ms": 4.9,
        "rerank_p50_ms": 25.1,
        "rerank_p95_ms": 120.9,
        "beam_p50_ms": 108.9,
        "next_words_p50_ms": 16.1,
        "footprint_peak_mb": 44.35
      },
      "v21_int8": {
        "predict_T128_p50_ms": 4.55,
        "rerank_p50_ms": 21.3,
        "rerank_p95_ms": 136.8,
        "beam_p50_ms": 101.8,
        "next_words_p50_ms": 13.2,
        "footprint_peak_mb": 49.95
      },
      "scope": "XCTest host process, not the keyboard extension"
    }
  },
  "deployment_status": "Research release. V2.1 remains the deployed default in Vime; V4.2 is an opt-in variant.",
  "source_reports_sha256": {
    "outputs/training-v42-20261020/remote/records/summary.json": "b15397eab9a11d732fe8465d2eecd17a840e25dad15714b6d6c35f59aba2d4ac",
    "outputs/evaluation-v42-20261020/standard-development/metrics.json": "17a03af37beaa494ab307bfa8199867dac5da8b601d416f500e73865abf23a28",
    "outputs/evaluation-v42-20261020/legacy-regression/metrics.json": "bbadc895779a62bd3b897efa8b8fcffa18ccd9563cb3d68b1211c7f636ef4864",
    "outputs/evaluation-v42-20261020/ajimee/metrics.json": "4c90bdea27b3876b58714eec204c5bef3a74c8030852bb8b622585d05eeab83d",
    "outputs/evaluation-v42-20261020/historical-development/metrics.json": "84d5e4008dcd372868c91713411be3bc3981a1f232f859e327bd32aaa32129e1",
    "outputs/evaluation-v42-20261020/plan.json": "d47df6154c193088b865782a1048794d542c2ecc2ffec938a3cd1a5eac2e579d",
    "outputs/evaluation-v42-reference-20261020/reference725/metrics.json": "5d7b2db2601317f68c7876f34ad19c41b17ca0c1ed0509cc419ceedd27dade3a",
    "outputs/evaluation-v42-reference-20261020/reference-test-plan.json": "76105753a0e65b08f59832b80db10ea549cf9a1c987ea3b98b83241b4bfd8de5",
    "outputs/evaluation-v42-reference-20261020/paired-v21.json": "0f824aa5a6513409d82655ff46e8a71750629d658311f45acc06dbecd5a0bf1f",
    "outputs/deployment/v42-mac-acceptance-v2/status.json": "f27981649fbaf85fffa99ce4d7f5a0337c6dd2c6fddfaa20da09e6e7c62f31df",
    "outputs/analysis-v42-regressions-20261020/summary.json": "c7d2e2d6c6edc24a3765d9761a11bca23d7a763ddde56b30ddbaf6cec4cb3ade"
  },
  "scope": "Aggregate metrics only. No corpus, benchmark text, candidate pools, per-case scores, case identifiers, user inputs or raw device traces are distributed."
}