| { |
| "format": "vimeml_public_evaluation_v2", |
| "model_version": "2.1-extend5-step40000-int8-b32-v1", |
| "checkpoint_sha256": "fa202f18d00cf5e77469bd3be5b8b264117fb73233ec9966fa3213257a470104", |
| "coreml_manifest_sha256": "1dda48c76eacfa3ba99764098fab721eed4ada413130b99c0324620365aca0b7", |
| "training": { |
| "full_validation_bpc_fp32": 3.0802686334, |
| "test_used_for_selection": false, |
| "v20_epochs": 4, |
| "restart1_epochs": 2, |
| "extend5_stopped_step": 161095, |
| "selected_extend5_step": 40000 |
| }, |
| "ranking": { |
| "ajimee": { |
| "original_order": { |
| "cases": 200, |
| "top1_correct": 87, |
| "top1_accuracy": 0.435, |
| "top5_correct": 143, |
| "top5_accuracy": 0.715, |
| "candidate_pool_covered": 162, |
| "candidate_pool_recall": 0.81, |
| "covered_top1_accuracy": 0.5370370370370371, |
| "mean_min_cer": 0.0821630611981915, |
| "mrr": 0.554232017982018, |
| "corrected_vs_azookey": 0, |
| "regressed_vs_azookey": 0, |
| "fallback_cases": 0 |
| }, |
| "fp32_lm": { |
| "cases": 200, |
| "top1_correct": 144, |
| "top1_accuracy": 0.72, |
| "top5_correct": 160, |
| "top5_accuracy": 0.8, |
| "candidate_pool_covered": 162, |
| "candidate_pool_recall": 0.81, |
| "covered_top1_accuracy": 0.8888888888888888, |
| "mean_min_cer": 0.030729146273316313, |
| "mrr": 0.7542142857142857, |
| "corrected_vs_azookey": 61, |
| "regressed_vs_azookey": 4, |
| "fallback_cases": 0 |
| }, |
| "int8_lm": { |
| "cases": 200, |
| "top1_correct": 144, |
| "top1_accuracy": 0.72, |
| "top5_correct": 160, |
| "top5_accuracy": 0.8, |
| "candidate_pool_covered": 162, |
| "candidate_pool_recall": 0.81, |
| "covered_top1_accuracy": 0.8888888888888888, |
| "mean_min_cer": 0.03116485517226051, |
| "mrr": 0.7533809523809524, |
| "corrected_vs_azookey": 62, |
| "regressed_vs_azookey": 5, |
| "fallback_cases": 0 |
| }, |
| "fp32_comparison": { |
| "order_changed_cases": 163, |
| "top1_changed_cases": 7, |
| "max_score_sum_abs": 1.8200132250785828, |
| "mean_score_sum_abs": 0.2882139610526506 |
| }, |
| "labels_formal_gold": null |
| }, |
| "development": { |
| "original_order": { |
| "cases": 137, |
| "top1_correct": 111, |
| "top1_accuracy": 0.8102189781021898, |
| "top5_correct": 130, |
| "top5_accuracy": 0.948905109489051, |
| "candidate_pool_covered": 136, |
| "candidate_pool_recall": 0.9927007299270073, |
| "covered_top1_accuracy": 0.8161764705882353, |
| "mean_min_cer": 0.04746998743349109, |
| "mrr": 0.8745925911071898, |
| "corrected_vs_azookey": 0, |
| "regressed_vs_azookey": 0, |
| "fallback_cases": 0 |
| }, |
| "fp32_lm": { |
| "cases": 137, |
| "top1_correct": 122, |
| "top1_accuracy": 0.8905109489051095, |
| "top5_correct": 136, |
| "top5_accuracy": 0.9927007299270073, |
| "candidate_pool_covered": 136, |
| "candidate_pool_recall": 0.9927007299270073, |
| "covered_top1_accuracy": 0.8970588235294118, |
| "mean_min_cer": 0.029006070976873896, |
| "mrr": 0.936374695863747, |
| "corrected_vs_azookey": 14, |
| "regressed_vs_azookey": 3, |
| "fallback_cases": 0 |
| }, |
| "int8_lm": { |
| "cases": 137, |
| "top1_correct": 122, |
| "top1_accuracy": 0.8905109489051095, |
| "top5_correct": 136, |
| "top5_accuracy": 0.9927007299270073, |
| "candidate_pool_covered": 136, |
| "candidate_pool_recall": 0.9927007299270073, |
| "covered_top1_accuracy": 0.8970588235294118, |
| "mean_min_cer": 0.029006070976873896, |
| "mrr": 0.9351581508515816, |
| "corrected_vs_azookey": 14, |
| "regressed_vs_azookey": 3, |
| "fallback_cases": 0 |
| }, |
| "fp32_comparison": { |
| "order_changed_cases": 121, |
| "top1_changed_cases": 1, |
| "max_score_sum_abs": 1.3734395503997803, |
| "mean_score_sum_abs": 0.19201541958957397 |
| }, |
| "labels_formal_gold": null |
| }, |
| "expanded-development": { |
| "original_order": { |
| "cases": 2000, |
| "top1_correct": 1224, |
| "top1_accuracy": 0.612, |
| "top5_correct": 1711, |
| "top5_accuracy": 0.8555, |
| "candidate_pool_covered": 1756, |
| "candidate_pool_recall": 0.878, |
| "covered_top1_accuracy": 0.6970387243735763, |
| "mean_min_cer": 0.049437279587549655, |
| "mrr": 0.714617032967033, |
| "corrected_vs_azookey": 0, |
| "regressed_vs_azookey": 0, |
| "fallback_cases": 0 |
| }, |
| "fp32_lm": { |
| "cases": 2000, |
| "top1_correct": 1487, |
| "top1_accuracy": 0.7435, |
| "top5_correct": 1748, |
| "top5_accuracy": 0.874, |
| "candidate_pool_covered": 1756, |
| "candidate_pool_recall": 0.878, |
| "covered_top1_accuracy": 0.8468109339407744, |
| "mean_min_cer": 0.029395604106192606, |
| "mrr": 0.8023303571428563, |
| "corrected_vs_azookey": 322, |
| "regressed_vs_azookey": 59, |
| "fallback_cases": 0 |
| }, |
| "int8_lm": { |
| "cases": 2000, |
| "top1_correct": 1481, |
| "top1_accuracy": 0.7405, |
| "top5_correct": 1748, |
| "top5_accuracy": 0.874, |
| "candidate_pool_covered": 1756, |
| "candidate_pool_recall": 0.878, |
| "covered_top1_accuracy": 0.8433940774487472, |
| "mean_min_cer": 0.02966218634430054, |
| "mrr": 0.800903769841269, |
| "corrected_vs_azookey": 319, |
| "regressed_vs_azookey": 62, |
| "fallback_cases": 0 |
| }, |
| "fp32_comparison": { |
| "order_changed_cases": 404, |
| "top1_changed_cases": 30, |
| "max_score_sum_abs": 1.790113139897585, |
| "mean_score_sum_abs": 0.22886438840285703 |
| }, |
| "labels_formal_gold": false |
| } |
| }, |
| "alignment": { |
| "fp32": { |
| "strict_passed": true, |
| "tolerances": { |
| "atol": 0.0003, |
| "rtol": 0.0003 |
| }, |
| "logits": [ |
| { |
| "max_abs": 9.5367431640625e-06, |
| "mean_abs": 1.2576065273606218e-06, |
| "rmse": 1.6289324748934612e-06, |
| "outside_tolerance": 0, |
| "values": 16384, |
| "id": "bos", |
| "length": 1, |
| "valid_prefix": 1, |
| "last_top1_equal": true |
| }, |
| { |
| "max_abs": 3.910064697265625e-05, |
| "mean_abs": 3.185306411523925e-06, |
| "rmse": 4.141091950346693e-06, |
| "outside_tolerance": 0, |
| "values": 262144, |
| "id": "short", |
| "length": 16, |
| "valid_prefix": 16, |
| "last_top1_equal": true |
| }, |
| { |
| "max_abs": 4.57763671875e-05, |
| "mean_abs": 4.321694975506236e-06, |
| "rmse": 5.538972619049361e-06, |
| "outside_tolerance": 0, |
| "values": 2097152, |
| "id": "long", |
| "length": 128, |
| "valid_prefix": 128, |
| "last_top1_equal": true |
| }, |
| { |
| "max_abs": 4.482269287109375e-05, |
| "mean_abs": 3.3942287953614425e-06, |
| "rmse": 4.3666045771710725e-06, |
| "outside_tolerance": 0, |
| "values": 262144, |
| "id": "right_pad", |
| "length": 128, |
| "valid_prefix": 16, |
| "last_top1_equal": true |
| }, |
| { |
| "max_abs": 4.482269287109375e-05, |
| "mean_abs": 3.3942287953614425e-06, |
| "rmse": 4.3666045771710725e-06, |
| "outside_tolerance": 0, |
| "values": 262144, |
| "id": "changed_future", |
| "length": 128, |
| "valid_prefix": 16, |
| "last_top1_equal": true |
| } |
| ], |
| "invariants": { |
| "right_pad": { |
| "max_abs": 0.0, |
| "mean_abs": 0.0, |
| "rmse": 0.0, |
| "outside_tolerance": 0, |
| "values": 262144 |
| }, |
| "changed_future": { |
| "max_abs": 0.0, |
| "mean_abs": 0.0, |
| "rmse": 0.0, |
| "outside_tolerance": 0, |
| "values": 262144 |
| } |
| } |
| }, |
| "int8": { |
| "strict_passed": false, |
| "tolerances": { |
| "atol": 0.0003, |
| "rtol": 0.0003 |
| }, |
| "logits": [ |
| { |
| "max_abs": 0.1717967987060547, |
| "mean_abs": 0.03132702315087954, |
| "rmse": 0.03977237435193969, |
| "outside_tolerance": 15692, |
| "values": 16384, |
| "id": "bos", |
| "length": 1, |
| "valid_prefix": 1, |
| "last_top1_equal": true |
| }, |
| { |
| "max_abs": 0.8956146240234375, |
| "mean_abs": 0.10476142495688912, |
| "rmse": 0.13775674684387576, |
| "outside_tolerance": 257685, |
| "values": 262144, |
| "id": "short", |
| "length": 16, |
| "valid_prefix": 16, |
| "last_top1_equal": true |
| }, |
| { |
| "max_abs": 1.0094914436340332, |
| "mean_abs": 0.10851713189305306, |
| "rmse": 0.1394378905311297, |
| "outside_tolerance": 2067332, |
| "values": 2097152, |
| "id": "long", |
| "length": 128, |
| "valid_prefix": 128, |
| "last_top1_equal": true |
| }, |
| { |
| "max_abs": 0.8956174850463867, |
| "mean_abs": 0.10476151736959416, |
| "rmse": 0.13775683790182477, |
| "outside_tolerance": 257684, |
| "values": 262144, |
| "id": "right_pad", |
| "length": 128, |
| "valid_prefix": 16, |
| "last_top1_equal": true |
| }, |
| { |
| "max_abs": 0.8956174850463867, |
| "mean_abs": 0.10476151736959416, |
| "rmse": 0.13775683790182477, |
| "outside_tolerance": 257684, |
| "values": 262144, |
| "id": "changed_future", |
| "length": 128, |
| "valid_prefix": 16, |
| "last_top1_equal": true |
| } |
| ], |
| "invariants": { |
| "right_pad": { |
| "max_abs": 0.0, |
| "mean_abs": 0.0, |
| "rmse": 0.0, |
| "outside_tolerance": 0, |
| "values": 262144 |
| }, |
| "changed_future": { |
| "max_abs": 0.0, |
| "mean_abs": 0.0, |
| "rmse": 0.0, |
| "outside_tolerance": 0, |
| "values": 262144 |
| } |
| } |
| } |
| }, |
| "fixed_generation": { |
| "next_token_top1_equal": 12, |
| "greedy_32_token_equal": 8, |
| "total": 12 |
| }, |
| "quantization_diagnostics": { |
| "format": "vimeml_quantization_analysis_v1", |
| "date": "2026-10-08", |
| "source_paths": [ |
| "outputs/ime-eval/expanded-v21-dev-draft-extend5-best/scores.jsonl", |
| "outputs/deployment/v21-int8-expanded-development-v1/scores.jsonl" |
| ], |
| "scope": "Existing frozen draft development scores; no model execution, blind data or label edits", |
| "cases": 2000, |
| "top1_changes": 30, |
| "wins": 8, |
| "losses": 14, |
| "paired_exact_two_sided_p": 0.28627872467041016, |
| "fp32_top1_margin_by_change": { |
| "changed": { |
| "cases": 30, |
| "median_logp_sum": 0.07003979850560427, |
| "maximum_logp_sum": 0.36277501191943884, |
| "count_margin_le_0_25": 27 |
| }, |
| "unchanged": { |
| "cases": 1910, |
| "median_logp_sum": 3.0718008121475577, |
| "maximum_logp_sum": 26.55984030291438, |
| "count_margin_le_0_25": 85 |
| } |
| }, |
| "interpretation": "No statistically significant accuracy decrease detected; this does not establish equivalence or formal gold quality." |
| }, |
| "physical_keyboard_extension": { |
| "device": "iPhone 16 Pro Max", |
| "os": "iOS 27.2 (24B5099f)", |
| "pid": 17014, |
| "model_version": "2.1-extend5-step40000-int8-b32-v1", |
| "instruments_duration_seconds": 201.29728, |
| "instruments_samples": 164, |
| "in_process_duration_seconds": 191.90217375, |
| "in_process_samples": 1920, |
| "kernel_lifetime_footprint_peak_mib": 34.720596, |
| "in_process_sampled_footprint_peak_mib": 28.298721, |
| "instruments_sampled_footprint_peak_mib": 28.001846, |
| "instruments_private_resident_peak_mib": 73.78125, |
| "lm_scoring_p95_ms": 53.915625, |
| "lm_next_words_p95_ms": 38.114333, |
| "measurement_enabled_after_completion": false, |
| "input_text_recorded": false, |
| "scope": "Bounded real extension workload; separate from test host, sampled resident values and lifetime physical footprint" |
| }, |
| "limitations": [ |
| { |
| "item": "INT8 strict frozen logits alignment", |
| "result": "failed", |
| "max_absolute_error": 1.0094914436340332, |
| "atol": 0.0003, |
| "rtol": 0.0003 |
| }, |
| { |
| "item": "Expanded development draft Top-1", |
| "fp32": 1487, |
| "int8": 1481, |
| "total": 2000, |
| "delta_percentage_points": -0.3 |
| }, |
| { |
| "item": "Actual extension result-to-UI long tail", |
| "maximum_ms": 1926.0947, |
| "cause": "unlocated; no per-event correlation timestamps" |
| }, |
| { |
| "item": "Late-window extension footprint growth and second model load", |
| "cause": "unlocated; insufficient evidence for leak/cache/host attribution" |
| }, |
| { |
| "item": "Fully cold startup", |
| "status": "not certified: measured first process load may use warm OS/Core ML caches" |
| }, |
| { |
| "item": "Long-term footprint plateau/system termination audit", |
| "status": "not certified by this bounded workload" |
| } |
| ], |
| "experiment_decision": "V2 series complete; current INT8 task-quality loss and observed UI tail accepted.", |
| "source_reports_sha256": { |
| "outputs/deployment/v21-int8-ajimee-v1/metrics.json": "7b136dbed85b2f354eeae1838eb7f32e34c8eb3e909fe4fcce1ac449f1480c63", |
| "outputs/deployment/v21-int8-development-v1/metrics.json": "f1384fa5d5b7c4da2503a9118300a0857bf2ef5c4ee2d7213817851839f2394c", |
| "outputs/deployment/v21-int8-expanded-development-v1/metrics.json": "0a07dbe706add83b3c3e917dce1334d0227f105d29347671bca539442f887226", |
| "outputs/deployment/v21-fp32-alignment-v1/alignment.json": "3c0a368f4836388733fcb21882e431b8b7548b1e9ea61fefeeb479ce51954b0b", |
| "outputs/deployment/v21-int8-alignment-v1/alignment.json": "791d047cf14d15025a33bc51c0e3e1ebaa986247832bdd42fcda027f8901db7f", |
| "outputs/deployment/v21-closeout-v1/acceptance.json": "1992372e8867f29c0d2add7d9d15ab0ace302555e42741c821bad8d6c0e20e3d", |
| "outputs/deployment/v21-quantization-analysis-v1/analysis.json": "0fd04daf4e8b86a6cb028d89f4c598312bdafcbca271f2b27d445b6db581a132" |
| }, |
| "scope": "Saved fixed-pool metrics and bounded device workload; draft expanded labels, no blind scoring. No benchmark texts, per-case scores, user inputs or raw device traces distributed." |
| } |
|
|
