| { |
| "format": "vimeml_public_evaluation_v4", |
| "model_version": "4.2-epoch20-step1081942", |
| "checkpoint_sha256": "0e83ea7417b9816b37183044468343b151f4e610da0764c31f1f780c5fa5796d", |
| "coreml_manifest_sha256": { |
| "stateless_int8": "0c3b6f3b809aaf3dd2f9ae18626c363f72dd34c59953dea48fe865ebbde9d6da", |
| "kvchunk_int8": "fd4bbb6c31fa2f71b2c9be6ab5feb0bbb03b7f51216907c7ff271171d26d1b75" |
| }, |
| "training": { |
| "epochs": 20, |
| "updates": 1081942, |
| "seed": 1219827974, |
| "supervised_targets": 12104389994, |
| "full_validation_lm_bpc_fp32": 3.0451515462768692, |
| "full_validation_chat_bpc_fp32": 3.122059834972166, |
| "selection": "best complete LM validation BPC (epoch 20), fixed before any IME scoring", |
| "test_or_blind_used": false |
| }, |
| "fixed_pool_fp32_cuda": { |
| "standard-development": { |
| "azookey": { |
| "cases": 728, |
| "top1_correct": 453, |
| "top1_accuracy": 0.6222527472527473, |
| "top5_correct": 608, |
| "top5_accuracy": 0.8351648351648352, |
| "candidate_pool_covered": 630, |
| "candidate_pool_recall": 0.8653846153846154, |
| "covered_top1_accuracy": 0.719047619047619, |
| "mean_min_cer": 0.05160664332397995, |
| "mrr": 0.7097974489912511, |
| "corrected_vs_azookey": 0, |
| "regressed_vs_azookey": 0, |
| "fallback_cases": 0 |
| }, |
| "lm_context_sum": { |
| "cases": 728, |
| "top1_correct": 523, |
| "top1_accuracy": 0.7184065934065934, |
| "top5_correct": 624, |
| "top5_accuracy": 0.8571428571428571, |
| "candidate_pool_covered": 630, |
| "candidate_pool_recall": 0.8653846153846154, |
| "covered_top1_accuracy": 0.8301587301587302, |
| "mean_min_cer": 0.034705386175760286, |
| "mrr": 0.7749688955732909, |
| "corrected_vs_azookey": 110, |
| "regressed_vs_azookey": 40, |
| "fallback_cases": 0 |
| }, |
| "lm_no_context_sum": { |
| "cases": 728, |
| "top1_correct": 510, |
| "top1_accuracy": 0.7005494505494505, |
| "top5_correct": 621, |
| "top5_accuracy": 0.853021978021978, |
| "candidate_pool_covered": 630, |
| "candidate_pool_recall": 0.8653846153846154, |
| "covered_top1_accuracy": 0.8095238095238095, |
| "mean_min_cer": 0.03638251311053943, |
| "mrr": 0.7641914137449851, |
| "corrected_vs_azookey": 108, |
| "regressed_vs_azookey": 51, |
| "fallback_cases": 0 |
| } |
| }, |
| "legacy-regression": { |
| "azookey": { |
| "cases": 500, |
| "top1_correct": 317, |
| "top1_accuracy": 0.634, |
| "top5_correct": 437, |
| "top5_accuracy": 0.874, |
| "candidate_pool_covered": 443, |
| "candidate_pool_recall": 0.886, |
| "covered_top1_accuracy": 0.7155756207674944, |
| "mean_min_cer": 0.04623841144643269, |
| "mrr": 0.7311617826617824, |
| "corrected_vs_azookey": 0, |
| "regressed_vs_azookey": 0, |
| "fallback_cases": 0 |
| }, |
| "lm_context_sum": { |
| "cases": 500, |
| "top1_correct": 370, |
| "top1_accuracy": 0.74, |
| "top5_correct": 442, |
| "top5_accuracy": 0.884, |
| "candidate_pool_covered": 443, |
| "candidate_pool_recall": 0.886, |
| "covered_top1_accuracy": 0.835214446952596, |
| "mean_min_cer": 0.028879998926345758, |
| "mrr": 0.8080666666666665, |
| "corrected_vs_azookey": 76, |
| "regressed_vs_azookey": 23, |
| "fallback_cases": 0 |
| }, |
| "lm_no_context_sum": { |
| "cases": 500, |
| "top1_correct": 369, |
| "top1_accuracy": 0.738, |
| "top5_correct": 442, |
| "top5_accuracy": 0.884, |
| "candidate_pool_covered": 443, |
| "candidate_pool_recall": 0.886, |
| "covered_top1_accuracy": 0.8329571106094809, |
| "mean_min_cer": 0.028316756130444452, |
| "mrr": 0.8037999999999997, |
| "corrected_vs_azookey": 77, |
| "regressed_vs_azookey": 25, |
| "fallback_cases": 0 |
| } |
| }, |
| "ajimee": { |
| "azookey": { |
| "cases": 200, |
| "top1_correct": 87, |
| "top1_accuracy": 0.435, |
| "top5_correct": 143, |
| "top5_accuracy": 0.715, |
| "candidate_pool_covered": 162, |
| "candidate_pool_recall": 0.81, |
| "covered_top1_accuracy": 0.5370370370370371, |
| "mean_min_cer": 0.0821630611981915, |
| "mrr": 0.554232017982018, |
| "corrected_vs_azookey": 0, |
| "regressed_vs_azookey": 0, |
| "fallback_cases": 0 |
| }, |
| "lm_context_sum": { |
| "cases": 200, |
| "top1_correct": 139, |
| "top1_accuracy": 0.695, |
| "top5_correct": 161, |
| "top5_accuracy": 0.805, |
| "candidate_pool_covered": 162, |
| "candidate_pool_recall": 0.81, |
| "covered_top1_accuracy": 0.8580246913580247, |
| "mean_min_cer": 0.03520357489337187, |
| "mrr": 0.7448809523809524, |
| "corrected_vs_azookey": 58, |
| "regressed_vs_azookey": 6, |
| "fallback_cases": 0 |
| }, |
| "lm_no_context_sum": { |
| "cases": 200, |
| "top1_correct": 130, |
| "top1_accuracy": 0.65, |
| "top5_correct": 161, |
| "top5_accuracy": 0.805, |
| "candidate_pool_covered": 162, |
| "candidate_pool_recall": 0.81, |
| "covered_top1_accuracy": 0.8024691358024691, |
| "mean_min_cer": 0.051505004672742855, |
| "mrr": 0.7155833333333333, |
| "corrected_vs_azookey": 52, |
| "regressed_vs_azookey": 9, |
| "fallback_cases": 0 |
| } |
| }, |
| "historical-development": { |
| "azookey": { |
| "cases": 137, |
| "top1_correct": 111, |
| "top1_accuracy": 0.8102189781021898, |
| "top5_correct": 130, |
| "top5_accuracy": 0.948905109489051, |
| "candidate_pool_covered": 136, |
| "candidate_pool_recall": 0.9927007299270073, |
| "covered_top1_accuracy": 0.8161764705882353, |
| "mean_min_cer": 0.04746998743349109, |
| "mrr": 0.8745925911071898, |
| "corrected_vs_azookey": 0, |
| "regressed_vs_azookey": 0, |
| "fallback_cases": 0 |
| }, |
| "lm_context_sum": { |
| "cases": 137, |
| "top1_correct": 123, |
| "top1_accuracy": 0.8978102189781022, |
| "top5_correct": 136, |
| "top5_accuracy": 0.9927007299270073, |
| "candidate_pool_covered": 136, |
| "candidate_pool_recall": 0.9927007299270073, |
| "covered_top1_accuracy": 0.9044117647058824, |
| "mean_min_cer": 0.02333226384321275, |
| "mrr": 0.9388077858880779, |
| "corrected_vs_azookey": 15, |
| "regressed_vs_azookey": 3, |
| "fallback_cases": 0 |
| }, |
| "lm_no_context_sum": { |
| "cases": 137, |
| "top1_correct": 124, |
| "top1_accuracy": 0.9051094890510949, |
| "top5_correct": 136, |
| "top5_accuracy": 0.9927007299270073, |
| "candidate_pool_covered": 136, |
| "candidate_pool_recall": 0.9927007299270073, |
| "covered_top1_accuracy": 0.9117647058823529, |
| "mean_min_cer": 0.019706327735524817, |
| "mrr": 0.9454987834549878, |
| "corrected_vs_azookey": 16, |
| "regressed_vs_azookey": 3, |
| "fallback_cases": 0 |
| } |
| } |
| }, |
| "fixed_reference_725": { |
| "plan_id": "81512f147d3749ef9a36e61470fce173", |
| "role": "reference_test_after_consumption", |
| "source_metrics": { |
| "by_source": { |
| "jmultiwoz": { |
| "cases": 250, |
| "top1_correct": 226, |
| "top1_accuracy": 0.904, |
| "top5_correct": 245, |
| "top5_accuracy": 0.98, |
| "candidate_pool_covered": 245, |
| "candidate_pool_recall": 0.98, |
| "covered_top1_accuracy": 0.9224489795918367, |
| "mean_min_cer": 0.009039811881542249, |
| "mrr": 0.9376666666666666, |
| "corrected_vs_azookey": 35, |
| "regressed_vs_azookey": 11, |
| "fallback_cases": 0 |
| }, |
| "wrime": { |
| "cases": 475, |
| "top1_correct": 315, |
| "top1_accuracy": 0.6631578947368421, |
| "top5_correct": 398, |
| "top5_accuracy": 0.8378947368421052, |
| "candidate_pool_covered": 405, |
| "candidate_pool_recall": 0.8526315789473684, |
| "covered_top1_accuracy": 0.7777777777777778, |
| "mean_min_cer": 0.05086094251484006, |
| "mrr": 0.7354361661730081, |
| "corrected_vs_azookey": 82, |
| "regressed_vs_azookey": 32, |
| "fallback_cases": 0 |
| } |
| }, |
| "independent_micro": { |
| "cases": 725, |
| "top1_correct": 541, |
| "top1_accuracy": 0.7462068965517241, |
| "top5_correct": 643, |
| "top5_accuracy": 0.886896551724138, |
| "candidate_pool_covered": 650, |
| "candidate_pool_recall": 0.896551724137931, |
| "covered_top1_accuracy": 0.8323076923076923, |
| "mean_min_cer": 0.03643986298611669, |
| "mrr": 0.805170821515649, |
| "corrected_vs_azookey": 117, |
| "regressed_vs_azookey": 43, |
| "fallback_cases": 0 |
| }, |
| "independent_equal_source_macro_top1": 0.783578947368421 |
| }, |
| "paired_vs_v21_fp32": { |
| "v1_wrong_v2_correct": 40, |
| "v1_correct_v2_wrong": 34, |
| "both_correct": 501, |
| "both_wrong": 150, |
| "net_correct": 6, |
| "exact_mcnemar_p_two_sided": 0.5613807442989819 |
| }, |
| "by_source_vs_v21": { |
| "wrime": { |
| "v1_wrong_v2_correct": 32, |
| "v1_correct_v2_wrong": 26, |
| "both_correct": 283, |
| "both_wrong": 134, |
| "net_correct": 6, |
| "exact_mcnemar_p_two_sided": 0.5118423084381689 |
| }, |
| "jmultiwoz": { |
| "v1_wrong_v2_correct": 8, |
| "v1_correct_v2_wrong": 8, |
| "both_correct": 218, |
| "both_wrong": 16, |
| "net_correct": 0, |
| "exact_mcnemar_p_two_sided": 1 |
| } |
| } |
| }, |
| "coreml": { |
| "top1": { |
| "cuda": { |
| "standard-development": 523, |
| "legacy-regression": 370, |
| "ajimee": 139, |
| "historical-development": 123 |
| }, |
| "mac_pytorch": { |
| "standard-development": 523, |
| "legacy-regression": 370, |
| "ajimee": 139, |
| "historical-development": 123 |
| }, |
| "coreml_fp32": { |
| "standard-development": 523, |
| "legacy-regression": 370, |
| "ajimee": 139, |
| "historical-development": 123 |
| }, |
| "coreml_int8": { |
| "standard-development": 522, |
| "legacy-regression": 371, |
| "ajimee": 139, |
| "historical-development": 123 |
| }, |
| "kv_fp32": { |
| "standard-development": 523, |
| "legacy-regression": 370, |
| "ajimee": 139, |
| "historical-development": 123 |
| }, |
| "kv_int8": { |
| "standard-development": 522, |
| "legacy-regression": 371, |
| "ajimee": 139, |
| "historical-development": 123 |
| }, |
| "kvchunk_fp32": { |
| "standard-development": 523, |
| "legacy-regression": 370, |
| "ajimee": 139, |
| "historical-development": 123 |
| }, |
| "kvchunk_int8": { |
| "standard-development": 522, |
| "legacy-regression": 371, |
| "ajimee": 139, |
| "historical-development": 123 |
| }, |
| "v21_coreml_fp32": { |
| "standard-development": 505, |
| "legacy-regression": 374, |
| "ajimee": 144, |
| "historical-development": 122 |
| }, |
| "v21_coreml_int8": { |
| "standard-development": 506, |
| "legacy-regression": 375, |
| "ajimee": 144, |
| "historical-development": 122 |
| } |
| }, |
| "checks": { |
| "1_mac_pytorch_fixtures_within_3e-4": true, |
| "1_mac_pytorch_top1_equals_recorded_cuda": true, |
| "1_mac_pytorch_orders_equal_cuda_all_methods": true, |
| "2_fp32_stateless_and_kv_converted": true, |
| "3_fp32_stateless_alignment_passed": true, |
| "3_fp32_kv_alignment_passed": true, |
| "3_coreml_fp32_top1_equals_mac_pytorch": true, |
| "3_coreml_fp32_orders_equal_mac_pytorch_all_methods": true, |
| "3_kv_fp32_top1_equals_mac_pytorch": true, |
| "3_kv_fp32_orders_equal_mac_pytorch_all_methods": true, |
| "4_int8_strict_logits_alignment_passed (expected false)": false, |
| "4_int8_kv_vs_same_weight_stateless_int8_passed": true, |
| "4_kv_int8_top1_equals_stateless_int8": true, |
| "4_kv_int8_orders_equal_stateless_int8_all_methods": false, |
| "7_kvchunk_fp32_alignment_passed": true, |
| "7_kvchunk_int8_vs_same_weight_stateless_int8_passed": true, |
| "7_kvchunk_int8_strict_vs_fp32_fixtures_passed (expected false)": false, |
| "7_kvchunk_fp32_top1_equals_stateless_fp32": true, |
| "7_kvchunk_fp32_orders_equal_stateless_fp32_all_methods": true, |
| "7_kvchunk_int8_top1_equals_stateless_int8": true, |
| "7_kvchunk_int8_orders_equal_stateless_int8_all_methods": false, |
| "8_v21_coreml_fp32_top1_equals_recorded": true, |
| "5_measurements_recorded": true, |
| "6_ios_resources_prepared": true |
| }, |
| "int8_vs_coreml_fp32": { |
| "standard-development": { |
| "fp32": 523, |
| "int8": 522, |
| "v1_wrong_v2_correct": 2, |
| "v1_correct_v2_wrong": 3, |
| "both_correct": 520, |
| "both_wrong": 203, |
| "net_correct": -1, |
| "exact_mcnemar_p_two_sided": 1, |
| "significant_at_0_05": false, |
| "order_changed_cases": 254, |
| "top1_changed_cases": 11, |
| "max_score_sum_abs": 0.7504582202527672, |
| "mean_score_sum_abs": 0.12163806840569316 |
| }, |
| "legacy-regression": { |
| "fp32": 370, |
| "int8": 371, |
| "v1_wrong_v2_correct": 1, |
| "v1_correct_v2_wrong": 0, |
| "both_correct": 370, |
| "both_wrong": 129, |
| "net_correct": 1, |
| "exact_mcnemar_p_two_sided": 1, |
| "significant_at_0_05": false, |
| "order_changed_cases": 46, |
| "top1_changed_cases": 2, |
| "max_score_sum_abs": 0.8114978885278106, |
| "mean_score_sum_abs": 0.11693049636118634 |
| }, |
| "ajimee": { |
| "fp32": 139, |
| "int8": 139, |
| "v1_wrong_v2_correct": 0, |
| "v1_correct_v2_wrong": 0, |
| "both_correct": 139, |
| "both_wrong": 61, |
| "net_correct": 0, |
| "exact_mcnemar_p_two_sided": 1.0, |
| "significant_at_0_05": false, |
| "order_changed_cases": 106, |
| "top1_changed_cases": 0, |
| "max_score_sum_abs": 0.9850609144195914, |
| "mean_score_sum_abs": 0.1471606100397155 |
| }, |
| "historical-development": { |
| "fp32": 123, |
| "int8": 123, |
| "v1_wrong_v2_correct": 0, |
| "v1_correct_v2_wrong": 0, |
| "both_correct": 123, |
| "both_wrong": 14, |
| "net_correct": 0, |
| "exact_mcnemar_p_two_sided": 1.0, |
| "significant_at_0_05": false, |
| "order_changed_cases": 88, |
| "top1_changed_cases": 0, |
| "max_score_sum_abs": 0.7235050201416016, |
| "mean_score_sum_abs": 0.09826715727665117 |
| } |
| }, |
| "v21_int8_vs_v42_int8": { |
| "standard-development": { |
| "v1_wrong_v2_correct": 44, |
| "v1_correct_v2_wrong": 28, |
| "both_correct": 478, |
| "both_wrong": 178, |
| "net_correct": 16, |
| "exact_mcnemar_p_two_sided": 0.07636915467183139 |
| }, |
| "legacy-regression": { |
| "v1_wrong_v2_correct": 13, |
| "v1_correct_v2_wrong": 17, |
| "both_correct": 358, |
| "both_wrong": 112, |
| "net_correct": -4, |
| "exact_mcnemar_p_two_sided": 0.584664711728692 |
| }, |
| "ajimee": { |
| "v1_wrong_v2_correct": 9, |
| "v1_correct_v2_wrong": 14, |
| "both_correct": 130, |
| "both_wrong": 47, |
| "net_correct": -5, |
| "exact_mcnemar_p_two_sided": 0.4048728942871094 |
| }, |
| "historical-development": { |
| "v1_wrong_v2_correct": 4, |
| "v1_correct_v2_wrong": 3, |
| "both_correct": 119, |
| "both_wrong": 11, |
| "net_correct": 1, |
| "exact_mcnemar_p_two_sided": 1 |
| } |
| }, |
| "alignment": { |
| "v42-fp32-alignment-v1": { |
| "passed": true, |
| "checks": 5, |
| "max_abs": 5.245208740234375e-05, |
| "failed_checks": 0 |
| }, |
| "v42-kv-fp32-alignment-v1": { |
| "passed": true, |
| "checks": 17, |
| "max_abs": 4.9591064453125e-05, |
| "failed_checks": 0 |
| }, |
| "v42-int8-alignment-v1": { |
| "passed": false, |
| "checks": 5, |
| "max_abs": 0.4774494171142578, |
| "failed_checks": 5 |
| }, |
| "v42-kv-int8-vs-stateless-int8-v1": { |
| "passed": true, |
| "checks": 16, |
| "max_abs": 4.00543212890625e-05, |
| "failed_checks": 0 |
| }, |
| "v42-kv-int8-vs-fp32-fixtures-v1": { |
| "passed": false, |
| "checks": 17, |
| "max_abs": 0.707888126373291, |
| "failed_checks": 17 |
| }, |
| "v42-kvchunk-fp32-alignment-v1": { |
| "passed": true, |
| "checks": 27, |
| "max_abs": 5.245208740234375e-05, |
| "failed_checks": 0 |
| }, |
| "v42-kvchunk-int8-vs-stateless-int8-v1": { |
| "passed": true, |
| "checks": 26, |
| "max_abs": 4.00543212890625e-05, |
| "failed_checks": 0 |
| }, |
| "v42-kvchunk-int8-vs-fp32-fixtures-v1": { |
| "passed": false, |
| "checks": 22, |
| "max_abs": 0.7078914642333984, |
| "failed_checks": 22 |
| } |
| } |
| }, |
| "notation_flip_analysis_vs_v21": { |
| "standard-development": { |
| "A": { |
| "improved": 28, |
| "regressed": 23, |
| "net": 5 |
| }, |
| "S": { |
| "improved": 15, |
| "regressed": 5, |
| "net": 10 |
| }, |
| "P": { |
| "improved": 3, |
| "regressed": 0, |
| "net": 3 |
| }, |
| "meaning_plus_proper_exact_p": 0.010622024536132812, |
| "flips_margin_lt_1nat": { |
| "all": 19, |
| "of": 74, |
| "A": 16 |
| } |
| }, |
| "legacy-regression": { |
| "A": { |
| "improved": 12, |
| "regressed": 17, |
| "net": -5 |
| }, |
| "S": { |
| "improved": 3, |
| "regressed": 2, |
| "net": 1 |
| }, |
| "P": { |
| "improved": 0, |
| "regressed": 0, |
| "net": 0 |
| }, |
| "meaning_plus_proper_exact_p": 1.0, |
| "flips_margin_lt_1nat": { |
| "all": 12, |
| "of": 34, |
| "A": 11 |
| } |
| }, |
| "ajimee": { |
| "A": { |
| "improved": 1, |
| "regressed": 2, |
| "net": -1 |
| }, |
| "S": { |
| "improved": 7, |
| "regressed": 11, |
| "net": -4 |
| }, |
| "P": { |
| "improved": 0, |
| "regressed": 0, |
| "net": 0 |
| }, |
| "meaning_plus_proper_exact_p": 0.480682373046875, |
| "flips_margin_lt_1nat": { |
| "all": 6, |
| "of": 21, |
| "A": 2 |
| } |
| }, |
| "historical-development": { |
| "A": { |
| "improved": 3, |
| "regressed": 1, |
| "net": 2 |
| }, |
| "S": { |
| "improved": 1, |
| "regressed": 2, |
| "net": -1 |
| }, |
| "P": { |
| "improved": 0, |
| "regressed": 0, |
| "net": 0 |
| }, |
| "meaning_plus_proper_exact_p": 1.0, |
| "flips_margin_lt_1nat": { |
| "all": 3, |
| "of": 7, |
| "A": 2 |
| } |
| }, |
| "reference725": { |
| "A": { |
| "improved": 27, |
| "regressed": 24, |
| "net": 3 |
| }, |
| "S": { |
| "improved": 11, |
| "regressed": 7, |
| "net": 4 |
| }, |
| "P": { |
| "improved": 2, |
| "regressed": 3, |
| "net": -1 |
| }, |
| "meaning_plus_proper_exact_p": 0.6776394844055176, |
| "flips_margin_lt_1nat": { |
| "all": 23, |
| "of": 74, |
| "A": 19 |
| } |
| } |
| }, |
| "measurements": { |
| "mac": { |
| "source_sha256": "66eeb948028365445bb1a8dad6e84e9f8d69806ac9702dd8293821b1a47e2a53", |
| "hardware": "Apple M3, macOS 27.2, CPU_ONLY", |
| "stateless_int8_warm_p50_ms": { |
| "T1": 1.3, |
| "T128": 6.07 |
| }, |
| "v21_int8_warm_p50_ms": { |
| "T1": 0.99, |
| "T128": 5.68 |
| }, |
| "kvchunk_int8_prefill_p50_ms": { |
| "P7": 2.49, |
| "P31": 3.52, |
| "P63": 4.55, |
| "P119": 6.56 |
| }, |
| "v21_kv_int8_prefill_p50_ms": { |
| "P7": 2.19, |
| "P31": 3.03, |
| "P63": 4.13, |
| "P119": 6.11 |
| } |
| }, |
| "iphone16promax_xctest_host": { |
| "source_sha256": "3c8566cef8ad78b11d5a9d1bf3eb55d6eadef8aaf3c1d0f5aa7680615a562cbb", |
| "conditions": "iOS 27.2, wired and charging, thermal state fair", |
| "v42_int8": { |
| "predict_T128_p50_ms": 4.9, |
| "rerank_p50_ms": 25.1, |
| "rerank_p95_ms": 120.9, |
| "beam_p50_ms": 108.9, |
| "next_words_p50_ms": 16.1, |
| "footprint_peak_mb": 44.35 |
| }, |
| "v21_int8": { |
| "predict_T128_p50_ms": 4.55, |
| "rerank_p50_ms": 21.3, |
| "rerank_p95_ms": 136.8, |
| "beam_p50_ms": 101.8, |
| "next_words_p50_ms": 13.2, |
| "footprint_peak_mb": 49.95 |
| }, |
| "scope": "XCTest host process, not the keyboard extension" |
| } |
| }, |
| "deployment_status": "Research release. V2.1 remains the deployed default in Vime; V4.2 is an opt-in variant.", |
| "source_reports_sha256": { |
| "outputs/training-v42-20261020/remote/records/summary.json": "b15397eab9a11d732fe8465d2eecd17a840e25dad15714b6d6c35f59aba2d4ac", |
| "outputs/evaluation-v42-20261020/standard-development/metrics.json": "17a03af37beaa494ab307bfa8199867dac5da8b601d416f500e73865abf23a28", |
| "outputs/evaluation-v42-20261020/legacy-regression/metrics.json": "bbadc895779a62bd3b897efa8b8fcffa18ccd9563cb3d68b1211c7f636ef4864", |
| "outputs/evaluation-v42-20261020/ajimee/metrics.json": "4c90bdea27b3876b58714eec204c5bef3a74c8030852bb8b622585d05eeab83d", |
| "outputs/evaluation-v42-20261020/historical-development/metrics.json": "84d5e4008dcd372868c91713411be3bc3981a1f232f859e327bd32aaa32129e1", |
| "outputs/evaluation-v42-20261020/plan.json": "d47df6154c193088b865782a1048794d542c2ecc2ffec938a3cd1a5eac2e579d", |
| "outputs/evaluation-v42-reference-20261020/reference725/metrics.json": "5d7b2db2601317f68c7876f34ad19c41b17ca0c1ed0509cc419ceedd27dade3a", |
| "outputs/evaluation-v42-reference-20261020/reference-test-plan.json": "76105753a0e65b08f59832b80db10ea549cf9a1c987ea3b98b83241b4bfd8de5", |
| "outputs/evaluation-v42-reference-20261020/paired-v21.json": "0f824aa5a6513409d82655ff46e8a71750629d658311f45acc06dbecd5a0bf1f", |
| "outputs/deployment/v42-mac-acceptance-v2/status.json": "f27981649fbaf85fffa99ce4d7f5a0337c6dd2c6fddfaa20da09e6e7c62f31df", |
| "outputs/analysis-v42-regressions-20261020/summary.json": "c7d2e2d6c6edc24a3765d9761a11bca23d7a763ddde56b30ddbaf6cec4cb3ade" |
| }, |
| "scope": "Aggregate metrics only. No corpus, benchmark text, candidate pools, per-case scores, case identifiers, user inputs or raw device traces are distributed." |
| } |
|
|
