| { |
| "format": "vimeml_public_evaluation_v1", |
| "bundle_manifest_sha256": "4284cdc5491513cc817633081677b363b14f59c0b4ffb78cdf7fcfb221f71b3c", |
| "coreml_manifest_sha256": "a9d516233fa57aaadd09477e2e909760972ff49b5577919f8a441c489f5c2245", |
| "training": { |
| "prediction_targets": 573295237, |
| "data_passes": 1.0, |
| "full_validation": { |
| "loss": 4.541570895102848, |
| "prediction_pairs": 5890326, |
| "perplexity": 93.83809420004854 |
| }, |
| "test_used_for_selection": false |
| }, |
| "ranking": { |
| "dev": { |
| "original_order": { |
| "cases": 137, |
| "top1_correct": 111, |
| "top1_accuracy": 0.8102189781021898, |
| "top5_correct": 130, |
| "top5_accuracy": 0.948905109489051, |
| "candidate_pool_covered": 136, |
| "candidate_pool_recall": 0.9927007299270073, |
| "covered_top1_accuracy": 0.8161764705882353, |
| "mean_min_cer": 0.04746998743349109, |
| "mrr": 0.8745925911071898, |
| "corrected_vs_azookey": 0, |
| "regressed_vs_azookey": 0, |
| "fallback_cases": 0 |
| }, |
| "fp32_lm": { |
| "cases": 137, |
| "top1_correct": 122, |
| "top1_accuracy": 0.8905109489051095, |
| "top5_correct": 134, |
| "top5_accuracy": 0.9781021897810219, |
| "candidate_pool_covered": 136, |
| "candidate_pool_recall": 0.9927007299270073, |
| "covered_top1_accuracy": 0.8970588235294118, |
| "mean_min_cer": 0.03045798483754688, |
| "mrr": 0.935349322210636, |
| "corrected_vs_azookey": 15, |
| "regressed_vs_azookey": 4, |
| "fallback_cases": 0 |
| }, |
| "int8_lm": { |
| "cases": 137, |
| "top1_correct": 122, |
| "top1_accuracy": 0.8905109489051095, |
| "top5_correct": 134, |
| "top5_accuracy": 0.9781021897810219, |
| "candidate_pool_covered": 136, |
| "candidate_pool_recall": 0.9927007299270073, |
| "covered_top1_accuracy": 0.8970588235294118, |
| "mean_min_cer": 0.03045798483754688, |
| "mrr": 0.935349322210636, |
| "corrected_vs_azookey": 15, |
| "regressed_vs_azookey": 4, |
| "fallback_cases": 0 |
| }, |
| "windows_fp32_comparison": { |
| "order_changed_cases": 119, |
| "top1_changed_cases": 0, |
| "max_score_sum_abs": 1.9853296279907227, |
| "mean_score_sum_abs": 0.20199488408841776 |
| }, |
| "original_report_comparison_baseline": "conservative Core ML", |
| "original_report_comparison": { |
| "order_changed_cases": 118, |
| "top1_changed_cases": 0, |
| "max_score_sum_abs": 1.994759202003479, |
| "mean_score_sum_abs": 0.2013680215637894 |
| }, |
| "benchmark_source_sha256": "a945f2117570530ef73b1dd4ef53b9dc69b07fa69ac98a6d67c9268b540eafa8", |
| "candidate_export_sha256": "ab192509dce1cb4f4b20273e623f7bfd4237b1c3691950438e6921b8a510ae21" |
| }, |
| "ajimee": { |
| "original_order": { |
| "cases": 200, |
| "top1_correct": 87, |
| "top1_accuracy": 0.435, |
| "top5_correct": 143, |
| "top5_accuracy": 0.715, |
| "candidate_pool_covered": 162, |
| "candidate_pool_recall": 0.81, |
| "covered_top1_accuracy": 0.5370370370370371, |
| "mean_min_cer": 0.0821630611981915, |
| "mrr": 0.554232017982018, |
| "corrected_vs_azookey": 0, |
| "regressed_vs_azookey": 0, |
| "fallback_cases": 0 |
| }, |
| "fp32_lm": { |
| "cases": 200, |
| "top1_correct": 124, |
| "top1_accuracy": 0.62, |
| "top5_correct": 151, |
| "top5_accuracy": 0.755, |
| "candidate_pool_covered": 162, |
| "candidate_pool_recall": 0.81, |
| "covered_top1_accuracy": 0.7654320987654321, |
| "mean_min_cer": 0.05657040945427151, |
| "mrr": 0.684061974789916, |
| "corrected_vs_azookey": 48, |
| "regressed_vs_azookey": 11, |
| "fallback_cases": 0 |
| }, |
| "int8_lm": { |
| "cases": 200, |
| "top1_correct": 124, |
| "top1_accuracy": 0.62, |
| "top5_correct": 151, |
| "top5_accuracy": 0.755, |
| "candidate_pool_covered": 162, |
| "candidate_pool_recall": 0.81, |
| "covered_top1_accuracy": 0.7654320987654321, |
| "mean_min_cer": 0.05657040945427151, |
| "mrr": 0.6841512605042016, |
| "corrected_vs_azookey": 48, |
| "regressed_vs_azookey": 11, |
| "fallback_cases": 0 |
| }, |
| "windows_fp32_comparison": { |
| "order_changed_cases": 163, |
| "top1_changed_cases": 2, |
| "max_score_sum_abs": 2.6928324103355408, |
| "mean_score_sum_abs": 0.33380034679328247 |
| }, |
| "original_report_comparison_baseline": "Windows FP32", |
| "original_report_comparison": { |
| "order_changed_cases": 163, |
| "top1_changed_cases": 2, |
| "max_score_sum_abs": 2.6928324103355408, |
| "mean_score_sum_abs": 0.33380034679328247 |
| }, |
| "benchmark_source_sha256": "e9eb668fd6aa14b1e26436f429b5550108af0a1dfd443b8cea0bcb3ab3028fca", |
| "candidate_export_sha256": "6b532af2e7b1536ded98e9d75cbba4fd7338b4204987fc4dafb16df7edfdf223" |
| } |
| }, |
| "alignment": { |
| "strict_passed": false, |
| "max_logit_abs": 3.6601905822753906, |
| "invariants": { |
| "right_pad": { |
| "max_abs": 2.8133392333984375e-05, |
| "mean_abs": 2.5544544911072975e-06, |
| "rmse": 3.420821628891477e-06, |
| "outside_tolerance": 0, |
| "values": 114688 |
| }, |
| "causal": { |
| "max_abs": 0.0, |
| "mean_abs": 0.0, |
| "rmse": 0.0, |
| "outside_tolerance": 0, |
| "values": 262144 |
| } |
| } |
| }, |
| "distribution": { |
| "original_report": { |
| "model": "artifacts/deployment/tiny-ja-v1-conservative-int8-b32-v1", |
| "compute_units": "CPU_ONLY", |
| "benchmark_text": { |
| "kl_coreml": { |
| "mean": 0.005891128288349137, |
| "max": 0.025890197604894638 |
| }, |
| "kl_simulated": { |
| "mean": 0.005977873008232564, |
| "max": 0.028363749384880066 |
| }, |
| "nll_fp32": { |
| "mean": 5.566977231502533, |
| "max": 8.531479835510254 |
| }, |
| "nll_coreml": { |
| "mean": 5.572893209457398, |
| "max": 8.633211135864258 |
| }, |
| "top1_agreement": { |
| "mean": 0.943656535744667, |
| "max": 1.0 |
| }, |
| "max_logprob_diff": { |
| "mean": 0.9077084493637085, |
| "max": 3.5512266159057617 |
| } |
| }, |
| "random_tokens": { |
| "kl_coreml": { |
| "mean": 0.015501508116722107, |
| "max": 0.01807662844657898 |
| }, |
| "kl_simulated": { |
| "mean": 0.015804098173975944, |
| "max": 0.01901794597506523 |
| }, |
| "nll_fp32": { |
| "mean": 11.800455474853516, |
| "max": 12.100545883178711 |
| }, |
| "nll_coreml": { |
| "mean": 11.795885848999024, |
| "max": 12.123295783996582 |
| }, |
| "top1_agreement": { |
| "mean": 0.9140625, |
| "max": 0.9296875 |
| }, |
| "max_logprob_diff": { |
| "mean": 2.206966495513916, |
| "max": 2.564694404602051 |
| } |
| } |
| }, |
| "aggregation": "Mean of per-example position means; not corpus token-weighted.", |
| "content_token_limit": 127, |
| "truncation_count_recorded": false, |
| "original_report_has_full_identity_metadata": false |
| }, |
| "source_reports_sha256": { |
| "outputs/deployment/conservative-int8-b32-dev-v1/metrics.json": "839e220c3024ec9798c921fb39d8862c5f874889c9dd9c2066cb339894658329", |
| "outputs/deployment/conservative-int8-b32-dev-v1/scores.jsonl": "f00a6b56b96e6be1db3ba7c58abe53bc91e7614c2f7f7959c27b0a8cce8db936", |
| "outputs/ime-eval/tiny-ja-v1-dev-v2-scores/manifest.json": "16bbd3b5b921e54896cfdca880d77c78fedd76b2aa8956fbaaa4c45bd4a4030e", |
| "outputs/ime-eval/tiny-ja-v1-dev-v2-scores/scores.jsonl": "2710b88428b5f73c6aeac5d45d442fa0d9bca1e038db2928c6059be993f3cfcb", |
| "outputs/deployment/conservative-dev-v3/metrics.json": "387d80e1b52a2c8dbcbb2196471ae548ae869b1e3f259dc1ee972c592e6b4863", |
| "outputs/deployment/conservative-dev-v3/scores.jsonl": "bb4fa9f52a074144a3cb432777ec6c94ad07637cdecd5b2126ad5d64a4e6a1d2", |
| "outputs/deployment/conservative-int8-b32-ajimee-v1/metrics.json": "3b59e396173328c78e8164f809da24764474685a2bd35765457fac5df644aac5", |
| "outputs/deployment/conservative-int8-b32-ajimee-v1/scores.jsonl": "9daf557818ab0aef12fb74e68b8cd4ae502806650d3d7e0cd23eeb2e1499e89e", |
| "outputs/ime-eval/tiny-ja-v1-ajimee/metrics.json": "219e15ec8ee54b09b455bd14ce60ab99bff239510ab9df344b69d2afc4b21196", |
| "outputs/ime-eval/tiny-ja-v1-ajimee/scores.jsonl": "247b29b59285551acaf73fabf9bee79c9617360623329e6bb60deb2d33ee7ddf", |
| "outputs/deployment/conservative-int8-b32-alignment-v1/alignment.json": "6773fafbd84e27dc1c349234fe601898e9f2af56ec5e2dcc6a32b12fbe14d018", |
| "outputs/deployment/conservative-int8-b32-distribution-v1/report.json": "6970eafdbb0d8e8b075d5b38b241cfa8fa46777961106ac32714755950722852", |
| "artifacts/models/tiny-ja-v1/summary.json": "4d9417b3bb8702b025b1ce4962d38ea36ad6d52928d19267dccdcaed3bf6c022" |
| }, |
| "scope": "Offline fixed pools and saved Mac observations; not app-wide accuracy or a Windows Core ML runtime rerun." |
| } |
|
|
