{
  "format": "vimeml_public_evaluation_v1",
  "bundle_manifest_sha256": "4284cdc5491513cc817633081677b363b14f59c0b4ffb78cdf7fcfb221f71b3c",
  "coreml_manifest_sha256": "a9d516233fa57aaadd09477e2e909760972ff49b5577919f8a441c489f5c2245",
  "training": {
    "prediction_targets": 573295237,
    "data_passes": 1.0,
    "full_validation": {
      "loss": 4.541570895102848,
      "prediction_pairs": 5890326,
      "perplexity": 93.83809420004854
    },
    "test_used_for_selection": false
  },
  "ranking": {
    "dev": {
      "original_order": {
        "cases": 137,
        "top1_correct": 111,
        "top1_accuracy": 0.8102189781021898,
        "top5_correct": 130,
        "top5_accuracy": 0.948905109489051,
        "candidate_pool_covered": 136,
        "candidate_pool_recall": 0.9927007299270073,
        "covered_top1_accuracy": 0.8161764705882353,
        "mean_min_cer": 0.04746998743349109,
        "mrr": 0.8745925911071898,
        "corrected_vs_azookey": 0,
        "regressed_vs_azookey": 0,
        "fallback_cases": 0
      },
      "fp32_lm": {
        "cases": 137,
        "top1_correct": 122,
        "top1_accuracy": 0.8905109489051095,
        "top5_correct": 134,
        "top5_accuracy": 0.9781021897810219,
        "candidate_pool_covered": 136,
        "candidate_pool_recall": 0.9927007299270073,
        "covered_top1_accuracy": 0.8970588235294118,
        "mean_min_cer": 0.03045798483754688,
        "mrr": 0.935349322210636,
        "corrected_vs_azookey": 15,
        "regressed_vs_azookey": 4,
        "fallback_cases": 0
      },
      "int8_lm": {
        "cases": 137,
        "top1_correct": 122,
        "top1_accuracy": 0.8905109489051095,
        "top5_correct": 134,
        "top5_accuracy": 0.9781021897810219,
        "candidate_pool_covered": 136,
        "candidate_pool_recall": 0.9927007299270073,
        "covered_top1_accuracy": 0.8970588235294118,
        "mean_min_cer": 0.03045798483754688,
        "mrr": 0.935349322210636,
        "corrected_vs_azookey": 15,
        "regressed_vs_azookey": 4,
        "fallback_cases": 0
      },
      "windows_fp32_comparison": {
        "order_changed_cases": 119,
        "top1_changed_cases": 0,
        "max_score_sum_abs": 1.9853296279907227,
        "mean_score_sum_abs": 0.20199488408841776
      },
      "original_report_comparison_baseline": "conservative Core ML",
      "original_report_comparison": {
        "order_changed_cases": 118,
        "top1_changed_cases": 0,
        "max_score_sum_abs": 1.994759202003479,
        "mean_score_sum_abs": 0.2013680215637894
      },
      "benchmark_source_sha256": "a945f2117570530ef73b1dd4ef53b9dc69b07fa69ac98a6d67c9268b540eafa8",
      "candidate_export_sha256": "ab192509dce1cb4f4b20273e623f7bfd4237b1c3691950438e6921b8a510ae21"
    },
    "ajimee": {
      "original_order": {
        "cases": 200,
        "top1_correct": 87,
        "top1_accuracy": 0.435,
        "top5_correct": 143,
        "top5_accuracy": 0.715,
        "candidate_pool_covered": 162,
        "candidate_pool_recall": 0.81,
        "covered_top1_accuracy": 0.5370370370370371,
        "mean_min_cer": 0.0821630611981915,
        "mrr": 0.554232017982018,
        "corrected_vs_azookey": 0,
        "regressed_vs_azookey": 0,
        "fallback_cases": 0
      },
      "fp32_lm": {
        "cases": 200,
        "top1_correct": 124,
        "top1_accuracy": 0.62,
        "top5_correct": 151,
        "top5_accuracy": 0.755,
        "candidate_pool_covered": 162,
        "candidate_pool_recall": 0.81,
        "covered_top1_accuracy": 0.7654320987654321,
        "mean_min_cer": 0.05657040945427151,
        "mrr": 0.684061974789916,
        "corrected_vs_azookey": 48,
        "regressed_vs_azookey": 11,
        "fallback_cases": 0
      },
      "int8_lm": {
        "cases": 200,
        "top1_correct": 124,
        "top1_accuracy": 0.62,
        "top5_correct": 151,
        "top5_accuracy": 0.755,
        "candidate_pool_covered": 162,
        "candidate_pool_recall": 0.81,
        "covered_top1_accuracy": 0.7654320987654321,
        "mean_min_cer": 0.05657040945427151,
        "mrr": 0.6841512605042016,
        "corrected_vs_azookey": 48,
        "regressed_vs_azookey": 11,
        "fallback_cases": 0
      },
      "windows_fp32_comparison": {
        "order_changed_cases": 163,
        "top1_changed_cases": 2,
        "max_score_sum_abs": 2.6928324103355408,
        "mean_score_sum_abs": 0.33380034679328247
      },
      "original_report_comparison_baseline": "Windows FP32",
      "original_report_comparison": {
        "order_changed_cases": 163,
        "top1_changed_cases": 2,
        "max_score_sum_abs": 2.6928324103355408,
        "mean_score_sum_abs": 0.33380034679328247
      },
      "benchmark_source_sha256": "e9eb668fd6aa14b1e26436f429b5550108af0a1dfd443b8cea0bcb3ab3028fca",
      "candidate_export_sha256": "6b532af2e7b1536ded98e9d75cbba4fd7338b4204987fc4dafb16df7edfdf223"
    }
  },
  "alignment": {
    "strict_passed": false,
    "max_logit_abs": 3.6601905822753906,
    "invariants": {
      "right_pad": {
        "max_abs": 2.8133392333984375e-05,
        "mean_abs": 2.5544544911072975e-06,
        "rmse": 3.420821628891477e-06,
        "outside_tolerance": 0,
        "values": 114688
      },
      "causal": {
        "max_abs": 0.0,
        "mean_abs": 0.0,
        "rmse": 0.0,
        "outside_tolerance": 0,
        "values": 262144
      }
    }
  },
  "distribution": {
    "original_report": {
      "model": "artifacts/deployment/tiny-ja-v1-conservative-int8-b32-v1",
      "compute_units": "CPU_ONLY",
      "benchmark_text": {
        "kl_coreml": {
          "mean": 0.005891128288349137,
          "max": 0.025890197604894638
        },
        "kl_simulated": {
          "mean": 0.005977873008232564,
          "max": 0.028363749384880066
        },
        "nll_fp32": {
          "mean": 5.566977231502533,
          "max": 8.531479835510254
        },
        "nll_coreml": {
          "mean": 5.572893209457398,
          "max": 8.633211135864258
        },
        "top1_agreement": {
          "mean": 0.943656535744667,
          "max": 1.0
        },
        "max_logprob_diff": {
          "mean": 0.9077084493637085,
          "max": 3.5512266159057617
        }
      },
      "random_tokens": {
        "kl_coreml": {
          "mean": 0.015501508116722107,
          "max": 0.01807662844657898
        },
        "kl_simulated": {
          "mean": 0.015804098173975944,
          "max": 0.01901794597506523
        },
        "nll_fp32": {
          "mean": 11.800455474853516,
          "max": 12.100545883178711
        },
        "nll_coreml": {
          "mean": 11.795885848999024,
          "max": 12.123295783996582
        },
        "top1_agreement": {
          "mean": 0.9140625,
          "max": 0.9296875
        },
        "max_logprob_diff": {
          "mean": 2.206966495513916,
          "max": 2.564694404602051
        }
      }
    },
    "aggregation": "Mean of per-example position means; not corpus token-weighted.",
    "content_token_limit": 127,
    "truncation_count_recorded": false,
    "original_report_has_full_identity_metadata": false
  },
  "source_reports_sha256": {
    "outputs/deployment/conservative-int8-b32-dev-v1/metrics.json": "839e220c3024ec9798c921fb39d8862c5f874889c9dd9c2066cb339894658329",
    "outputs/deployment/conservative-int8-b32-dev-v1/scores.jsonl": "f00a6b56b96e6be1db3ba7c58abe53bc91e7614c2f7f7959c27b0a8cce8db936",
    "outputs/ime-eval/tiny-ja-v1-dev-v2-scores/manifest.json": "16bbd3b5b921e54896cfdca880d77c78fedd76b2aa8956fbaaa4c45bd4a4030e",
    "outputs/ime-eval/tiny-ja-v1-dev-v2-scores/scores.jsonl": "2710b88428b5f73c6aeac5d45d442fa0d9bca1e038db2928c6059be993f3cfcb",
    "outputs/deployment/conservative-dev-v3/metrics.json": "387d80e1b52a2c8dbcbb2196471ae548ae869b1e3f259dc1ee972c592e6b4863",
    "outputs/deployment/conservative-dev-v3/scores.jsonl": "bb4fa9f52a074144a3cb432777ec6c94ad07637cdecd5b2126ad5d64a4e6a1d2",
    "outputs/deployment/conservative-int8-b32-ajimee-v1/metrics.json": "3b59e396173328c78e8164f809da24764474685a2bd35765457fac5df644aac5",
    "outputs/deployment/conservative-int8-b32-ajimee-v1/scores.jsonl": "9daf557818ab0aef12fb74e68b8cd4ae502806650d3d7e0cd23eeb2e1499e89e",
    "outputs/ime-eval/tiny-ja-v1-ajimee/metrics.json": "219e15ec8ee54b09b455bd14ce60ab99bff239510ab9df344b69d2afc4b21196",
    "outputs/ime-eval/tiny-ja-v1-ajimee/scores.jsonl": "247b29b59285551acaf73fabf9bee79c9617360623329e6bb60deb2d33ee7ddf",
    "outputs/deployment/conservative-int8-b32-alignment-v1/alignment.json": "6773fafbd84e27dc1c349234fe601898e9f2af56ec5e2dcc6a32b12fbe14d018",
    "outputs/deployment/conservative-int8-b32-distribution-v1/report.json": "6970eafdbb0d8e8b075d5b38b241cfa8fa46777961106ac32714755950722852",
    "artifacts/models/tiny-ja-v1/summary.json": "4d9417b3bb8702b025b1ce4962d38ea36ad6d52928d19267dccdcaed3bf6c022"
  },
  "scope": "Offline fixed pools and saved Mac observations; not app-wide accuracy or a Windows Core ML runtime rerun."
}