outputs/benchmark_v02_dev_incremental.json

{
  "schema_version": 2,
  "generated_at": "2026-08-10T10:35:16.437767+00:00",
  "status": "LOCAL_BENCHMARK",
  "stage": "DEV_SELECTION",
  "model": {
    "id": "ku-nlp/deberta-v2-tiny-japanese",
    "revision": "0427645e8cf44ee83b4a0b5f4498274d89e02adb",
    "task": "masked-language-model finite-candidate reranking",
    "candidate_generation": false
  },
  "candidate_source": {
    "kind": "Mozc OSS dictionary SQLite index",
    "manifest": {
      "schema_version": 1,
      "source_revision": "3f235b4eb6fcff7d14ef5f0fb8ee56de7ee4c732",
      "source_files": [
        {
          "name": "dictionary00.txt",
          "sha256": "9e07ce292932c4fff327ed8cb06274c09acca556bd8bdf7c7ef1f6173eae3b61",
          "size_bytes": 7157174
        },
        {
          "name": "dictionary01.txt",
          "sha256": "836fd5fd8399510de0d8ef7b33be913cca6bbaee7b712cc8045fba03f4c75a89",
          "size_bytes": 6428663
        },
        {
          "name": "dictionary02.txt",
          "sha256": "93943858d0cfd092b849a78def6901eaf228177e1ac5663544e865a3a1e45a9e",
          "size_bytes": 6480134
        },
        {
          "name": "dictionary03.txt",
          "sha256": "cb9f1b0b4a7550d682d86244a021cd17e0b15366328819e69135b5f183a1a24b",
          "size_bytes": 7434257
        },
        {
          "name": "dictionary04.txt",
          "sha256": "9fdc4e3948d03463de13ab5782adffbc11dd06e934cb80138c5920029cd3d2f6",
          "size_bytes": 5399172
        },
        {
          "name": "dictionary05.txt",
          "sha256": "199e3b0f050b691823ae7ba943090d437f51af6436b27c5f3dc69cafa12000e4",
          "size_bytes": 5316454
        },
        {
          "name": "dictionary06.txt",
          "sha256": "b95b14f7770ae2a824bc8e9fad8b03c1bc45c504f93a36554b8b56daa0368d2a",
          "size_bytes": 5373350
        },
        {
          "name": "dictionary07.txt",
          "sha256": "7f53649b529d4c8c495d3a8dd9c4d489dfe7696080f7423107ff01331174be48",
          "size_bytes": 5315756
        },
        {
          "name": "dictionary08.txt",
          "sha256": "50a31bb5fc1646721b00be371a5695dffab1292bda143e1e59cc70cbbdd8643f",
          "size_bytes": 4887114
        },
        {
          "name": "dictionary09.txt",
          "sha256": "4e5fcd4cbfbf11395166a37b506f547fc54d20edc0884f5184ecd3fc705f0d9a",
          "size_bytes": 6690572
        }
      ],
      "input_rows": 1289076,
      "indexed_entries": 1085365
    },
    "pool_size": 8,
    "prior_score": "-(cost - best_cost) / 1000"
  },
  "dataset": {
    "repository": "https://github.com/UniversalDependencies/UD_Japanese-GSD",
    "revision": "7bc20119f476b552635e0640644e577b6fd3606b",
    "license": "CC BY-SA 4.0",
    "file": {
      "url": "https://raw.githubusercontent.com/UniversalDependencies/UD_Japanese-GSD/7bc20119f476b552635e0640644e577b6fd3606b/ja_gsd-ud-dev.conllu",
      "sha256": "18d266e29336ef619787a928d89d20f2e64188be70d5b37829e3aadc5bb6b841",
      "size_bytes": 2961591,
      "cache_path": "work\\data\\ud-japanese-gsd\\ja_gsd-ud-dev-7bc20119f476.conllu"
    }
  },
  "selection": {
    "source": "GSD dev only",
    "dataset_revision": "7bc20119f476b552635e0640644e577b6fd3606b",
    "context_mode": "left_only",
    "prior_weights": [
      0.0,
      0.25,
      0.5,
      0.75,
      1.0,
      1.5,
      2.0,
      3.0,
      4.0,
      6.0,
      8.0
    ],
    "min_margins": [
      0.1,
      0.25,
      0.5,
      0.75,
      1.0,
      1.5,
      2.0,
      3.0
    ],
    "selected_prior_weight": 6.0,
    "selected_min_margin": 2.0
  },
  "sampling": {
    "limit": null,
    "seed": 20260810,
    "available": 7602,
    "evaluated": 7602
  },
  "coverage": {
    "eligible": 10656,
    "ambiguous": 8438,
    "oracle_in_pool": 7602,
    "oracle_miss": 836,
    "candidate_recall_at_k": 0.9009243896657976
  },
  "conditional_metrics": {
    "baseline": {
      "correct": 6503,
      "total": 7602,
      "accuracy": 0.8554327808471455
    },
    "model_only": {
      "correct": 4655,
      "total": 7602,
      "accuracy": 0.6123388581952118,
      "absolute_gain_vs_baseline": -0.24309392265193364,
      "changed": 3087,
      "improved": 429,
      "regressed": 2277,
      "both_correct": 4226,
      "both_wrong": 670
    },
    "selected": {
      "correct": 6663,
      "total": 7602,
      "accuracy": 0.8764798737174427,
      "absolute_gain_vs_baseline": 0.021047092870297268,
      "changed": 387,
      "improved": 245,
      "regressed": 85,
      "both_correct": 6418,
      "both_wrong": 854
    }
  },
  "end_to_end_ambiguous": {
    "denominator": 8438,
    "baseline_accuracy": 0.7706802559848305,
    "model_only_accuracy": 0.5516710120881726,
    "selected_accuracy": 0.7896420952832425
  },
  "performance": {
    "device": "cpu",
    "model_load_seconds": 0.881654700031504,
    "elapsed_seconds": 156.22223149996717,
    "examples_per_second": 48.66144803469663,
    "latency_ms_p50": 18.38885003235191,
    "latency_ms_p95": 45.48946499126031,
    "scoring_errors": 1323,
    "scoring_error_rate": 0.17403314917127072
  },
  "environment": {
    "python": "3.12.10",
    "platform": "Windows-11-10.0.26200-SP0",
    "torch": "2.13.0",
    "transformers": "5.14.1"
  },
  "claim_boundaries": [
    "This report selects hyperparameters and is not external evaluation evidence.",
    "Conditional accuracy requires the expected surface in Mozc top-k.",
    "PUD and AJIMEE are not read or scored by this function."
  ]
}