{
  "label": "",
  "bench": "../benchmarks/csn-3.12-licensed/bench.jsonl",
  "n": 600,
  "repos": 117,
  "oracle": "strict L1: byte-identical code object incl. docstrings and co_exceptiontable",
  "self_test": {
    "preflight_pct": 100.0,
    "mutation_kill_rate_pct": 100.0,
    "SOUND": true
  },
  "tuned_greedy": {
    "point": 84.33,
    "ci95_lo": 80.48,
    "ci95_hi": 87.94,
    "half_width_pp": 3.73,
    "binomial_ci95_lo": 81.42,
    "binomial_ci95_hi": 87.24,
    "design_effect": 1.65,
    "n": 600,
    "clusters": 117
  },
  "certified_at_32": {
    "point": 93.67,
    "ci95_lo": 90.86,
    "ci95_hi": 96.08,
    "half_width_pp": 2.61,
    "binomial_ci95_lo": 91.72,
    "binomial_ci95_hi": 95.62,
    "design_effect": 1.79,
    "n": 600,
    "clusters": 117
  },
  "base_greedy_UNTUNED_CONTROL": {
    "point": 0.67,
    "ci95_lo": 0.16,
    "ci95_hi": 1.35,
    "half_width_pp": 0.6,
    "binomial_ci95_lo": 0.02,
    "binomial_ci95_hi": 1.32,
    "design_effect": 0.84,
    "n": 600,
    "clusters": 117
  },
  "fine_tune_effect": {
    "tuned_certified": 506,
    "base_certified": 4,
    "absolute_gain_pp": 83.67,
    "tuned_only": 502,
    "base_only": 0,
    "note": "paired discordant counts; McNemar exact p below",
    "mcnemar_exact_p": "< 1e-12 (log10 p ~ -151)"
  },
  "certified_curve": {
    "1": {
      "certified": 506,
      "pct": 84.33,
      "budget": "1 greedy + 0 sampled"
    },
    "2": {
      "certified": 525,
      "pct": 87.5,
      "budget": "1 greedy + 1 sampled"
    },
    "4": {
      "certified": 546,
      "pct": 91.0,
      "budget": "1 greedy + 3 sampled"
    },
    "8": {
      "certified": 555,
      "pct": 92.5,
      "budget": "1 greedy + 7 sampled"
    },
    "16": {
      "certified": 558,
      "pct": 93.0,
      "budget": "1 greedy + 15 sampled"
    },
    "32": {
      "certified": 562,
      "pct": 93.67,
      "budget": "1 greedy + 31 sampled"
    }
  }
}