Spaces:

SlayerLab
/

Slayer1B-Benchmarks

Running

Benchmarks: first results bench50-v0.3.1 (results 1cf2b8bb, page b71d0c71)

2d3d73f verified 6 days ago

13.9 kB

"use strict";
/* Acceptance checks for the public bench50-v0.3.1 result packet (results.schema.json, schema_version 2).
   The exporter validates the full JSON Schema before publication; this file re-checks, in the browser,
   every rule the page relies on to decide what may be shown. Anything unexpected rejects the whole packet. */
(() => {
  const PINS = {
    tasks: "fcfc45d805243e7ea4aa12bf836940ea086ecafe9fe3e592bf8850890ca8cf6b",
    schema: "c133a37d4038912314b14d374cd3ae1ecc6d8bca3d88a465f7e1243437553eec",
    method: "8709632a394a3d69fa56567404e6543f7305d49fc97c9d381fbfad46d86628ae",
    fieldRoles: "de4280e9db5a947e4e569ad4e5dfcb048d6ee574014b640d254b6c8e1a604b35"
  };
  const HEX64 = /^[0-9a-f]{64}$/;
  const STATES = ["ok","not_run","blocked_access","blocked_runtime","blocked_security","blocked_judge","unsupported","failed","skipped"];
  const CONTAMINATION = ["clean","flagged","partially_scanned","unscanned","unknown"];
  const UNITS = ["percent","score","byte_ppl"];
  const VERDICTS = ["withheld","better","worse","inconclusive"];
  const SUBSETS = ["full","hit_free","hit_free_answer_fields"];
  const OBS_KEYS = ["task_id","model_id","protocol_id","metric","unit","higher_is_better","n_population","n_selected","n_scored","state","value","ci95",
    "source_id_digest","results_sha256","runner_sha256","reason","contamination","contamination_receipt_sha256","value_hit_free","n_hit_free","hit_rate",
    "value_hit_free_answer_fields","n_hit_free_answer_fields","hit_rate_answer_fields","ci95_hit_free","ci95_hit_free_answer_fields","reference"];
  const REQUIRED_OBS = ["task_id","model_id","protocol_id","metric","unit","higher_is_better","n_population","n_selected","n_scored","state","value","ci95",
    "source_id_digest","results_sha256","runner_sha256","contamination","contamination_receipt_sha256"];

  function fail(message) { throw new Error(message); }
  const isNum = v => typeof v === "number" && Number.isFinite(v);
  const isShare = v => isNum(v) && v >= 0 && v <= 1;
  function key(task, model, protocol) { return `${task}\u0000${model}\u0000${protocol}`; }

  function checkCI(ci, where) {
    if (ci === null) return;
    if (typeof ci !== "object") fail(`${where}: ci95 must be an object or null`);
    for (const k of Object.keys(ci)) if (!["lower","upper","level","method","replicates","seed"].includes(k)) fail(`${where}: unknown ci95 field ${k}`);
    if (!isNum(ci.lower) || !isNum(ci.upper) || ci.lower > ci.upper) fail(`${where}: invalid ci95 bounds`);
    if (ci.level !== 0.95 || ci.replicates !== 10000 || ci.seed !== 20261028) fail(`${where}: ci95 does not follow the protocol`);
    if (!["paired_bootstrap","paired_cluster_bootstrap"].includes(ci.method)) fail(`${where}: unknown ci95 method`);
  }

  function checkReference(ref, where) {
    if (ref === undefined || ref === null) return;
    if (typeof ref !== "object") fail(`${where}: reference must be an object or null`);
    for (const k of Object.keys(ref)) if (!["chance_value","majority_value","majority_prediction_share","constant_prediction"].includes(k)) fail(`${where}: unknown reference field ${k}`);
    for (const k of ["chance_value","majority_value"]) if (!(k in ref) || !(ref[k] === null || isNum(ref[k]))) fail(`${where}: reference.${k} must be a number or null`);
    if (!("majority_prediction_share" in ref) || !(ref.majority_prediction_share === null || isShare(ref.majority_prediction_share))) fail(`${where}: reference.majority_prediction_share must be 0..1 or null`);
    if (!("constant_prediction" in ref) || !(ref.constant_prediction === null || typeof ref.constant_prediction === "boolean")) fail(`${where}: reference.constant_prediction must be boolean or null`);
  }

  function checkObservation(o, i, tasks, models, protocols) {
    const where = `observation ${i}`;
    if (!o || typeof o !== "object") fail(`${where}: not an object`);
    for (const k of Object.keys(o)) if (!OBS_KEYS.includes(k)) fail(`${where}: unknown field ${k}`);
    for (const k of REQUIRED_OBS) if (!(k in o)) fail(`${where}: missing ${k}`);
    if (!tasks.has(o.task_id)) fail(`${where}: task ${o.task_id} is not in the benchmark list`);
    if (!models.has(o.model_id)) fail(`${where}: model ${o.model_id} is not declared`);
    if (!protocols.includes(o.protocol_id)) fail(`${where}: unknown protocol ${o.protocol_id}`);
    if (!STATES.includes(o.state)) fail(`${where}: unknown state`);
    if (!CONTAMINATION.includes(o.contamination)) fail(`${where}: unknown contamination state`);
    if (!UNITS.includes(o.unit)) fail(`${where}: unknown unit`);
    if (typeof o.higher_is_better !== "boolean") fail(`${where}: higher_is_better must be boolean`);
    const task = tasks.get(o.task_id);
    if (o.unit !== task.unit || o.higher_is_better !== task.higher_is_better) fail(`${where}: unit or direction differs from the benchmark list`);
    if (!Number.isInteger(o.n_scored) || o.n_scored < 0) fail(`${where}: invalid n_scored`);
    for (const k of ["n_population","n_selected"]) if (!(o[k] === null || (Number.isInteger(o[k]) && o[k] >= 0))) fail(`${where}: invalid ${k}`);
    checkCI(o.ci95, where);
    checkReference(o.reference, where);
    if (o.state === "ok") {
      if (!isNum(o.value) || o.n_scored < 1) fail(`${where}: ok without a value`);
      for (const k of ["source_id_digest","results_sha256","runner_sha256"]) if (!HEX64.test(o[k] || "")) fail(`${where}: ok without ${k}`);
      if (o.ci95 !== null && (o.value < o.ci95.lower || o.value > o.ci95.upper)) fail(`${where}: value outside its interval`);
    } else if (o.value !== null || o.ci95 !== null) fail(`${where}: a non-ok state carries a value`);
    if (["clean","flagged","partially_scanned"].includes(o.contamination) && !HEX64.test(o.contamination_receipt_sha256 || "")) fail(`${where}: scan state without a receipt`);
    const variants = [["value_hit_free","n_hit_free","hit_rate","ci95_hit_free"], ["value_hit_free_answer_fields","n_hit_free_answer_fields","hit_rate_answer_fields","ci95_hit_free_answer_fields"]];
    for (const [v, n, r, c] of variants) {
      const ci = o[c] === undefined ? null : o[c];
      checkCI(ci, `${where} ${c}`);
      const given = [o[v], o[n], o[r]].some(x => x !== undefined && x !== null);
      if (o.contamination === "flagged" && o.state === "ok" && !given) fail(`${where}: flagged without ${n}`);
      if (!given) { if (ci !== null) fail(`${where}: ${c} without ${v}`); continue; }
      if (!Number.isInteger(o[n]) || o[n] < 0 || !isShare(o[r])) fail(`${where}: invalid ${n} or ${r}`);
      if ((o[n] === 0) !== (o[v] === null || o[v] === undefined)) fail(`${where}: ${v} must be empty exactly when ${n} is 0`);
      if (o[n] > 0 && !isNum(o[v])) fail(`${where}: invalid ${v}`);
      if (o.n_scored && o[n] > o.n_scored) fail(`${where}: ${n} exceeds n_scored`);
      if (ci !== null && (o[n] === 0 || o[v] < ci.lower || o[v] > ci.upper)) fail(`${where}: ${v} outside ${c}`);
    }
  }

  function validate(packet, taskList) {
    if (!packet || typeof packet !== "object") fail("packet is not an object");
    for (const k of Object.keys(packet)) if (!["schema_version","protocol_id","run_id","models","tasks","observations","contrasts","provenance"].includes(k)) fail(`unknown top-level field ${k}`);
    if (packet.schema_version !== 2) fail("schema_version must be 2");
    const protocols = ["bench50-v0.3.1-panel200","bench50-v0.3.1-fullB"];
    if (!protocols.includes(packet.protocol_id)) fail("unknown protocol_id");
    if (typeof packet.run_id !== "string" || !packet.run_id) fail("missing run_id");
    const p = packet.provenance;
    if (!p || p.kind !== "measurement") fail("only measurement packets are shown");
    if (!/^[0-9a-f]{40}$/.test(p.inventory_commit || "")) fail("provenance.inventory_commit missing");
    if (!Array.isArray(p.sources) || !p.sources.length || !p.sources.every(s => s && typeof s.run_id === "string" && HEX64.test(s.results_sha256 || ""))) fail("provenance.sources missing");
    if (!Array.isArray(packet.models) || !packet.models.length) fail("no models");
    const models = new Map();
    for (const m of packet.models) {
      if (!m || typeof m.model_id !== "string" || !m.model_id || typeof m.repo !== "string" || !m.repo) fail("invalid model entry");
      if (models.has(m.model_id)) fail(`duplicate model ${m.model_id}`);
      if (!(m.weights_sha256 === null || HEX64.test(m.weights_sha256))) fail(`invalid weights_sha256 for ${m.model_id}`);
      models.set(m.model_id, m);
    }
    const tasks = new Map(taskList.rows.map(r => [r.id, r]));
    if (!Array.isArray(packet.tasks)) fail("tasks must be a list");
    for (const t of packet.tasks) {
      const row = tasks.get(t && t.task_id);
      if (!row) fail(`task ${t && t.task_id} is not in the benchmark list`);
      if (t.language !== row.language) fail(`task ${t.task_id}: language differs from the benchmark list`);
    }
    if (!Array.isArray(packet.observations)) fail("observations must be a list");
    const observations = new Map();
    packet.observations.forEach((o, i) => {
      checkObservation(o, i, tasks, models, protocols);
      const k = key(o.task_id, o.model_id, o.protocol_id);
      if (observations.has(k)) fail(`duplicate observation ${o.task_id} / ${o.model_id}`);
      observations.set(k, o);
    });
    if (!Array.isArray(packet.contrasts)) fail("contrasts must be a list");
    const contrasts = [];
    packet.contrasts.forEach((c, i) => {
      const where = `contrast ${i}`;
      if (!c || !VERDICTS.includes(c.verdict)) fail(`${where}: unknown verdict`);
      if (!SUBSETS.includes(c.subset)) fail(`${where}: unknown subset`);
      if (!(c.n === null || (Number.isInteger(c.n) && c.n >= 0))) fail(`${where}: invalid n`);
      if (typeof c.metric !== "string" || !c.metric || !UNITS.includes(c.unit)) fail(`${where}: missing metric or unit`);
      const left = observations.get(key(c.task_id, c.model_id, c.protocol_id));
      const right = observations.get(key(c.task_id, c.model_ref, c.protocol_id));
      if (!left || !right) fail(`${where}: refers to a missing observation`);
      if (c.unit !== left.unit) fail(`${where}: unit differs from its observation`);
      checkCI(c.ci95, where);
      if (c.verdict !== "withheld") {
        const r = c.review || {};
        if (r.status !== "accepted" || typeof r.actor !== "string" || !r.actor || !HEX64.test(r.artifact_sha256 || "")) fail(`${where}: verdict without an accepted review`);
        if (!isNum(c.delta) || !isShare(c.p_holm) || !Number.isInteger(c.family_size) || c.family_size < 1) fail(`${where}: verdict without paired test numbers`);
        if (!Number.isInteger(c.n) || c.n < 1) fail(`${where}: verdict without n`);
        if (left.state !== "ok" || right.state !== "ok") fail(`${where}: verdict on a non-ok observation`);
        checkVerdict(c, left, right, observations, where);
      }
      contrasts.push(c);
    });
    return {models, tasks, observations, contrasts};
  }

  // Hit-free values carry a number only with at least this many items (method rule, 2026-10-29).
  const MIN_HIT_FREE_N = 50;
  const ALPHA = 0.05;
  const enough = n => Number.isInteger(n) && n >= MIN_HIT_FREE_N;
  const SHOWN = ["clean","flagged","unknown"];
  // Highest hit rate in a row where some flagged model keeps fewer than MIN_HIT_FREE_N items without any hits ("most items
  // found in training data"); null outside that class.
  function mostlyTrained(observations, taskId, protocolId) {
    let rate = null;
    for (const o of observations.values()) {
      if (o.task_id === taskId && o.protocol_id === protocolId && o.state === "ok" && o.contamination === "flagged" && !enough(o.n_hit_free)) rate = Math.max(rate ?? 0, o.hit_rate);
    }
    return rate;
  }
  // One rule set with the exporter (_check_verdict, agents 31b34ca6), applied to every non-withheld verdict: no verdict
  // against a constant answer or a cell without a number (the exporter publishes no contrast for such pairs); the
  // verdict-bearing subset is full, or hit_free_answer_fields in the "most items in training data" row class, any other
  // subset stays withheld; directional exactly when p_holm <= ALPHA and n >= MIN_HIT_FREE_N (on the answer-field subset
  // also n_hit_free_answer_fields of model_id), with the sign of delta (model_id minus model_ref) matching the verdict and
  // higher_is_better; otherwise inconclusive.
  function checkVerdict(c, left, right, observations, where) {
    for (const o of [left, right]) {
      if (!SHOWN.includes(o.contamination)) fail(`${where}: verdict against a cell without a number`);
      if (o.reference && o.reference.constant_prediction === true) fail(`${where}: verdict against a constant answer`);
    }
    const bearing = mostlyTrained(observations, c.task_id, c.protocol_id) !== null ? "hit_free_answer_fields" : "full";
    if (c.subset !== bearing) fail(`${where}: subset ${c.subset} is a diagnostic in this row (verdicts only on ${bearing})`);
    const nOk = enough(c.n) && (c.subset !== "hit_free_answer_fields" || enough(left.n_hit_free_answer_fields));
    const significant = c.p_holm <= ALPHA && nOk;
    if (c.verdict === "inconclusive") {
      if (significant) fail(`${where}: inconclusive although p_holm ${c.p_holm} <= ${ALPHA} with n >= ${MIN_HIT_FREE_N}`);
      return;
    }
    if (!significant) fail(`${where}: verdict ${c.verdict} needs p_holm <= ${ALPHA} and n >= ${MIN_HIT_FREE_N}`);
    if (c.delta === 0) fail(`${where}: verdict without a delta`);
    const better = (c.delta > 0) === left.higher_is_better;
    if (c.verdict !== (better ? "better" : "worse")) fail(`${where}: verdict ${c.verdict} contradicts delta ${c.delta}`);
  }

  const api = {PINS, MIN_HIT_FREE_N, key, validate, mostlyTrained};
  if (typeof module !== "undefined" && module.exports) module.exports = api; else window.BenchContract = api;
})();

	"use strict";
	/* Acceptance checks for the public bench50-v0.3.1 result packet (results.schema.json, schema_version 2).
	The exporter validates the full JSON Schema before publication; this file re-checks, in the browser,
	every rule the page relies on to decide what may be shown. Anything unexpected rejects the whole packet. */
	(() => {
	const PINS = {
	tasks: "fcfc45d805243e7ea4aa12bf836940ea086ecafe9fe3e592bf8850890ca8cf6b",
	schema: "c133a37d4038912314b14d374cd3ae1ecc6d8bca3d88a465f7e1243437553eec",
	method: "8709632a394a3d69fa56567404e6543f7305d49fc97c9d381fbfad46d86628ae",
	fieldRoles: "de4280e9db5a947e4e569ad4e5dfcb048d6ee574014b640d254b6c8e1a604b35"
	};
	const HEX64 = /^[0-9a-f]{64}$/;
	const STATES = ["ok","not_run","blocked_access","blocked_runtime","blocked_security","blocked_judge","unsupported","failed","skipped"];
	const CONTAMINATION = ["clean","flagged","partially_scanned","unscanned","unknown"];
	const UNITS = ["percent","score","byte_ppl"];
	const VERDICTS = ["withheld","better","worse","inconclusive"];
	const SUBSETS = ["full","hit_free","hit_free_answer_fields"];
	const OBS_KEYS = ["task_id","model_id","protocol_id","metric","unit","higher_is_better","n_population","n_selected","n_scored","state","value","ci95",
	"source_id_digest","results_sha256","runner_sha256","reason","contamination","contamination_receipt_sha256","value_hit_free","n_hit_free","hit_rate",
	"value_hit_free_answer_fields","n_hit_free_answer_fields","hit_rate_answer_fields","ci95_hit_free","ci95_hit_free_answer_fields","reference"];
	const REQUIRED_OBS = ["task_id","model_id","protocol_id","metric","unit","higher_is_better","n_population","n_selected","n_scored","state","value","ci95",
	"source_id_digest","results_sha256","runner_sha256","contamination","contamination_receipt_sha256"];

	function fail(message) { throw new Error(message); }
	const isNum = v => typeof v === "number" && Number.isFinite(v);
	const isShare = v => isNum(v) && v >= 0 && v <= 1;
	function key(task, model, protocol) { return `${task}\u0000${model}\u0000${protocol}`; }

	function checkCI(ci, where) {
	if (ci === null) return;
	if (typeof ci !== "object") fail(`${where}: ci95 must be an object or null`);
	for (const k of Object.keys(ci)) if (!["lower","upper","level","method","replicates","seed"].includes(k)) fail(`${where}: unknown ci95 field ${k}`);
	if (!isNum(ci.lower) \|\| !isNum(ci.upper) \|\| ci.lower > ci.upper) fail(`${where}: invalid ci95 bounds`);
	if (ci.level !== 0.95 \|\| ci.replicates !== 10000 \|\| ci.seed !== 20261028) fail(`${where}: ci95 does not follow the protocol`);
	if (!["paired_bootstrap","paired_cluster_bootstrap"].includes(ci.method)) fail(`${where}: unknown ci95 method`);
	}

	function checkReference(ref, where) {
	if (ref === undefined \|\| ref === null) return;
	if (typeof ref !== "object") fail(`${where}: reference must be an object or null`);
	for (const k of Object.keys(ref)) if (!["chance_value","majority_value","majority_prediction_share","constant_prediction"].includes(k)) fail(`${where}: unknown reference field ${k}`);
	for (const k of ["chance_value","majority_value"]) if (!(k in ref) \|\| !(ref[k] === null \|\| isNum(ref[k]))) fail(`${where}: reference.${k} must be a number or null`);
	if (!("majority_prediction_share" in ref) \|\| !(ref.majority_prediction_share === null \|\| isShare(ref.majority_prediction_share))) fail(`${where}: reference.majority_prediction_share must be 0..1 or null`);
	if (!("constant_prediction" in ref) \|\| !(ref.constant_prediction === null \|\| typeof ref.constant_prediction === "boolean")) fail(`${where}: reference.constant_prediction must be boolean or null`);
	}

	function checkObservation(o, i, tasks, models, protocols) {
	const where = `observation ${i}`;
	if (!o \|\| typeof o !== "object") fail(`${where}: not an object`);
	for (const k of Object.keys(o)) if (!OBS_KEYS.includes(k)) fail(`${where}: unknown field ${k}`);
	for (const k of REQUIRED_OBS) if (!(k in o)) fail(`${where}: missing ${k}`);
	if (!tasks.has(o.task_id)) fail(`${where}: task ${o.task_id} is not in the benchmark list`);
	if (!models.has(o.model_id)) fail(`${where}: model ${o.model_id} is not declared`);
	if (!protocols.includes(o.protocol_id)) fail(`${where}: unknown protocol ${o.protocol_id}`);
	if (!STATES.includes(o.state)) fail(`${where}: unknown state`);
	if (!CONTAMINATION.includes(o.contamination)) fail(`${where}: unknown contamination state`);
	if (!UNITS.includes(o.unit)) fail(`${where}: unknown unit`);
	if (typeof o.higher_is_better !== "boolean") fail(`${where}: higher_is_better must be boolean`);
	const task = tasks.get(o.task_id);
	if (o.unit !== task.unit \|\| o.higher_is_better !== task.higher_is_better) fail(`${where}: unit or direction differs from the benchmark list`);
	if (!Number.isInteger(o.n_scored) \|\| o.n_scored < 0) fail(`${where}: invalid n_scored`);
	for (const k of ["n_population","n_selected"]) if (!(o[k] === null \|\| (Number.isInteger(o[k]) && o[k] >= 0))) fail(`${where}: invalid ${k}`);
	checkCI(o.ci95, where);
	checkReference(o.reference, where);
	if (o.state === "ok") {
	if (!isNum(o.value) \|\| o.n_scored < 1) fail(`${where}: ok without a value`);
	for (const k of ["source_id_digest","results_sha256","runner_sha256"]) if (!HEX64.test(o[k] \|\| "")) fail(`${where}: ok without ${k}`);
	if (o.ci95 !== null && (o.value < o.ci95.lower \|\| o.value > o.ci95.upper)) fail(`${where}: value outside its interval`);
	} else if (o.value !== null \|\| o.ci95 !== null) fail(`${where}: a non-ok state carries a value`);
	if (["clean","flagged","partially_scanned"].includes(o.contamination) && !HEX64.test(o.contamination_receipt_sha256 \|\| "")) fail(`${where}: scan state without a receipt`);
	const variants = [["value_hit_free","n_hit_free","hit_rate","ci95_hit_free"], ["value_hit_free_answer_fields","n_hit_free_answer_fields","hit_rate_answer_fields","ci95_hit_free_answer_fields"]];
	for (const [v, n, r, c] of variants) {
	const ci = o[c] === undefined ? null : o[c];
	checkCI(ci, `${where} ${c}`);
	const given = [o[v], o[n], o[r]].some(x => x !== undefined && x !== null);
	if (o.contamination === "flagged" && o.state === "ok" && !given) fail(`${where}: flagged without ${n}`);
	if (!given) { if (ci !== null) fail(`${where}: ${c} without ${v}`); continue; }
	if (!Number.isInteger(o[n]) \|\| o[n] < 0 \|\| !isShare(o[r])) fail(`${where}: invalid ${n} or ${r}`);
	if ((o[n] === 0) !== (o[v] === null \|\| o[v] === undefined)) fail(`${where}: ${v} must be empty exactly when ${n} is 0`);
	if (o[n] > 0 && !isNum(o[v])) fail(`${where}: invalid ${v}`);
	if (o.n_scored && o[n] > o.n_scored) fail(`${where}: ${n} exceeds n_scored`);
	if (ci !== null && (o[n] === 0 \|\| o[v] < ci.lower \|\| o[v] > ci.upper)) fail(`${where}: ${v} outside ${c}`);
	}
	}

	function validate(packet, taskList) {
	if (!packet \|\| typeof packet !== "object") fail("packet is not an object");
	for (const k of Object.keys(packet)) if (!["schema_version","protocol_id","run_id","models","tasks","observations","contrasts","provenance"].includes(k)) fail(`unknown top-level field ${k}`);
	if (packet.schema_version !== 2) fail("schema_version must be 2");
	const protocols = ["bench50-v0.3.1-panel200","bench50-v0.3.1-fullB"];
	if (!protocols.includes(packet.protocol_id)) fail("unknown protocol_id");
	if (typeof packet.run_id !== "string" \|\| !packet.run_id) fail("missing run_id");
	const p = packet.provenance;
	if (!p \|\| p.kind !== "measurement") fail("only measurement packets are shown");
	if (!/^[0-9a-f]{40}$/.test(p.inventory_commit \|\| "")) fail("provenance.inventory_commit missing");
	if (!Array.isArray(p.sources) \|\| !p.sources.length \|\| !p.sources.every(s => s && typeof s.run_id === "string" && HEX64.test(s.results_sha256 \|\| ""))) fail("provenance.sources missing");
	if (!Array.isArray(packet.models) \|\| !packet.models.length) fail("no models");
	const models = new Map();
	for (const m of packet.models) {
	if (!m \|\| typeof m.model_id !== "string" \|\| !m.model_id \|\| typeof m.repo !== "string" \|\| !m.repo) fail("invalid model entry");
	if (models.has(m.model_id)) fail(`duplicate model ${m.model_id}`);
	if (!(m.weights_sha256 === null \|\| HEX64.test(m.weights_sha256))) fail(`invalid weights_sha256 for ${m.model_id}`);
	models.set(m.model_id, m);
	}
	const tasks = new Map(taskList.rows.map(r => [r.id, r]));
	if (!Array.isArray(packet.tasks)) fail("tasks must be a list");
	for (const t of packet.tasks) {
	const row = tasks.get(t && t.task_id);
	if (!row) fail(`task ${t && t.task_id} is not in the benchmark list`);
	if (t.language !== row.language) fail(`task ${t.task_id}: language differs from the benchmark list`);
	}
	if (!Array.isArray(packet.observations)) fail("observations must be a list");
	const observations = new Map();
	packet.observations.forEach((o, i) => {
	checkObservation(o, i, tasks, models, protocols);
	const k = key(o.task_id, o.model_id, o.protocol_id);
	if (observations.has(k)) fail(`duplicate observation ${o.task_id} / ${o.model_id}`);
	observations.set(k, o);
	});
	if (!Array.isArray(packet.contrasts)) fail("contrasts must be a list");
	const contrasts = [];
	packet.contrasts.forEach((c, i) => {
	const where = `contrast ${i}`;
	if (!c \|\| !VERDICTS.includes(c.verdict)) fail(`${where}: unknown verdict`);
	if (!SUBSETS.includes(c.subset)) fail(`${where}: unknown subset`);
	if (!(c.n === null \|\| (Number.isInteger(c.n) && c.n >= 0))) fail(`${where}: invalid n`);
	if (typeof c.metric !== "string" \|\| !c.metric \|\| !UNITS.includes(c.unit)) fail(`${where}: missing metric or unit`);
	const left = observations.get(key(c.task_id, c.model_id, c.protocol_id));
	const right = observations.get(key(c.task_id, c.model_ref, c.protocol_id));
	if (!left \|\| !right) fail(`${where}: refers to a missing observation`);
	if (c.unit !== left.unit) fail(`${where}: unit differs from its observation`);
	checkCI(c.ci95, where);
	if (c.verdict !== "withheld") {
	const r = c.review \|\| {};
	if (r.status !== "accepted" \|\| typeof r.actor !== "string" \|\| !r.actor \|\| !HEX64.test(r.artifact_sha256 \|\| "")) fail(`${where}: verdict without an accepted review`);
	if (!isNum(c.delta) \|\| !isShare(c.p_holm) \|\| !Number.isInteger(c.family_size) \|\| c.family_size < 1) fail(`${where}: verdict without paired test numbers`);
	if (!Number.isInteger(c.n) \|\| c.n < 1) fail(`${where}: verdict without n`);
	if (left.state !== "ok" \|\| right.state !== "ok") fail(`${where}: verdict on a non-ok observation`);
	checkVerdict(c, left, right, observations, where);
	}
	contrasts.push(c);
	});
	return {models, tasks, observations, contrasts};
	}

	// Hit-free values carry a number only with at least this many items (method rule, 2026-10-29).
	const MIN_HIT_FREE_N = 50;
	const ALPHA = 0.05;
	const enough = n => Number.isInteger(n) && n >= MIN_HIT_FREE_N;
	const SHOWN = ["clean","flagged","unknown"];
	// Highest hit rate in a row where some flagged model keeps fewer than MIN_HIT_FREE_N items without any hits ("most items
	// found in training data"); null outside that class.
	function mostlyTrained(observations, taskId, protocolId) {
	let rate = null;
	for (const o of observations.values()) {
	if (o.task_id === taskId && o.protocol_id === protocolId && o.state === "ok" && o.contamination === "flagged" && !enough(o.n_hit_free)) rate = Math.max(rate ?? 0, o.hit_rate);
	}
	return rate;
	}
	// One rule set with the exporter (_check_verdict, agents 31b34ca6), applied to every non-withheld verdict: no verdict
	// against a constant answer or a cell without a number (the exporter publishes no contrast for such pairs); the
	// verdict-bearing subset is full, or hit_free_answer_fields in the "most items in training data" row class, any other
	// subset stays withheld; directional exactly when p_holm <= ALPHA and n >= MIN_HIT_FREE_N (on the answer-field subset
	// also n_hit_free_answer_fields of model_id), with the sign of delta (model_id minus model_ref) matching the verdict and
	// higher_is_better; otherwise inconclusive.
	function checkVerdict(c, left, right, observations, where) {
	for (const o of [left, right]) {
	if (!SHOWN.includes(o.contamination)) fail(`${where}: verdict against a cell without a number`);
	if (o.reference && o.reference.constant_prediction === true) fail(`${where}: verdict against a constant answer`);
	}
	const bearing = mostlyTrained(observations, c.task_id, c.protocol_id) !== null ? "hit_free_answer_fields" : "full";
	if (c.subset !== bearing) fail(`${where}: subset ${c.subset} is a diagnostic in this row (verdicts only on ${bearing})`);
	const nOk = enough(c.n) && (c.subset !== "hit_free_answer_fields" \|\| enough(left.n_hit_free_answer_fields));
	const significant = c.p_holm <= ALPHA && nOk;
	if (c.verdict === "inconclusive") {
	if (significant) fail(`${where}: inconclusive although p_holm ${c.p_holm} <= ${ALPHA} with n >= ${MIN_HIT_FREE_N}`);
	return;
	}
	if (!significant) fail(`${where}: verdict ${c.verdict} needs p_holm <= ${ALPHA} and n >= ${MIN_HIT_FREE_N}`);
	if (c.delta === 0) fail(`${where}: verdict without a delta`);
	const better = (c.delta > 0) === left.higher_is_better;
	if (c.verdict !== (better ? "better" : "worse")) fail(`${where}: verdict ${c.verdict} contradicts delta ${c.delta}`);
	}

	const api = {PINS, MIN_HIT_FREE_N, key, validate, mostlyTrained};
	if (typeof module !== "undefined" && module.exports) module.exports = api; else window.BenchContract = api;
	})();