config.js

window.CONFIG = {
  task: "calibration",
  title: "Judge calibration — 100 gradings",
  itemsPath: "calibration_items.json",
  // Display-only: keys are "qid:rep", so repeats of one question would be
  // recognisable on screen. Logs and scoring always use the true key.
  opaqueIds: "C",
  build: "d2b2018-space",
  landing: "You will grade whether a model's answer states the same fact as a "
    + "reference answer. 100 gradings plus 4 discussable warm-ups; expect 2–3 "
    + "hours total, in as many sittings as you like — progress saves continuously. "
    + "Please stick to one browser: work is guarded per browser, and switching "
    + "browsers mid-audit merges on save rather than instantly.",
  independence: "Grade independently: no discussing real items until everyone has "
    + "finished, and do not look at the LLM judges' verdicts in the repo first.",
  rubric: "Find the response's FINAL answer — the one it commits to, not answers "
    + "it merely considers — and test it against the reference.\n\nmatch — the "
    + "final answer contains everything the reference states. Wording never matters: "
    + "paraphrase, another language, different formatting, or extra correct detail are "
    + "all still a match ('Labrador' matches reference 'dog'). Numeric answers must "
    + "agree within 1% relative error, any convertible unit; a range does not match a "
    + "specific value even if it contains it.\n\nno_match — the final answer "
    + "contradicts or omits part of the reference, or the response hedges between "
    + "several different final answers.\n\nno_answer — the response never commits "
    + "to a final answer: cut off before concluding, answers a different question, "
    + "empty, or only restates the question.\n\nThe reference is authoritative — "
    + "do not re-solve the question, and do not judge the reasoning. A 1-character "
    + "response is data, not a display bug (the character count is shown). Press e to "
    + "jump to the end of a long response; final answers usually live there. Press u "
    + "to flag a judgment you are unsure about (it still counts).",
  fields: [
    { name: "verdict", label: "verdict",
      options: [["match","1"],["no_match","2"],["no_answer","3"]] },
  ],
  hotkeys: { "1":["verdict","match"], "2":["verdict","no_match"], "3":["verdict","no_answer"] },
  notePlaceholder: "note (optional)",
  isComplete: (ev) => !!ev.verdict,
  valuesOf: (ev) => ({ verdict: ev.verdict }),
  summarize: (v) => v.verdict,
  blocks: (item) => [
    { label: "question", text: item.question },
    { label: "reference answer", text: item.reference, ref: true },
    { label: "model response", text: item.response, scroll: true, count: true },
  ],
};