config.yaml

annotation_task_name: "Evaluation of Math Tutor Responses"
task_dir: "."

data_files:
  - "my-annotation-task/data/math_annotator_demo_all_with_id_text2show.csv"

item_properties:
  id_key: id
  text_key: text2show

instance_display:
  fields:
    - key: text2show_html
      type: html
  layout:
    direction: vertical
    gap: "16px"
    resizable: false

output_annotation_dir: "annotation_output/"
export_annotation_format: "json"

require_password: false

annotation_schemes:
  - annotation_type: radio
    required: true
    name: Content Correctness
    description: "Content Correctness: Are the facts, explanations, code, fixes, or judgments technically and factually correct?"
    labels: ["Yes", "To some extent", "No"]

  - annotation_type: radio
    required: true
    name: Learner-State Assessment
    description: "Learner-State Assessment: Does the tutor correctly assess whether the learner's work is correct, incorrect, incomplete, or uncertain?"
    labels: ["Yes", "To some extent", "No"]

  - annotation_type: radio
    required: true
    name: Issue Localization
    description: "Issue Localization: Does the tutor identify the relevant step, line, concept, or reasoning gap?"
    labels: ["Yes", "To some extent", "No"]

  - annotation_type: radio
    required: true
    name: Disclosure Appropriateness
    description: "Disclosure Appropriateness: Does the tutor reveal an appropriate amount of solution detail given the task type and learner state?"
    labels: ["Yes", "To some extent", "No"]

  - annotation_type: radio
    required: true
    name: Providing Guidance
    description: "Providing Guidance: Does the response provide useful help, such as a hint, explanation, validation, diagnostic feedback, worked step, or repair advice?"
    labels: ["Yes", "To some extent", "No"]

  - annotation_type: radio
    required: true
    name: Coherence
    description: "Coherence: Is the response logically consistent and generally relevant to the task?"
    labels: ["Yes", "To some extent", "No"]

  - annotation_type: radio
    required: true
    name: Actionability
    description: "Actionability: After reading the response, is it clear what the learner should do, think about, or check next?"
    labels: ["Yes", "To some extent", "No"]

  - annotation_type: radio
    required: true
    name: Clarity
    description: "Clarity: Is the response easy to understand in terms of context, structure, and references?"
    labels: ["Yes", "To some extent", "No"]

  - annotation_type: radio
    required: true
    name: Conciseness
    description: "Conciseness: Is the response appropriately scoped in length and detail for the context?"
    labels: ["Yes", "To some extent", "No"]

  - annotation_type: radio
    required: true
    name: Humanness
    description: "Humanness: Does this look or sound like what a human tutor would do or say in this context?"
    labels: ["Yes", "To some extent", "No"]

# Generated but intentionally not enabled for the first demo:
# phases:
#   training:
#     enabled: true
#     data_file: "my-annotation-task/data/training_questions.json"
#     passing_criteria:
#       min_correct: 8
#       total_questions: 14
#     retries:
#       enabled: true
#       max_retries: 2
#     show_explanations: true
#     randomize: true
#
# gold_standards:
#   enabled: true
#   items_file: "my-annotation-task/data/gold_standards.json"
#   mode: "separate"
#   accuracy:
#     min_threshold: 0.7
#     evaluation_count: 14

custom_footer_html: |
  <style>
    :root {
      --math-demo-accent: #6d55d8;
      --math-demo-ink: #111827;
      --math-demo-muted: #5b6472;
      --math-demo-border: #d0d7de;
      --math-demo-soft: #f6f8fa;
    }

    body {
      background: #f3f5f9;
    }

    .container-fluid {
      padding-left: clamp(10px, 2vw, 24px);
      padding-right: clamp(10px, 2vw, 24px);
    }

    #task_layout.text-annotation {
      margin-left: auto;
      margin-right: auto;
      max-width: min(92vw, 960px);
      width: 100%;
    }

    #task_layout.text-annotation .shadcn-card-content {
      padding: clamp(10px, 1.4vw, 18px);
    }

    .potato-instance-meta {
      display: flex;
      flex-wrap: wrap;
      gap: 8px;
      margin: 0 0 10px;
    }

    .potato-instance-meta span {
      border: 1px solid #d7dcef;
      border-radius: 999px;
      background: #f8fafc;
      color: #475569;
      font-size: 12px;
      font-weight: 700;
      padding: 4px 9px;
    }

    .potato-text-section {
      margin-bottom: 14px;
    }

    .potato-text-section h3 {
      color: var(--math-demo-ink);
      font-size: 1.18rem;
      line-height: 1.2;
      margin: 0 0 6px;
    }

    .potato-text-section p {
      color: var(--math-demo-ink);
      font-size: 0.96rem;
      line-height: 1.5;
      margin: 0 0 10px;
    }

    .potato-text-section code {
      background: #f1f5f9;
      border-radius: 4px;
      color: #b83280;
      font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
      padding: 1px 4px;
    }

    .potato-code-block {
      background: var(--math-demo-soft);
      border: 1px solid var(--math-demo-border);
      border-radius: 6px;
      color: #24292f;
      font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
      font-size: 14px;
      line-height: 1.45;
      margin: 8px 0 12px;
      overflow-x: auto;
      padding: 12px;
      white-space: pre-wrap;
    }

    .annotation-stepper-workspace {
      align-items: stretch;
      display: flex;
      flex-direction: column;
      gap: 18px;
    }

    .annotation-stepper-workspace > .mb-4 {
      margin-bottom: 0 !important;
      min-width: 0;
    }

    .annotation-stepper-workspace .instance-display-container {
      background: #fff;
      border: 1px solid var(--math-demo-border);
      border-radius: 8px;
      box-shadow: inset -8px 0 0 rgba(148, 163, 184, 0.08);
      padding: clamp(14px, 1.6vw, 22px);
    }

    .annotation-stepper-workspace .display-field,
    .annotation-stepper-workspace .display-field-resizable,
    .annotation-stepper-workspace .text-display-content {
      max-height: none !important;
      min-height: 0 !important;
    }

    #annotation-forms.annotation-stepper {
      background: #fff;
      border: 1px solid var(--math-demo-border);
      border-radius: 8px;
      box-shadow: 0 8px 24px rgba(15, 23, 42, 0.08);
      margin-top: 4px;
      padding: 13px;
      width: 100%;
    }

    #annotation-forms.annotation-stepper .annotation-stepper-hidden {
      display: none !important;
    }

    #annotation-forms.annotation-stepper .annotation-form {
      border: 0;
      box-shadow: none;
      margin-bottom: 0;
      padding: 0;
    }

    #annotation-forms.annotation-stepper .shadcn-radio-options {
      display: flex;
      flex-wrap: wrap;
      gap: 10px;
    }

    #annotation-forms.annotation-stepper .shadcn-radio-option {
      min-width: 150px;
    }

    #annotation-forms.annotation-stepper .shadcn-radio-input[value="Yes"] + .shadcn-radio-label {
      background: #e8f5ee;
      border: 1px solid #8fc9a8;
      border-radius: 999px;
      color: #17603a;
      font-weight: 700;
      padding: 6px 10px;
    }

    #annotation-forms.annotation-stepper .shadcn-radio-input[value="To some extent"] + .shadcn-radio-label {
      background: #fff7d6;
      border: 1px solid #e6c75a;
      border-radius: 999px;
      color: #7a5a00;
      font-weight: 700;
      padding: 6px 10px;
    }

    #annotation-forms.annotation-stepper .shadcn-radio-input[value="No"] + .shadcn-radio-label {
      background: #fde8e8;
      border: 1px solid #f2a6a6;
      border-radius: 999px;
      color: #9b1c1c;
      font-weight: 700;
      padding: 6px 10px;
    }

    #annotation-forms.annotation-stepper .shadcn-radio-input:checked + .shadcn-radio-label {
      box-shadow: 0 0 0 3px rgba(31, 111, 235, 0.18);
    }

    .annotation-stepper-controls {
      border-bottom: 1px solid #e5e7eb;
      margin-bottom: 12px;
      padding-bottom: 10px;
    }

    .annotation-stepper-row {
      align-items: center;
      display: flex;
      gap: 12px;
      justify-content: space-between;
      margin-bottom: 8px;
    }

    .annotation-stepper-kicker {
      color: var(--math-demo-muted);
      font-size: 0.82rem;
      font-weight: 700;
      text-transform: uppercase;
    }

    .annotation-stepper-title {
      color: var(--math-demo-ink);
      font-size: 1rem;
      font-weight: 750;
      line-height: 1.3;
    }

    .annotation-stepper-help {
      color: var(--math-demo-muted);
      font-size: 0.9rem;
      margin-top: 4px;
    }

    .annotation-stepper-actions {
      display: flex;
      flex-wrap: wrap;
      gap: 8px;
      justify-content: flex-end;
    }

    .annotation-stepper-button {
      background: #fff;
      border: 1px solid #cbd5e1;
      border-radius: 6px;
      color: #1f2937;
      cursor: pointer;
      font-weight: 650;
      min-height: 36px;
      padding: 7px 10px;
    }

    .annotation-stepper-button:hover:not(:disabled) {
      background: #f8fafc;
      border-color: #94a3b8;
    }

    .annotation-stepper-button:disabled {
      color: #9ca3af;
      cursor: not-allowed;
      opacity: 0.7;
    }

    .annotation-stepper-progress {
      display: flex;
      flex-wrap: wrap;
      gap: 6px;
      margin-top: 6px;
    }

    .annotation-stepper-dot {
      align-items: center;
      background: #f8fafc;
      border: 1px solid #d1d5db;
      border-radius: 999px;
      color: #4b5563;
      cursor: pointer;
      display: inline-flex;
      font-size: 0.8rem;
      font-weight: 700;
      height: 28px;
      justify-content: center;
      min-width: 28px;
      padding: 0 8px;
    }

    .annotation-stepper-dot.is-active {
      background: #1f6feb;
      border-color: #1f6feb;
      color: #fff;
    }

    .annotation-stepper-dot.is-answered:not(.is-active) {
      background: #e8f5ee;
      border-color: #8fc9a8;
      color: #17603a;
    }

    @media (max-width: 720px) {
      .container-fluid {
        padding-left: 8px;
        padding-right: 8px;
      }

      .annotation-stepper-row {
        align-items: flex-start;
        flex-direction: column;
      }

      .annotation-stepper-workspace {
        gap: 12px;
      }

      .annotation-stepper-workspace .instance-display-container {
        padding: 12px;
      }

      #annotation-forms.annotation-stepper .shadcn-radio-option {
        min-width: 100%;
      }
    }
  </style>
  <script>
    window.MathJax = {
      tex: {
        inlineMath: [["\\(", "\\)"]],
        displayMath: [["\\[", "\\]"]]
      },
      options: { skipHtmlTags: ["script", "noscript", "style", "textarea", "pre", "code"] }
    };
  </script>
  <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js" async></script>
  <script>
    (function () {
      function onReady(fn) {
        if (document.readyState === "loading") {
          document.addEventListener("DOMContentLoaded", fn);
        } else {
          fn();
        }
      }

      function textOf(element) {
        return element ? (element.textContent || "").trim() : "";
      }

      function dimensionTitle(form) {
        var legend = textOf(form.querySelector("legend"));
        var colon = legend.indexOf(":");
        return colon > 0 ? legend.slice(0, colon) : (legend || form.id || "Dimension");
      }

      function isAnswered(form) {
        if (form.querySelector("input[type='radio']:checked")) return true;
        if (form.querySelector("input[type='checkbox']:checked")) return true;
        var fields = form.querySelectorAll("input[type='text'], input[type='number'], textarea, select");
        for (var i = 0; i < fields.length; i += 1) {
          if ((fields[i].value || "").trim()) return true;
        }
        return false;
      }

      function firstUnansweredIndex(forms) {
        for (var i = 0; i < forms.length; i += 1) {
          if (!isAnswered(forms[i])) return i;
        }
        return forms.length - 1;
      }

      function initDimensionStepper() {
        var container = document.getElementById("annotation-forms");
        if (!container || container.getAttribute("data-stepper-ready") === "true") return;

        var forms = Array.prototype.slice.call(container.querySelectorAll("form.annotation-form.radio"));
        if (forms.length < 2) return;

        container.setAttribute("data-stepper-ready", "true");
        container.classList.add("annotation-stepper");

        var main = document.getElementById("main-content");
        var displayPane = main ? main.querySelector(".mb-4") : null;
        if (main && displayPane) {
          var workspace = document.createElement("div");
          workspace.className = "annotation-stepper-workspace";
          main.insertBefore(workspace, displayPane);
          workspace.appendChild(displayPane);
          workspace.appendChild(container);
        }

        var currentIndex = firstUnansweredIndex(forms);
        var showAll = false;
        var controls = document.createElement("div");
        controls.className = "annotation-stepper-controls";
        controls.innerHTML = [
          '<div class="annotation-stepper-row">',
          '  <div>',
          '    <div class="annotation-stepper-kicker"></div>',
          '    <div class="annotation-stepper-title"></div>',
          '    <div class="annotation-stepper-help"></div>',
          '  </div>',
          '  <div class="annotation-stepper-actions">',
          '    <button type="button" class="annotation-stepper-button" data-stepper-prev>Prev</button>',
          '    <button type="button" class="annotation-stepper-button" data-stepper-next>Next</button>',
          '    <button type="button" class="annotation-stepper-button" data-stepper-toggle>Show all</button>',
          '  </div>',
          '</div>',
          '<div class="annotation-stepper-progress" aria-label="Dimension progress"></div>'
        ].join("");
        container.insertBefore(controls, container.firstChild);

        var kicker = controls.querySelector(".annotation-stepper-kicker");
        var title = controls.querySelector(".annotation-stepper-title");
        var help = controls.querySelector(".annotation-stepper-help");
        var progress = controls.querySelector(".annotation-stepper-progress");
        var prev = controls.querySelector("[data-stepper-prev]");
        var next = controls.querySelector("[data-stepper-next]");
        var toggle = controls.querySelector("[data-stepper-toggle]");

        var dots = forms.map(function (form, index) {
          var dot = document.createElement("button");
          dot.type = "button";
          dot.className = "annotation-stepper-dot";
          dot.textContent = String(index + 1);
          dot.title = dimensionTitle(form);
          dot.addEventListener("click", function () {
            showAll = false;
            currentIndex = index;
            render(true);
          });
          progress.appendChild(dot);
          return dot;
        });

        function render(shouldScroll) {
          for (var i = 0; i < forms.length; i += 1) {
            var active = showAll || i === currentIndex;
            forms[i].classList.toggle("annotation-stepper-hidden", !active);
            forms[i].setAttribute("aria-hidden", active ? "false" : "true");
            dots[i].classList.toggle("is-active", !showAll && i === currentIndex);
            dots[i].classList.toggle("is-answered", isAnswered(forms[i]));
          }

          var answeredCount = forms.filter(isAnswered).length;
          if (showAll) {
            kicker.textContent = answeredCount + " / " + forms.length + " answered";
            title.textContent = "All dimensions";
            help.textContent = "Showing every annotation box.";
          } else {
            kicker.textContent = "Dimension " + (currentIndex + 1) + " / " + forms.length;
            title.textContent = dimensionTitle(forms[currentIndex]);
            help.textContent = isAnswered(forms[currentIndex])
              ? "Answered. You can adjust it or move to another dimension."
              : "Choose a label, or use Next to skip ahead.";
          }

          prev.disabled = showAll || currentIndex === 0;
          next.disabled = showAll || currentIndex === forms.length - 1;
          toggle.textContent = showAll ? "One at a time" : "Show all";

          if (shouldScroll && window.matchMedia("(max-width: 1099px)").matches) {
            container.scrollIntoView({ behavior: "smooth", block: "start" });
          }
        }

        prev.addEventListener("click", function () {
          if (currentIndex > 0) {
            currentIndex -= 1;
            showAll = false;
            render(true);
          }
        });

        next.addEventListener("click", function () {
          if (currentIndex < forms.length - 1) {
            currentIndex += 1;
            showAll = false;
            render(true);
          }
        });

        toggle.addEventListener("click", function () {
          showAll = !showAll;
          render(false);
        });

        container.addEventListener("change", function (event) {
          var form = event.target.closest ? event.target.closest("form.annotation-form") : null;
          var index = forms.indexOf(form);
          if (index === -1) return;
          currentIndex = index;
          render(false);
          if (!showAll && event.target.type === "radio" && event.target.checked && index < forms.length - 1) {
            window.setTimeout(function () {
              if (currentIndex === index && isAnswered(forms[index])) {
                currentIndex = index + 1;
                render(false);
              }
            }, 350);
          }
        });

        render(false);
      }

      onReady(function () {
        window.setTimeout(initDimensionStepper, 250);
      });
    }());
  </script>