eval.yaml

name: MATH-Vision
description: >
  MATH-Vision (MATH-V) is a meticulously curated collection of 3,040
  high-quality mathematical problems with visual contexts sourced from real
  math competitions. Spanning 16 distinct mathematical disciplines and graded
  across 5 levels of difficulty, it evaluates the multimodal mathematical
  reasoning abilities of Large Multimodal Models (LMMs).
evaluation_framework: "inspect-ai"

tasks:
  - id: mathvision
    config: default
    split: test

    field_spec:
      input: question
      input_image: decoded_image
      target: answer
      choices: options

    solvers:
      - name: system_message
        args:
          template: |
            Solve the following mathematics problem, analyzing the accompanying
            image. If answer options are provided, choose the single best one.
            Reason step by step, then on the final line output ONLY your answer
            in the form:
            ANSWER: <your final answer>
      - name: generate

    scorers:
      - name: match
        args:
          location: end

  - id: mathvision_testmini
    config: default
    split: testmini

    field_spec:
      input: question
      input_image: decoded_image
      target: answer
      choices: options

    solvers:
      - name: system_message
        args:
          template: |
            Solve the following mathematics problem, analyzing the accompanying
            image. If answer options are provided, choose the single best one.
            Reason step by step, then on the final line output ONLY your answer
            in the form:
            ANSWER: <your final answer>
      - name: generate

    scorers:
      - name: match
        args:
          location: end