eval.yaml

# yaml file for compatibility with inspect-ai

name: GSM8K
description: >
  GSM8K is a dataset of 8,000+ high-quality, single-step arithmetic word problems.
evaluation_framework: inspect-ai

tasks:
  - id: gsm8k
    config: main
    split: test

    epochs: 4
    epoch_reducer: pass_at_1

    field_spec:
      input: question
      target: answer

    solvers:
      - name: prompt_template
        args:
          template: >
            Solve the following math problem efficiently and clearly.  The last line
            of your response should be of the following format: 'Therefore, the final
            answer is: $\\boxed{ANSWER}$. I hope it is correct' (without quotes)
            where ANSWER is just the final number or expression that solves the
            problem. Think step by step before answering.

            {prompt}
      - name: generate
          
    scorers:
      - name: model_graded_fact