reference/code/dMoE/evaluations/evaluation/arc/prompt.py

import os
from collections import defaultdict
from typing import Any

import io
import numpy as np
from tqdm import tqdm

QUERY_TEMPLATE_MULTICHOICE = """
Answer the following multiple choice question. Think step by step and then give the final answer of the following format: \\boxed{{$LETTER}} where LETTER is one of ABCD.

{Question}

A) {A}
B) {B}
C) {C}
D) {D}
""".strip()

def format_prompt(row: dict):
    choices = row["choices"]["text"]

    choices = choices + ["NA"] * (4 - len(choices))

    prompt_messages = QUERY_TEMPLATE_MULTICHOICE.format(
        Question=row["question"],
        A=choices[0],
        B=choices[1],
        C=choices[2],
        D=choices[3],
    )
    return prompt_messages, row["answerKey"]