File size: 4,694 Bytes
b0c0df0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 |
import json
import os
import random
import re
import numpy as np
from PIL import Image
MULTI_CHOICE_DIRECT_PROMPT = "Answer the question with the option's letter from the given choices directly."
COT_PROMPT = "Solve the multiple-choice question and then answer with the option letter from the given choices. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering."
PROMPTS = {"MULTI_CHOICE_DIRECT_PROMPT": MULTI_CHOICE_DIRECT_PROMPT, "COT_PROMPT": COT_PROMPT}
def VisualPuzzles_doc_to_visual(doc):
return [doc["image"]]
def VisualPuzzles_doc_to_text(doc, lmms_eval_specific_kwargs):
question = "Question: " + doc["question"].strip()
options = doc["options"]
if options != None:
question += "\nOptions:\n(A) " + options[0] + "\n(B) " + options[1] + "\n(C) " + options[2] + "\n(D) " + options[3]
else:
question += "\nOptions: Choose from (A) (B) (C) (D) in the image."
question += "\n" + PROMPTS[lmms_eval_specific_kwargs["prompt"]]
return question
def parse_response(response, all_choices, index2ans):
"""
Return the last letter appearing after 'ANSWER:' in the input text.
If there's no match, return None.
"""
pattern = r"Answer:\s*\(([A-Za-z])\)" # Answer: (A)
matches = re.findall(pattern, response)
if matches:
for match in matches[::-1]:
if match in all_choices or match.upper() in all_choices:
return match
pattern = r"(?<!Final )Answer:\s*([A-Za-z])" # Answer: A
matches = re.findall(pattern, response)
if matches:
for match in matches[::-1]:
if match in all_choices or match.upper() in all_choices:
return match
pattern = r"Answer:\s*([A-Za-z])" # Answer: A
matches = re.findall(pattern, response)
if matches:
for match in matches[::-1]:
if match in all_choices or match.upper() in all_choices:
return match
pattern = r"\s*\(([A-Za-z])\)" # e.g., (A) (B) (C) (D)
matches = re.findall(pattern, response)
if matches:
for match in matches[::-1]:
if match in all_choices or match.upper() in all_choices:
return match
response = " " + response.strip()
pattern = r"\s*([A-Za-z])\)" # e.g., A) B) C) D)
matches = re.findall(pattern, response)
if matches:
for match in matches[::-1]:
if match in all_choices or match.upper() in all_choices:
return match
pattern = r"\s*\{([A-Za-z])\}" # e.g., {A} {B} {C} {D}
matches = re.findall(pattern, response)
if matches:
for match in matches[::-1]:
if match in all_choices or match.upper() in all_choices:
return match
pattern = r"\s*\$([A-Za-z])\$" # e.g., $A$, $B$, $C$, $D$
matches = re.findall(pattern, response)
if matches:
for match in matches[::-1]:
if match in all_choices or match.upper() in all_choices:
return match
pattern = r" ([A-Da-d])\." # e.g., A. B. C. D.
matches = re.findall(pattern, response)
if matches:
for match in matches[::-1]:
if match in all_choices or match.upper() in all_choices:
return match
pattern = r" ([A-Da-d])" # e.g., A B C D
matches = re.findall(pattern, response)
if matches and len(response) <= 5:
for match in matches[::-1]:
if match in all_choices or match.upper() in all_choices:
return match
if index2ans != None:
for index in all_choices:
ans = index2ans[index]
if f"answer: {ans}" in response.lower():
return index
if f"answer:{ans}" in response.lower():
return index
last_found = None
last_index = -1
for index in all_choices:
ans = index2ans[index]
idx = response.rfind(ans)
if idx > last_index:
last_found = index
last_index = idx
if last_found:
return last_found
return random.choice(all_choices)
def VisualPuzzles_process_result(doc, results):
print(f"results: {results}")
pred = results[0].strip()
all_choices = ["A", "B", "C", "D"]
if doc["options"] == None:
index2ans = None
else:
index2ans = {all_choices[i]: doc["options"][i] for i in range(4)}
pred = parse_response(pred, all_choices, index2ans)
target = doc["answer"]
if pred.lower() == target.lower():
return {"exact_match": 1.0}
return {"exact_match": 0.0}
|