File size: 1,232 Bytes
01f199c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | from .Dataset import Dataset
from evaluations.func_evaluate import evaluate_io, evaluate_functional_correctness
from constants.paths import *
class MBPPDataset(Dataset):
def __init__(
self,
path: str = MBPP_DATA_PATH,
):
super().__init__(path)
self.id_key = "name"
def evaluate(
self,
item: dict,
cur_imp: str,
language: str,
):
# result, _ = evaluate_io(item['test_list'],cur_imp,5,True)
# return result
result = evaluate_functional_correctness(
problem=item,
completion=cur_imp
)
return result == "passed"
def evaluate_sample_io(
self,
item: dict,
cur_imp: str,
language: str,
):
if "sample_io" not in item:
return True, ""
if len(item["sample_io"]) == 0:
return True, ""
return evaluate_io(
sample_io=item["sample_io"],
completion=cur_imp,
)
@staticmethod
def get_prompt(item):
# function_signature = item['code'].split('\n')[0].strip()
# return f"{item['text']}\nFunction Signature: {function_signature}"
return item["prompt"]
|