File size: 1,070 Bytes
01f199c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | from .Dataset import Dataset
from evaluations.func_evaluate import evaluate_functional_correctness, evaluate_io
from constants.paths import *
class HumanDataset(Dataset):
def __init__(
self,
path: str = HUMAN_WST_DATA_PATH,
):
super().__init__(path)
self.id_key = "task_id"
def evaluate(
self,
item: dict,
cur_imp: str,
language: str,
):
result = evaluate_functional_correctness(
problem=item,
completion=cur_imp
)
return result == "passed"
def evaluate_sample_io(
self,
item: dict,
cur_imp: str,
language: str,
):
return evaluate_io(
sample_io=item["sample_io"],
completion=cur_imp,
)
@staticmethod
def get_prompt(item):
if "prompt" in item:
return f"{item['prompt']}"
elif "text" in item:
return f"{item['text']}"
else:
raise Exception("No prompt or text in item")
|