|
|
|
|
|
|
|
|
from ...evaluators.aflow_evaluator import AFlowEvaluator |
|
|
from ...core.callbacks import suppress_logger_info |
|
|
from ...core.logging import logger |
|
|
|
|
|
class EvaluationUtils: |
|
|
|
|
|
def __init__(self, root_path: str): |
|
|
self.root_path = root_path |
|
|
|
|
|
async def evaluate_graph_async(self, optimizer, validation_n, data, initial=False): |
|
|
|
|
|
evaluator = AFlowEvaluator(llm=optimizer.executor_llm) |
|
|
sum_score = 0 |
|
|
|
|
|
for _ in range(validation_n): |
|
|
|
|
|
with suppress_logger_info(): |
|
|
score, avg_cost, total_cost, all_failed = await evaluator.graph_evaluate_async(optimizer.benchmark, optimizer.graph, is_test=False) |
|
|
cur_round = optimizer.round + 1 if initial is False else optimizer.round |
|
|
new_data = optimizer.data_utils.create_result_data(cur_round, score, avg_cost, total_cost) |
|
|
data.append(new_data) |
|
|
|
|
|
result_path = optimizer.data_utils.get_results_file_path(self.root_path) |
|
|
optimizer.data_utils.save_results(result_path, data) |
|
|
|
|
|
sum_score += score |
|
|
|
|
|
if all_failed: |
|
|
logger.warning(f"All test cases failed in round {cur_round}. Stopping evaluation for this round.") |
|
|
break |
|
|
|
|
|
return sum_score / validation_n |
|
|
|
|
|
async def evaluate_graph_test_async(self, optimizer): |
|
|
|
|
|
evaluator = AFlowEvaluator(llm=optimizer.executor_llm) |
|
|
with suppress_logger_info(): |
|
|
score, avg_cost, total_cost, all_failed = await evaluator.graph_evaluate_async(optimizer.benchmark, optimizer.graph, is_test=True) |
|
|
return score, avg_cost, total_cost |
