File size: 3,419 Bytes
5374a2d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | import os
from dotenv import load_dotenv
from typing import Any, Callable
from evoagentx.benchmark import HotPotQA,PubMedQA
from evoagentx.optimizers import AFlowOptimizer
from evoagentx.models import LiteLLMConfig, LiteLLM, OpenAILLMConfig, OpenAILLM
load_dotenv()
api_key = "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"
# Function to encode the image
import os
os.environ["OPENAI_API_KEY"] = api_key
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
EXPERIMENTAL_CONFIG = {
"humaneval": {
"question_type": "code",
"operators": ["Custom", "CustomCodeGenerate", "Test", "ScEnsemble"]
},
"mbpp": {
"question_type": "code",
"operators": ["Custom", "CustomCodeGenerate", "Test", "ScEnsemble"]
},
"hotpotqa": {
"question_type": "qa",
"operators": ["Custom", "AnswerGenerate", "QAScEnsemble"]
},
"gsm8k": {
"question_type": "math",
"operators": ["Custom", "ScEnsemble", "Programmer"]
},
"math": {
"question_type": "math",
"operators": ["Custom", "ScEnsemble", "Programmer"]
}
}
class HotPotQASplits(PubMedQA):
def _load_data(self):
# load the original test data
super()._load_data()
# split the data into train, dev and test
import numpy as np
np.random.seed(42)
permutation = np.random.permutation(len(self._dev_data))
full_test_data = self._dev_data
# randomly select 10 samples for train, 40 for dev, and 100 for test
self._train_data = [full_test_data[idx] for idx in permutation[:50]]
self._dev_data = [full_test_data[idx] for idx in permutation[:50]]
self._test_data =self._test_data[0:500]
self._fulldata = full_test_data
async def async_evaluate(self, graph: Callable, example: Any) -> float:
prompt = example["question"]
paragraphs = example["context"]["contexts"]
context_str = "\n".join(paragraphs)
inputs = f"Context: {context_str}\n\nQuestion: {prompt}\n\nAnswer:"
solution = await graph(inputs)
label = self._get_label(example)
metrics = await super().async_evaluate(prediction=solution, label=label)
outlist.append(metrics)
return metrics["acc"]
def main():
llm_config = OpenAILLMConfig(model="gpt-4o-mini-2024-07-18", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)
executor_llm = OpenAILLM(config=llm_config)
optimizer_llm = OpenAILLM(config=llm_config)
# load benchmark
hotpotqa = HotPotQASplits()
# create optimizer
optimizer = AFlowOptimizer(
graph_path = "examples/aflow/pubmedqa",
optimized_path = "examples/aflow/pubmedqa/optimized",
optimizer_llm=optimizer_llm,
executor_llm=executor_llm,
validation_rounds=3,
eval_rounds=3,
max_rounds=20,
**EXPERIMENTAL_CONFIG["hotpotqa"]
)
# # run optimization
optimizer.optimize(hotpotqa)
# run test
optimizer.test(hotpotqa) # use `test_rounds: List[int]` to specify the rounds to test
if __name__ == "__main__":
outlist = []
main()
import pandas as pd
dfnew = pd.DataFrame(outlist)
dfnew.to_csv("./pubmedqa_save.csv") |