Spaces:
Sleeping
Sleeping
File size: 2,158 Bytes
0279c66 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | import logging
import json
from param_mem.memory.parametric import ParametricMemory
from param_mem.agent.agent_loop import ParamAgent
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def run_evaluation(model_name: str = "TinyLlama/TinyLlama-1.1B-Chat-v1.0", lora_path: str = None, domain: str = "humaneval"):
"""
Evaluates the ParamMem agent dynamically on a given benchmark domain.
Domains supported (mock implementation): humaneval, gsm8k, hotpotqa
"""
logger.info(f"Starting Evaluation on {domain.upper()} domain...")
memory = ParametricMemory(base_model_name=model_name, lora_path=lora_path)
agent = ParamAgent(memory_module=memory)
# Mock dataset for demonstration
mock_datasets = {
"humaneval": ["def add(a, b):", "def is_prime(n):"],
"gsm8k": ["John has 5 apples, gives 2 to Mary. How many?", "A train travels 60mph for 2 hours."],
"hotpotqa": ["Who was the director of the movie starring Tom Hanks in 1994?"]
}
dataset = mock_datasets.get(domain, [])
results = []
for problem in dataset:
logger.info(f"Evaluating Problem: {problem}")
result = agent.solve_task(problem, max_iterations=3)
results.append({
"problem": problem,
"success": result["success"],
"iterations": result["iterations"]
})
success_rate = sum([1 for r in results if r["success"]]) / len(results) if results else 0
logger.info(f"Evaluation Complete. Success Rate: {success_rate * 100:.2f}%")
with open(f"./data/eval_results_{domain}.json", "w") as f:
json.dump(results, f, indent=4)
return success_rate
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--model", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct")
parser.add_argument("--lora", type=str, default=None)
parser.add_argument("--domain", type=str, choices=["humaneval", "gsm8k", "hotpotqa"], default="humaneval")
args = parser.parse_args()
run_evaluation(args.model, args.lora, args.domain)
|