Spaces:
Running on Zero
Running on Zero
File size: 3,353 Bytes
9936912 9e637cd 9936912 9e637cd 9936912 9e637cd 9936912 9e637cd 9936912 9e637cd 9936912 9e637cd 9936912 9e637cd 9936912 9e637cd 9936912 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | #!/usr/bin/env python3
"""Run benchmark evaluation using the ControlAI Tool-Calling Agent."""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from controlai_agent.agent import ControlAgent
from controlai_agent.engine import LocalEngine
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--benchmark",
type=Path,
default=Path("benchmarks/smoke_30.jsonl"),
help="Benchmark JSONL file path",
)
parser.add_argument(
"--model",
type=str,
default="mlx-community/Qwen3-4B-Instruct-2507-4bit",
help="Base model path or HuggingFace repo",
)
parser.add_argument(
"--adapter-path",
type=str,
default=None,
help="Optional adapter checkpoint path",
)
parser.add_argument(
"--output",
type=Path,
default=Path("benchmarks/responses/agent_smoke_30.jsonl"),
help="Output response JSONL file",
)
parser.add_argument(
"--max-tokens",
type=int,
default=768,
help="Maximum generation tokens per step",
)
args = parser.parse_args()
print(f"Loading benchmark from {args.benchmark}...")
items = [
json.loads(line)
for line in args.benchmark.read_text(encoding="utf-8").splitlines()
if line.strip()
]
print(f"Total benchmark items: {len(items)}")
print(f"Initializing ControlAI Agent ({args.model})...")
agent = ControlAgent(engine=LocalEngine(model_id=args.model, adapter_path=args.adapter_path))
print("Agent ready.")
responses = []
args.output.parent.mkdir(parents=True, exist_ok=True)
start_time = time.time()
for idx, item in enumerate(items, 1):
item_id = item.get("id", f"item_{idx:03d}")
prompt = item.get("prompt", "")
print(f"[{idx}/{len(items)}] Evaluating {item_id}...", end=" ", flush=True)
item_start = time.time()
result = agent.run(prompt, max_tokens=args.max_tokens)
elapsed = time.time() - item_start
tool_names = [t.name for t in result.traces]
print(f"done in {elapsed:.2f}s | Steps: {len(result.traces)} | Tools: {tool_names}")
record = {
"benchmark_id": item_id,
"id": item_id,
"family": item.get("family", ""),
"prompt": prompt,
"response": result.answer,
"tool_calls": [
{
"name": t.name,
"arguments": t.arguments,
"result": t.result,
}
for t in result.traces
],
"total_steps": len(result.traces),
"finish_reason": "stop",
}
responses.append(record)
with args.output.open("w", encoding="utf-8") as stream:
for resp in responses:
stream.write(json.dumps(resp, ensure_ascii=False) + "\n")
total_time = time.time() - start_time
print(f"\nSaved {len(responses)} agent responses to {args.output} (Took {total_time:.2f}s)")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|