File size: 2,002 Bytes
221ca5c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | #!/usr/bin/env python3
"""Benchmark Qwen uncensored Q4_K_M speed at different contexts."""
import json, urllib.request, time
URL = "http://localhost:8081/v1/chat/completions"
TOOLS = [{"type": "function", "function": {"name": "browser_use", "description": "Browser",
"parameters": {"type": "object", "properties": {"action": {"type": "string"}, "url": {"type": "string"}}, "required": ["action"]}}}]
def test(name, messages, tools=True):
data = {"model": "test", "stream": False, "messages": messages}
if tools:
data["tools"] = TOOLS
req = urllib.request.Request(URL, data=json.dumps(data).encode(),
headers={"Content-Type": "application/json"})
try:
resp = urllib.request.urlopen(req, timeout=120)
result = json.loads(resp.read())
t = result.get("timings", {})
m = result["choices"][0]["message"]
tc = bool(m.get("tool_calls"))
print(f"{name:30s} {t.get('prompt_per_second',0):7.1f} p/s {t.get('predicted_per_second',0):6.1f} g/s tc={tc} in={t.get('prompt_n',0)} out={t.get('predicted_n',0)}")
except Exception as e:
print(f"{name:30s} ERROR: {e}")
print(f"{'Test':30s} {'Prompt':>7s} {'Gen':>6s} {'Tool':>4s} {'In':>4s} {'Out':>4s}")
print("-" * 80)
test("Short + tools", [{"role": "user", "content": "Navigate to google.com"}])
test("Medium + tools", [{"role": "system", "content": "You are a browser agent. Use tools."}, {"role": "user", "content": "Go to duckduckgo and search weather Dubai"}])
FILLER = "This is padding text for benchmarking context length impact. " * 100
test("Long ~2K + tools", [{"role": "system", "content": "Agent. " + FILLER[:2000]}, {"role": "user", "content": "Navigate to google.com"}])
test("Very long ~4K + tools", [{"role": "system", "content": "Agent. " + FILLER}, {"role": "user", "content": "Navigate to google.com"}])
test("Short no tools", [{"role": "user", "content": "What is 2+2? Answer briefly."}], tools=False)
|