File size: 1,733 Bytes
386d766 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | import json, time, urllib.request, concurrent.futures, sys
U="http://127.0.0.1:8888/v1/chat/completions"
MODEL="hy3-nvfp4"
def chat(prompt, max_tokens=128, temp=0):
body=json.dumps({"model":MODEL,"messages":[{"role":"user","content":prompt}],
"max_tokens":max_tokens,"temperature":temp}).encode()
t0=time.time()
r=urllib.request.urlopen(urllib.request.Request(U,body,{"Content-Type":"application/json"}),timeout=600)
d=json.load(r); dt=time.time()-t0
return d["usage"]["completion_tokens"], dt
chat("hello", 8) # warmup
P="Write a detailed, technical paragraph about the history and architecture of modern GPUs."
print("=== concurrency sweep (128 out tok, temp 0) ===")
for C in [1,4,8,16,32]:
t0=time.time()
with concurrent.futures.ThreadPoolExecutor(max_workers=C) as ex:
res=list(ex.map(lambda _: chat(P,128), range(C)))
wall=time.time()-t0
tot=sum(ct for ct,_ in res)
perreq=sum(ct/dt for ct,dt in res)/len(res)
print(f" C={C:2d}: {tot:5d} tok / {wall:5.1f}s = {tot/wall:6.1f} tok/s aggregate | {perreq:5.1f} tok/s per-req")
print("=== prefix cache (repeat 8k-tok prompt, small output) ===")
long=" ".join(["The archive holds structured records and metadata."]*900)+" Reply with OK."
n,d1=chat(long,4); _,d2=chat(long,4)
print(f" ~{len(long.split())}w prompt: run1={d1:.2f}s run2={d2:.2f}s ({100*(d1-d2)/d1:.0f}% faster on cache hit)")
print("=== long-context prefill (~120k tok) ===")
big=" ".join(["Section entry with descriptive filler content here."]*13000)+" How many words roughly? one number."
try:
_,dt=chat(big,8); import re
print(f" big prompt latency {dt:.1f}s")
except Exception as e:
print(" big-ctx err:", str(e)[:120])
|