flozi00 commited on
Commit
86d2239
·
verified ·
1 Parent(s): 9199b01

zero-gpu: probe reports its own latency and allows for the think block

Browse files
BUILD.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "built_at": "2026-10-29T13:57:47",
3
  "commit": "c778551",
4
  "files": 53,
5
  "with_store": false,
 
1
  {
2
+ "built_at": "2026-10-29T14:00:39",
3
  "commit": "c778551",
4
  "files": 53,
5
  "with_store": false,
traceforge/space/app.py CHANGED
@@ -221,8 +221,10 @@ def build_app(store: Optional[TraceStore] = None,
221
  from traceforge.space.local_llm import probe
222
  r = probe()
223
  if r.get("ok"):
224
- return f"**local model answered:** `{r['text']}`"
225
- return f"**local model failed:** `{r['error']}`"
 
 
226
 
227
  def on_debug():
228
  from traceforge.triage import _PARSE_DEBUG, _TRIAGE_DEBUG
 
221
  from traceforge.space.local_llm import probe
222
  r = probe()
223
  if r.get("ok"):
224
+ return (f"**local model answered in {r.get('seconds')} s:** "
225
+ f"`{r['text']}`")
226
+ return (f"**local model failed after {r.get('seconds')} s:** "
227
+ f"`{r['error']}`")
228
 
229
  def on_debug():
230
  from traceforge.triage import _PARSE_DEBUG, _TRIAGE_DEBUG
traceforge/space/local_llm.py CHANGED
@@ -506,7 +506,9 @@ def probe() -> dict:
506
  "model": LOCAL_MODEL,
507
  "messages": [{"role": "user",
508
  "content": "Reply with the single word: ok"}],
509
- "max_tokens": 8,
 
 
510
  "temperature": 0.0,
511
  }
512
  req = urllib.request.Request(
@@ -515,13 +517,16 @@ def probe() -> dict:
515
  headers={"Content-Type": "application/json"},
516
  method="POST",
517
  )
 
518
  try:
519
  with urllib.request.urlopen(req, timeout=180) as r:
520
  data = json.loads(r.read().decode("utf-8"))
521
  return {"ok": True,
 
522
  "text": (data["choices"][0]["message"]["content"] or "")[:200]}
523
  except Exception as e:
524
- return {"ok": False, "error": f"{type(e).__name__}: {e}"}
 
525
 
526
 
527
  def _volume_read_mbps(mb: int = 256) -> dict:
 
506
  "model": LOCAL_MODEL,
507
  "messages": [{"role": "user",
508
  "content": "Reply with the single word: ok"}],
509
+ # The model always emits a thinking block before answering, so a
510
+ # handful of tokens would only ever return reasoning.
511
+ "max_tokens": 200,
512
  "temperature": 0.0,
513
  }
514
  req = urllib.request.Request(
 
517
  headers={"Content-Type": "application/json"},
518
  method="POST",
519
  )
520
+ t0 = time.time()
521
  try:
522
  with urllib.request.urlopen(req, timeout=180) as r:
523
  data = json.loads(r.read().decode("utf-8"))
524
  return {"ok": True,
525
+ "seconds": round(time.time() - t0, 1),
526
  "text": (data["choices"][0]["message"]["content"] or "")[:200]}
527
  except Exception as e:
528
+ return {"ok": False, "seconds": round(time.time() - t0, 1),
529
+ "error": f"{type(e).__name__}: {e}"}
530
 
531
 
532
  def _volume_read_mbps(mb: int = 256) -> dict: