CJHauser commited on
Commit
f15f0f3
·
verified ·
1 Parent(s): 4f64f5f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +238 -49
app.py CHANGED
@@ -1,17 +1,22 @@
1
  import os, sys, re, json, math, time, threading, subprocess, glob, html, logging, collections
2
- import datetime, shutil, platform
3
- from http.server import BaseHTTPRequestHandler, HTTPServer
4
  from huggingface_hub import HfApi, hf_hub_download
5
 
6
  DATA = "/data"
7
  BUCKET = os.environ.get("BUCKET", "")
 
8
  MAX_PARAMS_B = 1.5
9
  MAX_GB = 3.0 # file size cap when the repo reports a param count
10
  UNKNOWN_MAX_GB = 1.6 # tighter cap when params are unknown (verified after load)
11
  TIMEOUT = 900
12
  HEARTBEAT_S = 30
 
13
  SUMMARY_EVERY = 50
14
  N_QUESTIONS = int(os.environ.get("N_QUESTIONS", "50"))
 
 
 
15
  BENCH = 3 # scoring version: 3 = next-token letter logits, logits_all=True
16
  DONE_FILE = "state/done-v3.txt"
17
  api = HfApi()
@@ -48,6 +53,12 @@ STATUS = {"current": "starting", "queue": 0, "model_started": 0.0, "stage": "ini
48
  SESSION = {"ok": 0, "failed": 0, "skipped": 0, "started": time.time()}
49
  CACHE = {"t": 0, "rows": [], "counts": {}}
50
 
 
 
 
 
 
 
51
  # ---------- logging ----------
52
  LOG_BUF = collections.deque(maxlen=3000)
53
  log = logging.getLogger("lb")
@@ -58,20 +69,58 @@ class BufHandler(logging.Handler):
58
  LOG_BUF.append(self.format(record))
59
 
60
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  def setup_log():
62
  log.setLevel(logging.DEBUG)
63
  fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%H:%M:%S")
64
- handlers = [logging.StreamHandler(sys.stdout), BufHandler()]
65
- try:
66
- os.makedirs(f"{DATA}/logs", exist_ok=True)
67
- day = datetime.date.today().isoformat()
68
- handlers.append(logging.FileHandler(f"{DATA}/logs/run-{day}.log"))
69
- except Exception as e:
70
- print("file logging disabled:", e, flush=True)
71
  for h in handlers:
72
  h.setFormatter(fmt)
73
  h.setLevel(logging.DEBUG)
74
  log.addHandler(h)
 
75
 
76
 
77
  def mem_avail_gb():
@@ -92,7 +141,7 @@ def disk_free_gb(path):
92
 
93
 
94
  def startup_report():
95
- log.info("=== startup (bench v%d) ===", BENCH)
96
  log.info("python %s on %s, cpus=%s", platform.python_version(), platform.platform(), os.cpu_count())
97
  try:
98
  log.info("affinity cpus=%d, cgroup quota=%s, using THREADS=%d",
@@ -121,9 +170,11 @@ def startup_report():
121
  log.info("cpu: %s | simd flags: %s", model.group(1) if model else "?", wanted)
122
  except Exception:
123
  pass
124
- log.info("config: MAX_PARAMS_B=%s MAX_GB=%s UNKNOWN_MAX_GB=%s TIMEOUT=%ss N_QUESTIONS=%s BUCKET=%s",
125
- MAX_PARAMS_B, MAX_GB, UNKNOWN_MAX_GB, TIMEOUT, N_QUESTIONS, BUCKET or "(unset)")
126
- log.info("HF_TOKEN set: %s, HF_HOME=%s", bool(os.environ.get("HF_TOKEN")), os.environ.get("HF_HOME"))
 
 
127
  log.info("mem available: %.1f GB, disk free (cwd): %.1f GB, disk free (%s): %.1f GB",
128
  mem_avail_gb(), disk_free_gb("."), DATA, disk_free_gb(DATA))
129
  try:
@@ -253,6 +304,7 @@ def run_worker(path):
253
  """Run worker subprocess, streaming its output into the log live."""
254
  proc = subprocess.Popen([sys.executable, __file__, "--worker", path],
255
  stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
 
256
  timed_out = {"v": False}
257
 
258
  def kill():
@@ -291,6 +343,7 @@ def run_worker(path):
291
  proc.wait()
292
  finally:
293
  timer.cancel()
 
294
  for h in hints:
295
  log.warning(" [llama.cpp] %s", h)
296
  return proc.returncode, result, list(tail), timed_out["v"], hints, n_params
@@ -322,7 +375,7 @@ def pick_file(siblings, max_gb):
322
  # ---------- heartbeat ----------
323
  def heartbeat():
324
  while True:
325
- time.sleep(HEARTBEAT_S)
326
  el = time.time() - STATUS["model_started"] if STATUS["model_started"] else 0
327
  log.info("heartbeat: model=%s stage=%s elapsed=%.0fs | session ok=%d failed=%d skipped=%d | "
328
  "mem_avail=%.1fGB disk_free=%.1fGB uptime=%.0fmin",
@@ -338,7 +391,7 @@ def loop():
338
  STATUS["stage"] = "listing"
339
  t = time.time()
340
  try:
341
- ids = sorted((m.id for m in api.list_models(filter="gguf", limit=5000)), key=str.lower)
342
  except Exception:
343
  log.exception("list_models failed")
344
  STATUS["current"] = "error: list_models failed"
@@ -348,6 +401,9 @@ def loop():
348
  log.info("listed %d gguf repos in %.1fs (%d remaining)", len(ids), time.time() - t, len(todo))
349
 
350
  for n, mid in enumerate(todo, 1):
 
 
 
351
  STATUS.update(current=mid, model_started=time.time(), stage="metadata")
352
  result, path, t0 = {"model": mid, "ts": time.time(), "bench": BENCH}, None, time.time()
353
  log.info("[%d/%d] %s: start", n, len(todo), mid)
@@ -392,33 +448,36 @@ def loop():
392
  mid, dl, f.size / 1e6 / dl, mem_avail_gb(), disk_free_gb("."))
393
  result.update(file=f.rfilename, arch=arch)
394
 
395
- STATUS["stage"] = "benchmark"
396
- rc, res, tail, timed_out, hints, n_params = run_worker(path)
397
- if n_params:
398
- result["params_b"] = round(n_params / 1e9, 2)
399
- if timed_out:
400
- result.update(status="failed", error=f"timeout after {TIMEOUT}s")
401
- elif rc == 0 and res:
402
- res.pop("n_params", None)
403
- result.update(res, status="ok")
404
- elif rc == 3:
405
- unknown = None
406
- for h in hints:
407
- mm = UNKNOWN_ARCH.search(h)
408
- if mm:
409
- unknown = mm.group(1)
410
- if unknown:
411
- result.update(status="skipped_unsupported_arch", error=f"unknown architecture '{unknown}'")
 
 
 
 
 
 
 
 
 
412
  else:
413
- result.update(status="skipped_load_failed", error=" | ".join(hints or tail)[-500:])
414
- elif rc == 4:
415
- result.update(status="skipped_too_slow", error="projected benchmark time over limit")
416
- elif rc == 5:
417
- result.update(status="skipped_too_big", error="actual param count over limit after load")
418
- elif rc == 6:
419
- result.update(status="failed", error="degenerate logits: label token scores were identical")
420
  else:
421
- result.update(status="failed", error=f"exit code {rc}: " + " | ".join(tail)[-500:])
422
  except Exception as e:
423
  log.exception("%s: exception during processing", mid)
424
  result.update(status="failed", error=f"{type(e).__name__}: {e}"[:500])
@@ -430,6 +489,10 @@ def loop():
430
  except Exception as e:
431
  log.warning("%s: cleanup failed: %s", mid, e)
432
 
 
 
 
 
433
  s = result["status"]
434
  dt = time.time() - t0
435
  if s == "ok":
@@ -459,7 +522,67 @@ def loop():
459
  n, len(todo), up, SESSION["ok"], SESSION["failed"], SESSION["skipped"])
460
 
461
  STATUS.update(current="finished", stage="idle", model_started=0.0)
462
- log.info("all models processed")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
463
 
464
 
465
  # ---------- status page (reads full history from the mount) ----------
@@ -487,7 +610,44 @@ def refresh():
487
  CACHE.update(t=time.time(), rows=rows[:200], counts=counts)
488
 
489
 
 
 
 
490
  class H(BaseHTTPRequestHandler):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
491
  def do_GET(self):
492
  if self.path.startswith("/logs"):
493
  n = 300
@@ -502,6 +662,25 @@ class H(BaseHTTPRequestHandler):
502
  self.end_headers()
503
  self.wfile.write(text.encode())
504
  return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
505
  refresh()
506
  c = CACHE["counts"]
507
  body = "".join(
@@ -510,7 +689,21 @@ class H(BaseHTTPRequestHandler):
510
  f"<td>{html.escape(r.get('file', ''))}</td></tr>" for r in CACHE["rows"])
511
  link = f'<a href="https://huggingface.co/buckets/{BUCKET}">bucket</a>' if BUCKET else "bucket"
512
  el = time.time() - STATUS["model_started"] if STATUS["model_started"] else 0
513
- page = f"""<html><body style="font-family:sans-serif;max-width:900px;margin:2em auto">
 
 
 
 
 
 
 
 
 
 
 
 
 
 
514
  <h2>Tiny model leaderboard</h2>
515
  <p>Now: <b>{html.escape(STATUS['current'])}</b> ({STATUS['stage']}, {el:.0f}s)<br>
516
  all-time (bench v{BENCH}): ok {c.get('ok', 0)} · failed {c.get('failed', 0)} · skipped {c.get('skipped', 0)}
@@ -520,11 +713,7 @@ class H(BaseHTTPRequestHandler):
520
  Raw results: {link} (top 200 shown, refreshed every 60s) ·
521
  <a href="/logs">recent logs</a> (<a href="/logs?n=1000">1000 lines</a>)</p>
522
  <table border=1 cellpadding=4><tr><th>model</th><th>params</th><th>score</th><th>n</th>
523
- <th>prompt tok/s</th><th>file</th></tr>{body}</table></body></html>"""
524
- self.send_response(200)
525
- self.send_header("Content-Type", "text/html")
526
- self.end_headers()
527
- self.wfile.write(page.encode())
528
 
529
  def log_message(self, *a):
530
  pass
@@ -538,6 +727,6 @@ if __name__ == "__main__":
538
  startup_report()
539
  os.makedirs(f"{DATA}/results", exist_ok=True)
540
  threading.Thread(target=heartbeat, daemon=True).start()
541
- threading.Thread(target=loop, daemon=True).start()
542
  log.info("status page listening on :7860")
543
- HTTPServer(("0.0.0.0", 7860), H).serve_forever()
 
1
  import os, sys, re, json, math, time, threading, subprocess, glob, html, logging, collections
2
+ import datetime, shutil, platform, hmac, urllib.parse, concurrent.futures
3
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
4
  from huggingface_hub import HfApi, hf_hub_download
5
 
6
  DATA = "/data"
7
  BUCKET = os.environ.get("BUCKET", "")
8
+ ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "")
9
  MAX_PARAMS_B = 1.5
10
  MAX_GB = 3.0 # file size cap when the repo reports a param count
11
  UNKNOWN_MAX_GB = 1.6 # tighter cap when params are unknown (verified after load)
12
  TIMEOUT = 900
13
  HEARTBEAT_S = 30
14
+ IDLE_HEARTBEAT_S = 1800
15
  SUMMARY_EVERY = 50
16
  N_QUESTIONS = int(os.environ.get("N_QUESTIONS", "50"))
17
+ MAX_REPOS = int(os.environ.get("MAX_REPOS", "5000")) # 0 = every gguf repo on the Hub
18
+ LOG_FLUSH_S = int(os.environ.get("LOG_FLUSH_S", "60"))
19
+ RUN_ID = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
20
  BENCH = 3 # scoring version: 3 = next-token letter logits, logits_all=True
21
  DONE_FILE = "state/done-v3.txt"
22
  api = HfApi()
 
53
  SESSION = {"ok": 0, "failed": 0, "skipped": 0, "started": time.time()}
54
  CACHE = {"t": 0, "rows": [], "counts": {}}
55
 
56
+ # loop control + reset
57
+ STOP = threading.Event()
58
+ LOOP = {"thread": None, "proc": None}
59
+ RESET = {"active": False, "msg": "", "done": 0, "total": 0, "last": ""}
60
+ RESET_LOCK = threading.Lock()
61
+
62
  # ---------- logging ----------
63
  LOG_BUF = collections.deque(maxlen=3000)
64
  log = logging.getLogger("lb")
 
69
  LOG_BUF.append(self.format(record))
70
 
71
 
72
+ class BucketLogHandler(logging.Handler):
73
+ """Keeps lines in memory and rewrites a closed log file on the mounted bucket every
74
+ LOG_FLUSH_S seconds. A file held open on the mount does not sync until it is closed."""
75
+ MAX_LINES = 60000
76
+
77
+ def __init__(self):
78
+ super().__init__()
79
+ self.lines, self.part, self.dirty = [], 1, False
80
+ self.mem_lock = threading.Lock()
81
+
82
+ def path(self):
83
+ suffix = "" if self.part == 1 else f"-p{self.part}"
84
+ return f"{DATA}/logs/run-{RUN_ID}{suffix}.log"
85
+
86
+ def write_out(self):
87
+ try:
88
+ os.makedirs(f"{DATA}/logs", exist_ok=True)
89
+ with open(self.path(), "w") as f:
90
+ f.write("\n".join(self.lines) + "\n")
91
+ self.dirty = False
92
+ except Exception as e:
93
+ print("log sync failed:", e, flush=True)
94
+
95
+ def emit(self, record):
96
+ with self.mem_lock:
97
+ if len(self.lines) >= self.MAX_LINES:
98
+ self.write_out()
99
+ self.part += 1
100
+ self.lines = []
101
+ self.lines.append(self.format(record))
102
+ self.dirty = True
103
+
104
+ def flush_loop(self):
105
+ delay = 5 # first sync soon after startup, then every LOG_FLUSH_S
106
+ while True:
107
+ time.sleep(delay)
108
+ delay = LOG_FLUSH_S
109
+ with self.mem_lock:
110
+ if self.dirty:
111
+ self.write_out()
112
+
113
+
114
  def setup_log():
115
  log.setLevel(logging.DEBUG)
116
  fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%H:%M:%S")
117
+ bucket_handler = BucketLogHandler()
118
+ handlers = [logging.StreamHandler(sys.stdout), BufHandler(), bucket_handler]
 
 
 
 
 
119
  for h in handlers:
120
  h.setFormatter(fmt)
121
  h.setLevel(logging.DEBUG)
122
  log.addHandler(h)
123
+ threading.Thread(target=bucket_handler.flush_loop, daemon=True).start()
124
 
125
 
126
  def mem_avail_gb():
 
141
 
142
 
143
  def startup_report():
144
+ log.info("=== startup (bench v%d, run %s) ===", BENCH, RUN_ID)
145
  log.info("python %s on %s, cpus=%s", platform.python_version(), platform.platform(), os.cpu_count())
146
  try:
147
  log.info("affinity cpus=%d, cgroup quota=%s, using THREADS=%d",
 
170
  log.info("cpu: %s | simd flags: %s", model.group(1) if model else "?", wanted)
171
  except Exception:
172
  pass
173
+ log.info("config: MAX_PARAMS_B=%s MAX_GB=%s UNKNOWN_MAX_GB=%s TIMEOUT=%ss N_QUESTIONS=%s MAX_REPOS=%s BUCKET=%s",
174
+ MAX_PARAMS_B, MAX_GB, UNKNOWN_MAX_GB, TIMEOUT, N_QUESTIONS, MAX_REPOS or "all", BUCKET or "(unset)")
175
+ log.info("log file: logs/run-%s.log in the bucket, synced every %ss", RUN_ID, LOG_FLUSH_S)
176
+ log.info("HF_TOKEN set: %s, ADMIN_PASSWORD set (reset enabled): %s, HF_HOME=%s",
177
+ bool(os.environ.get("HF_TOKEN")), bool(ADMIN_PASSWORD), os.environ.get("HF_HOME"))
178
  log.info("mem available: %.1f GB, disk free (cwd): %.1f GB, disk free (%s): %.1f GB",
179
  mem_avail_gb(), disk_free_gb("."), DATA, disk_free_gb(DATA))
180
  try:
 
304
  """Run worker subprocess, streaming its output into the log live."""
305
  proc = subprocess.Popen([sys.executable, __file__, "--worker", path],
306
  stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
307
+ LOOP["proc"] = proc
308
  timed_out = {"v": False}
309
 
310
  def kill():
 
343
  proc.wait()
344
  finally:
345
  timer.cancel()
346
+ LOOP["proc"] = None
347
  for h in hints:
348
  log.warning(" [llama.cpp] %s", h)
349
  return proc.returncode, result, list(tail), timed_out["v"], hints, n_params
 
375
  # ---------- heartbeat ----------
376
  def heartbeat():
377
  while True:
378
+ time.sleep(IDLE_HEARTBEAT_S if STATUS["stage"] == "idle" else HEARTBEAT_S)
379
  el = time.time() - STATUS["model_started"] if STATUS["model_started"] else 0
380
  log.info("heartbeat: model=%s stage=%s elapsed=%.0fs | session ok=%d failed=%d skipped=%d | "
381
  "mem_avail=%.1fGB disk_free=%.1fGB uptime=%.0fmin",
 
391
  STATUS["stage"] = "listing"
392
  t = time.time()
393
  try:
394
+ ids = sorted((m.id for m in api.list_models(filter="gguf", limit=MAX_REPOS or None)), key=str.lower)
395
  except Exception:
396
  log.exception("list_models failed")
397
  STATUS["current"] = "error: list_models failed"
 
401
  log.info("listed %d gguf repos in %.1fs (%d remaining)", len(ids), time.time() - t, len(todo))
402
 
403
  for n, mid in enumerate(todo, 1):
404
+ if STOP.is_set():
405
+ log.info("loop stopped (reset requested)")
406
+ return
407
  STATUS.update(current=mid, model_started=time.time(), stage="metadata")
408
  result, path, t0 = {"model": mid, "ts": time.time(), "bench": BENCH}, None, time.time()
409
  log.info("[%d/%d] %s: start", n, len(todo), mid)
 
448
  mid, dl, f.size / 1e6 / dl, mem_avail_gb(), disk_free_gb("."))
449
  result.update(file=f.rfilename, arch=arch)
450
 
451
+ if not STOP.is_set():
452
+ STATUS["stage"] = "benchmark"
453
+ rc, res, tail, timed_out, hints, n_params = run_worker(path)
454
+ if n_params:
455
+ result["params_b"] = round(n_params / 1e9, 2)
456
+ if timed_out:
457
+ result.update(status="failed", error=f"timeout after {TIMEOUT}s")
458
+ elif rc == 0 and res:
459
+ res.pop("n_params", None)
460
+ result.update(res, status="ok")
461
+ elif rc == 3:
462
+ unknown = None
463
+ for h in hints:
464
+ mm = UNKNOWN_ARCH.search(h)
465
+ if mm:
466
+ unknown = mm.group(1)
467
+ if unknown:
468
+ result.update(status="skipped_unsupported_arch", error=f"unknown architecture '{unknown}'")
469
+ else:
470
+ result.update(status="skipped_load_failed", error=" | ".join(hints or tail)[-500:])
471
+ elif rc == 4:
472
+ result.update(status="skipped_too_slow", error="projected benchmark time over limit")
473
+ elif rc == 5:
474
+ result.update(status="skipped_too_big", error="actual param count over limit after load")
475
+ elif rc == 6:
476
+ result.update(status="failed", error="degenerate logits: label token scores were identical")
477
  else:
478
+ result.update(status="failed", error=f"exit code {rc}: " + " | ".join(tail)[-500:])
 
 
 
 
 
 
479
  else:
480
+ result["status"] = "failed"
481
  except Exception as e:
482
  log.exception("%s: exception during processing", mid)
483
  result.update(status="failed", error=f"{type(e).__name__}: {e}"[:500])
 
489
  except Exception as e:
490
  log.warning("%s: cleanup failed: %s", mid, e)
491
 
492
+ if STOP.is_set():
493
+ log.info("%s: reset requested, discarding this result", mid)
494
+ return
495
+
496
  s = result["status"]
497
  dt = time.time() - t0
498
  if s == "ok":
 
522
  n, len(todo), up, SESSION["ok"], SESSION["failed"], SESSION["skipped"])
523
 
524
  STATUS.update(current="finished", stage="idle", model_started=0.0)
525
+ log.info("all models processed: session ok=%d failed=%d skipped=%d, %d done in total",
526
+ SESSION["ok"], SESSION["failed"], SESSION["skipped"], len(done))
527
+ log.info("idle now. Set MAX_REPOS=0 (Space variable) and restart to cover every gguf repo on the Hub.")
528
+
529
+
530
+ def start_loop():
531
+ STOP.clear()
532
+ t = threading.Thread(target=loop, daemon=True)
533
+ LOOP["thread"] = t
534
+ t.start()
535
+
536
+
537
+ # ---------- reset ("fresh start") ----------
538
+ def do_reset(delete_logs):
539
+ try:
540
+ RESET.update(active=True, done=0, total=0, msg="stopping the current run")
541
+ log.warning("RESET requested: stopping the run and deleting results and state (old logs too: %s)", delete_logs)
542
+ STOP.set()
543
+ t = LOOP["thread"]
544
+ while t and t.is_alive():
545
+ RESET["msg"] = "waiting for the current model to stop"
546
+ p = LOOP["proc"]
547
+ if p and p.poll() is None:
548
+ p.kill()
549
+ t.join(2)
550
+
551
+ files = glob.glob(f"{DATA}/results/*") + glob.glob(f"{DATA}/state/*")
552
+ if delete_logs:
553
+ keep = f"run-{RUN_ID}" # the current run's log is rewritten from memory, so leave it
554
+ files += [f for f in glob.glob(f"{DATA}/logs/*") if not os.path.basename(f).startswith(keep)]
555
+ RESET.update(total=len(files), done=0, msg="deleting files")
556
+
557
+ def rm(f):
558
+ try:
559
+ os.remove(f)
560
+ return True
561
+ except Exception as e:
562
+ log.warning("could not delete %s: %s", f, e)
563
+ return False
564
+
565
+ failed = 0
566
+ with concurrent.futures.ThreadPoolExecutor(8) as ex:
567
+ for i, ok in enumerate(ex.map(rm, files), 1):
568
+ RESET["done"] = i
569
+ failed += (not ok)
570
+ if i % 1000 == 0:
571
+ log.info("reset: deleted %d/%d", i, len(files))
572
+
573
+ SESSION.update(ok=0, failed=0, skipped=0, started=time.time())
574
+ CACHE.update(t=0, rows=[], counts={})
575
+ STATUS.update(current="starting", queue=0, model_started=0.0, stage="init")
576
+ RESET["last"] = f"Last reset deleted {len(files) - failed} files" + (f" ({failed} failed)" if failed else "")
577
+ log.warning("RESET done: deleted %d files (%d failed). Starting a fresh run.", len(files) - failed, failed)
578
+ except Exception:
579
+ log.exception("reset failed")
580
+ RESET["last"] = "Last reset failed, see logs"
581
+ finally:
582
+ RESET["active"] = False
583
+ RESET["msg"] = ""
584
+ RESET_LOCK.release()
585
+ start_loop()
586
 
587
 
588
  # ---------- status page (reads full history from the mount) ----------
 
610
  CACHE.update(t=time.time(), rows=rows[:200], counts=counts)
611
 
612
 
613
+ PAGE_STYLE = "font-family:sans-serif;max-width:900px;margin:2em auto;padding:0 12px"
614
+
615
+
616
  class H(BaseHTTPRequestHandler):
617
+ def send_html(self, body, code=200, extra_head=""):
618
+ page = f"<html><head><meta charset='utf-8'>{extra_head}</head><body style=\"{PAGE_STYLE}\">{body}</body></html>"
619
+ self.send_response(code)
620
+ self.send_header("Content-Type", "text/html; charset=utf-8")
621
+ self.end_headers()
622
+ self.wfile.write(page.encode())
623
+
624
+ def redirect(self, to="/"):
625
+ self.send_response(303)
626
+ self.send_header("Location", to)
627
+ self.end_headers()
628
+
629
+ def do_POST(self):
630
+ if not self.path.startswith("/reset"):
631
+ return self.send_error(404)
632
+ n = int(self.headers.get("Content-Length") or 0)
633
+ form = urllib.parse.parse_qs(self.rfile.read(min(n, 4096)).decode())
634
+ if (form.get("confirm") or [""])[0] != "yes":
635
+ return self.redirect("/")
636
+ if not ADMIN_PASSWORD:
637
+ return self.send_html("<h2>Reset is disabled</h2><p>Set the ADMIN_PASSWORD secret on the Space to enable it.</p>"
638
+ "<p><a href='/'>Back</a></p>", 403)
639
+ pw = (form.get("password") or [""])[0]
640
+ if not hmac.compare_digest(pw.encode(), ADMIN_PASSWORD.encode()):
641
+ time.sleep(1.5)
642
+ log.warning("reset attempt with a wrong password")
643
+ return self.send_html("<h2>Wrong password</h2><p>Nothing was deleted.</p>"
644
+ "<p><a href='/reset'>Try again</a> · <a href='/'>Back</a></p>", 403)
645
+ if not RESET_LOCK.acquire(blocking=False):
646
+ return self.send_html("<h2>A reset is already running</h2><p><a href='/'>Back</a></p>", 409)
647
+ delete_logs = (form.get("logs") or [""])[0] == "yes"
648
+ threading.Thread(target=do_reset, args=(delete_logs,), daemon=True).start()
649
+ self.redirect("/")
650
+
651
  def do_GET(self):
652
  if self.path.startswith("/logs"):
653
  n = 300
 
662
  self.end_headers()
663
  self.wfile.write(text.encode())
664
  return
665
+
666
+ if self.path.startswith("/reset"):
667
+ if not ADMIN_PASSWORD:
668
+ return self.send_html("<h2>Reset is disabled</h2><p>Add a Space secret named ADMIN_PASSWORD to enable it.</p>"
669
+ "<p><a href='/'>Back</a></p>", 403)
670
+ n_res = len(glob.glob(f"{DATA}/results/*"))
671
+ n_state = len(glob.glob(f"{DATA}/state/*"))
672
+ n_logs = len(glob.glob(f"{DATA}/logs/*"))
673
+ return self.send_html(f"""<h2>Delete everything and start fresh?</h2>
674
+ <p>This stops the current run, then permanently deletes:</p>
675
+ <ul><li>{n_res} result files</li><li>{n_state} progress files (the "already done" lists)</li></ul>
676
+ <p>The benchmark then restarts from the first model. This cannot be undone.</p>
677
+ <form method="post" action="/reset">
678
+ <p><label><input type="checkbox" name="logs" value="yes"> Also delete old logs ({n_logs} files; the current run's log is kept)</label></p>
679
+ <p><label>Admin password <input type="password" name="password" required autocomplete="off"></label></p>
680
+ <button name="confirm" value="yes" style="padding:6px 14px;background:#b42318;color:#fff;border:0">Yes, delete everything</button>
681
+ <button name="confirm" value="no" formnovalidate style="padding:6px 14px">No, keep my results</button>
682
+ </form>""")
683
+
684
  refresh()
685
  c = CACHE["counts"]
686
  body = "".join(
 
689
  f"<td>{html.escape(r.get('file', ''))}</td></tr>" for r in CACHE["rows"])
690
  link = f'<a href="https://huggingface.co/buckets/{BUCKET}">bucket</a>' if BUCKET else "bucket"
691
  el = time.time() - STATUS["model_started"] if STATUS["model_started"] else 0
692
+
693
+ if RESET["active"]:
694
+ top = (f"<div style='border:2px solid #b42318;padding:10px;margin-bottom:14px'><b>Reset in progress:</b> "
695
+ f"{html.escape(RESET['msg'])} ({RESET['done']}/{RESET['total']})</div>")
696
+ head = "<meta http-equiv='refresh' content='5'>"
697
+ else:
698
+ if ADMIN_PASSWORD:
699
+ action = "<a href='/reset'>Fresh start: delete all results…</a>"
700
+ else:
701
+ action = "<span style='color:#888'>Fresh start is disabled (add an ADMIN_PASSWORD secret to enable)</span>"
702
+ last = f" · {html.escape(RESET['last'])}" if RESET["last"] else ""
703
+ top = f"<div style='border:1px solid #ccc;padding:8px 10px;margin-bottom:14px'>{action}{last}</div>"
704
+ head = ""
705
+
706
+ self.send_html(f"""{top}
707
  <h2>Tiny model leaderboard</h2>
708
  <p>Now: <b>{html.escape(STATUS['current'])}</b> ({STATUS['stage']}, {el:.0f}s)<br>
709
  all-time (bench v{BENCH}): ok {c.get('ok', 0)} · failed {c.get('failed', 0)} · skipped {c.get('skipped', 0)}
 
713
  Raw results: {link} (top 200 shown, refreshed every 60s) ·
714
  <a href="/logs">recent logs</a> (<a href="/logs?n=1000">1000 lines</a>)</p>
715
  <table border=1 cellpadding=4><tr><th>model</th><th>params</th><th>score</th><th>n</th>
716
+ <th>prompt tok/s</th><th>file</th></tr>{body}</table>""", extra_head=head)
 
 
 
 
717
 
718
  def log_message(self, *a):
719
  pass
 
727
  startup_report()
728
  os.makedirs(f"{DATA}/results", exist_ok=True)
729
  threading.Thread(target=heartbeat, daemon=True).start()
730
+ start_loop()
731
  log.info("status page listening on :7860")
732
+ ThreadingHTTPServer(("0.0.0.0", 7860), H).serve_forever()