import os, sys, re, json, math, time, threading, subprocess, glob, html, logging, collections import datetime, shutil, platform, hmac, urllib.parse, concurrent.futures from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from huggingface_hub import HfApi, hf_hub_download DATA = "/data" BUCKET = os.environ.get("BUCKET", "") ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "") MAX_PARAMS_B = 1.5 MAX_GB = 3.0 # file size cap when the repo reports a param count UNKNOWN_MAX_GB = 1.6 # tighter cap when params are unknown (verified after load) TIMEOUT = 900 HEARTBEAT_S = 30 IDLE_HEARTBEAT_S = 1800 SUMMARY_EVERY = 50 N_QUESTIONS = int(os.environ.get("N_QUESTIONS", "100")) MAX_REPOS = int(os.environ.get("MAX_REPOS", "7000")) # 0 = every gguf repo on the Hub LOG_FLUSH_S = int(os.environ.get("LOG_FLUSH_S", "60")) CACHE_EVERY_S = 120 RUN_ID = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") BENCH = 5 # scoring version: 4 = letter logits (variants merged), space-token handling, format check SHOW_BENCH = (3, 4) # result versions shown in the table UNRELIABLE_MASS = 0.02 # mean share of probability on answer letters below this = unreliable score DONE_FILE = f"state/done-v{BENCH}.txt" api = HfApi() def cpu_quota(): """CPUs this container may actually use (cgroup quota), or None if unknown.""" try: q, p = open("/sys/fs/cgroup/cpu.max").read().split() if q != "max": return max(1, round(int(q) / int(p))) except Exception: pass try: q = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read()) p = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read()) if q > 0: return max(1, round(q / p)) except Exception: pass return None THREADS = int(os.environ.get("THREADS", "0") or 0) or cpu_quota() or 2 # files that are never a standalone language model BAD_NAME = re.compile(r"mmproj|lora|adapter|imatrix|draft", re.I) SHARD = re.compile(r"-\d{5}-of-\d{5}", re.I) NON_TEXT_TASKS = {"text-to-image", "image-to-image", "image-to-3d", "text-to-3d", "text-to-video", "image-to-video", "unconditional-image-generation", "text-to-speech", "automatic-speech-recognition"} STATUS = {"current": "starting", "queue": 0, "model_started": 0.0, "stage": "init"} SESSION = {"ok": 0, "failed": 0, "skipped": 0, "started": time.time()} CACHE = {"t": 0, "rows": [], "counts": {}} # loop control + reset STOP = threading.Event() LOOP = {"thread": None, "proc": None} RESET = {"active": False, "msg": "", "done": 0, "total": 0, "last": ""} RESET_LOCK = threading.Lock() # ---------- logging ---------- LOG_BUF = collections.deque(maxlen=3000) log = logging.getLogger("lb") class BufHandler(logging.Handler): def emit(self, record): LOG_BUF.append(self.format(record)) class BucketLogHandler(logging.Handler): """Keeps lines in memory and rewrites a closed log file on the mounted bucket every LOG_FLUSH_S seconds. A file held open on the mount does not sync until it is closed.""" MAX_LINES = 60000 def __init__(self): super().__init__() self.lines, self.part, self.dirty = [], 1, False self.mem_lock = threading.Lock() def path(self): suffix = "" if self.part == 1 else f"-p{self.part}" return f"{DATA}/logs/run-{RUN_ID}{suffix}.log" def write_out(self): try: os.makedirs(f"{DATA}/logs", exist_ok=True) with open(self.path(), "w") as f: f.write("\n".join(self.lines) + "\n") self.dirty = False except Exception as e: print("log sync failed:", e, flush=True) def emit(self, record): with self.mem_lock: if len(self.lines) >= self.MAX_LINES: self.write_out() self.part += 1 self.lines = [] self.lines.append(self.format(record)) self.dirty = True def flush_loop(self): delay = 5 # first sync soon after startup, then every LOG_FLUSH_S while True: time.sleep(delay) delay = LOG_FLUSH_S with self.mem_lock: if self.dirty: self.write_out() def setup_log(): log.setLevel(logging.DEBUG) fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%H:%M:%S") bucket_handler = BucketLogHandler() handlers = [logging.StreamHandler(sys.stdout), BufHandler(), bucket_handler] for h in handlers: h.setFormatter(fmt) h.setLevel(logging.DEBUG) log.addHandler(h) threading.Thread(target=bucket_handler.flush_loop, daemon=True).start() def mem_avail_gb(): try: for line in open("/proc/meminfo"): if line.startswith("MemAvailable"): return int(line.split()[1]) / 1e6 except Exception: pass return -1.0 def disk_free_gb(path): try: return shutil.disk_usage(path).free / 1e9 except Exception: return -1.0 def startup_report(): log.info("=== startup (bench v%d, run %s) ===", BENCH, RUN_ID) log.info("python %s on %s, cpus=%s", platform.python_version(), platform.platform(), os.cpu_count()) try: log.info("affinity cpus=%d, cgroup quota=%s, using THREADS=%d", len(os.sched_getaffinity(0)), cpu_quota(), THREADS) except Exception: log.info("cgroup quota=%s, using THREADS=%d", cpu_quota(), THREADS) for p in ("/sys/fs/cgroup/cpu.max", "/sys/fs/cgroup/memory.max"): try: log.info("%s = %s", p, open(p).read().strip()) except Exception: pass try: import llama_cpp log.info("llama_cpp version %s", llama_cpp.__version__) try: log.info("llama.cpp system info: %s", llama_cpp.llama_print_system_info().decode().strip()) except Exception as e: log.warning("could not read llama.cpp system info: %s", e) except Exception as e: log.error("llama_cpp import failed: %s", e) try: cpuinfo = open("/proc/cpuinfo").read() model = re.search(r"model name\s*:\s*(.+)", cpuinfo) flags = re.search(r"flags\s*:\s*(.+)", cpuinfo) wanted = [f for f in ("avx", "avx2", "avx512f", "fma", "f16c") if flags and f" {f} " in f" {flags.group(1)} "] log.info("cpu: %s | simd flags: %s", model.group(1) if model else "?", wanted) except Exception: pass log.info("config: MAX_PARAMS_B=%s MAX_GB=%s UNKNOWN_MAX_GB=%s TIMEOUT=%ss N_QUESTIONS=%s MAX_REPOS=%s BUCKET=%s", MAX_PARAMS_B, MAX_GB, UNKNOWN_MAX_GB, TIMEOUT, N_QUESTIONS, MAX_REPOS or "all", BUCKET or "(unset)") log.info("log file: logs/run-%s.log in the bucket, synced every %ss", RUN_ID, LOG_FLUSH_S) log.info("HF_TOKEN set: %s, ADMIN_PASSWORD set (reset enabled): %s, HF_HOME=%s", bool(os.environ.get("HF_TOKEN")), bool(ADMIN_PASSWORD), os.environ.get("HF_HOME")) log.info("mem available: %.1f GB, disk free (cwd): %.1f GB, disk free (%s): %.1f GB", mem_avail_gb(), disk_free_gb("."), DATA, disk_free_gb(DATA)) try: t = f"{DATA}/.write_test" open(t, "w").write("ok") os.remove(t) log.info("%s is writable", DATA) except Exception as e: log.error("%s is NOT writable (%s): results will not persist!", DATA, e) log.info("existing result files: %d, done in v%d list: %d", len(glob.glob(f"{DATA}/results/*.json")), BENCH, len(load_done())) # ---------- worker (separate process) ---------- # exit codes: 0 ok, 3 model failed to load, 4 projected too slow, 5 actually too big, 6 degenerate logits def worker(path): def say(msg): print(msg, flush=True) t_start = time.time() say(f"worker start: {os.path.basename(path)} ({os.path.getsize(path) / 1e9:.2f} GB) threads={THREADS}") t = time.time() import numpy as np from llama_cpp import Llama from datasets import load_dataset, disable_progress_bars disable_progress_bars() say(f"imports done in {time.time() - t:.1f}s") t = time.time() ds = load_dataset("allenai/ai2_arc", "ARC-Easy", split="test").shuffle(seed=0).select(range(N_QUESTIONS)) say(f"dataset loaded in {time.time() - t:.1f}s ({len(ds)} questions)") t = time.time() try: llm = Llama(model_path=path, n_ctx=512, n_threads=THREADS, n_threads_batch=THREADS, logits_all=True, verbose=False) except Exception as e: say(f"LOAD_FAILED {type(e).__name__}: {e}") say("LOAD_RETRY_VERBOSE (re-running loader with llama.cpp output to capture the reason)") try: Llama(model_path=path, n_ctx=512, n_threads=THREADS, n_threads_batch=THREADS, verbose=True) except Exception: pass sys.exit(3) say(f"model loaded in {time.time() - t:.1f}s") n_params = None try: n_params = int(llm._model.n_params()) except Exception as e: say(f"could not read actual param count: {e}") if n_params: say(f"N_PARAMS {n_params} ({n_params / 1e9:.3f}B actual)") if n_params / 1e9 > MAX_PARAMS_B * 1.1: say(f"TOO_BIG {n_params / 1e9:.2f}B > {MAX_PARAMS_B}B limit") sys.exit(5) # warm-up: page the weights in so the first question isn't penalised try: t = time.time() warm = llm.tokenize(("Question: what is the capital of France? " * 8).encode())[:64] llm.reset() llm.eval(warm) say(f"warm-up ({len(warm)} tokens) in {time.time() - t:.1f}s") except Exception as e: say(f"warm-up failed (ignored): {e}") def lse(a): a = np.asarray(a, dtype=np.float64) m = a.max() return float(m + np.log(np.exp(a - m).sum())) def tok_str(i): return llm.detokenize([int(i)]).decode("utf-8", errors="replace") def letter_ids(lab): """All single-token forms of an answer label ('A', ' A', ...). Their probabilities are summed.""" ids = set() for s in (f"Answer: {lab}", lab, f" {lab}"): try: toks = llm.tokenize(s.encode(), add_bos=False) if toks: ids.add(int(toks[-1])) except Exception: pass return sorted(ids) label_ids = {} correct, deg, toks, t0 = 0, 0, 0, time.time() masses = [] for i, ex in enumerate(ds): qt = time.time() labels = ex["choices"]["label"] opts = "\n".join(f"{l}. {t}" for l, t in zip(labels, ex["choices"]["text"])) prompt = f"Question: {ex['question']}\n{opts}\nAnswer:" for lab in labels: if lab not in label_ids: label_ids[lab] = letter_ids(lab) if i == 0: say("label tokens: " + "; ".join(f"{l}=" + "|".join(repr(tok_str(t)) for t in label_ids[l]) for l in labels)) ptoks = llm.tokenize(prompt.encode())[-500:] llm.reset() llm.eval(ptoks) row = np.asarray(llm.scores[llm.n_tokens - 1], dtype=np.float64) top_id = int(np.argmax(row)) top_tok = tok_str(top_id) note = "" if top_tok.strip() == "": # the model wants a space/newline token before the answer: feed it, then read the next row llm.eval([top_id]) row = np.asarray(llm.scores[llm.n_tokens - 1], dtype=np.float64) note = " +ws" vals = [lse(row[label_ids[l]]) for l in labels] mass = math.exp(lse(row[[t for l in labels for t in label_ids[l]]]) - lse(row)) masses.append(mass) m = max(vals) exps = [math.exp(v - m) for v in vals] probs = [e / sum(exps) for e in exps] best = max(range(len(labels)), key=lambda k: vals[k]) pred = labels[best] ok = pred == ex["answerKey"] correct += ok toks += len(ptoks) degenerate = (max(vals) - min(vals)) < 1e-6 deg += degenerate say(f"Q{i + 1}/{len(ds)} expected={ex['answerKey']} pred={pred} conf={probs[best]:.2f} mass={mass:.2f} " f"{'OK' if ok else 'x'} top={top_tok!r}{note} logits={[round(v, 1) for v in vals]} " f"prompt_tok={len(ptoks)} {time.time() - qt:.1f}s running={correct}/{i + 1}") if i == 4 and deg == 5: say("DEGENERATE_LOGITS first 5 questions had identical label logits; scores would be meaningless") sys.exit(6) if i == 2: proj = (time.time() - t0) / 3 * len(ds) say(f"projected benchmark time: {proj:.0f}s (limit {TIMEOUT}s)") if proj > TIMEOUT * 0.9: say(f"TOO_SLOW projected {proj:.0f}s") sys.exit(4) elapsed = time.time() - t0 mean_mass = sum(masses) / len(masses) unreliable = mean_mass < UNRELIABLE_MASS say(f"benchmark done in {elapsed:.1f}s ({toks / elapsed:.1f} prompt tok/s), worker total {time.time() - t_start:.1f}s, " f"degenerate questions: {deg}/{len(ds)}, mean answer-letter mass: {mean_mass:.3f}") if unreliable: say(f"LOW_FORMAT_MASS mean mass {mean_mass:.4f} < {UNRELIABLE_MASS}: the model does not answer with a letter, score is unreliable") if deg > len(ds) // 2: say("DEGENERATE_LOGITS majority of questions had identical label logits") sys.exit(6) say("RESULT " + json.dumps({"score": correct / len(ds), "n_questions": len(ds), "tok_per_s": round(toks / elapsed, 1), "n_params": n_params, "format_mass": round(mean_mass, 4), "unreliable": unreliable})) LOADER_HINT = re.compile(r"error|unknown|fail|unsupported|arch|invalid|mismatch|missing|not found", re.I) UNKNOWN_ARCH = re.compile(r"unknown model architecture: '([^']+)'") def run_worker(path): """Run worker subprocess, streaming its output into the log live.""" proc = subprocess.Popen([sys.executable, __file__, "--worker", path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) LOOP["proc"] = proc timed_out = {"v": False} def kill(): timed_out["v"] = True log.error(" worker exceeded %ss, killing pid %s", TIMEOUT, proc.pid) proc.kill() timer = threading.Timer(TIMEOUT, kill) timer.start() result, n_params = None, None tail, verbose_mode, hints = collections.deque(maxlen=15), False, [] try: for line in proc.stdout: line = line.rstrip() if not line: continue if "unauthenticated requests" in line or line.startswith("Generating "): continue if line.startswith("RESULT "): result = json.loads(line[7:]) continue if line.startswith("N_PARAMS "): try: n_params = int(line.split()[1]) except ValueError: pass tail.append(line) if line.startswith("LOAD_RETRY_VERBOSE"): verbose_mode = True if verbose_mode and not line.startswith(("LOAD_", "worker")): # raw llama.cpp loader chatter: keep only the lines that explain the failure if LOADER_HINT.search(line) and "- kv" not in line and len(hints) < 8: hints.append(line[:300]) continue log.info(" [worker] %s", line[:500]) proc.wait() finally: timer.cancel() LOOP["proc"] = None for h in hints: log.warning(" [llama.cpp] %s", h) return proc.returncode, result, list(tail), timed_out["v"], hints, n_params # ---------- storage (mounted bucket) ---------- def load_done(): p = f"{DATA}/{DONE_FILE}" return set(open(p).read().split()) if os.path.exists(p) else set() def push(text, remote): p = f"{DATA}/{remote}" os.makedirs(os.path.dirname(p), exist_ok=True) with open(p, "w") as f: f.write(text) # ---------- file selection ---------- def pick_file(siblings, max_gb): cands = [s for s in siblings if s.rfilename.endswith(".gguf") and s.size and s.size / 1e9 <= max_gb and not BAD_NAME.search(s.rfilename) and not SHARD.search(s.rfilename)] if not cands: return None, cands q4 = [s for s in cands if "q4_k_m" in s.rfilename.lower()] return min(q4 or cands, key=lambda s: s.size), cands # ---------- heartbeat ---------- def heartbeat(): while True: time.sleep(IDLE_HEARTBEAT_S if STATUS["stage"] == "idle" else HEARTBEAT_S) el = time.time() - STATUS["model_started"] if STATUS["model_started"] else 0 log.info("heartbeat: model=%s stage=%s elapsed=%.0fs | session ok=%d failed=%d skipped=%d | " "mem_avail=%.1fGB disk_free=%.1fGB uptime=%.0fmin", STATUS["current"], STATUS["stage"], el, SESSION["ok"], SESSION["failed"], SESSION["skipped"], mem_avail_gb(), disk_free_gb("."), (time.time() - SESSION["started"]) / 60) # ---------- main loop ---------- def loop(): done = load_done() log.info("resuming: %d models already done under bench v%d", len(done), BENCH) STATUS["stage"] = "listing" t = time.time() try: ids = sorted((m.id for m in api.list_models(filter="gguf", limit=MAX_REPOS or None)), key=str.lower) except Exception: log.exception("list_models failed") STATUS["current"] = "error: list_models failed" return todo = [i for i in ids if i not in done] STATUS["queue"] = len(ids) log.info("listed %d gguf repos in %.1fs (%d remaining)", len(ids), time.time() - t, len(todo)) for n, mid in enumerate(todo, 1): if STOP.is_set(): log.info("loop stopped (reset requested)") return STATUS.update(current=mid, model_started=time.time(), stage="metadata") result, path, t0 = {"model": mid, "ts": time.time(), "bench": BENCH}, None, time.time() log.info("[%d/%d] %s: start", n, len(todo), mid) try: t = time.time() info = api.model_info(mid, files_metadata=True) gguf = getattr(info, "gguf", None) or {} params_b = (gguf.get("total") or 0) / 1e9 known = params_b > 0 arch = gguf.get("architecture") task = getattr(info, "pipeline_tag", None) result["task"] = task all_gguf = [s for s in info.siblings if s.rfilename.endswith(".gguf")] log.debug("%s: metadata in %.1fs, arch=%s task=%s params=%.3fB gguf_files=%d", mid, time.time() - t, arch, task, params_b, len(all_gguf)) if task in NON_TEXT_TASKS: result["status"] = "skipped_not_text" log.info("%s: skip, pipeline_tag=%s", mid, task) elif known and params_b > MAX_PARAMS_B: result.update(status="skipped_too_big", params_b=round(params_b, 2)) log.info("%s: skip, %.2fB params > %.1fB limit", mid, params_b, MAX_PARAMS_B) else: cap = MAX_GB if known else UNKNOWN_MAX_GB f, cands = pick_file(info.siblings, cap) if not known: log.info("%s: no param count in metadata, will verify after load (file cap %.1f GB)", mid, cap) if f is None: result["status"] = "skipped_no_usable_file" names = [f"{s.rfilename}={((s.size or 0) / 1e9):.2f}GB" for s in all_gguf[:5]] log.info("%s: skip, no usable model file <= %.1f GB (excluded mmproj/lora/draft/shards); found %d: %s", mid, cap, len(all_gguf), names) else: if known: result["params_b"] = round(params_b, 2) log.info("%s: params=%s arch=%s, %d candidate files, picked %s (%.2f GB)", mid, f"{params_b:.2f}B" if known else "unknown", arch, len(cands), f.rfilename, f.size / 1e9) STATUS["stage"] = "download" t = time.time() path = hf_hub_download(mid, f.rfilename) dl = max(time.time() - t, 0.01) log.info("%s: downloaded in %.1fs (%.1f MB/s), mem_avail=%.1fGB disk_free=%.1fGB", mid, dl, f.size / 1e6 / dl, mem_avail_gb(), disk_free_gb(".")) result.update(file=f.rfilename, arch=arch) if not STOP.is_set(): STATUS["stage"] = "benchmark" rc, res, tail, timed_out, hints, n_params = run_worker(path) if n_params: result["params_b"] = round(n_params / 1e9, 2) if timed_out: result.update(status="failed", error=f"timeout after {TIMEOUT}s") elif rc == 0 and res: res.pop("n_params", None) result.update(res, status="ok") elif rc == 3: unknown = None for h in hints: mm = UNKNOWN_ARCH.search(h) if mm: unknown = mm.group(1) if unknown: result.update(status="skipped_unsupported_arch", error=f"unknown architecture '{unknown}'") else: result.update(status="skipped_load_failed", error=" | ".join(hints or tail)[-500:]) elif rc == 4: result.update(status="skipped_too_slow", error="projected benchmark time over limit") elif rc == 5: result.update(status="skipped_too_big", error="actual param count over limit after load") elif rc == 6: result.update(status="failed", error="degenerate logits: label token scores were identical") else: result.update(status="failed", error=f"exit code {rc}: " + " | ".join(tail)[-500:]) else: result["status"] = "failed" except Exception as e: log.exception("%s: exception during processing", mid) result.update(status="failed", error=f"{type(e).__name__}: {e}"[:500]) finally: if path and os.path.exists(path): try: os.remove(os.path.realpath(path)) log.debug("%s: deleted downloaded file", mid) except Exception as e: log.warning("%s: cleanup failed: %s", mid, e) if STOP.is_set(): log.info("%s: reset requested, discarding this result", mid) return s = result["status"] dt = time.time() - t0 if s == "ok": SESSION["ok"] += 1 log.info("%s: OK score=%.2f (%d q) fmt=%.2f%s prompt-tok/s=%s params=%sB (%.0fs total)", mid, result["score"], result["n_questions"], result.get("format_mass", 0), " UNRELIABLE" if result.get("unreliable") else "", result["tok_per_s"], result.get("params_b", "?"), dt) elif s == "failed": SESSION["failed"] += 1 log.error("%s: FAILED after %.0fs: %s", mid, dt, result.get("error", "").replace("\n", " | ")[:400]) else: SESSION["skipped"] += 1 if result.get("error"): log.warning("%s: %s after %.0fs: %s", mid, s, dt, result["error"].replace("\n", " | ")[:300]) STATUS["stage"] = "saving" try: push(json.dumps(result), f"results/{mid.replace('/', '__')}.json") done.add(mid) push("\n".join(sorted(done)), DONE_FILE) log.debug("%s: saved result (%d done total)", mid, len(done)) except Exception: log.exception("%s: write failed", mid) if n % SUMMARY_EVERY == 0: up = (time.time() - SESSION["started"]) / 60 log.info("=== summary: %d/%d processed this session in %.0f min | ok=%d failed=%d skipped=%d ===", n, len(todo), up, SESSION["ok"], SESSION["failed"], SESSION["skipped"]) STATUS.update(current="finished", stage="idle", model_started=0.0) log.info("all models processed: session ok=%d failed=%d skipped=%d, %d done in total", SESSION["ok"], SESSION["failed"], SESSION["skipped"], len(done)) log.info("idle now. Set MAX_REPOS=0 (Space variable) and restart to cover every gguf repo on the Hub.") def start_loop(): STOP.clear() t = threading.Thread(target=loop, daemon=True) LOOP["thread"] = t t.start() # ---------- reset ("fresh start") ---------- def do_reset(delete_logs): try: RESET.update(active=True, done=0, total=0, msg="stopping the current run") log.warning("RESET requested: stopping the run and deleting results and state (old logs too: %s)", delete_logs) STOP.set() t = LOOP["thread"] while t and t.is_alive(): RESET["msg"] = "waiting for the current model to stop" p = LOOP["proc"] if p and p.poll() is None: p.kill() t.join(2) files = glob.glob(f"{DATA}/results/*") + glob.glob(f"{DATA}/state/*") if delete_logs: keep = f"run-{RUN_ID}" # the current run's log is rewritten from memory, so leave it files += [f for f in glob.glob(f"{DATA}/logs/*") if not os.path.basename(f).startswith(keep)] RESET.update(total=len(files), done=0, msg="deleting files") def rm(f): try: os.remove(f) return True except Exception as e: log.warning("could not delete %s: %s", f, e) return False failed = 0 with concurrent.futures.ThreadPoolExecutor(8) as ex: for i, ok in enumerate(ex.map(rm, files), 1): RESET["done"] = i failed += (not ok) if i % 1000 == 0: log.info("reset: deleted %d/%d", i, len(files)) SESSION.update(ok=0, failed=0, skipped=0, started=time.time()) CACHE.update(t=0, rows=[], counts={}) STATUS.update(current="starting", queue=0, model_started=0.0, stage="init") RESET["last"] = f"Last reset deleted {len(files) - failed} files" + (f" ({failed} failed)" if failed else "") log.warning("RESET done: deleted %d files (%d failed). Starting a fresh run.", len(files) - failed, failed) except Exception: log.exception("reset failed") RESET["last"] = "Last reset failed, see logs" finally: RESET["active"] = False RESET["msg"] = "" RESET_LOCK.release() start_loop() # ---------- results table (built in the background so the page never waits on the mount) ---------- def refresh(): t0 = time.time() rows, counts = [], {"ok": 0, "failed": 0, "skipped": 0, "old": 0, "unreliable": 0} for f in glob.glob(f"{DATA}/results/*.json"): try: r = json.load(open(f)) except Exception: continue if r.get("bench") not in SHOW_BENCH: counts["old"] += 1 continue s = r.get("status", "") if s == "ok": counts["ok"] += 1 if r.get("unreliable"): counts["unreliable"] += 1 rows.append(r) elif s == "failed": counts["failed"] += 1 else: counts["skipped"] += 1 rows.sort(key=lambda r: (bool(r.get("unreliable")), -r["score"], -r["tok_per_s"])) CACHE.update(t=time.time(), rows=rows[:300], counts=counts) log.debug("table refreshed in %.1fs (%d ok rows)", time.time() - t0, counts["ok"]) def cache_loop(): while True: try: refresh() except Exception: log.exception("table refresh failed") time.sleep(CACHE_EVERY_S) PAGE_STYLE = "font-family:sans-serif;max-width:1100px;margin:2em auto;padding:0 12px" class H(BaseHTTPRequestHandler): def send_html(self, body, code=200, extra_head=""): page = f"{extra_head}{body}" self.send_response(code) self.send_header("Content-Type", "text/html; charset=utf-8") self.end_headers() self.wfile.write(page.encode()) def redirect(self, to="/"): self.send_response(303) self.send_header("Location", to) self.end_headers() def do_POST(self): if not self.path.startswith("/reset"): return self.send_error(404) n = int(self.headers.get("Content-Length") or 0) form = urllib.parse.parse_qs(self.rfile.read(min(n, 4096)).decode()) if (form.get("confirm") or [""])[0] != "yes": return self.redirect("/") if not ADMIN_PASSWORD: return self.send_html("

Reset is disabled

Set the ADMIN_PASSWORD secret on the Space to enable it.

" "

Back

", 403) pw = (form.get("password") or [""])[0] if not hmac.compare_digest(pw.encode(), ADMIN_PASSWORD.encode()): time.sleep(1.5) log.warning("reset attempt with a wrong password") return self.send_html("

Wrong password

Nothing was deleted.

" "

Try again · Back

", 403) if not RESET_LOCK.acquire(blocking=False): return self.send_html("

A reset is already running

Back

", 409) delete_logs = (form.get("logs") or [""])[0] == "yes" threading.Thread(target=do_reset, args=(delete_logs,), daemon=True).start() self.redirect("/") def do_GET(self): if self.path.startswith("/logs"): n = 300 if "n=" in self.path: try: n = min(int(self.path.split("n=")[1].split("&")[0]), 3000) except ValueError: pass text = "\n".join(list(LOG_BUF)[-n:]) self.send_response(200) self.send_header("Content-Type", "text/plain; charset=utf-8") self.end_headers() self.wfile.write(text.encode()) return if self.path.startswith("/reset"): if not ADMIN_PASSWORD: return self.send_html("

Reset is disabled

Add a Space secret named ADMIN_PASSWORD to enable it.

" "

Back

", 403) n_res = len(glob.glob(f"{DATA}/results/*")) n_state = len(glob.glob(f"{DATA}/state/*")) n_logs = len(glob.glob(f"{DATA}/logs/*")) return self.send_html(f"""

Delete everything and start fresh?

This stops the current run, then permanently deletes:

The benchmark then restarts from the first model. This cannot be undone.

""") c = CACHE["counts"] rows_html = [] for r in CACHE["rows"]: style = " style='color:#999'" if r.get("unreliable") else "" rows_html.append( f"{html.escape(r['model'])}{r.get('params_b', '?')}B" f"{r['score']:.2f}{'?' if r.get('unreliable') else ''}{r.get('n_questions', '?')}" f"{r.get('format_mass', '')}{r['tok_per_s']}" f"{html.escape(str(r.get('task') or ''))}{html.escape(r.get('file', ''))}" f"v{r.get('bench', '?')}") body = "".join(rows_html) if CACHE["t"] == 0: body = "Building the results table in the background, refresh in a minute…" link = f'bucket' if BUCKET else "bucket" el = time.time() - STATUS["model_started"] if STATUS["model_started"] else 0 if RESET["active"]: top = (f"
Reset in progress: " f"{html.escape(RESET['msg'])} ({RESET['done']}/{RESET['total']})
") head = "" else: if ADMIN_PASSWORD: action = "Fresh start: delete all results…" else: action = "Fresh start is disabled (add an ADMIN_PASSWORD secret to enable)" last = f" · {html.escape(RESET['last'])}" if RESET["last"] else "" top = f"
{action}{last}
" head = "" self.send_html(f"""{top}

Tiny model leaderboard

Now: {html.escape(STATUS['current'])} ({STATUS['stage']}, {el:.0f}s)
in table (v{'/v'.join(str(v) for v in SHOW_BENCH)}): ok {c.get('ok', 0)} (unreliable {c.get('unreliable', 0)}) · failed {c.get('failed', 0)} · skipped {c.get('skipped', 0)} · ignored old formats {c.get('old', 0)} · queue {STATUS['queue']}
this session: ok {SESSION['ok']} · failed {SESSION['failed']} · skipped {SESSION['skipped']}

Score = ARC-Easy, {N_QUESTIONS} questions, zero-shot, answer picked by next-token letter logits (chance = 0.25; one score is only good to about ±10 points at this size). fmt = share of probability the model puts on the answer letters; grey rows with “?” are below {UNRELIABLE_MASS} (the model doesn't answer with a letter, so the score is unreliable). Rows marked v3 use the previous scoring and are replaced as the v{BENCH} run reaches them. Raw results: {link} (top 300 shown, table rebuilt every {CACHE_EVERY_S}s) · recent logs (1000 lines)

{body}
modelparamsscorenfmt prompt tok/staskfilev
""", extra_head=head) def log_message(self, *a): pass if __name__ == "__main__": if len(sys.argv) > 2 and sys.argv[1] == "--worker": worker(sys.argv[2]) else: setup_log() startup_report() os.makedirs(f"{DATA}/results", exist_ok=True) threading.Thread(target=heartbeat, daemon=True).start() threading.Thread(target=cache_loop, daemon=True).start() start_loop() log.info("status page listening on :7860") ThreadingHTTPServer(("0.0.0.0", 7860), H).serve_forever()