Download app.py from CJHauser/eeeeeee: direct link, hf CLI and curl.
- Browser
- Download file 35.9 kB
-
https://huggingface.co/spaces/CJHauser/eeeeeee/resolve/main/app.py
- Command line
-
hf download hf://spaces/CJHauser/eeeeeee/app.py
-
curl -L -o app.py https://huggingface.co/spaces/CJHauser/eeeeeee/resolve/main/app.py
35.9 kB
| import os, sys, re, json, math, time, threading, subprocess, glob, html, logging, collections | |
| import datetime, shutil, platform, hmac, urllib.parse, concurrent.futures | |
| from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| from huggingface_hub import HfApi, hf_hub_download | |
| DATA = "/data" | |
| BUCKET = os.environ.get("BUCKET", "") | |
| ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "") | |
| MAX_PARAMS_B = 1.5 | |
| MAX_GB = 3.0 # file size cap when the repo reports a param count | |
| UNKNOWN_MAX_GB = 1.6 # tighter cap when params are unknown (verified after load) | |
| TIMEOUT = 900 | |
| HEARTBEAT_S = 30 | |
| IDLE_HEARTBEAT_S = 1800 | |
| SUMMARY_EVERY = 50 | |
| N_QUESTIONS = int(os.environ.get("N_QUESTIONS", "100")) | |
| MAX_REPOS = int(os.environ.get("MAX_REPOS", "7000")) # 0 = every gguf repo on the Hub | |
| LOG_FLUSH_S = int(os.environ.get("LOG_FLUSH_S", "60")) | |
| CACHE_EVERY_S = 120 | |
| RUN_ID = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") | |
| BENCH = 5 # scoring version: 4 = letter logits (variants merged), space-token handling, format check | |
| SHOW_BENCH = (3, 4) # result versions shown in the table | |
| UNRELIABLE_MASS = 0.02 # mean share of probability on answer letters below this = unreliable score | |
| DONE_FILE = f"state/done-v{BENCH}.txt" | |
| api = HfApi() | |
| def cpu_quota(): | |
| """CPUs this container may actually use (cgroup quota), or None if unknown.""" | |
| try: | |
| q, p = open("/sys/fs/cgroup/cpu.max").read().split() | |
| if q != "max": | |
| return max(1, round(int(q) / int(p))) | |
| except Exception: | |
| pass | |
| try: | |
| q = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read()) | |
| p = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read()) | |
| if q > 0: | |
| return max(1, round(q / p)) | |
| except Exception: | |
| pass | |
| return None | |
| THREADS = int(os.environ.get("THREADS", "0") or 0) or cpu_quota() or 2 | |
| # files that are never a standalone language model | |
| BAD_NAME = re.compile(r"mmproj|lora|adapter|imatrix|draft", re.I) | |
| SHARD = re.compile(r"-\d{5}-of-\d{5}", re.I) | |
| NON_TEXT_TASKS = {"text-to-image", "image-to-image", "image-to-3d", "text-to-3d", "text-to-video", | |
| "image-to-video", "unconditional-image-generation", "text-to-speech", | |
| "automatic-speech-recognition"} | |
| STATUS = {"current": "starting", "queue": 0, "model_started": 0.0, "stage": "init"} | |
| SESSION = {"ok": 0, "failed": 0, "skipped": 0, "started": time.time()} | |
| CACHE = {"t": 0, "rows": [], "counts": {}} | |
| # loop control + reset | |
| STOP = threading.Event() | |
| LOOP = {"thread": None, "proc": None} | |
| RESET = {"active": False, "msg": "", "done": 0, "total": 0, "last": ""} | |
| RESET_LOCK = threading.Lock() | |
| # ---------- logging ---------- | |
| LOG_BUF = collections.deque(maxlen=3000) | |
| log = logging.getLogger("lb") | |
| class BufHandler(logging.Handler): | |
| def emit(self, record): | |
| LOG_BUF.append(self.format(record)) | |
| class BucketLogHandler(logging.Handler): | |
| """Keeps lines in memory and rewrites a closed log file on the mounted bucket every | |
| LOG_FLUSH_S seconds. A file held open on the mount does not sync until it is closed.""" | |
| MAX_LINES = 60000 | |
| def __init__(self): | |
| super().__init__() | |
| self.lines, self.part, self.dirty = [], 1, False | |
| self.mem_lock = threading.Lock() | |
| def path(self): | |
| suffix = "" if self.part == 1 else f"-p{self.part}" | |
| return f"{DATA}/logs/run-{RUN_ID}{suffix}.log" | |
| def write_out(self): | |
| try: | |
| os.makedirs(f"{DATA}/logs", exist_ok=True) | |
| with open(self.path(), "w") as f: | |
| f.write("\n".join(self.lines) + "\n") | |
| self.dirty = False | |
| except Exception as e: | |
| print("log sync failed:", e, flush=True) | |
| def emit(self, record): | |
| with self.mem_lock: | |
| if len(self.lines) >= self.MAX_LINES: | |
| self.write_out() | |
| self.part += 1 | |
| self.lines = [] | |
| self.lines.append(self.format(record)) | |
| self.dirty = True | |
| def flush_loop(self): | |
| delay = 5 # first sync soon after startup, then every LOG_FLUSH_S | |
| while True: | |
| time.sleep(delay) | |
| delay = LOG_FLUSH_S | |
| with self.mem_lock: | |
| if self.dirty: | |
| self.write_out() | |
| def setup_log(): | |
| log.setLevel(logging.DEBUG) | |
| fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%H:%M:%S") | |
| bucket_handler = BucketLogHandler() | |
| handlers = [logging.StreamHandler(sys.stdout), BufHandler(), bucket_handler] | |
| for h in handlers: | |
| h.setFormatter(fmt) | |
| h.setLevel(logging.DEBUG) | |
| log.addHandler(h) | |
| threading.Thread(target=bucket_handler.flush_loop, daemon=True).start() | |
| def mem_avail_gb(): | |
| try: | |
| for line in open("/proc/meminfo"): | |
| if line.startswith("MemAvailable"): | |
| return int(line.split()[1]) / 1e6 | |
| except Exception: | |
| pass | |
| return -1.0 | |
| def disk_free_gb(path): | |
| try: | |
| return shutil.disk_usage(path).free / 1e9 | |
| except Exception: | |
| return -1.0 | |
| def startup_report(): | |
| log.info("=== startup (bench v%d, run %s) ===", BENCH, RUN_ID) | |
| log.info("python %s on %s, cpus=%s", platform.python_version(), platform.platform(), os.cpu_count()) | |
| try: | |
| log.info("affinity cpus=%d, cgroup quota=%s, using THREADS=%d", | |
| len(os.sched_getaffinity(0)), cpu_quota(), THREADS) | |
| except Exception: | |
| log.info("cgroup quota=%s, using THREADS=%d", cpu_quota(), THREADS) | |
| for p in ("/sys/fs/cgroup/cpu.max", "/sys/fs/cgroup/memory.max"): | |
| try: | |
| log.info("%s = %s", p, open(p).read().strip()) | |
| except Exception: | |
| pass | |
| try: | |
| import llama_cpp | |
| log.info("llama_cpp version %s", llama_cpp.__version__) | |
| try: | |
| log.info("llama.cpp system info: %s", llama_cpp.llama_print_system_info().decode().strip()) | |
| except Exception as e: | |
| log.warning("could not read llama.cpp system info: %s", e) | |
| except Exception as e: | |
| log.error("llama_cpp import failed: %s", e) | |
| try: | |
| cpuinfo = open("/proc/cpuinfo").read() | |
| model = re.search(r"model name\s*:\s*(.+)", cpuinfo) | |
| flags = re.search(r"flags\s*:\s*(.+)", cpuinfo) | |
| wanted = [f for f in ("avx", "avx2", "avx512f", "fma", "f16c") if flags and f" {f} " in f" {flags.group(1)} "] | |
| log.info("cpu: %s | simd flags: %s", model.group(1) if model else "?", wanted) | |
| except Exception: | |
| pass | |
| log.info("config: MAX_PARAMS_B=%s MAX_GB=%s UNKNOWN_MAX_GB=%s TIMEOUT=%ss N_QUESTIONS=%s MAX_REPOS=%s BUCKET=%s", | |
| MAX_PARAMS_B, MAX_GB, UNKNOWN_MAX_GB, TIMEOUT, N_QUESTIONS, MAX_REPOS or "all", BUCKET or "(unset)") | |
| log.info("log file: logs/run-%s.log in the bucket, synced every %ss", RUN_ID, LOG_FLUSH_S) | |
| log.info("HF_TOKEN set: %s, ADMIN_PASSWORD set (reset enabled): %s, HF_HOME=%s", | |
| bool(os.environ.get("HF_TOKEN")), bool(ADMIN_PASSWORD), os.environ.get("HF_HOME")) | |
| log.info("mem available: %.1f GB, disk free (cwd): %.1f GB, disk free (%s): %.1f GB", | |
| mem_avail_gb(), disk_free_gb("."), DATA, disk_free_gb(DATA)) | |
| try: | |
| t = f"{DATA}/.write_test" | |
| open(t, "w").write("ok") | |
| os.remove(t) | |
| log.info("%s is writable", DATA) | |
| except Exception as e: | |
| log.error("%s is NOT writable (%s): results will not persist!", DATA, e) | |
| log.info("existing result files: %d, done in v%d list: %d", | |
| len(glob.glob(f"{DATA}/results/*.json")), BENCH, len(load_done())) | |
| # ---------- worker (separate process) ---------- | |
| # exit codes: 0 ok, 3 model failed to load, 4 projected too slow, 5 actually too big, 6 degenerate logits | |
| def worker(path): | |
| def say(msg): | |
| print(msg, flush=True) | |
| t_start = time.time() | |
| say(f"worker start: {os.path.basename(path)} ({os.path.getsize(path) / 1e9:.2f} GB) threads={THREADS}") | |
| t = time.time() | |
| import numpy as np | |
| from llama_cpp import Llama | |
| from datasets import load_dataset, disable_progress_bars | |
| disable_progress_bars() | |
| say(f"imports done in {time.time() - t:.1f}s") | |
| t = time.time() | |
| ds = load_dataset("allenai/ai2_arc", "ARC-Easy", split="test").shuffle(seed=0).select(range(N_QUESTIONS)) | |
| say(f"dataset loaded in {time.time() - t:.1f}s ({len(ds)} questions)") | |
| t = time.time() | |
| try: | |
| llm = Llama(model_path=path, n_ctx=512, n_threads=THREADS, n_threads_batch=THREADS, | |
| logits_all=True, verbose=False) | |
| except Exception as e: | |
| say(f"LOAD_FAILED {type(e).__name__}: {e}") | |
| say("LOAD_RETRY_VERBOSE (re-running loader with llama.cpp output to capture the reason)") | |
| try: | |
| Llama(model_path=path, n_ctx=512, n_threads=THREADS, n_threads_batch=THREADS, verbose=True) | |
| except Exception: | |
| pass | |
| sys.exit(3) | |
| say(f"model loaded in {time.time() - t:.1f}s") | |
| n_params = None | |
| try: | |
| n_params = int(llm._model.n_params()) | |
| except Exception as e: | |
| say(f"could not read actual param count: {e}") | |
| if n_params: | |
| say(f"N_PARAMS {n_params} ({n_params / 1e9:.3f}B actual)") | |
| if n_params / 1e9 > MAX_PARAMS_B * 1.1: | |
| say(f"TOO_BIG {n_params / 1e9:.2f}B > {MAX_PARAMS_B}B limit") | |
| sys.exit(5) | |
| # warm-up: page the weights in so the first question isn't penalised | |
| try: | |
| t = time.time() | |
| warm = llm.tokenize(("Question: what is the capital of France? " * 8).encode())[:64] | |
| llm.reset() | |
| llm.eval(warm) | |
| say(f"warm-up ({len(warm)} tokens) in {time.time() - t:.1f}s") | |
| except Exception as e: | |
| say(f"warm-up failed (ignored): {e}") | |
| def lse(a): | |
| a = np.asarray(a, dtype=np.float64) | |
| m = a.max() | |
| return float(m + np.log(np.exp(a - m).sum())) | |
| def tok_str(i): | |
| return llm.detokenize([int(i)]).decode("utf-8", errors="replace") | |
| def letter_ids(lab): | |
| """All single-token forms of an answer label ('A', ' A', ...). Their probabilities are summed.""" | |
| ids = set() | |
| for s in (f"Answer: {lab}", lab, f" {lab}"): | |
| try: | |
| toks = llm.tokenize(s.encode(), add_bos=False) | |
| if toks: | |
| ids.add(int(toks[-1])) | |
| except Exception: | |
| pass | |
| return sorted(ids) | |
| label_ids = {} | |
| correct, deg, toks, t0 = 0, 0, 0, time.time() | |
| masses = [] | |
| for i, ex in enumerate(ds): | |
| qt = time.time() | |
| labels = ex["choices"]["label"] | |
| opts = "\n".join(f"{l}. {t}" for l, t in zip(labels, ex["choices"]["text"])) | |
| prompt = f"Question: {ex['question']}\n{opts}\nAnswer:" | |
| for lab in labels: | |
| if lab not in label_ids: | |
| label_ids[lab] = letter_ids(lab) | |
| if i == 0: | |
| say("label tokens: " + "; ".join(f"{l}=" + "|".join(repr(tok_str(t)) for t in label_ids[l]) for l in labels)) | |
| ptoks = llm.tokenize(prompt.encode())[-500:] | |
| llm.reset() | |
| llm.eval(ptoks) | |
| row = np.asarray(llm.scores[llm.n_tokens - 1], dtype=np.float64) | |
| top_id = int(np.argmax(row)) | |
| top_tok = tok_str(top_id) | |
| note = "" | |
| if top_tok.strip() == "": | |
| # the model wants a space/newline token before the answer: feed it, then read the next row | |
| llm.eval([top_id]) | |
| row = np.asarray(llm.scores[llm.n_tokens - 1], dtype=np.float64) | |
| note = " +ws" | |
| vals = [lse(row[label_ids[l]]) for l in labels] | |
| mass = math.exp(lse(row[[t for l in labels for t in label_ids[l]]]) - lse(row)) | |
| masses.append(mass) | |
| m = max(vals) | |
| exps = [math.exp(v - m) for v in vals] | |
| probs = [e / sum(exps) for e in exps] | |
| best = max(range(len(labels)), key=lambda k: vals[k]) | |
| pred = labels[best] | |
| ok = pred == ex["answerKey"] | |
| correct += ok | |
| toks += len(ptoks) | |
| degenerate = (max(vals) - min(vals)) < 1e-6 | |
| deg += degenerate | |
| say(f"Q{i + 1}/{len(ds)} expected={ex['answerKey']} pred={pred} conf={probs[best]:.2f} mass={mass:.2f} " | |
| f"{'OK' if ok else 'x'} top={top_tok!r}{note} logits={[round(v, 1) for v in vals]} " | |
| f"prompt_tok={len(ptoks)} {time.time() - qt:.1f}s running={correct}/{i + 1}") | |
| if i == 4 and deg == 5: | |
| say("DEGENERATE_LOGITS first 5 questions had identical label logits; scores would be meaningless") | |
| sys.exit(6) | |
| if i == 2: | |
| proj = (time.time() - t0) / 3 * len(ds) | |
| say(f"projected benchmark time: {proj:.0f}s (limit {TIMEOUT}s)") | |
| if proj > TIMEOUT * 0.9: | |
| say(f"TOO_SLOW projected {proj:.0f}s") | |
| sys.exit(4) | |
| elapsed = time.time() - t0 | |
| mean_mass = sum(masses) / len(masses) | |
| unreliable = mean_mass < UNRELIABLE_MASS | |
| say(f"benchmark done in {elapsed:.1f}s ({toks / elapsed:.1f} prompt tok/s), worker total {time.time() - t_start:.1f}s, " | |
| f"degenerate questions: {deg}/{len(ds)}, mean answer-letter mass: {mean_mass:.3f}") | |
| if unreliable: | |
| say(f"LOW_FORMAT_MASS mean mass {mean_mass:.4f} < {UNRELIABLE_MASS}: the model does not answer with a letter, score is unreliable") | |
| if deg > len(ds) // 2: | |
| say("DEGENERATE_LOGITS majority of questions had identical label logits") | |
| sys.exit(6) | |
| say("RESULT " + json.dumps({"score": correct / len(ds), "n_questions": len(ds), | |
| "tok_per_s": round(toks / elapsed, 1), "n_params": n_params, | |
| "format_mass": round(mean_mass, 4), "unreliable": unreliable})) | |
| LOADER_HINT = re.compile(r"error|unknown|fail|unsupported|arch|invalid|mismatch|missing|not found", re.I) | |
| UNKNOWN_ARCH = re.compile(r"unknown model architecture: '([^']+)'") | |
| def run_worker(path): | |
| """Run worker subprocess, streaming its output into the log live.""" | |
| proc = subprocess.Popen([sys.executable, __file__, "--worker", path], | |
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) | |
| LOOP["proc"] = proc | |
| timed_out = {"v": False} | |
| def kill(): | |
| timed_out["v"] = True | |
| log.error(" worker exceeded %ss, killing pid %s", TIMEOUT, proc.pid) | |
| proc.kill() | |
| timer = threading.Timer(TIMEOUT, kill) | |
| timer.start() | |
| result, n_params = None, None | |
| tail, verbose_mode, hints = collections.deque(maxlen=15), False, [] | |
| try: | |
| for line in proc.stdout: | |
| line = line.rstrip() | |
| if not line: | |
| continue | |
| if "unauthenticated requests" in line or line.startswith("Generating "): | |
| continue | |
| if line.startswith("RESULT "): | |
| result = json.loads(line[7:]) | |
| continue | |
| if line.startswith("N_PARAMS "): | |
| try: | |
| n_params = int(line.split()[1]) | |
| except ValueError: | |
| pass | |
| tail.append(line) | |
| if line.startswith("LOAD_RETRY_VERBOSE"): | |
| verbose_mode = True | |
| if verbose_mode and not line.startswith(("LOAD_", "worker")): | |
| # raw llama.cpp loader chatter: keep only the lines that explain the failure | |
| if LOADER_HINT.search(line) and "- kv" not in line and len(hints) < 8: | |
| hints.append(line[:300]) | |
| continue | |
| log.info(" [worker] %s", line[:500]) | |
| proc.wait() | |
| finally: | |
| timer.cancel() | |
| LOOP["proc"] = None | |
| for h in hints: | |
| log.warning(" [llama.cpp] %s", h) | |
| return proc.returncode, result, list(tail), timed_out["v"], hints, n_params | |
| # ---------- storage (mounted bucket) ---------- | |
| def load_done(): | |
| p = f"{DATA}/{DONE_FILE}" | |
| return set(open(p).read().split()) if os.path.exists(p) else set() | |
| def push(text, remote): | |
| p = f"{DATA}/{remote}" | |
| os.makedirs(os.path.dirname(p), exist_ok=True) | |
| with open(p, "w") as f: | |
| f.write(text) | |
| # ---------- file selection ---------- | |
| def pick_file(siblings, max_gb): | |
| cands = [s for s in siblings if s.rfilename.endswith(".gguf") and s.size and s.size / 1e9 <= max_gb | |
| and not BAD_NAME.search(s.rfilename) and not SHARD.search(s.rfilename)] | |
| if not cands: | |
| return None, cands | |
| q4 = [s for s in cands if "q4_k_m" in s.rfilename.lower()] | |
| return min(q4 or cands, key=lambda s: s.size), cands | |
| # ---------- heartbeat ---------- | |
| def heartbeat(): | |
| while True: | |
| time.sleep(IDLE_HEARTBEAT_S if STATUS["stage"] == "idle" else HEARTBEAT_S) | |
| el = time.time() - STATUS["model_started"] if STATUS["model_started"] else 0 | |
| log.info("heartbeat: model=%s stage=%s elapsed=%.0fs | session ok=%d failed=%d skipped=%d | " | |
| "mem_avail=%.1fGB disk_free=%.1fGB uptime=%.0fmin", | |
| STATUS["current"], STATUS["stage"], el, SESSION["ok"], SESSION["failed"], | |
| SESSION["skipped"], mem_avail_gb(), disk_free_gb("."), | |
| (time.time() - SESSION["started"]) / 60) | |
| # ---------- main loop ---------- | |
| def loop(): | |
| done = load_done() | |
| log.info("resuming: %d models already done under bench v%d", len(done), BENCH) | |
| STATUS["stage"] = "listing" | |
| t = time.time() | |
| try: | |
| ids = sorted((m.id for m in api.list_models(filter="gguf", limit=MAX_REPOS or None)), key=str.lower) | |
| except Exception: | |
| log.exception("list_models failed") | |
| STATUS["current"] = "error: list_models failed" | |
| return | |
| todo = [i for i in ids if i not in done] | |
| STATUS["queue"] = len(ids) | |
| log.info("listed %d gguf repos in %.1fs (%d remaining)", len(ids), time.time() - t, len(todo)) | |
| for n, mid in enumerate(todo, 1): | |
| if STOP.is_set(): | |
| log.info("loop stopped (reset requested)") | |
| return | |
| STATUS.update(current=mid, model_started=time.time(), stage="metadata") | |
| result, path, t0 = {"model": mid, "ts": time.time(), "bench": BENCH}, None, time.time() | |
| log.info("[%d/%d] %s: start", n, len(todo), mid) | |
| try: | |
| t = time.time() | |
| info = api.model_info(mid, files_metadata=True) | |
| gguf = getattr(info, "gguf", None) or {} | |
| params_b = (gguf.get("total") or 0) / 1e9 | |
| known = params_b > 0 | |
| arch = gguf.get("architecture") | |
| task = getattr(info, "pipeline_tag", None) | |
| result["task"] = task | |
| all_gguf = [s for s in info.siblings if s.rfilename.endswith(".gguf")] | |
| log.debug("%s: metadata in %.1fs, arch=%s task=%s params=%.3fB gguf_files=%d", | |
| mid, time.time() - t, arch, task, params_b, len(all_gguf)) | |
| if task in NON_TEXT_TASKS: | |
| result["status"] = "skipped_not_text" | |
| log.info("%s: skip, pipeline_tag=%s", mid, task) | |
| elif known and params_b > MAX_PARAMS_B: | |
| result.update(status="skipped_too_big", params_b=round(params_b, 2)) | |
| log.info("%s: skip, %.2fB params > %.1fB limit", mid, params_b, MAX_PARAMS_B) | |
| else: | |
| cap = MAX_GB if known else UNKNOWN_MAX_GB | |
| f, cands = pick_file(info.siblings, cap) | |
| if not known: | |
| log.info("%s: no param count in metadata, will verify after load (file cap %.1f GB)", mid, cap) | |
| if f is None: | |
| result["status"] = "skipped_no_usable_file" | |
| names = [f"{s.rfilename}={((s.size or 0) / 1e9):.2f}GB" for s in all_gguf[:5]] | |
| log.info("%s: skip, no usable model file <= %.1f GB (excluded mmproj/lora/draft/shards); found %d: %s", | |
| mid, cap, len(all_gguf), names) | |
| else: | |
| if known: | |
| result["params_b"] = round(params_b, 2) | |
| log.info("%s: params=%s arch=%s, %d candidate files, picked %s (%.2f GB)", | |
| mid, f"{params_b:.2f}B" if known else "unknown", arch, len(cands), f.rfilename, f.size / 1e9) | |
| STATUS["stage"] = "download" | |
| t = time.time() | |
| path = hf_hub_download(mid, f.rfilename) | |
| dl = max(time.time() - t, 0.01) | |
| log.info("%s: downloaded in %.1fs (%.1f MB/s), mem_avail=%.1fGB disk_free=%.1fGB", | |
| mid, dl, f.size / 1e6 / dl, mem_avail_gb(), disk_free_gb(".")) | |
| result.update(file=f.rfilename, arch=arch) | |
| if not STOP.is_set(): | |
| STATUS["stage"] = "benchmark" | |
| rc, res, tail, timed_out, hints, n_params = run_worker(path) | |
| if n_params: | |
| result["params_b"] = round(n_params / 1e9, 2) | |
| if timed_out: | |
| result.update(status="failed", error=f"timeout after {TIMEOUT}s") | |
| elif rc == 0 and res: | |
| res.pop("n_params", None) | |
| result.update(res, status="ok") | |
| elif rc == 3: | |
| unknown = None | |
| for h in hints: | |
| mm = UNKNOWN_ARCH.search(h) | |
| if mm: | |
| unknown = mm.group(1) | |
| if unknown: | |
| result.update(status="skipped_unsupported_arch", error=f"unknown architecture '{unknown}'") | |
| else: | |
| result.update(status="skipped_load_failed", error=" | ".join(hints or tail)[-500:]) | |
| elif rc == 4: | |
| result.update(status="skipped_too_slow", error="projected benchmark time over limit") | |
| elif rc == 5: | |
| result.update(status="skipped_too_big", error="actual param count over limit after load") | |
| elif rc == 6: | |
| result.update(status="failed", error="degenerate logits: label token scores were identical") | |
| else: | |
| result.update(status="failed", error=f"exit code {rc}: " + " | ".join(tail)[-500:]) | |
| else: | |
| result["status"] = "failed" | |
| except Exception as e: | |
| log.exception("%s: exception during processing", mid) | |
| result.update(status="failed", error=f"{type(e).__name__}: {e}"[:500]) | |
| finally: | |
| if path and os.path.exists(path): | |
| try: | |
| os.remove(os.path.realpath(path)) | |
| log.debug("%s: deleted downloaded file", mid) | |
| except Exception as e: | |
| log.warning("%s: cleanup failed: %s", mid, e) | |
| if STOP.is_set(): | |
| log.info("%s: reset requested, discarding this result", mid) | |
| return | |
| s = result["status"] | |
| dt = time.time() - t0 | |
| if s == "ok": | |
| SESSION["ok"] += 1 | |
| log.info("%s: OK score=%.2f (%d q) fmt=%.2f%s prompt-tok/s=%s params=%sB (%.0fs total)", | |
| mid, result["score"], result["n_questions"], result.get("format_mass", 0), | |
| " UNRELIABLE" if result.get("unreliable") else "", result["tok_per_s"], | |
| result.get("params_b", "?"), dt) | |
| elif s == "failed": | |
| SESSION["failed"] += 1 | |
| log.error("%s: FAILED after %.0fs: %s", mid, dt, result.get("error", "").replace("\n", " | ")[:400]) | |
| else: | |
| SESSION["skipped"] += 1 | |
| if result.get("error"): | |
| log.warning("%s: %s after %.0fs: %s", mid, s, dt, result["error"].replace("\n", " | ")[:300]) | |
| STATUS["stage"] = "saving" | |
| try: | |
| push(json.dumps(result), f"results/{mid.replace('/', '__')}.json") | |
| done.add(mid) | |
| push("\n".join(sorted(done)), DONE_FILE) | |
| log.debug("%s: saved result (%d done total)", mid, len(done)) | |
| except Exception: | |
| log.exception("%s: write failed", mid) | |
| if n % SUMMARY_EVERY == 0: | |
| up = (time.time() - SESSION["started"]) / 60 | |
| log.info("=== summary: %d/%d processed this session in %.0f min | ok=%d failed=%d skipped=%d ===", | |
| n, len(todo), up, SESSION["ok"], SESSION["failed"], SESSION["skipped"]) | |
| STATUS.update(current="finished", stage="idle", model_started=0.0) | |
| log.info("all models processed: session ok=%d failed=%d skipped=%d, %d done in total", | |
| SESSION["ok"], SESSION["failed"], SESSION["skipped"], len(done)) | |
| log.info("idle now. Set MAX_REPOS=0 (Space variable) and restart to cover every gguf repo on the Hub.") | |
| def start_loop(): | |
| STOP.clear() | |
| t = threading.Thread(target=loop, daemon=True) | |
| LOOP["thread"] = t | |
| t.start() | |
| # ---------- reset ("fresh start") ---------- | |
| def do_reset(delete_logs): | |
| try: | |
| RESET.update(active=True, done=0, total=0, msg="stopping the current run") | |
| log.warning("RESET requested: stopping the run and deleting results and state (old logs too: %s)", delete_logs) | |
| STOP.set() | |
| t = LOOP["thread"] | |
| while t and t.is_alive(): | |
| RESET["msg"] = "waiting for the current model to stop" | |
| p = LOOP["proc"] | |
| if p and p.poll() is None: | |
| p.kill() | |
| t.join(2) | |
| files = glob.glob(f"{DATA}/results/*") + glob.glob(f"{DATA}/state/*") | |
| if delete_logs: | |
| keep = f"run-{RUN_ID}" # the current run's log is rewritten from memory, so leave it | |
| files += [f for f in glob.glob(f"{DATA}/logs/*") if not os.path.basename(f).startswith(keep)] | |
| RESET.update(total=len(files), done=0, msg="deleting files") | |
| def rm(f): | |
| try: | |
| os.remove(f) | |
| return True | |
| except Exception as e: | |
| log.warning("could not delete %s: %s", f, e) | |
| return False | |
| failed = 0 | |
| with concurrent.futures.ThreadPoolExecutor(8) as ex: | |
| for i, ok in enumerate(ex.map(rm, files), 1): | |
| RESET["done"] = i | |
| failed += (not ok) | |
| if i % 1000 == 0: | |
| log.info("reset: deleted %d/%d", i, len(files)) | |
| SESSION.update(ok=0, failed=0, skipped=0, started=time.time()) | |
| CACHE.update(t=0, rows=[], counts={}) | |
| STATUS.update(current="starting", queue=0, model_started=0.0, stage="init") | |
| RESET["last"] = f"Last reset deleted {len(files) - failed} files" + (f" ({failed} failed)" if failed else "") | |
| log.warning("RESET done: deleted %d files (%d failed). Starting a fresh run.", len(files) - failed, failed) | |
| except Exception: | |
| log.exception("reset failed") | |
| RESET["last"] = "Last reset failed, see logs" | |
| finally: | |
| RESET["active"] = False | |
| RESET["msg"] = "" | |
| RESET_LOCK.release() | |
| start_loop() | |
| # ---------- results table (built in the background so the page never waits on the mount) ---------- | |
| def refresh(): | |
| t0 = time.time() | |
| rows, counts = [], {"ok": 0, "failed": 0, "skipped": 0, "old": 0, "unreliable": 0} | |
| for f in glob.glob(f"{DATA}/results/*.json"): | |
| try: | |
| r = json.load(open(f)) | |
| except Exception: | |
| continue | |
| if r.get("bench") not in SHOW_BENCH: | |
| counts["old"] += 1 | |
| continue | |
| s = r.get("status", "") | |
| if s == "ok": | |
| counts["ok"] += 1 | |
| if r.get("unreliable"): | |
| counts["unreliable"] += 1 | |
| rows.append(r) | |
| elif s == "failed": | |
| counts["failed"] += 1 | |
| else: | |
| counts["skipped"] += 1 | |
| rows.sort(key=lambda r: (bool(r.get("unreliable")), -r["score"], -r["tok_per_s"])) | |
| CACHE.update(t=time.time(), rows=rows[:300], counts=counts) | |
| log.debug("table refreshed in %.1fs (%d ok rows)", time.time() - t0, counts["ok"]) | |
| def cache_loop(): | |
| while True: | |
| try: | |
| refresh() | |
| except Exception: | |
| log.exception("table refresh failed") | |
| time.sleep(CACHE_EVERY_S) | |
| PAGE_STYLE = "font-family:sans-serif;max-width:1100px;margin:2em auto;padding:0 12px" | |
| class H(BaseHTTPRequestHandler): | |
| def send_html(self, body, code=200, extra_head=""): | |
| page = f"<html><head><meta charset='utf-8'>{extra_head}</head><body style=\"{PAGE_STYLE}\">{body}</body></html>" | |
| self.send_response(code) | |
| self.send_header("Content-Type", "text/html; charset=utf-8") | |
| self.end_headers() | |
| self.wfile.write(page.encode()) | |
| def redirect(self, to="/"): | |
| self.send_response(303) | |
| self.send_header("Location", to) | |
| self.end_headers() | |
| def do_POST(self): | |
| if not self.path.startswith("/reset"): | |
| return self.send_error(404) | |
| n = int(self.headers.get("Content-Length") or 0) | |
| form = urllib.parse.parse_qs(self.rfile.read(min(n, 4096)).decode()) | |
| if (form.get("confirm") or [""])[0] != "yes": | |
| return self.redirect("/") | |
| if not ADMIN_PASSWORD: | |
| return self.send_html("<h2>Reset is disabled</h2><p>Set the ADMIN_PASSWORD secret on the Space to enable it.</p>" | |
| "<p><a href='/'>Back</a></p>", 403) | |
| pw = (form.get("password") or [""])[0] | |
| if not hmac.compare_digest(pw.encode(), ADMIN_PASSWORD.encode()): | |
| time.sleep(1.5) | |
| log.warning("reset attempt with a wrong password") | |
| return self.send_html("<h2>Wrong password</h2><p>Nothing was deleted.</p>" | |
| "<p><a href='/reset'>Try again</a> · <a href='/'>Back</a></p>", 403) | |
| if not RESET_LOCK.acquire(blocking=False): | |
| return self.send_html("<h2>A reset is already running</h2><p><a href='/'>Back</a></p>", 409) | |
| delete_logs = (form.get("logs") or [""])[0] == "yes" | |
| threading.Thread(target=do_reset, args=(delete_logs,), daemon=True).start() | |
| self.redirect("/") | |
| def do_GET(self): | |
| if self.path.startswith("/logs"): | |
| n = 300 | |
| if "n=" in self.path: | |
| try: | |
| n = min(int(self.path.split("n=")[1].split("&")[0]), 3000) | |
| except ValueError: | |
| pass | |
| text = "\n".join(list(LOG_BUF)[-n:]) | |
| self.send_response(200) | |
| self.send_header("Content-Type", "text/plain; charset=utf-8") | |
| self.end_headers() | |
| self.wfile.write(text.encode()) | |
| return | |
| if self.path.startswith("/reset"): | |
| if not ADMIN_PASSWORD: | |
| return self.send_html("<h2>Reset is disabled</h2><p>Add a Space secret named ADMIN_PASSWORD to enable it.</p>" | |
| "<p><a href='/'>Back</a></p>", 403) | |
| n_res = len(glob.glob(f"{DATA}/results/*")) | |
| n_state = len(glob.glob(f"{DATA}/state/*")) | |
| n_logs = len(glob.glob(f"{DATA}/logs/*")) | |
| return self.send_html(f"""<h2>Delete everything and start fresh?</h2> | |
| <p>This stops the current run, then permanently deletes:</p> | |
| <ul><li>{n_res} result files</li><li>{n_state} progress files (the "already done" lists)</li></ul> | |
| <p>The benchmark then restarts from the first model. This cannot be undone.</p> | |
| <form method="post" action="/reset"> | |
| <p><label><input type="checkbox" name="logs" value="yes"> Also delete old logs ({n_logs} files; the current run's log is kept)</label></p> | |
| <p><label>Admin password <input type="password" name="password" required autocomplete="off"></label></p> | |
| <button name="confirm" value="yes" style="padding:6px 14px;background:#b42318;color:#fff;border:0">Yes, delete everything</button> | |
| <button name="confirm" value="no" formnovalidate style="padding:6px 14px">No, keep my results</button> | |
| </form>""") | |
| c = CACHE["counts"] | |
| rows_html = [] | |
| for r in CACHE["rows"]: | |
| style = " style='color:#999'" if r.get("unreliable") else "" | |
| rows_html.append( | |
| f"<tr{style}><td>{html.escape(r['model'])}</td><td>{r.get('params_b', '?')}B</td>" | |
| f"<td>{r['score']:.2f}{'?' if r.get('unreliable') else ''}</td><td>{r.get('n_questions', '?')}</td>" | |
| f"<td>{r.get('format_mass', '')}</td><td>{r['tok_per_s']}</td>" | |
| f"<td>{html.escape(str(r.get('task') or ''))}</td><td>{html.escape(r.get('file', ''))}</td>" | |
| f"<td>v{r.get('bench', '?')}</td></tr>") | |
| body = "".join(rows_html) | |
| if CACHE["t"] == 0: | |
| body = "<tr><td colspan=9>Building the results table in the background, refresh in a minute…</td></tr>" | |
| link = f'<a href="https://huggingface.co/buckets/{BUCKET}">bucket</a>' if BUCKET else "bucket" | |
| el = time.time() - STATUS["model_started"] if STATUS["model_started"] else 0 | |
| if RESET["active"]: | |
| top = (f"<div style='border:2px solid #b42318;padding:10px;margin-bottom:14px'><b>Reset in progress:</b> " | |
| f"{html.escape(RESET['msg'])} ({RESET['done']}/{RESET['total']})</div>") | |
| head = "<meta http-equiv='refresh' content='5'>" | |
| else: | |
| if ADMIN_PASSWORD: | |
| action = "<a href='/reset'>Fresh start: delete all results…</a>" | |
| else: | |
| action = "<span style='color:#888'>Fresh start is disabled (add an ADMIN_PASSWORD secret to enable)</span>" | |
| last = f" · {html.escape(RESET['last'])}" if RESET["last"] else "" | |
| top = f"<div style='border:1px solid #ccc;padding:8px 10px;margin-bottom:14px'>{action}{last}</div>" | |
| head = "" | |
| self.send_html(f"""{top} | |
| <h2>Tiny model leaderboard</h2> | |
| <p>Now: <b>{html.escape(STATUS['current'])}</b> ({STATUS['stage']}, {el:.0f}s)<br> | |
| in table (v{'/v'.join(str(v) for v in SHOW_BENCH)}): ok {c.get('ok', 0)} (unreliable {c.get('unreliable', 0)}) | |
| · failed {c.get('failed', 0)} · skipped {c.get('skipped', 0)} | |
| · ignored old formats {c.get('old', 0)} · queue {STATUS['queue']}<br> | |
| this session: ok {SESSION['ok']} · failed {SESSION['failed']} · skipped {SESSION['skipped']}</p> | |
| <p>Score = ARC-Easy, {N_QUESTIONS} questions, zero-shot, answer picked by next-token letter logits (chance = 0.25; | |
| one score is only good to about ±10 points at this size). <b>fmt</b> = share of probability the model puts on the answer | |
| letters; grey rows with “?” are below {UNRELIABLE_MASS} (the model doesn't answer with a letter, so the score is unreliable). | |
| Rows marked v3 use the previous scoring and are replaced as the v{BENCH} run reaches them. | |
| Raw results: {link} (top 300 shown, table rebuilt every {CACHE_EVERY_S}s) · | |
| <a href="/logs">recent logs</a> (<a href="/logs?n=1000">1000 lines</a>)</p> | |
| <table border=1 cellpadding=4><tr><th>model</th><th>params</th><th>score</th><th>n</th><th>fmt</th> | |
| <th>prompt tok/s</th><th>task</th><th>file</th><th>v</th></tr>{body}</table>""", extra_head=head) | |
| def log_message(self, *a): | |
| pass | |
| if __name__ == "__main__": | |
| if len(sys.argv) > 2 and sys.argv[1] == "--worker": | |
| worker(sys.argv[2]) | |
| else: | |
| setup_log() | |
| startup_report() | |
| os.makedirs(f"{DATA}/results", exist_ok=True) | |
| threading.Thread(target=heartbeat, daemon=True).start() | |
| threading.Thread(target=cache_loop, daemon=True).start() | |
| start_loop() | |
| log.info("status page listening on :7860") | |
| ThreadingHTTPServer(("0.0.0.0", 7860), H).serve_forever() |