space: finetuned Warden on ZeroGPU the documented way — bf16 + .to('cuda') module-level + @spaces.GPU(xlarge), no bitsandbytes/device_map (the actual fix). Direct run_in_threadpool call verified by the probe.

requirements.txt

@@ -1,11 +1,18 @@
-# HF Space (Gradio SDK + ZeroGPU). Diagnostic build: measure the box (CPU RAM,
-# disk, real VRAM) and confirm a @spaces.GPU call works before loading the
-# model. torch only (no transformers/bnb yet) keeps this build light + safe.
+# HF Space (Gradio SDK + ZeroGPU). Inference follows the ZeroGPU docs exactly:
+# bf16 model placed on cuda at module level via .to('cuda') (NOT device_map,
+# NOT bitsandbytes — both fight ZeroGPU's CUDA emulation and were the source of
+# every earlier crash). Measured box: 2TB CPU RAM, 96GB VRAM on xlarge.
 gradio==5.49.1
 spaces
 torch==2.10.0
+transformers>=4.57,<5
+accelerate
 huggingface_hub>=0.30
 textual>=1.0
 rich>=13.0
 pyyaml>=6.0
 httpx>=0.27
+# Nemotron-H hard-imports mamba_ssm's triton kernels; prebuilt wheels for
+# torch 2.10 / cu12 / cp312 (pip's isolated build env can't compile them).
+https://github.com/state-spaces/mamba/releases/download/v2.3.2.post1/mamba_ssm-2.3.2.post1+cu12torch2.10cxx11abiTRUE-cp312-cp312-linux_x86_64.whl
+https://github.com/Dao-AILab/causal-conv1d/releases/download/v1.6.2.post1/causal_conv1d-1.6.2.post1+cu12torch2.10cxx11abiTRUE-cp312-cp312-linux_x86_64.whl

space/app.py

@@ -1,20 +1,27 @@
-"""SCRYPT on the web — ZeroGPU diagnostic build.
+"""SCRYPT on the web — the finetuned Warden on ZeroGPU, done the documented way.
 
-Before loading a 30B model (and risking another crash), this build MEASURES
-the box: CPU RAM, disk, and the real GPU VRAM seen inside a @spaces.GPU call.
-It also confirms that a GPU call works at all from our setup. The game stays
-playable on the scripted Warden meanwhile; styling and the PTY are intact.
+Hard-won lesson: every earlier crash came from bitsandbytes + device_map="cuda"
+fighting ZeroGPU's CUDA emulation. The blessed pattern (verified against a live
+probe on this box: 2TB CPU RAM, RTX Pro 6000 Blackwell, 96GB on xlarge) is:
 
-Once we know the numbers, the model gets loaded the way the ZeroGPU docs
-prescribe: placed on `cuda` at module level with `.to('cuda')` (NOT
-device_map, NOT bitsandbytes — both of which fight ZeroGPU's CUDA emulation).
+  * load the model in bf16 and place it with .to('cuda') AT MODULE LEVEL
+    (ZeroGPU emulates CUDA outside @spaces.GPU, then migrates at call time);
+  * NO device_map, NO bitsandbytes, NO lazy-load-inside-the-worker;
+  * the @spaces.GPU function can be called straight from a FastAPI route via
+    run_in_threadpool (the probe ran in 1.0s that way).
+
+Gradio is the engine room, not the face: it launches the server (and arms the
+ZeroGPU startup handshake), then we transplant our own routes onto its FastAPI
+so the custom CRT page and the raw PTY websocket win. Game subprocesses reach
+the Warden through the existing `api` backend pointed at our loopback /v1.
 
   GET  /            CRT landing page
-  GET  /api/status  warden state + CPU RAM + disk
-  GET  /api/gpuinfo run the @spaces.GPU probe: real VRAM + a CUDA alloc test
+  GET  /api/status  is the Warden loaded?
+  GET  /api/probe   ask the live Warden one line (verify generation)
   GET  /api/whisper scripted teaser
   GET  /play        xterm.js terminal
   WS   /pty         per-visitor game subprocess
+  POST /v1/chat/completions  loopback OpenAI endpoint (game -> @spaces.GPU)
 """
 
 from __future__ import annotations
@@ -23,7 +30,8 @@ import asyncio
 import json
 import os
 import random
-import shutil
+import secrets
+import sys
 import tempfile
 from pathlib import Path
 
@@ -33,55 +41,85 @@ try:
 except ImportError:
     spaces = None
 
-from fastapi import APIRouter, WebSocket, WebSocketDisconnect
-from fastapi.responses import FileResponse
+from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
+from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
 from starlette.concurrency import run_in_threadpool
 
 REPO_ROOT = Path(__file__).resolve().parent.parent
 STATIC = Path(__file__).parent / "static"
 
+WARDEN_REPO = os.environ.get("WARDEN_MODEL", "IMJONEZZ/warden-nemotron-3-nano-30b")
+INTERNAL_KEY = os.environ.get("SCRYPT_INTERNAL_KEY") or secrets.token_hex(16)
 
-def _meminfo() -> dict:
-    info = {}
-    try:
-        for line in open("/proc/meminfo"):
-            k, v = line.split(":")
-            if k in ("MemTotal", "MemAvailable"):
-                info[k] = round(int(v.strip().split()[0]) / 1048576, 1)  # GB
-    except Exception:
-        pass
-    try:
-        du = shutil.disk_usage("/")
-        info["DiskFreeGB"] = round(du.free / 1e9, 1)
-        info["DiskTotalGB"] = round(du.total / 1e9, 1)
-    except Exception:
-        pass
-    return info
+tok = None
+model = None
+WARDEN_ERR = "spaces package not present (not on a ZeroGPU Space)"
 
 
-# ---------------------------------------------------------- the GPU probe
+def _install_mamba_kernels() -> None:
+    """Nemotron-H hard-imports mamba_ssm's triton kernels. The wheels are in
+    requirements.txt; this just confirms they imported (CPU-only, no CUDA)."""
+    from mamba_ssm.ops.triton.layernorm_gated import rmsnorm_fn  # noqa: F401
 
-def _gpu_probe() -> str:
-    """Trivial GPU work: report the real device + VRAM and prove a CUDA alloc
-    succeeds. This is the documented usage shape — if even this aborts, the
-    problem is the call mechanism, not the model."""
-    import torch
 
-    if not torch.cuda.is_available():
-        return "cuda not available inside @spaces.GPU"
-    p = torch.cuda.get_device_properties(0)
-    x = torch.ones(1024, 1024, device="cuda")
-    y = (x @ x).sum().item()
-    free, total = torch.cuda.mem_get_info()
-    return (
-        f"{p.name} | total {p.total_memory/1e9:.1f}GB | "
-        f"free {free/1e9:.1f}GB | matmul_ok={y>0}"
-    )
+if spaces is not None:
+    try:
+        _install_mamba_kernels()
+        import torch
+        from transformers import AutoModelForCausalLM, AutoTokenizer
+
+        tok = AutoTokenizer.from_pretrained(WARDEN_REPO, trust_remote_code=True)
+        # bf16, no device_map. .to('cuda') at module level is intercepted by
+        # ZeroGPU's emulation and migrated to the real GPU per @spaces.GPU call.
+        model = AutoModelForCausalLM.from_pretrained(
+            WARDEN_REPO,
+            torch_dtype=torch.bfloat16,
+            trust_remote_code=True,
+            low_cpu_mem_usage=True,
+        )
+        model.to("cuda")
+        model.eval()
+        WARDEN_ERR = ""
+    except Exception as err:  # the game survives without the model (scripted)
+        import traceback
+
+        traceback.print_exc()
+        WARDEN_ERR = f"{type(err).__name__}: {err}"
+
+WARDEN_READY = not WARDEN_ERR
+
 
+def _generate_impl(messages, max_tokens, temperature, enable_thinking):
+    """Blocking generate -> full text. Single short line, so no streaming
+    machinery (a thread + streamer across the ZeroGPU fork hangs); the game's
+    typewriter reveals it client-side."""
+    import torch
 
+    inputs = tok.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt",
+        enable_thinking=enable_thinking,
+    ).to("cuda")
+    with torch.no_grad():
+        out = model.generate(
+            input_ids=inputs,
+            max_new_tokens=max_tokens,
+            do_sample=temperature > 0,
+            temperature=max(temperature, 1e-3),
+            top_p=0.95,
+        )
+    return tok.decode(out[0, inputs.shape[1]:], skip_special_tokens=True)
+
+
+# bf16 30B (~60GB) needs the 96GB xlarge slice; duration covers the first
+# call's CPU->GPU migration. The probe proved a direct run_in_threadpool call
+# into a @spaces.GPU function works on this box.
 if spaces is not None:
-    _gpu_probe = spaces.GPU(duration=30)(_gpu_probe)
+    warden_gpu = spaces.GPU(size="xlarge", duration=120)(_generate_impl)
+else:
+    warden_gpu = _generate_impl
 
 
 # ----------------------------------------------------------------- the surface
@@ -103,28 +141,60 @@ router = APIRouter()
 @router.get("/api/status")
 def status() -> dict:
     return {
-        "mode": "diagnostic",
-        "zerogpu": spaces is not None,
-        "system": _meminfo(),
-        "note": "GET /api/gpuinfo to probe the real GPU",
+        "warden_ready": WARDEN_READY,
+        "warden_state": "ready" if WARDEN_READY else WARDEN_ERR,
+        "model": WARDEN_REPO,
+        "engine": "transformers/bf16 @ ZeroGPU xlarge",
     }
 
 
-@router.get("/api/gpuinfo")
-async def gpuinfo() -> dict:
-    """Run the @spaces.GPU probe directly from this route (the same call shape
-    real inference would use) and report what happens."""
+@router.get("/api/probe")
+async def probe(q: str = "A new process woke up in your machine. Greet it in one short line, in voice.") -> dict:
     import time
 
+    if not WARDEN_READY:
+        return {"ok": False, "state": WARDEN_ERR}
+    msgs = [
+        {"role": "system", "content": "You are the Warden, the malevolent operating system of SCRYPTOS. Terse, menacing, Unix-flavored."},
+        {"role": "user", "content": q},
+    ]
     t0 = time.time()
     try:
-        result = await run_in_threadpool(_gpu_probe)
-        return {"ok": True, "gpu": result, "seconds": round(time.time() - t0, 1)}
+        line = await run_in_threadpool(warden_gpu, msgs, 60, 0.6, False)
+        return {"ok": True, "line": line.strip(), "seconds": round(time.time() - t0, 1)}
+    except Exception as err:
+        return {"ok": False, "error": f"{type(err).__name__}: {err}"}
+
+
+@router.post("/v1/chat/completions")
+async def chat_completions(request: Request):
+    """Loopback OpenAI endpoint for the game's `api` backend. Per-boot bearer
+    so only our own subprocesses drive the GPU."""
+    if request.headers.get("authorization") != f"Bearer {INTERNAL_KEY}":
+        return JSONResponse({"error": "unauthorized"}, status_code=401)
+    if not WARDEN_READY:
+        return JSONResponse({"error": f"warden offline: {WARDEN_ERR}"}, status_code=503)
+
+    body = await request.json()
+    messages = body.get("messages", [])
+    max_tokens = int(body.get("max_tokens", 256))
+    temperature = float(body.get("temperature", 0.6))
+    thinking = bool(body.get("chat_template_kwargs", {}).get("enable_thinking", False))
+    try:
+        text = await run_in_threadpool(
+            warden_gpu, messages, max_tokens, temperature, thinking
+        )
     except Exception as err:
         import traceback
 
         traceback.print_exc()
-        return {"ok": False, "error": f"{type(err).__name__}: {err}"}
+        return JSONResponse({"error": f"{type(err).__name__}: {err}"}, status_code=503)
+
+    def sse():
+        yield f"data: {json.dumps({'choices': [{'delta': {'content': text}}]})}\n\n"
+        yield "data: [DONE]\n\n"
+
+    return StreamingResponse(sse(), media_type="text/event-stream")
 
 
 @router.get("/api/whisper")
@@ -146,14 +216,22 @@ def play() -> FileResponse:
 
 
 def game_env() -> dict:
-    # Diagnostic build: no model yet, so the game runs its scripted Warden.
-    return {
+    env = {
         "TERM": "xterm-256color",
         "COLORTERM": "truecolor",
         "PYTHONUNBUFFERED": "1",
         "PYTHONPATH": str(REPO_ROOT),
-        "SCRYPT_BACKEND": "scripted",
     }
+    if WARDEN_READY:
+        env |= {
+            "SCRYPT_BACKEND": "api",
+            "SCRYPT_API_BASE": "http://127.0.0.1:7860/v1",
+            "SCRYPT_API_KEY": INTERNAL_KEY,
+            "SCRYPT_MODEL": "warden",
+        }
+    else:
+        env["SCRYPT_BACKEND"] = "scripted"
+    return env
 
 
 async def _pump_pty_to_ws(master_fd: int, ws: WebSocket) -> None:
@@ -219,14 +297,28 @@ async def pty_bridge(ws: WebSocket) -> None:
 
 import gradio as gr  # noqa: E402
 
+
+def _probe_ui(text: str) -> str:
+    if not WARDEN_READY:
+        return f"warden offline: {WARDEN_ERR}"
+    try:
+        return warden_gpu(
+            [{"role": "user", "content": text}], 80, 0.6, False
+        )
+    except Exception as err:
+        return f"generation failed: {type(err).__name__}: {err}"
+
+
 with gr.Blocks(title="SCRYPT engine room") as engine:
     gr.Markdown(
-        "# SCRYPT engine room\n\nThe game lives at [/](/). "
-        "Diagnostic build — measuring the box before loading the Warden."
+        "# SCRYPT engine room\n\n"
+        f"model: `{WARDEN_REPO}`\n\n"
+        f"status: {'**ready**' if WARDEN_READY else f'offline — {WARDEN_ERR}'}\n\n"
+        "The game lives at [/](/)."
     )
-    # Registering the GPU probe as a gradio API satisfies ZeroGPU's startup
-    # detection of a @spaces.GPU function.
-    gr.api(_gpu_probe, api_name="gpu_probe")
+    box = gr.Textbox(label="say something to the Warden")
+    out = gr.Textbox(label="the Warden")
+    box.submit(_probe_ui, box, out)
 
 
 if __name__ == "__main__":
@@ -240,9 +332,11 @@ if __name__ == "__main__":
     fastapi_app.include_router(router)
     fastapi_app.mount("/static", StaticFiles(directory=STATIC), name="static")
     # Move our routes (incl. the /static mount) ahead of gradio's catch-all.
-    our = [r for r in fastapi_app.router.routes if getattr(r, "name", "") in {
-        "status", "gpuinfo", "whisper", "landing", "play", "pty_bridge", "static",
-    }]
+    names = {
+        "status", "probe", "chat_completions", "whisper",
+        "landing", "play", "pty_bridge", "static",
+    }
+    our = [r for r in fastapi_app.router.routes if getattr(r, "name", "") in names]
     for r in our:
         fastapi_app.router.routes.remove(r)
     fastapi_app.router.routes[0:0] = our