gr.Code(language='rust') is not supported in gradio 5.50 -- drop it

app.py

@@ -1,193 +1,196 @@
-"""THOX Rust Coder — hosted Qwen3-Coder-REAP-25B-A3B-Rust (GGUF).
-
-SIZING: "3B active" IS NOT A MEMORY BUDGET
-------------------------------------------
-This is a Mixture-of-Experts model: 25B total parameters, ~3B active per token.
-Those two numbers govern different resources and it is easy to conflate them:
-
-    active params (~3B)  -> COMPUTE per token.  Decode is as cheap as a 3B dense.
-    total params (25B)   -> MEMORY.  Every expert must be resident, because the
-                            router may select any of them on any token.
-
-So a Q4_K_M GGUF is **15.1 GB of RAM/VRAM**, not "3B-worth". You cannot fit this
-on a tier sized for a 3B model. What MoE buys you here is speed-per-byte, not a
-smaller footprint -- which is exactly why a CPU tier is viable at all: we pay
-25B-sized memory but only 3B-sized arithmetic.
-
-TIER
-----
-Starts on `cpu-upgrade` (8 vCPU / 32 GB, ~$0.03/hr). That fits Q4_K_M with room
-for the KV cache, and the 3B active path keeps CPU decode tolerable. If measured
-throughput is too slow to be a useful coding assistant, escalate to a GPU tier --
-but escalate on a MEASUREMENT, not on the assumption that 25B implies a GPU.
-"""
-
-from __future__ import annotations
-
-import os
-import time
-import uuid
-
-import gradio as gr
-from fastapi import FastAPI
-from huggingface_hub import hf_hub_download
-from pydantic import BaseModel
-
-MODEL_REPO = os.environ.get("THOX_MODEL_REPO", "Em-80/Qwen3-coder-REAP-25B-A3B-Rust-GGUF")
-MODEL_FILE = os.environ.get("THOX_MODEL_FILE", "Qwen3-Coder-REAP-25B-A3B-Rust-Q4_K_M.gguf")
-N_CTX = int(os.environ.get("THOX_N_CTX", "8192"))
-
-SYSTEM = (
-    "You are THOX Rust Coder. You write correct, idiomatic Rust. Prefer showing "
-    "compiling code over prose. If a request is ambiguous, state the assumption "
-    "you made in one line, then give the code."
-)
-
-_llm = None
-
-
-def _usable_cpus() -> int:
-    """Threads from the cgroup quota, not the host.
-
-    `os.cpu_count()` reports the HOST's core count inside a container. On a
-    sibling Space this oversubscribed a 2-vCPU cgroup ~8x and cost ~290x
-    throughput -- the model ran slower than the edge device it was meant to
-    offload. Read the quota.
-    """
-    try:
-        quota, period = open("/sys/fs/cgroup/cpu.max").read().split()
-        if quota != "max":
-            return max(1, int(int(quota) / int(period)))
-    except Exception:
-        pass
-    try:
-        q = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read())
-        p = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read())
-        if q > 0:
-            return max(1, q // p)
-    except Exception:
-        pass
-    try:
-        return max(1, len(os.sched_getaffinity(0)))
-    except Exception:
-        return os.cpu_count() or 2
-
-
-def llm():
-    global _llm
-    if _llm is None:
-        from llama_cpp import Llama
-
-        path = hf_hub_download(MODEL_REPO, MODEL_FILE,
-                               token=os.environ.get("HF_TOKEN"))
-        _llm = Llama(
-            model_path=path,
-            n_ctx=N_CTX,
-            n_threads=_usable_cpus(),
-            # -1 offloads every layer when a GPU is present, and is simply
-            # ignored on a CPU build -- so the same image works on both tiers.
-            n_gpu_layers=int(os.environ.get("THOX_GPU_LAYERS", "-1")),
-            verbose=False,
-        )
-    return _llm
-
-
-def generate(messages, max_tokens=512, temperature=0.2):
-    t0 = time.time()
-    # create_chat_completion uses the chat template embedded in the GGUF, rather
-    # than a hand-rolled one. Qwen3-Coder is ChatML, but reading it from the file
-    # means a re-quant with a different template does not silently break output.
-    out = llm().create_chat_completion(
-        messages=messages, max_tokens=max_tokens, temperature=temperature,
-    )
-    dt = time.time() - t0
-    text = out["choices"][0]["message"]["content"]
-    n = out.get("usage", {}).get("completion_tokens") or 0
-    return text, n, dt
-
-
-api = FastAPI()
-
-
-class Msg(BaseModel):
-    role: str
-    content: str
-
-
-class ChatRequest(BaseModel):
-    model: str | None = None
-    messages: list[Msg]
-    max_tokens: int | None = 512
-    temperature: float | None = 0.2
-
-
-@api.get("/healthz")
-def healthz():
-    return {
-        "status": "ok",
-        "model": MODEL_REPO,
-        "file": MODEL_FILE,
-        "role": "thox-rust-coder",
-        "n_ctx": N_CTX,
-        "threads": _usable_cpus(),
-        "loaded": _llm is not None,
-    }
-
-
-@api.post("/v1/chat/completions")
-def chat_completions(req: ChatRequest):
-    msgs = [m.model_dump() for m in req.messages]
-    if not any(m["role"] == "system" for m in msgs):
-        msgs = [{"role": "system", "content": SYSTEM}] + msgs
-    text, n, dt = generate(msgs, req.max_tokens or 512, req.temperature or 0.2)
-    return {
-        "id": "chatcmpl-" + uuid.uuid4().hex[:12],
-        "object": "chat.completion",
-        "created": int(time.time()),
-        "model": "thox-rust-coder",
-        "choices": [{"index": 0, "finish_reason": "stop",
-                     "message": {"role": "assistant", "content": text}}],
-        "usage": {"completion_tokens": n},
-        "thox_perf": {"tok_per_s": round(n / dt, 1) if dt else None,
-                      "seconds": round(dt, 2)},
-    }
-
-
-def ui(prompt, max_tokens):
-    text, n, dt = generate(
-        [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}],
-        int(max_tokens),
-    )
-    tps = n / dt if dt else 0
-    return text, f"{n} tok in {dt:.1f}s = {tps:.1f} tok/s"
-
-
-demo = gr.Interface(
-    fn=ui,
-    inputs=[gr.Textbox(label="Prompt", lines=4,
-                       value="Write a Rust function that parses a semver string "
-                             "into (major, minor, patch), returning Result."),
-            gr.Slider(64, 2048, value=512, step=64, label="max tokens")],
-    outputs=[gr.Code(label="THOX Rust Coder", language="rust"),
-             gr.Textbox(label="Measured")],
-    title="THOX Rust Coder — Qwen3-Coder-REAP-25B-A3B-Rust",
-    description=(
-        "Rust-specialised MoE coder, Apache-2.0, served as GGUF.\n\n"
-        "`POST /v1/chat/completions` (OpenAI-shaped, ThoxRoute-registerable) · "
-        "`GET /healthz`\n\n"
-        "**25B total / ~3B active.** Memory is sized by the 25B (all experts stay "
-        "resident); speed is sized by the 3B. That combination is what makes a "
-        "CPU tier viable."
-    ),
-)
-
-app = gr.mount_gradio_app(api, demo, path="/")
-
-
-if __name__ == "__main__":
-    # Defining `app` does not serve it -- without this the process exits 0 and
-    # the Space reports RUNTIME_ERROR with no traceback to read.
-    import uvicorn
-
-    uvicorn.run(app, host="0.0.0.0",
-                port=int(os.environ.get("GRADIO_SERVER_PORT", 7860)))
+"""THOX Rust Coder — hosted Qwen3-Coder-REAP-25B-A3B-Rust (GGUF).
+
+SIZING: "3B active" IS NOT A MEMORY BUDGET
+------------------------------------------
+This is a Mixture-of-Experts model: 25B total parameters, ~3B active per token.
+Those two numbers govern different resources and it is easy to conflate them:
+
+    active params (~3B)  -> COMPUTE per token.  Decode is as cheap as a 3B dense.
+    total params (25B)   -> MEMORY.  Every expert must be resident, because the
+                            router may select any of them on any token.
+
+So a Q4_K_M GGUF is **15.1 GB of RAM/VRAM**, not "3B-worth". You cannot fit this
+on a tier sized for a 3B model. What MoE buys you here is speed-per-byte, not a
+smaller footprint -- which is exactly why a CPU tier is viable at all: we pay
+25B-sized memory but only 3B-sized arithmetic.
+
+TIER
+----
+Starts on `cpu-upgrade` (8 vCPU / 32 GB, ~$0.03/hr). That fits Q4_K_M with room
+for the KV cache, and the 3B active path keeps CPU decode tolerable. If measured
+throughput is too slow to be a useful coding assistant, escalate to a GPU tier --
+but escalate on a MEASUREMENT, not on the assumption that 25B implies a GPU.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+import uuid
+
+import gradio as gr
+from fastapi import FastAPI
+from huggingface_hub import hf_hub_download
+from pydantic import BaseModel
+
+MODEL_REPO = os.environ.get("THOX_MODEL_REPO", "Em-80/Qwen3-coder-REAP-25B-A3B-Rust-GGUF")
+MODEL_FILE = os.environ.get("THOX_MODEL_FILE", "Qwen3-Coder-REAP-25B-A3B-Rust-Q4_K_M.gguf")
+N_CTX = int(os.environ.get("THOX_N_CTX", "8192"))
+
+SYSTEM = (
+    "You are THOX Rust Coder. You write correct, idiomatic Rust. Prefer showing "
+    "compiling code over prose. If a request is ambiguous, state the assumption "
+    "you made in one line, then give the code."
+)
+
+_llm = None
+
+
+def _usable_cpus() -> int:
+    """Threads from the cgroup quota, not the host.
+
+    `os.cpu_count()` reports the HOST's core count inside a container. On a
+    sibling Space this oversubscribed a 2-vCPU cgroup ~8x and cost ~290x
+    throughput -- the model ran slower than the edge device it was meant to
+    offload. Read the quota.
+    """
+    try:
+        quota, period = open("/sys/fs/cgroup/cpu.max").read().split()
+        if quota != "max":
+            return max(1, int(int(quota) / int(period)))
+    except Exception:
+        pass
+    try:
+        q = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read())
+        p = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read())
+        if q > 0:
+            return max(1, q // p)
+    except Exception:
+        pass
+    try:
+        return max(1, len(os.sched_getaffinity(0)))
+    except Exception:
+        return os.cpu_count() or 2
+
+
+def llm():
+    global _llm
+    if _llm is None:
+        from llama_cpp import Llama
+
+        path = hf_hub_download(MODEL_REPO, MODEL_FILE,
+                               token=os.environ.get("HF_TOKEN"))
+        _llm = Llama(
+            model_path=path,
+            n_ctx=N_CTX,
+            n_threads=_usable_cpus(),
+            # -1 offloads every layer when a GPU is present, and is simply
+            # ignored on a CPU build -- so the same image works on both tiers.
+            n_gpu_layers=int(os.environ.get("THOX_GPU_LAYERS", "-1")),
+            verbose=False,
+        )
+    return _llm
+
+
+def generate(messages, max_tokens=512, temperature=0.2):
+    t0 = time.time()
+    # create_chat_completion uses the chat template embedded in the GGUF, rather
+    # than a hand-rolled one. Qwen3-Coder is ChatML, but reading it from the file
+    # means a re-quant with a different template does not silently break output.
+    out = llm().create_chat_completion(
+        messages=messages, max_tokens=max_tokens, temperature=temperature,
+    )
+    dt = time.time() - t0
+    text = out["choices"][0]["message"]["content"]
+    n = out.get("usage", {}).get("completion_tokens") or 0
+    return text, n, dt
+
+
+api = FastAPI()
+
+
+class Msg(BaseModel):
+    role: str
+    content: str
+
+
+class ChatRequest(BaseModel):
+    model: str | None = None
+    messages: list[Msg]
+    max_tokens: int | None = 512
+    temperature: float | None = 0.2
+
+
+@api.get("/healthz")
+def healthz():
+    return {
+        "status": "ok",
+        "model": MODEL_REPO,
+        "file": MODEL_FILE,
+        "role": "thox-rust-coder",
+        "n_ctx": N_CTX,
+        "threads": _usable_cpus(),
+        "loaded": _llm is not None,
+    }
+
+
+@api.post("/v1/chat/completions")
+def chat_completions(req: ChatRequest):
+    msgs = [m.model_dump() for m in req.messages]
+    if not any(m["role"] == "system" for m in msgs):
+        msgs = [{"role": "system", "content": SYSTEM}] + msgs
+    text, n, dt = generate(msgs, req.max_tokens or 512, req.temperature or 0.2)
+    return {
+        "id": "chatcmpl-" + uuid.uuid4().hex[:12],
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "thox-rust-coder",
+        "choices": [{"index": 0, "finish_reason": "stop",
+                     "message": {"role": "assistant", "content": text}}],
+        "usage": {"completion_tokens": n},
+        "thox_perf": {"tok_per_s": round(n / dt, 1) if dt else None,
+                      "seconds": round(dt, 2)},
+    }
+
+
+def ui(prompt, max_tokens):
+    text, n, dt = generate(
+        [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}],
+        int(max_tokens),
+    )
+    tps = n / dt if dt else 0
+    return text, f"{n} tok in {dt:.1f}s = {tps:.1f} tok/s"
+
+
+demo = gr.Interface(
+    fn=ui,
+    inputs=[gr.Textbox(label="Prompt", lines=4,
+                       value="Write a Rust function that parses a semver string "
+                             "into (major, minor, patch), returning Result."),
+            gr.Slider(64, 2048, value=512, step=64, label="max tokens")],
+    # gradio 5.50's gr.Code has a fixed language allow-list and "rust" is NOT
+    # on it -- passing it raises ValueError at import and the Space exits 1.
+    # No highlighting is better than no Space.
+    outputs=[gr.Code(label="THOX Rust Coder"),
+             gr.Textbox(label="Measured")],
+    title="THOX Rust Coder — Qwen3-Coder-REAP-25B-A3B-Rust",
+    description=(
+        "Rust-specialised MoE coder, Apache-2.0, served as GGUF.\n\n"
+        "`POST /v1/chat/completions` (OpenAI-shaped, ThoxRoute-registerable) · "
+        "`GET /healthz`\n\n"
+        "**25B total / ~3B active.** Memory is sized by the 25B (all experts stay "
+        "resident); speed is sized by the 3B. That combination is what makes a "
+        "CPU tier viable."
+    ),
+)
+
+app = gr.mount_gradio_app(api, demo, path="/")
+
+
+if __name__ == "__main__":
+    # Defining `app` does not serve it -- without this the process exits 0 and
+    # the Space reports RUNTIME_ERROR with no traceback to read.
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0",
+                port=int(os.environ.get("GRADIO_SERVER_PORT", 7860)))