Use lower-latency interactive Q4_0

Exact source: ttracx/thoxroute@cf9bdd16421e23a5f11fc363d420cf0344b573ad

Dockerfile

@@ -22,7 +22,7 @@ ENV HF_HOME=/home/mambauser/.cache/huggingface \
     PORT=7860 \
     THOX_FAST_MODEL_REPO=Qwen/Qwen2.5-0.5B-Instruct-GGUF \
     THOX_FAST_MODEL_REVISION=9217f5db79a29953eb74d5343926648285ec7e67 \
-    THOX_FAST_MODEL_FILE=qwen2.5-0.5b-instruct-q4_k_m.gguf \
+    THOX_FAST_MODEL_FILE=qwen2.5-0.5b-instruct-q4_0.gguf \
     THOX_FAST_POOL_SIZE=2 \
     THOX_FAST_QUEUE_TIMEOUT_S=6 \
     THOX_FAST_MAX_OUTPUT_TOKENS=16 \
@@ -31,7 +31,7 @@ ENV HF_HOME=/home/mambauser/.cache/huggingface \
 
 # Bake the immutable public interactive model into the image. A RUNNING Space
 # therefore cannot report ready while still waiting on a first-request download.
-RUN python -c "from huggingface_hub import hf_hub_download; hf_hub_download('Qwen/Qwen2.5-0.5B-Instruct-GGUF', 'qwen2.5-0.5b-instruct-q4_k_m.gguf', revision='9217f5db79a29953eb74d5343926648285ec7e67')"
+RUN python -c "from huggingface_hub import hf_hub_download; hf_hub_download('Qwen/Qwen2.5-0.5B-Instruct-GGUF', 'qwen2.5-0.5b-instruct-q4_0.gguf', revision='9217f5db79a29953eb74d5343926648285ec7e67')"
 
 WORKDIR /app
 COPY --chown=$MAMBA_USER:$MAMBA_USER app.py /app/app.py

README.md

@@ -25,4 +25,5 @@ work; two-second SSE comments keep the stream live during synchronous prompt
 evaluation. The interactive provider itself caps every request at 16 output
 tokens, so malformed or older callers cannot leave minutes of abandoned work.
 The interactive model is pinned to immutable model revision
-`9217f5db79a29953eb74d5343926648285ec7e67` and baked into the image.
+`9217f5db79a29953eb74d5343926648285ec7e67`; its simpler Q4_0 artifact is
+baked into the image to minimize sustained CPU prompt-evaluation latency.

app.py

@@ -32,7 +32,7 @@ FAST_MODEL_REVISION = os.environ.get(
     "THOX_FAST_MODEL_REVISION", "9217f5db79a29953eb74d5343926648285ec7e67"
 )
 FAST_MODEL_FILE = os.environ.get(
-    "THOX_FAST_MODEL_FILE", "qwen2.5-0.5b-instruct-q4_k_m.gguf"
+    "THOX_FAST_MODEL_FILE", "qwen2.5-0.5b-instruct-q4_0.gguf"
 )
 CODER_MODEL_ID = "thox-rust-coder"
 CODER_MODEL_REPO = os.environ.get(
@@ -244,6 +244,7 @@ def healthz() -> dict[str, Any]:
         "interactive_model": FAST_MODEL_ID,
         "interactive_model_repo": FAST_MODEL_REPO,
         "interactive_model_revision": FAST_MODEL_REVISION,
+        "interactive_model_file": FAST_MODEL_FILE,
         "interactive_pool_size": FAST_POOL_SIZE,
         "interactive_available": active.interactive_available,
         "specialist_model": CODER_MODEL_ID,