Replace Gradio SmolVLM app with DeepSeek-7B-base GGUF OpenAI-compatible API (Docker)

main.py

@@ -166,6 +166,44 @@ def _require_model() -> Llama:
     return _llm
 
 
+def _messages_to_raw_prompt(messages: List["ChatMessage"]) -> str:
+    """Flatten chat messages into a plain-text prompt for BASE (non-chat) models.
+
+    A base model was never trained on a chat template — wrapping its input in
+    one (e.g. llama-2 [INST] tags) makes it emit only <s>/</s> tokens. Instead
+    we join the message contents into plain text and let the model continue it.
+    """
+    parts = [m.content.strip() for m in messages if m.content and m.content.strip()]
+    return "\n\n".join(parts)
+
+
+def _raw_chat_completion(llm: Llama, req: "ChatCompletionRequest") -> Dict[str, Any]:
+    """Text-completion path used when CHAT_FORMAT is empty (base model)."""
+    prompt = _messages_to_raw_prompt(req.messages)
+    result = llm.create_completion(
+        prompt=prompt,
+        temperature=req.temperature,
+        top_p=req.top_p,
+        max_tokens=req.max_tokens if req.max_tokens is not None else DEFAULT_MAX_TOKENS,
+        stop=req.stop,
+    )
+    text = result["choices"][0]["text"]
+    # Re-shape the completion result into an OpenAI chat-completion response so
+    # clients (which always call /v1/chat/completions) need no changes.
+    return {
+        "id": f"chatcmpl-{uuid.uuid4().hex}",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": MODEL_ID,
+        "choices": [{
+            "index": 0,
+            "message": {"role": "assistant", "content": text},
+            "finish_reason": result["choices"][0].get("finish_reason", "stop"),
+        }],
+        "usage": result.get("usage", {}),
+    }
+
+
 # ── Endpoints ─────────────────────────────────────────────────────────────────
 @app.get("/health")
 def health() -> Dict[str, Any]:
@@ -201,6 +239,15 @@ def chat_completions(req: ChatCompletionRequest) -> Dict[str, Any]:
     if not req.messages:
         raise HTTPException(status_code=400, detail="`messages` must not be empty.")
 
+    # Base model (no chat template): use raw text completion instead of a chat
+    # template the model was never trained on.
+    if not CHAT_FORMAT:
+        try:
+            with _llm_lock:
+                return _raw_chat_completion(llm, req)
+        except Exception as exc:
+            raise HTTPException(status_code=500, detail=f"generation failed: {exc}") from exc
+
     kwargs: Dict[str, Any] = dict(
         messages=[m.model_dump() for m in req.messages],
         temperature=req.temperature,
@@ -232,11 +279,20 @@ def generate(req: GenerateRequest) -> Dict[str, Any]:
     t0 = time.time()
     try:
         with _llm_lock:
-            result = llm.create_chat_completion(
-                messages=[{"role": "user", "content": req.prompt}],
-                temperature=req.temperature,
-                max_tokens=req.max_tokens,
-            )
+            if not CHAT_FORMAT:  # base model -> raw continuation of the prompt
+                raw = llm.create_completion(prompt=req.prompt,
+                                            temperature=req.temperature,
+                                            max_tokens=req.max_tokens)
+                result = {
+                    "choices": [{"message": {"content": raw["choices"][0]["text"]}}],
+                    "usage": raw.get("usage", {}),
+                }
+            else:
+                result = llm.create_chat_completion(
+                    messages=[{"role": "user", "content": req.prompt}],
+                    temperature=req.temperature,
+                    max_tokens=req.max_tokens,
+                )
     except Exception as exc:
         raise HTTPException(status_code=500, detail=f"generation failed: {exc}") from exc