Upload 3 files

download_model.py

@@ -0,0 +1,25 @@
+"""
+Downloads the GGUF model at Docker build time.
+Model: Qwen3-0.6B (Q4_K_M quantized) — ~400MB, runs well on CPU
+"""
+from huggingface_hub import hf_hub_download
+import os
+
+MODEL_REPO = "Qwen/Qwen3-0.6B-GGUF"
+MODEL_FILE = "qwen3-0.6b-q4_k_m.gguf"
+SAVE_PATH  = "/app/model.gguf"
+
+print(f"Downloading {MODEL_FILE} from {MODEL_REPO}...")
+
+path = hf_hub_download(
+    repo_id=MODEL_REPO,
+    filename=MODEL_FILE,
+    local_dir="/app",
+    local_dir_use_symlinks=False,
+)
+
+# Rename to a fixed path for server.py
+if path != SAVE_PATH:
+    os.rename(path, SAVE_PATH)
+
+print(f"Model saved to {SAVE_PATH}")

requirements.txt

@@ -1,2 +1,5 @@
-huggingface_hub
-llama-cpp-python[server]
+llama-cpp-python==0.3.4
+fastapi==0.115.0
+uvicorn==0.30.6
+huggingface-hub==0.24.6
+pydantic==2.8.2

server.py

@@ -0,0 +1,152 @@
+"""
+openclaw-api — OpenAI-compatible LLM API running locally on CPU
+Uses llama-cpp-python with Qwen3-0.6B GGUF model
+"""
+import time
+import uuid
+import os
+from fastapi import FastAPI, HTTPException, Depends, Header
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+from typing import List, Optional, AsyncGenerator
+from llama_cpp import Llama
+import json
+
+# ─── CONFIG ────────────────────────────────────────────────────────────────
+MODEL_PATH = "/app/model.gguf"
+API_KEY    = os.environ.get("API_KEY", "")   # optional: set in HF Secrets
+N_CTX      = 2048                             # context window
+N_THREADS  = 4                                # CPU threads
+# ───────────────────────────────────────────────────────────────────────────
+
+app = FastAPI(title="openclaw-api", version="1.0.0")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Load model once at startup
+print("Loading model...")
+llm = Llama(
+    model_path=MODEL_PATH,
+    n_ctx=N_CTX,
+    n_threads=N_THREADS,
+    verbose=False,
+)
+print("Model loaded!")
+
+# ─── Auth ───────────────────────────────────────────────────────────────────
+def verify_key(authorization: Optional[str] = Header(None)):
+    if not API_KEY:
+        return  # no key set = open
+    if authorization != f"Bearer {API_KEY}":
+        raise HTTPException(status_code=401, detail="Unauthorized")
+
+# ─── Schemas ────────────────────────────────────────────────────────────────
+class Message(BaseModel):
+    role: str
+    content: str
+
+class ChatRequest(BaseModel):
+    model: Optional[str] = "qwen3-0.6b"
+    messages: List[Message]
+    max_tokens: Optional[int] = 512
+    temperature: Optional[float] = 0.7
+    stream: Optional[bool] = False
+
+class CompletionRequest(BaseModel):
+    model: Optional[str] = "qwen3-0.6b"
+    prompt: str
+    max_tokens: Optional[int] = 512
+    temperature: Optional[float] = 0.7
+    stream: Optional[bool] = False
+
+# ─── Routes ─────────────────────────────────────────────────────────────────
+
+@app.get("/")
+def root():
+    return {"status": "openclaw-api is running", "model": "qwen3-0.6b", "backend": "llama-cpp-python (CPU)"}
+
+@app.get("/v1/models", dependencies=[Depends(verify_key)])
+def list_models():
+    return {
+        "object": "list",
+        "data": [{
+            "id": "qwen3-0.6b",
+            "object": "model",
+            "created": int(time.time()),
+            "owned_by": "local",
+        }]
+    }
+
+@app.post("/v1/chat/completions", dependencies=[Depends(verify_key)])
+def chat_completions(req: ChatRequest):
+    messages = [{"role": m.role, "content": m.content} for m in req.messages]
+
+    if req.stream:
+        def generate():
+            stream = llm.create_chat_completion(
+                messages=messages,
+                max_tokens=req.max_tokens,
+                temperature=req.temperature,
+                stream=True,
+            )
+            for chunk in stream:
+                delta = chunk["choices"][0].get("delta", {})
+                data = {
+                    "id": f"chatcmpl-{uuid.uuid4().hex}",
+                    "object": "chat.completion.chunk",
+                    "created": int(time.time()),
+                    "model": req.model,
+                    "choices": [{"delta": delta, "index": 0, "finish_reason": None}],
+                }
+                yield f"data: {json.dumps(data)}\n\n"
+            yield "data: [DONE]\n\n"
+
+        return StreamingResponse(generate(), media_type="text/event-stream")
+
+    result = llm.create_chat_completion(
+        messages=messages,
+        max_tokens=req.max_tokens,
+        temperature=req.temperature,
+    )
+
+    return {
+        "id": f"chatcmpl-{uuid.uuid4().hex}",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": req.model,
+        "choices": [{
+            "index": 0,
+            "message": {
+                "role": "assistant",
+                "content": result["choices"][0]["message"]["content"],
+            },
+            "finish_reason": result["choices"][0].get("finish_reason", "stop"),
+        }],
+        "usage": result.get("usage", {}),
+    }
+
+@app.post("/v1/completions", dependencies=[Depends(verify_key)])
+def completions(req: CompletionRequest):
+    result = llm(
+        req.prompt,
+        max_tokens=req.max_tokens,
+        temperature=req.temperature,
+    )
+    return {
+        "id": f"cmpl-{uuid.uuid4().hex}",
+        "object": "text_completion",
+        "created": int(time.time()),
+        "model": req.model,
+        "choices": [{
+            "text": result["choices"][0]["text"],
+            "index": 0,
+            "finish_reason": result["choices"][0].get("finish_reason", "stop"),
+        }],
+        "usage": result.get("usage", {}),
+    }