Upload app.py with huggingface_hub

app.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+"""POCKET-35B CPU chat — FastAPI + llama-cpp-python (no GPU).
+Serves a 35B MoE model (Q2_K) answering on a CPU-only Hugging Face Space."""
+import os, json, threading
+from fastapi import FastAPI, Request
+from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse
+from huggingface_hub import hf_hub_download
+
+MODEL_REPO = os.environ.get("MODEL_REPO", "FINAL-Bench/POCKET-35B-GGUF")
+MODEL_FILE = os.environ.get("MODEL_FILE", "POCKET-35B-Q2_K.gguf")
+N_THREADS  = int(os.environ.get("N_THREADS", str(os.cpu_count() or 8)))
+N_CTX      = int(os.environ.get("N_CTX", "4096"))
+
+app = FastAPI(title="POCKET-35B CPU chat")
+_llm = None
+_lock = threading.Lock()
+_status = {"state": "loading", "detail": "downloading model…"}
+
+def get_llm():
+    global _llm
+    if _llm is None:
+        with _lock:
+            if _llm is None:
+                from llama_cpp import Llama
+                _status.update(state="loading", detail="downloading %s…" % MODEL_FILE)
+                path = hf_hub_download(MODEL_REPO, MODEL_FILE)
+                _status.update(detail="loading into RAM…")
+                _llm = Llama(model_path=path, n_ctx=N_CTX, n_threads=N_THREADS,
+                             n_gpu_layers=0, verbose=False, chat_format=None)
+                _status.update(state="ready", detail="POCKET-35B on CPU · ready")
+    return _llm
+
+@app.on_event("startup")
+def _warm():
+    threading.Thread(target=get_llm, daemon=True).start()
+
+@app.get("/api/status")
+def status():
+    return JSONResponse(_status)
+
+@app.post("/api/chat")
+async def chat(req: Request):
+    body = await req.json()
+    messages = body.get("messages") or [{"role": "user", "content": body.get("prompt", "Hi")}]
+    temperature = float(body.get("temperature", 0.7))
+    max_tokens = int(body.get("max_tokens", 512))
+    llm = get_llm()
+
+    def gen():
+        try:
+            stream = llm.create_chat_completion(
+                messages=messages, temperature=temperature, top_p=0.95,
+                max_tokens=max_tokens, stream=True)
+            for chunk in stream:
+                delta = chunk["choices"][0]["delta"].get("content")
+                if delta:
+                    yield "data: " + json.dumps({"t": delta}) + "\n\n"
+            yield "data: " + json.dumps({"done": True}) + "\n\n"
+        except Exception as e:
+            yield "data: " + json.dumps({"error": str(e)[:200]}) + "\n\n"
+
+    return StreamingResponse(gen(), media_type="text/event-stream")
+
+@app.get("/", response_class=HTMLResponse)
+def index():
+    with open(os.path.join(os.path.dirname(__file__), "index.html"), encoding="utf-8") as f:
+        return f.read()