Create app.py

app.py

@@ -0,0 +1,285 @@
+import os
+import json
+import time
+import subprocess
+
+import requests
+from huggingface_hub import hf_hub_download
+
+# ----------------------- config (override via Space variables) -----------------------
+# Serves the copied 0.8B GGUF from anon334test/qwopus.
+MODEL_REPO  = os.environ.get("MODEL_REPO", "anon334test/qwopus")
+GGUF_FILE   = os.environ.get("GGUF_FILE", "Qwen3.5-0.8B.Q4_K_M.gguf")
+HF_TOKEN    = os.environ.get("HF_TOKEN")  # optional; model is public
+PORT        = int(os.environ.get("PORT", "7860"))
+LLAMA_PORT  = int(os.environ.get("LLAMA_PORT", "8080"))
+# IMPORTANT: cpu-basic = 2 physical vCPU, but os.cpu_count() reports the HOST core
+# count (e.g. 32/64) inside the cgroup -> launching -t with that number oversubscribes
+# the 2 vCPU and collapses throughput (~0.1 tok/s). Default to 2; override via NUM_THREADS.
+NUM_THREADS = os.environ.get("NUM_THREADS", "2")
+BIN         = "/opt/llamabin"
+# Custom chat template that adds an enable_thinking toggle (the model's own template
+# ALWAYS opens a <think> block; this lets "Fast (no thinking)" actually skip reasoning).
+CHAT_TEMPLATE_FILE = os.environ.get("CHAT_TEMPLATE_FILE", "/home/user/app/chat_template.jinja")
+LLAMA       = f"http://127.0.0.1:{LLAMA_PORT}"
+# Qwen3.5-0.8B native context = 262144. We default to a generous 32768 window (good for
+# large files / long chats) while keeping startup + memory reasonable on CPU; raise N_CTX up
+# to 262144 via a Space variable if you really need it (much slower prefill on CPU).
+N_CTX       = os.environ.get("N_CTX", "32768")
+
+
+def _env():
+    e = os.environ.copy()
+    e["LD_LIBRARY_PATH"] = BIN + ":" + e.get("LD_LIBRARY_PATH", "")
+    return e
+
+
+# ----------------------- step 1: download the GGUF (no conversion) -----------------------
+def ensure_gguf():
+    print(f"[init] downloading {GGUF_FILE} from {MODEL_REPO} ...", flush=True)
+    path = hf_hub_download(MODEL_REPO, GGUF_FILE, repo_type="model", token=HF_TOKEN)
+    print(f"[init] model ready: {path}", flush=True)
+    return path
+
+
+# ----------------------- step 2: launch internal llama-server -----------------------
+def start_llama(model_path):
+    cmd = [
+        os.path.join(BIN, "llama-server"),
+        "-m", model_path, "--host", "127.0.0.1", "--port", str(LLAMA_PORT),
+        "-c", N_CTX, "-t", NUM_THREADS, "-b", "256", "--no-mmap",
+        "--parallel", os.environ.get("PARALLEL", "1"),
+        # Apply our chat template (adds enable_thinking toggle for true fast mode).
+        "--jinja",
+    ]
+    if CHAT_TEMPLATE_FILE and os.path.exists(CHAT_TEMPLATE_FILE):
+        cmd += ["--chat-template-file", CHAT_TEMPLATE_FILE]
+    # Separate the <think> chain-of-thought into reasoning_content so the answer stays clean.
+    cmd += ["--reasoning-format", "auto"]
+    print(f"[init] cpu_count={os.cpu_count()} threads={NUM_THREADS}", flush=True)
+    # Optional extra flags (e.g. "-fa on") via LLAMA_EXTRA_ARGS, space separated.
+    extra = os.environ.get("LLAMA_EXTRA_ARGS", "").split()
+    if extra:
+        cmd += extra
+    print("[init] starting internal llama-server: " + " ".join(cmd), flush=True)
+    subprocess.Popen(cmd, env=_env())
+    for _ in range(900):
+        try:
+            r = requests.get(LLAMA + "/health", timeout=3)
+            if r.status_code == 200 and r.json().get("status") == "ok":
+                print("[init] internal llama-server is healthy.", flush=True)
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    raise RuntimeError("internal llama-server did not become healthy in time")
+
+
+# ============================================================================
+#  Serving: thin PASS-THROUGH proxy to llama-server's native OpenAI endpoints.
+#  llama-server applies the model's embedded chat template (--jinja) itself, so
+#  what the model sees is exactly the messages you send -- nothing injected.
+# ============================================================================
+from fastapi import FastAPI, Request
+from fastapi.responses import StreamingResponse, JSONResponse, HTMLResponse, Response
+from fastapi.middleware.cors import CORSMiddleware
+
+app = FastAPI()
+app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=False,
+                   allow_methods=["*"], allow_headers=["*"])
+
+# Sensible defaults (only applied when the caller doesn't set them). Overridable per request.
+# max_tokens = -1 -> UNLIMITED output (generate until EOS or the context window is full).
+DEFAULTS = {"temperature": 0.3, "top_p": 0.9, "top_k": 20, "repeat_penalty": 1.05,
+            "max_tokens": int(os.environ.get("MAX_TOKENS", "-1"))}
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok"}
+
+
+@app.get("/v1/models")
+def models():
+    try:
+        return JSONResponse(requests.get(LLAMA + "/v1/models", timeout=15).json())
+    except Exception:
+        return {"object": "list", "data": [{"id": GGUF_FILE, "object": "model", "owned_by": "anon334test"}]}
+
+
+# ----- Lightweight branding: short identity prompt that does NOT suppress reasoning -----
+# Kept short on purpose: long/defensive prompts make small models reason worse (see report.md).
+# Empty by default for a 0.8B model so nothing dilutes its limited attention; set SYSTEM_PROMPT
+# as a Space variable to enable an identity line.
+SYSTEM_PROMPT = os.environ.get("SYSTEM_PROMPT", "").strip()
+
+
+def _prep(body):
+    """Apply defaults and translate convenience fields, then leave everything else untouched
+    so all OpenAI / llama.cpp params pass straight through to the model.
+
+    Thinking control (Qwen3.5): accept a top-level `enable_thinking` bool or a friendly
+    `thinking: "on"|"off"`. Both map to chat_template_kwargs.enable_thinking, honored by the
+    model's own chat template. If neither is given, the model's default applies.
+    """
+    for k, v in DEFAULTS.items():
+        body.setdefault(k, v)
+
+    think = None
+    if "enable_thinking" in body:
+        think = bool(body.pop("enable_thinking"))
+    if "thinking" in body:
+        t = str(body.pop("thinking")).lower()
+        think = t in ("on", "true", "1", "yes", "smart")
+    if think is not None:
+        ctk = dict(body.get("chat_template_kwargs") or {})
+        ctk["enable_thinking"] = think
+        body["chat_template_kwargs"] = ctk
+
+    # Optional identity injection (only if SYSTEM_PROMPT is set).
+    if SYSTEM_PROMPT:
+        msgs = body.get("messages")
+        if isinstance(msgs, list):
+            msgs = [m for m in msgs
+                    if not (isinstance(m, dict) and m.get("role") == "system")]
+            msgs.insert(0, {"role": "system", "content": SYSTEM_PROMPT})
+            body["messages"] = msgs
+    return body
+
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: Request):
+    body = _prep(await request.json())
+    stream = bool(body.get("stream", False))
+    if stream:
+        def gen():
+            with requests.post(LLAMA + "/v1/chat/completions", json=body, stream=True, timeout=900) as r:
+                for chunk in r.iter_content(chunk_size=None):
+                    if chunk:
+                        yield chunk
+        return StreamingResponse(gen(), media_type="text/event-stream")
+    r = requests.post(LLAMA + "/v1/chat/completions", json=body, timeout=900)
+    return Response(content=r.content, media_type="application/json", status_code=r.status_code)
+
+
+@app.post("/v1/completions")
+async def completions(request: Request):
+    body = _prep(await request.json())
+    stream = bool(body.get("stream", False))
+    if stream:
+        def gen():
+            with requests.post(LLAMA + "/v1/completions", json=body, stream=True, timeout=900) as r:
+                for chunk in r.iter_content(chunk_size=None):
+                    if chunk:
+                        yield chunk
+        return StreamingResponse(gen(), media_type="text/event-stream")
+    r = requests.post(LLAMA + "/v1/completions", json=body, timeout=900)
+    return Response(content=r.content, media_type="application/json", status_code=r.status_code)
+
+
+@app.get("/", response_class=HTMLResponse)
+def index():
+    return INDEX_HTML
+
+
+INDEX_HTML = """<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8"/>
+<meta name="viewport" content="width=device-width, initial-scale=1"/>
+<title>Qwopus3.5-0.8B Chat</title>
+<style>
+  :root { color-scheme: light dark; }
+  * { box-sizing: border-box; }
+  body { margin:0; font-family: ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,sans-serif;
+         background:#0b0d12; color:#e7e9ee; display:flex; flex-direction:column; height:100vh; }
+  header { padding:12px 18px; border-bottom:1px solid #1e2430; font-weight:600; font-size:15px;
+           display:flex; align-items:center; gap:8px; }
+  header .dot { width:8px; height:8px; border-radius:50%; background:#33d17a; }
+  header small { font-weight:400; opacity:.55; }
+  #chat { flex:1; overflow-y:auto; padding:18px; display:flex; flex-direction:column; gap:14px; }
+  .msg { max-width:820px; width:100%; margin:0 auto; }
+  .who { font-size:12px; opacity:.6; margin-bottom:4px; }
+  .bubble { padding:10px 14px; border-radius:12px; white-space:pre-wrap; line-height:1.55;
+            font-size:14.5px; word-wrap:break-word; overflow-wrap:anywhere; }
+  .user { display:flex; justify-content:flex-end; }
+  .user .bubble { background:#1d4ed8; color:#fff; }
+  .bot .bubble { background:#161b24; border:1px solid #232a36; }
+  pre { background:#0f131b; border:1px solid #232a36; border-radius:8px; padding:10px;
+        overflow-x:auto; font-size:13px; }
+  footer { border-top:1px solid #1e2430; padding:12px; }
+  form { max-width:820px; margin:0 auto; display:flex; gap:8px; }
+  textarea { flex:1; resize:none; background:#11151d; color:#e7e9ee; border:1px solid #232a36;
+             border-radius:10px; padding:11px 12px; font-size:14.5px; max-height:160px; }
+  button { background:#1d4ed8; color:#fff; border:0; border-radius:10px; padding:0 18px;
+           font-weight:600; cursor:pointer; }
+  button:disabled { opacity:.5; cursor:default; }
+  .hint { text-align:center; opacity:.45; font-size:12px; margin-top:8px; }
+  .typing { opacity:.5; font-style:italic; }
+  .row { max-width:820px; margin:0 auto 8px; display:flex; gap:8px; }
+  .row button { background:#232a36; font-weight:500; font-size:12px; padding:4px 10px; }
+  .think { max-width:820px; width:100%; margin:0 auto 6px; font-size:13px; opacity:.75;
+           background:#0f131b; border:1px dashed #2a3340; border-radius:10px; padding:6px 12px; }
+  .think summary { cursor:pointer; user-select:none; }
+  .think-body { white-space:pre-wrap; margin-top:6px; line-height:1.5; }
+</style>
+</head>
+<body>
+<header><span class="dot"></span> Qwopus3.5-0.8B <small>&middot; Qwen3.5 0.8B &middot; fast chat / reasoning &middot; live GGUF</small>
+  <label style="margin-left:auto; font-weight:400; font-size:13px; display:flex; align-items:center; gap:6px;">Mode
+    <select id="mode" style="background:#11151d; color:#e7e9ee; border:1px solid #232a36; border-radius:8px; padding:4px 8px; font-size:13px;">
+      <option value="off" selected>&#9889; Fast (no thinking)</option>
+      <option value="on">&#129504; Smart (thinking)</option>
+    </select>
+  </label>
+</header>
+<div id="chat"></div>
+<footer>
+  <div class="row"><button id="reset" type="button">New chat</button></div>
+  <form id="f">
+    <textarea id="t" rows="1" placeholder="Ask anything..." autofocus></textarea>
+    <button id="send" type="submit">Send</button>
+  </form>
+  <div class="hint">Pure model &middot; OpenAI-compatible API at <code>/v1/chat/completions</code></div>
+</footer>
+<script>
+const chat=document.getElementById('chat'),form=document.getElementById('f'),ta=document.getElementById('t'),sendBtn=document.getElementById('send');
+let history=[];
+function esc(s){return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');}
+function render(t){return esc(t).replace(/```([\\s\\S]*?)```/g,(m,c)=>'<pre><code>'+c.replace(/^\\w*\\n/,'')+'</code></pre>');}
+function stripThink(t){return t.replace(/<think>[\\s\\S]*?<\\/think>/gi,'').replace(/^[\\s\\S]*?<\\/think>/i, m=>m.includes('<think>')?'':m).trim();}
+function addUser(t){const w=document.createElement('div');w.className='msg user';w.innerHTML='<div class="bubble">'+render(t)+'</div>';chat.appendChild(w);chat.scrollTop=chat.scrollHeight;}
+function addBot(){const w=document.createElement('div');w.className='msg bot';w.innerHTML='<div class="who">Assistant</div><details class="think" style="display:none"><summary>&#129504; Thinking</summary><div class="think-body"></div></details><div class="bubble"><span class="typing">&hellip;</span></div>';chat.appendChild(w);chat.scrollTop=chat.scrollHeight;return {think:w.querySelector('.think'),thinkBody:w.querySelector('.think-body'),bubble:w.querySelector('.bubble')};}
+document.getElementById('reset').onclick=()=>{history=[];chat.innerHTML='';ta.focus();};
+ta.addEventListener('input',()=>{ta.style.height='auto';ta.style.height=Math.min(ta.scrollHeight,160)+'px';});
+ta.addEventListener('keydown',e=>{if(e.key==='Enter'&&!e.shiftKey){e.preventDefault();form.requestSubmit();}});
+form.addEventListener('submit',async e=>{
+  e.preventDefault();const text=ta.value.trim();if(!text)return;
+  ta.value='';ta.style.height='auto';addUser(text);history.push({role:'user',content:text});
+  sendBtn.disabled=true;const {think,thinkBody,bubble}=addBot();let acc='';let rc='';
+  try{
+    const resp=await fetch('/v1/chat/completions',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({messages:history,stream:true,thinking:document.getElementById('mode').value})});
+    const reader=resp.body.getReader(),dec=new TextDecoder();let buf='';
+    while(true){const {value,done}=await reader.read();if(done)break;buf+=dec.decode(value,{stream:true});let idx;
+      while((idx=buf.indexOf('\\n\\n'))>=0){const line=buf.slice(0,idx).trim();buf=buf.slice(idx+2);
+        if(!line.startsWith('data:'))continue;const data=line.slice(5).trim();if(data==='[DONE]')continue;
+        try{const o=JSON.parse(data);const dl=o.choices?.[0]?.delta||{};
+          const rd=dl.reasoning_content||'';if(rd){rc+=rd;think.style.display='block';thinkBody.textContent=rc;chat.scrollTop=chat.scrollHeight;}
+          const d=dl.content||'';if(d){acc+=d;bubble.innerHTML=render(stripThink(acc));chat.scrollTop=chat.scrollHeight;}
+        }catch(_){}}}
+  }catch(err){acc=acc||('[error] '+err);bubble.innerHTML=render(acc);}
+  const clean=stripThink(acc);if(!clean)bubble.innerHTML=render('(no response)');
+  history.push({role:'assistant',content:clean});sendBtn.disabled=false;ta.focus();
+});
+</script>
+</body>
+</html>"""
+
+
+# ----------------------- main -----------------------
+if __name__ == "__main__":
+    model_path = ensure_gguf()
+    start_llama(model_path)
+    import uvicorn
+    print(f"[init] starting public proxy on port {PORT} ...", flush=True)
+    uvicorn.run(app, host="0.0.0.0", port=PORT, log_level="info")