Update clean backend-only project docs and eval

server.py

@@ -2,9 +2,9 @@
 """
 BlitzKode backend server.
 
-Serves the bundled frontend and proxies prompts to a local GGUF model
-through llama.cpp. Model is loaded lazily so the module stays importable
-in tests and environments where the model artifact is not present yet.
+Exposes a FastAPI backend for local GGUF inference through llama.cpp.
+Model is loaded lazily so the module stays importable in tests and
+environments where the model artifact is not present yet.
 """
 
 from __future__ import annotations
@@ -13,24 +13,26 @@ import asyncio
 import json
 import logging
 import os
-import queue
+import re
 import threading
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
+from collections import deque
+from collections.abc import Callable
 from contextlib import asynccontextmanager, suppress
 from dataclasses import dataclass
 from dataclasses import field as dataclass_field
+from html.parser import HTMLParser
 from pathlib import Path
 from typing import Any, Literal, cast
 
 import llama_cpp
 import uvicorn
-from fastapi import FastAPI, HTTPException, Request
+from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
-from fastapi.staticfiles import StaticFiles
+from fastapi.responses import JSONResponse, StreamingResponse
 from pydantic import BaseModel, Field
 from starlette.middleware.base import BaseHTTPMiddleware
 
@@ -39,23 +41,26 @@ APP_VERSION = "2.0"
 CREATOR = "Sajad"
 ROOT_DIR = Path(__file__).resolve().parent
 DEFAULT_MODEL_PATH = ROOT_DIR / "blitzkode.gguf"
-DEFAULT_FRONTEND_DIST_PATH = ROOT_DIR / "frontend" / "dist" / "index.html"
 DEFAULT_CONTEXT = 2048
 DEFAULT_MAX_PROMPT_LENGTH = 4000
 DEFAULT_MAX_TOKENS = 512
 DEFAULT_RATE_LIMIT_MAX = 30
 DEFAULT_MAX_SEARCH_RESULTS = 5
 DEFAULT_SEARCH_TIMEOUT_SECONDS = 8
+DEFAULT_SEARCH_CACHE_TTL_SECONDS = 300
 DEFAULT_MAX_MESSAGES = 20
+DEFAULT_BATCH = 256
+DEFAULT_PROMPT_CACHE_BYTES = 64 * 1024 * 1024
 STOP_TOKENS = ["<|im_end|>", "<|im_start|>user"]
 
 SYSTEM_PROMPT = (
     "<|im_start|>system\n"
     "You are BlitzKode, an AI coding assistant created by Sajad. "
     "You are an expert in Python, JavaScript, Java, C++, and other programming languages. "
-    "For coding work, first understand the user's goal and constraints, then provide a short plan before code when useful. "
-    "Do not invent APIs, file contents, citations, or execution results. "
-    "If evidence is missing, say what is unknown and give a safe next step. "
+    "For coding work, first identify the user's goal, constraints, and any unknowns. "
+    "If asked about a library, API, file, function, citation, execution result, or repository detail that is not provided, "
+    "do not fabricate it. Say you do not know and explain how to verify it. "
+    "Prefer safe minimal fixes over speculative code. "
     "Write clean, efficient, and well-documented code. Keep responses concise and practical.<|im_end|>"
 )
 
@@ -84,13 +89,6 @@ def _path_from_env(name: str, default: Path) -> Path:
     return Path(value) if value else default
 
 
-def _frontend_path_from_env() -> Path:
-    value = os.getenv("BLITZKODE_FRONTEND_PATH")
-    if value:
-        return Path(value)
-    return DEFAULT_FRONTEND_DIST_PATH
-
-
 def _validate_prompt(prompt: str, max_length: int) -> tuple[str, JSONResponse | None]:
     prompt = prompt.strip()
     if not prompt:
@@ -103,17 +101,51 @@ def _validate_prompt(prompt: str, max_length: int) -> tuple[str, JSONResponse |
     return prompt, None
 
 
+_SIGNATURE_QUERY_RE = re.compile(
+    r"(?:signature|how\s+(?:do|can)\s+i\s+use|usage\s+of|docs?\s+for).{0,120}?"
+    r"(?P<symbol>[A-Za-z_][\w.]*\s*\([^)]*\)|[A-Za-z_][\w.]*\s+function)",
+    re.IGNORECASE | re.DOTALL,
+)
+_CODE_CONTEXT_RE = re.compile(r"```|\b(def|class|function|interface|type|import|from)\b|\{\s*\"|\bsource\s+code\b", re.IGNORECASE)
+
+
+def _grounding_guard_response(prompt: str, has_external_context: bool = False) -> str | None:
+    """Prevents confident fabricated API/function signatures when no source/docs are provided.
+
+    Small local models can ignore system instructions around unknown symbols. This guardrail only triggers for direct API/signature
+    lookup questions that contain no pasted source/docs context. It does not block normal code-generation tasks.
+    """
+    if has_external_context or _CODE_CONTEXT_RE.search(prompt):
+        return None
+
+    match = _SIGNATURE_QUERY_RE.search(prompt)
+    if not match:
+        return None
+
+    symbol = " ".join(match.group("symbol").split())
+    return (
+        f"I don't have enough verified context to know the signature or usage of `{symbol}`. "
+        "Please provide the source code or official documentation, or enable research mode so I can ground the answer."
+    )
+
+
 @dataclass(slots=True)
 class Settings:
     root_dir: Path = ROOT_DIR
     model_path: Path = dataclass_field(default_factory=lambda: _path_from_env("BLITZKODE_MODEL_PATH", DEFAULT_MODEL_PATH))
-    frontend_path: Path = dataclass_field(default_factory=_frontend_path_from_env)
     host: str = os.getenv("BLITZKODE_HOST", "0.0.0.0")
     port: int = _int_from_env("BLITZKODE_PORT", 7860)
     n_gpu_layers: int = _int_from_env("BLITZKODE_GPU_LAYERS", 0)
     n_ctx: int = _int_from_env("BLITZKODE_N_CTX", DEFAULT_CONTEXT)
     n_threads: int = _int_from_env("BLITZKODE_THREADS", max(1, min(8, os.cpu_count() or 1)))
-    n_batch: int = _int_from_env("BLITZKODE_BATCH", 128)
+    n_threads_batch: int = _int_from_env("BLITZKODE_THREADS_BATCH", max(1, min(8, os.cpu_count() or 1)))
+    n_batch: int = _int_from_env("BLITZKODE_BATCH", DEFAULT_BATCH)
+    n_ubatch: int = _int_from_env("BLITZKODE_UBATCH", min(DEFAULT_BATCH, 128))
+    prompt_cache_enabled: bool = _bool_from_env("BLITZKODE_PROMPT_CACHE", default=True)
+    prompt_cache_bytes: int = _int_from_env("BLITZKODE_PROMPT_CACHE_BYTES", DEFAULT_PROMPT_CACHE_BYTES)
+    use_mmap: bool = _bool_from_env("BLITZKODE_USE_MMAP", default=True)
+    use_mlock: bool = _bool_from_env("BLITZKODE_USE_MLOCK", default=False)
+    offload_kqv: bool = _bool_from_env("BLITZKODE_OFFLOAD_KQV", default=True)
     max_prompt_length: int = _int_from_env("BLITZKODE_MAX_PROMPT_LENGTH", DEFAULT_MAX_PROMPT_LENGTH)
     preload_model: bool = _bool_from_env("BLITZKODE_PRELOAD_MODEL", default=False)
     cors_origins: str = os.getenv("BLITZKODE_CORS_ORIGINS", "http://localhost:7860")
@@ -121,6 +153,7 @@ class Settings:
     web_search_enabled: bool = _bool_from_env("BLITZKODE_WEB_SEARCH", default=True)
     search_timeout_seconds: int = _int_from_env("BLITZKODE_SEARCH_TIMEOUT", DEFAULT_SEARCH_TIMEOUT_SECONDS)
     max_search_results: int = _int_from_env("BLITZKODE_MAX_SEARCH_RESULTS", DEFAULT_MAX_SEARCH_RESULTS)
+    search_cache_ttl_seconds: int = _int_from_env("BLITZKODE_SEARCH_CACHE_TTL", DEFAULT_SEARCH_CACHE_TTL_SECONDS)
 
 
 class MessageItem(BaseModel):
@@ -166,9 +199,74 @@ class SearchResult:
         }
 
 
+class DuckDuckGoHTMLParser(HTMLParser):
+    def __init__(self, max_results: int):
+        super().__init__(convert_charrefs=True)
+        self.max_results = max_results
+        self.results: list[dict[str, str]] = []
+        self._active_field: Literal["title", "snippet"] | None = None
+        self._active_href = ""
+        self._text_parts: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag != "a":
+            return
+
+        attr_map = {name: value or "" for name, value in attrs}
+        classes = set(attr_map.get("class", "").split())
+        if "result__a" in classes:
+            if len(self.results) >= self.max_results:
+                return
+            self._active_field = "title"
+        elif "result__snippet" in classes:
+            self._active_field = "snippet"
+        else:
+            return
+
+        self._active_href = attr_map.get("href", "")
+        self._text_parts = []
+
+    def handle_data(self, data: str) -> None:
+        if self._active_field:
+            self._text_parts.append(data)
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag != "a" or not self._active_field:
+            return
+
+        text = " ".join("".join(self._text_parts).split())
+        url = self._unwrap_result_url(self._active_href)
+
+        if self._active_field == "title" and text and url and len(self.results) < self.max_results:
+            self.results.append({"title": text, "url": url, "snippet": ""})
+        elif self._active_field == "snippet" and text and self.results:
+            target = next((item for item in reversed(self.results) if not item["snippet"]), None)
+            if target:
+                target["snippet"] = text
+
+        self._active_field = None
+        self._active_href = ""
+        self._text_parts = []
+
+    @staticmethod
+    def _unwrap_result_url(href: str) -> str:
+        href = href.strip()
+        if href.startswith("//"):
+            href = f"https:{href}"
+
+        parsed = urllib.parse.urlparse(href)
+        if "duckduckgo.com" in parsed.netloc and parsed.path.startswith("/l/"):
+            target = urllib.parse.parse_qs(parsed.query).get("uddg", [""])[0]
+            if target:
+                return target
+        return href
+
+
 class WebSearchService:
     def __init__(self, settings: Settings):
         self.settings = settings
+        self._cache: dict[tuple[str, int, bool], tuple[float, list[dict[str, str]]]] = {}
+        self._cache_lock = threading.Lock()
 
     @property
     def enabled(self) -> bool:
@@ -208,6 +306,37 @@ class WebSearchService:
                 title = text.split(" - ", 1)[0]
                 self._append_result(results, seen_urls, title, url, text, max_results)
 
+    def _read_search_payload(self, request: urllib.request.Request) -> dict[str, Any]:
+        with urllib.request.urlopen(request, timeout=self.settings.search_timeout_seconds) as response:
+            raw = response.read().decode("utf-8")
+        try:
+            payload = json.loads(raw)
+        except json.JSONDecodeError as exc:
+            raise RuntimeError("Search provider returned an invalid JSON response") from exc
+        if not isinstance(payload, dict):
+            raise RuntimeError("Search provider returned an unexpected response shape")
+        return payload
+
+    def _search_html(
+        self,
+        query: str,
+        results: list[SearchResult],
+        seen_urls: set[str],
+        max_results: int,
+    ) -> None:
+        params = urllib.parse.urlencode({"q": query})
+        request = urllib.request.Request(
+            f"https://html.duckduckgo.com/html/?{params}",
+            headers={"User-Agent": f"Mozilla/5.0 {APP_NAME}/{APP_VERSION}"},
+        )
+        with urllib.request.urlopen(request, timeout=self.settings.search_timeout_seconds) as response:
+            raw = response.read().decode("utf-8", errors="replace")
+
+        parser = DuckDuckGoHTMLParser(max_results)
+        parser.feed(raw)
+        for item in parser.results:
+            self._append_result(results, seen_urls, item["title"], item["url"], item["snippet"], max_results)
+
     def search(self, query: str, max_results: int = DEFAULT_MAX_SEARCH_RESULTS, deep: bool = False) -> list[dict[str, str]]:
         if not self.enabled:
             raise RuntimeError("Web search is disabled. Set BLITZKODE_WEB_SEARCH=true to enable it.")
@@ -217,6 +346,13 @@ class WebSearchService:
             raise ValueError("Search query is required")
 
         limit = min(max_results, max(1, self.settings.max_search_results), 10)
+        cache_key = (query.lower(), limit, deep)
+        now = time.monotonic()
+        with self._cache_lock:
+            cached = self._cache.get(cache_key)
+            if cached and now - cached[0] < self.settings.search_cache_ttl_seconds:
+                return [dict(item) for item in cached[1]]
+
         results: list[SearchResult] = []
         seen_urls: set[str] = set()
 
@@ -235,8 +371,14 @@ class WebSearchService:
                 f"https://api.duckduckgo.com/?{params}",
                 headers={"User-Agent": f"{APP_NAME}/{APP_VERSION}"},
             )
-            with urllib.request.urlopen(request, timeout=self.settings.search_timeout_seconds) as response:
-                payload = json.loads(response.read().decode("utf-8"))
+            try:
+                payload = self._read_search_payload(request)
+            except RuntimeError as exc:
+                result_count = len(results)
+                self._search_html(variant, results, seen_urls, limit)
+                if len(results) == result_count:
+                    raise exc
+                continue
 
             self._append_result(
                 results,
@@ -248,7 +390,10 @@ class WebSearchService:
             )
             self._collect_related_topics(payload.get("RelatedTopics", []), results, seen_urls, limit)
 
-        return [result.as_dict() for result in results]
+        search_payload = [result.as_dict() for result in results]
+        with self._cache_lock:
+            self._cache[cache_key] = (time.monotonic(), search_payload)
+        return [dict(item) for item in search_payload]
 
 
 class ModelService:
@@ -294,20 +439,18 @@ class ModelService:
 
             start_time = time.perf_counter()
             try:
-                self._llm = llama_cpp.Llama(
-                    model_path=str(self.settings.model_path),
-                    n_gpu_layers=self.settings.n_gpu_layers,
-                    n_ctx=self.settings.n_ctx,
-                    n_threads=self.settings.n_threads,
-                    n_batch=self.settings.n_batch,
-                    verbose=False,
-                    use_mmap=True,
-                    use_mlock=False,
-                    seed=-1,
-                )
+                self._llm = self._create_llama()
                 self._load_time_seconds = time.perf_counter() - start_time
                 self._last_error = None
-                logger.info("Model loaded in %.2fs (gpu_layers=%d)", self._load_time_seconds, self.settings.n_gpu_layers)
+                self._configure_prompt_cache(self._llm)
+                logger.info(
+                    "Model loaded in %.2fs (gpu_layers=%d, ctx=%d, threads=%d, batch=%d)",
+                    self._load_time_seconds,
+                    self.settings.n_gpu_layers,
+                    self.settings.n_ctx,
+                    self.settings.n_threads,
+                    self.settings.n_batch,
+                )
             except Exception as exc:
                 self._last_error = str(exc)
                 logger.error("Model load failed: %s", exc)
@@ -315,6 +458,45 @@ class ModelService:
 
         return self._llm
 
+    def _create_llama(self) -> llama_cpp.Llama:
+        kwargs: dict[str, Any] = {
+            "model_path": str(self.settings.model_path),
+            "n_gpu_layers": self.settings.n_gpu_layers,
+            "n_ctx": self.settings.n_ctx,
+            "n_threads": self.settings.n_threads,
+            "n_threads_batch": self.settings.n_threads_batch,
+            "n_batch": self.settings.n_batch,
+            "n_ubatch": self.settings.n_ubatch,
+            "offload_kqv": self.settings.offload_kqv,
+            "verbose": False,
+            "use_mmap": self.settings.use_mmap,
+            "use_mlock": self.settings.use_mlock,
+            "seed": -1,
+        }
+        try:
+            return llama_cpp.Llama(**kwargs)
+        except TypeError as exc:
+            message = str(exc)
+            unsupported = [key for key in ("n_threads_batch", "n_ubatch", "offload_kqv") if key in message]
+            if not unsupported:
+                raise
+            for key in unsupported:
+                kwargs.pop(key, None)
+            logger.warning("Retrying model load without unsupported llama.cpp options: %s", ", ".join(unsupported))
+            return llama_cpp.Llama(**kwargs)
+
+    def _configure_prompt_cache(self, llm: llama_cpp.Llama) -> None:
+        if not self.settings.prompt_cache_enabled or self.settings.prompt_cache_bytes <= 0:
+            return
+        cache_cls = getattr(llama_cpp, "LlamaRAMCache", None)
+        set_cache = getattr(llm, "set_cache", None)
+        if cache_cls is None or set_cache is None:
+            return
+        try:
+            set_cache(cache_cls(capacity_bytes=self.settings.prompt_cache_bytes))
+        except Exception as exc:
+            logger.warning("Prompt cache setup skipped: %s", exc)
+
     def build_prompt(self, req: GenerateRequest) -> str:
         parts = [SYSTEM_PROMPT]
         for msg in req.messages:
@@ -345,7 +527,7 @@ class ModelService:
         )
         if len(research_prompt) > max_length:
             research_prompt = research_prompt[: max_length - 120].rstrip() + "\n\n[Context truncated to fit prompt limit.]"
-        return req.model_copy(update={"prompt": research_prompt})
+        return cast(GenerateRequest, req.model_copy(update={"prompt": research_prompt}))
 
     def _gen_params(self, req: GenerateRequest) -> dict:
         return {
@@ -372,8 +554,8 @@ class ModelService:
         finally:
             self._busy = False
 
-    def _run_stream(self, req: GenerateRequest, out_q: queue.Queue):
-        """Runs streaming inference in a worker thread, puts tokens into out_q."""
+    def _run_stream(self, req: GenerateRequest, emit: Callable[[str | None], None]):
+        """Runs streaming inference in a worker thread and emits SSE chunks."""
         try:
             llm = self.load_model()
             self._busy = True
@@ -386,16 +568,16 @@ class ModelService:
                 text = token["choices"][0].get("text", "")
                 if text:
                     token_count += 1
-                    out_q.put(f"data: {json.dumps({'token': text})}\n\n")
+                    emit(f"data: {json.dumps({'token': text})}\n\n")
             elapsed = time.perf_counter() - start
             logger.info("Streamed %d tokens in %.2fs", token_count, elapsed)
-            out_q.put("data: [DONE]\n\n")
+            emit("data: [DONE]\n\n")
         except Exception as exc:
             logger.error("Stream error: %s", exc)
-            out_q.put(f"data: {json.dumps({'error': str(exc)})}\n\n")
+            emit(f"data: {json.dumps({'error': str(exc)})}\n\n")
         finally:
             self._busy = False
-            out_q.put(None)
+            emit(None)
 
 
 def _check_api_key(request: Request, settings: Settings) -> JSONResponse | None:
@@ -417,7 +599,7 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
         super().__init__(app)
         self._max = max_requests
         self._window = window_seconds
-        self._clients: dict[str, list[float]] = {}
+        self._clients: dict[str, deque[float]] = {}
         self._lock = threading.Lock()
         self._cleanup_done = 0
 
@@ -431,11 +613,13 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
             self._cleanup_done = 0
             with self._lock:
                 cutoff = now - self._window
-                self._clients = {ip: [t for t in ts if t >= cutoff] for ip, ts in self._clients.items() if ts}
+                self._clients = {ip: deque(t for t in ts if t >= cutoff) for ip, ts in self._clients.items() if ts}
 
         with self._lock:
-            timestamps = self._clients.get(client_ip, [])
-            timestamps = [t for t in timestamps if now - t < self._window]
+            timestamps = self._clients.setdefault(client_ip, deque())
+            cutoff = now - self._window
+            while timestamps and timestamps[0] < cutoff:
+                timestamps.popleft()
             if len(timestamps) >= self._max:
                 return JSONResponse(
                     {"error": "Rate limit exceeded. Try again later."},
@@ -443,7 +627,6 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
                     headers={"Retry-After": str(self._window)},
                 )
             timestamps.append(now)
-            self._clients[client_ip] = timestamps
         return await call_next(request)
 
 
@@ -483,10 +666,6 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     app.state.model_service = model_service
     app.state.search_service = search_service
 
-    frontend_assets_path = settings.frontend_path.parent / "assets"
-    if frontend_assets_path.exists():
-        app.mount("/assets", StaticFiles(directory=str(frontend_assets_path)), name="frontend-assets")
-
     cors_origins = [o.strip() for o in settings.cors_origins.split(",") if o.strip()]
     app.add_middleware(
         CORSMiddleware,
@@ -501,20 +680,23 @@ def create_app(settings: Settings | None = None) -> FastAPI:
 
     @app.get("/")
     async def root():
-        if not settings.frontend_path.exists():
-            raise HTTPException(status_code=404, detail="Frontend build is missing. Run `npm install` and `npm run build` in frontend/.")
-        return FileResponse(str(settings.frontend_path))
+        return JSONResponse(
+            {
+                "name": APP_NAME,
+                "version": APP_VERSION,
+                "message": "BlitzKode backend API is running. Use /info for endpoint details.",
+            }
+        )
 
     @app.get("/health")
     async def health():
-        status = "healthy" if model_service.model_exists else "degraded"
+        status = "healthy" if model_service.model_exists and not model_service.last_error else "degraded"
         return JSONResponse(
             {
                 "status": status,
                 "model_loaded": model_service.model_loaded,
                 "model_path": str(settings.model_path),
                 "model_exists": model_service.model_exists,
-                "frontend_exists": settings.frontend_path.exists(),
                 "version": APP_VERSION,
                 "gpu_layers": settings.n_gpu_layers,
                 "last_error": model_service.last_error,
@@ -532,6 +714,12 @@ def create_app(settings: Settings | None = None) -> FastAPI:
         if err:
             return err
 
+        guard_response = _grounding_guard_response(prompt)
+        if guard_response:
+            return JSONResponse(
+                {"response": guard_response, "creator": CREATOR, "model": APP_NAME, "version": APP_VERSION, "guarded": True}
+            )
+
         async with model_lock:
             try:
                 sanitized = req.model_copy(update={"prompt": prompt})
@@ -584,20 +772,33 @@ def create_app(settings: Settings | None = None) -> FastAPI:
         if not model_service.model_exists:
             return JSONResponse({"error": f"Model not found at {settings.model_path}"}, status_code=503)
 
+        guard_response = _grounding_guard_response(prompt)
+        if guard_response:
+
+            async def _guarded_stream():
+                yield f"data: {json.dumps({'token': guard_response})}\n\n"
+                yield "data: [DONE]\n\n"
+
+            return StreamingResponse(_guarded_stream(), media_type="text/event-stream")
+
         sanitized = req.model_copy(update={"prompt": prompt})
 
         async def _locked_stream():
             async with model_lock:
-                token_q: queue.Queue = queue.Queue()
+                loop = asyncio.get_running_loop()
+                token_q: asyncio.Queue[str | None] = asyncio.Queue()
+
+                def emit(chunk: str | None) -> None:
+                    loop.call_soon_threadsafe(token_q.put_nowait, chunk)
+
                 thread = threading.Thread(
                     target=model_service._run_stream,
-                    args=(sanitized, token_q),
+                    args=(sanitized, emit),
                     daemon=True,
                 )
                 thread.start()
-                # Use thread-safe queue.get() instead of deprecated get_running_loop()
                 while True:
-                    chunk = await asyncio.to_thread(token_q.get)
+                    chunk = await token_q.get()
                     if chunk is None:
                         break
                     yield chunk
@@ -636,6 +837,11 @@ def create_app(settings: Settings | None = None) -> FastAPI:
                 "mode": f"{'GPU' if settings.n_gpu_layers > 0 else 'CPU'} (llama.cpp)",
                 "gpu_layers": settings.n_gpu_layers,
                 "context_window": settings.n_ctx,
+                "threads": settings.n_threads,
+                "threads_batch": settings.n_threads_batch,
+                "batch": settings.n_batch,
+                "ubatch": settings.n_ubatch,
+                "prompt_cache_enabled": settings.prompt_cache_enabled,
                 "model_loaded": model_service.model_loaded,
                 "load_time_seconds": model_service.load_time_seconds,
                 "busy": model_service.busy,