formscout/serving/llama_cpp.py

"""
llama.cpp HTTP client wrapper for FormScout.

Wraps the llama.cpp server's /completion and /embedding endpoints.
Falls back gracefully when the server is unavailable.

Model: Qwen3-VL-8B-Instruct (Q4_K_M GGUF) for VLM inference.
Model: Qwen3-VL-Embedding-8B (Q4_K_M GGUF) for embeddings.
Params: 8B each (shared backbone).
License: Apache-2.0.
"""
from __future__ import annotations

import base64
import json
import logging
from pathlib import Path
from typing import Any

import requests

from formscout import config

logger = logging.getLogger(__name__)

_TIMEOUT = 120  # seconds — VLM can be slow


class LlamaCppClient:
    """HTTP client for a llama.cpp server instance."""

    def __init__(self, host: str | None = None, port: int | None = None):
        self.host = host or config.LLAMA_CPP_HOST
        self.port = port or config.LLAMA_CPP_PORT_VLM
        self.base_url = f"http://{self.host}:{self.port}"

    @property
    def available(self) -> bool:
        """Check if the server is reachable."""
        try:
            r = requests.get(f"{self.base_url}/health", timeout=5)
            return r.status_code == 200
        except (requests.ConnectionError, requests.Timeout):
            return False

    def complete(
        self,
        prompt: str,
        images: list[str] | None = None,
        max_tokens: int = 512,
        temperature: float = 0.1,
        stop: list[str] | None = None,
    ) -> dict[str, Any]:
        """
        Send a completion request. Returns parsed JSON if the response is JSON,
        otherwise returns {"text": raw_text}.

        Args:
            prompt: The text prompt (system + user combined).
            images: Optional list of base64-encoded images or file paths.
            max_tokens: Max generation tokens.
            temperature: Sampling temperature.
            stop: Stop sequences.
        """
        payload: dict[str, Any] = {
            "prompt": prompt,
            "n_predict": max_tokens,
            "temperature": temperature,
            "stop": stop or ["\n\n"],
        }

        # Add images for multimodal (Qwen3-VL via llama.cpp mmproj)
        if images:
            image_data = []
            for img in images:
                if Path(img).exists():
                    with open(img, "rb") as f:
                        image_data.append({"data": base64.b64encode(f.read()).decode()})
                else:
                    # Assume already base64
                    image_data.append({"data": img})
            payload["image_data"] = image_data

        try:
            r = requests.post(
                f"{self.base_url}/completion",
                json=payload,
                timeout=_TIMEOUT,
            )
            r.raise_for_status()
            result = r.json()
            content = result.get("content", "")
            # Try to parse as JSON
            try:
                return json.loads(content)
            except (json.JSONDecodeError, TypeError):
                return {"text": content}
        except requests.ConnectionError:
            return {"error": "llama.cpp server not available", "text": ""}
        except requests.Timeout:
            return {"error": "llama.cpp server timeout", "text": ""}
        except Exception as e:
            return {"error": str(e), "text": ""}


class EmbeddingClient:
    """HTTP client for the llama.cpp embedding server."""

    def __init__(self, host: str | None = None, port: int | None = None):
        self.host = host or config.LLAMA_CPP_HOST
        self.port = port or config.LLAMA_CPP_PORT_EMBED
        self.base_url = f"http://{self.host}:{self.port}"

    @property
    def available(self) -> bool:
        try:
            r = requests.get(f"{self.base_url}/health", timeout=5)
            return r.status_code == 200
        except (requests.ConnectionError, requests.Timeout):
            return False

    def embed(self, text: str) -> list[float] | None:
        """Get embedding vector for text. Returns None on failure."""
        try:
            r = requests.post(
                f"{self.base_url}/embedding",
                json={"content": text},
                timeout=30,
            )
            r.raise_for_status()
            data = r.json()
            return data.get("embedding")
        except Exception:
            return None