server/llamacpp-client.mjs

import { buildModelContext } from "./agent-core.mjs";
import { SYSTEM_PROMPT, parseModelJson } from "./model-prompt.mjs";

// llama.cpp ships an OpenAI-compatible server (`llama-server`) that exposes
// /v1/chat/completions. This lets the agent run any small GGUF locally — e.g. an
// OpenBMB MiniCPM build — which is the self-contained, hackathon-compliant path.
//   llama-server -hf openbmb/MiniCPM4-8B-GGUF --port 8080
const DEFAULT_LLAMACPP_URL = "http://127.0.0.1:8080/v1/chat/completions";

// llama.cpp enforces this GBNF grammar during sampling, guaranteeing syntactically valid
// JSON even from tiny local models (small GGUFs ignore the loose `json_object` flag and
// happily return prose). The agent's normaliser fills any missing fields, so a constrained
// object is always parseable end-to-end.
const JSON_GRAMMAR = `root   ::= object
value  ::= object | array | string | number | ("true" | "false" | "null") ws
object ::= "{" ws ( string ":" ws value ("," ws string ":" ws value)* )? "}" ws
array  ::= "[" ws ( value ("," ws value)* )? "]" ws
string ::= "\\"" ( [^"\\\\] | "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) )* "\\"" ws
number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
ws ::= [ \\t\\n]*`;

export async function runLlamaCppAgent(localRun, env = process.env) {
  if (env.AGENT_OFFLINE === "1") {
    return {
      status: "skipped",
      reason: "AGENT_OFFLINE is enabled",
      model: configuredLlamaCppModel(env),
      result: null,
    };
  }

  if (!llamaCppEnabled(env)) {
    return {
      status: "skipped",
      reason: "LLAMACPP_ENABLED is disabled",
      model: configuredLlamaCppModel(env),
      result: null,
    };
  }

  const model = configuredLlamaCppModel(env);
  const context = buildModelContext(localRun);
  const basePayload = {
    model,
    messages: [
      { role: "system", content: SYSTEM_PROMPT },
      { role: "user", content: JSON.stringify(context) },
    ],
    temperature: Number(env.LLAMACPP_TEMPERATURE ?? 0.2),
    max_tokens: Number(env.LLAMACPP_MAX_TOKENS ?? 900),
    stream: false,
  };

  const richPayload = {
    ...basePayload,
    grammar: JSON_GRAMMAR,
  };

  try {
    const body = await postChat(richPayload, env);
    return { status: "done", reason: null, model, result: parseModelJson(body, "llama.cpp") };
  } catch (error) {
    if (isRetryableProviderError(error)) {
      try {
        const body = await postChat(basePayload, env);
        return {
          status: "done",
          reason: "Retried without JSON response_format",
          model,
          result: parseModelJson(body, "llama.cpp"),
        };
      } catch (retryError) {
        return errorResult(retryError, model);
      }
    }
    return errorResult(error, model);
  }
}

export function llamaCppEnabled(env = process.env) {
  // Off by default so `auto` does not stall on a missing local server; turn on
  // with LLAMACPP_ENABLED=1 once `llama-server` is running.
  return env.LLAMACPP_ENABLED === "1";
}

export function configuredLlamaCppModel(env = process.env) {
  return env.LLAMACPP_MODEL || "local-gguf";
}

export function llamaCppChatCompletionsUrl(env = process.env) {
  if (env.LLAMACPP_CHAT_COMPLETIONS_URL) return env.LLAMACPP_CHAT_COMPLETIONS_URL;
  if (env.LLAMACPP_BASE_URL) {
    return `${env.LLAMACPP_BASE_URL.replace(/\/$/, "")}/chat/completions`;
  }
  return DEFAULT_LLAMACPP_URL;
}

async function postChat(payload, env) {
  const timeoutMs = Number(env.LLAMACPP_TIMEOUT_MS ?? 60000);
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), timeoutMs);
  const apiKey = env.LLAMACPP_API_KEY || "";

  try {
    const response = await fetch(llamaCppChatCompletionsUrl(env), {
      method: "POST",
      headers: {
        ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
        "Content-Type": "application/json",
      },
      body: JSON.stringify(payload),
      signal: controller.signal,
    });

    const text = await response.text();
    if (!response.ok) {
      throw new LlamaCppRequestError(response.status, conciseError(text));
    }

    try {
      return JSON.parse(text);
    } catch {
      throw new Error("llama.cpp returned a non-JSON chat response");
    }
  } catch (error) {
    if (error?.name === "AbortError") {
      throw new Error(`llama.cpp request timed out after ${timeoutMs}ms`);
    }
    throw error;
  } finally {
    clearTimeout(timeout);
  }
}

function isRetryableProviderError(error) {
  return error instanceof LlamaCppRequestError && (error.status === 400 || error.status === 422);
}

function errorResult(error, model) {
  return {
    status: "error",
    reason: error instanceof Error ? error.message : "Unknown llama.cpp error",
    model,
    result: null,
  };
}

function conciseError(text) {
  if (!text) return "empty error body";
  try {
    const parsed = JSON.parse(text);
    return parsed.error?.message || parsed.error || parsed.message || JSON.stringify(parsed).slice(0, 500);
  } catch {
    return text.slice(0, 500);
  }
}

class LlamaCppRequestError extends Error {
  constructor(status, message) {
    super(`llama.cpp request failed with HTTP ${status}: ${message}`);
    this.name = "LlamaCppRequestError";
    this.status = status;
  }
}