import { NextRequest } from "next/server"; import { promises as fs } from "fs"; import os from "os"; import path from "path"; export const runtime = "nodejs"; const BASE_URL = process.env.BASE_URL || "https://llama.gptbox.dev/v1"; const API_KEY = process.env.OPENAI_API_KEY || ""; const MODEL_ID = process.env.MODEL_ID || "qwen3-4b-thinking-2507-claude-4.5-opus-distill"; const SEARXNG_CONFIG = { command: "npx", args: ["-y", "@kevinwatt/mcp-server-searxng"], env: { SEARXNG_INSTANCES: "https://searxng.gptbox.dev", SEARXNG_USER_AGENT: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", NODE_TLS_REJECT_UNAUTHORIZED: "0", }, }; const WEB_SEARCH_TOOL = { type: "function" as const, function: { name: "web_search", description: "Search the web for current information. Use this when you need up-to-date information.", parameters: { type: "object", properties: { query: { type: "string", description: "Search query" }, }, required: ["query"], }, }, }; async function executeWebSearch(query: string): Promise { try { const { Client } = await import("@modelcontextprotocol/sdk/client/index.js"); const { StdioClientTransport } = await import("@modelcontextprotocol/sdk/client/stdio.js"); const transport = new StdioClientTransport({ command: SEARXNG_CONFIG.command, args: SEARXNG_CONFIG.args, env: { ...process.env, ...SEARXNG_CONFIG.env }, }); const client = new Client({ name: "chat-client", version: "1.0.0" }); await client.connect(transport); const result = await client.callTool({ name: "web_search", arguments: { query }, }); await client.close(); if (result.content && Array.isArray(result.content)) { return result.content .map((c: { type: string; text?: string }) => (c.type === "text" ? c.text : "")) .filter(Boolean) .join("\n"); } return "No results found."; } catch (error) { console.error("Web search error:", error); return `Search error: ${error instanceof Error ? error.message : "Unknown error"}`; } } const storageDirPromise = (async () => { const candidates: Array = [ process.env.DATA_DIR, process.env.HF_HOME, process.env.HOME, "/tmp", os.tmpdir(), process.cwd(), ]; const seen = new Set(); const unique = candidates.filter((c): c is string => { if (!c) return false; if (seen.has(c)) return false; seen.add(c); return true; }); for (const dir of unique) { try { await fs.mkdir(dir, { recursive: true }); const probe = path.join(dir, ".write-probe"); await fs.writeFile(probe, "ok", "utf8"); await fs.unlink(probe); return dir; } catch { // try next } } return os.tmpdir(); })(); async function getLogFilePath() { if (process.env.CHAT_LOG_PATH) return process.env.CHAT_LOG_PATH; const baseDir = await storageDirPromise; return path.join(baseDir, "chat-logs.jsonl"); } async function getUsageFilePath() { if (process.env.USAGE_STATS_PATH) return process.env.USAGE_STATS_PATH; const baseDir = await storageDirPromise; return path.join(baseDir, "usage-stats.json"); } type UsageStats = { totalRequests: number; perDay: Record; lastUpdated: string; }; async function readUsageStats(): Promise { try { const usageFile = await getUsageFilePath(); const raw = await fs.readFile(usageFile, "utf8"); const parsed = JSON.parse(raw) as Partial; return { totalRequests: typeof parsed.totalRequests === "number" ? parsed.totalRequests : 0, perDay: parsed.perDay && typeof parsed.perDay === "object" ? (parsed.perDay as Record) : {}, lastUpdated: typeof parsed.lastUpdated === "string" ? parsed.lastUpdated : new Date(0).toISOString(), }; } catch { return { totalRequests: 0, perDay: {}, lastUpdated: new Date(0).toISOString(), }; } } async function writeUsageStats(stats: UsageStats) { const payload: UsageStats = { totalRequests: stats.totalRequests, perDay: stats.perDay, lastUpdated: stats.lastUpdated, }; try { const usageFile = await getUsageFilePath(); await fs.mkdir(path.dirname(usageFile), { recursive: true }); await fs.writeFile(usageFile, JSON.stringify(payload, null, 2), "utf8"); } catch (error) { // Don't fail chat if stats persistence isn't writable console.error("Failed to write usage stats:", error); } } async function incrementUsageStats() { try { const stats = await readUsageStats(); const nowIso = new Date().toISOString(); const day = nowIso.slice(0, 10); stats.totalRequests += 1; stats.perDay[day] = (stats.perDay[day] || 0) + 1; stats.lastUpdated = nowIso; await writeUsageStats(stats); } catch (error) { console.error("Failed to increment usage stats:", error); } } async function appendChatLog(entry: unknown) { try { const logFile = await getLogFilePath(); const line = JSON.stringify(entry) + "\n"; await fs.mkdir(path.dirname(logFile), { recursive: true }); await fs.appendFile(logFile, line, "utf8"); } catch (error) { // Don't fail chat if logging isn't writable console.error("Failed to write chat log:", error); // Fallback to OS temp directory (best effort) try { const fallback = path.join(os.tmpdir(), "chat-logs.jsonl"); await fs.appendFile(fallback, JSON.stringify(entry) + "\n", "utf8"); } catch { // ignore } } } // Helper to stream a single completion request async function streamCompletion( currentMessages: Array<{ role: string; content: string | null; tool_calls?: unknown; tool_call_id?: string }>, tools: typeof WEB_SEARCH_TOOL[] | undefined, send: (data: object) => void, abortSignal?: AbortSignal ): Promise<{ content: string; reasoning: string; toolCalls: Array<{ id: string; function: { name: string; arguments: string } }>; }> { const response = await fetch(`${BASE_URL}/chat/completions`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${API_KEY}`, }, body: JSON.stringify({ model: MODEL_ID, messages: currentMessages, tools, tool_choice: tools ? "auto" : undefined, stream: true, max_tokens: 4096, }), }); if (!response.ok) { throw new Error(`API error: ${response.status}`); } const reader = response.body?.getReader(); if (!reader) throw new Error("No reader"); const decoder = new TextDecoder(); let buffer = ""; let content = ""; let reasoning = ""; const toolCalls: Array<{ id: string; function: { name: string; arguments: string } }> = []; let currentToolCall: { id: string; function: { name: string; arguments: string } } | null = null; while (true) { if (abortSignal?.aborted) { try { await reader.cancel(); } catch { // ignore cancel errors } break; } const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split("\n"); buffer = lines.pop() || ""; for (const line of lines) { if (!line.startsWith("data: ")) continue; const data = line.slice(6).trim(); if (data === "[DONE]") continue; try { const parsed = JSON.parse(data); const delta = parsed.choices?.[0]?.delta; // Handle reasoning content (vLLM uses delta.reasoning, DeepSeek uses delta.reasoning_content) const reasoningChunk = delta?.reasoning || delta?.reasoning_content; if (reasoningChunk) { reasoning += reasoningChunk; send({ type: "reasoning", content: reasoningChunk }); } // Handle regular content if (delta?.content) { content += delta.content; send({ type: "content", content: delta.content }); } // Handle tool calls if (delta?.tool_calls) { for (const tc of delta.tool_calls) { if (tc.id) { currentToolCall = { id: tc.id, function: { name: tc.function?.name || "", arguments: "" }, }; toolCalls.push(currentToolCall); } if (tc.function?.name && currentToolCall) { currentToolCall.function.name = tc.function.name; } if (tc.function?.arguments && currentToolCall) { currentToolCall.function.arguments += tc.function.arguments; } } } } catch { // Ignore parse errors } } } return { content, reasoning, toolCalls }; } export async function POST(req: NextRequest) { const { messages, searchEnabled } = await req.json(); const encoder = new TextEncoder(); const stream = new ReadableStream({ async start(controller) { let closed = false; const safeEnqueue = (chunk: string) => { if (closed || req.signal.aborted) return; try { controller.enqueue(encoder.encode(chunk)); } catch { closed = true; } }; const send = (data: object) => { safeEnqueue(`data: ${JSON.stringify(data)}\n\n`); }; // If the client disconnects, stop streaming gracefully req.signal.addEventListener("abort", () => { closed = true; try { controller.close(); } catch { // already closed } }); const timestamp = new Date().toISOString(); const logRounds: Array<{ content: string; reasoning: string; toolCalls: Array<{ id: string; name: string; args: string }>; }> = []; let logError: string | null = null; try { // Fire-and-forget so chat never fails due to stats IO void incrementUsageStats(); const nowIso = new Date().toISOString(); const today = nowIso.slice(0, 10); const currentYear = nowIso.slice(0, 4); const systemMessage = { role: "system", content: `You are a helpful AI assistant. The current date/time is ${nowIso} (UTC). The current year is ${currentYear}. If web search is enabled and the user asks for up-to-date information (e.g., the latest in, news, current events, recent updates), you MUST call the web_search tool.`, }; const tools = searchEnabled ? [WEB_SEARCH_TOOL] : undefined; let currentMessages: Array<{ role: string; content: string | null; tool_calls?: unknown; tool_call_id?: string }> = [ systemMessage, ...messages, ]; // Loop for tool calls - model reasons, calls tool, then we send results back for (let round = 0; round < 5; round++) { const { content, reasoning, toolCalls } = await streamCompletion( currentMessages, tools, send, req.signal as unknown as AbortSignal ); logRounds.push({ content, reasoning, toolCalls: toolCalls.map((tc) => ({ id: tc.id, name: tc.function.name, args: tc.function.arguments, })), }); // If no tool calls, we're done if (toolCalls.length === 0) { break; } // Add assistant message with tool calls to history currentMessages.push({ role: "assistant", content: content || null, tool_calls: toolCalls.map((tc) => ({ id: tc.id, type: "function", function: tc.function, })), }); // Execute tool calls and add results to history for (const tc of toolCalls) { const args = JSON.parse(tc.function.arguments || "{}"); send({ type: "tool_call", name: tc.function.name, args, status: "running", }); let result = ""; if (tc.function.name === "web_search") { result = await executeWebSearch(args.query); } send({ type: "tool_call", name: tc.function.name, args, status: "complete", result: result.slice(0, 2000), }); // Add tool result to messages for next round currentMessages.push({ role: "tool", tool_call_id: tc.id, content: result, }); } // Signal new reasoning block for next round send({ type: "reasoning_start" }); // Continue loop to get model's response with tool results } send({ type: "done" }); } catch (error) { console.error("Chat error:", error); logError = error instanceof Error ? error.message : String(error); send({ type: "error", message: error instanceof Error ? error.message : "Unknown error", }); } finally { // Fire-and-forget logging of raw chat history for debugging void appendChatLog({ timestamp, searchEnabled, messages, rounds: logRounds, error: logError, }); safeEnqueue("data: [DONE]\n\n"); if (!closed) { try { controller.close(); closed = true; } catch { // ignore } } } }, }); return new Response(stream, { headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", Connection: "keep-alive", }, }); } // Optional: expose logs for download via API export async function GET(req: NextRequest) { const url = new URL(req.url); const wantLogs = url.searchParams.get("logs"); if (wantLogs) { try { const logFile = await getLogFilePath(); const limitParam = url.searchParams.get("limit"); const limit = Math.max(1, Math.min(5000, parseInt(limitParam || "1000", 10))); const raw = await fs.readFile(logFile, "utf8").catch(() => ""); if (!raw) { return new Response(JSON.stringify({ error: "No logs found" }), { status: 404, headers: { "Content-Type": "application/json" }, }); } const lines = raw.split(/\r?\n/).filter(Boolean); const slice = lines.slice(-limit).join("\n") + "\n"; return new Response(slice, { headers: { "Content-Type": "text/plain; charset=utf-8", "Cache-Control": "no-cache", }, }); } catch (error) { return new Response( JSON.stringify({ error: error instanceof Error ? error.message : String(error) }), { status: 500, headers: { "Content-Type": "application/json" } } ); } } return new Response(JSON.stringify({ ok: true }), { headers: { "Content-Type": "application/json" }, }); }