"use client"; import { useEffect, useMemo, useRef, useState } from "react"; import { AnimatePresence, motion } from "framer-motion"; import { Bot, Check, Copy, Loader2, Play, Square, X } from "lucide-react"; import { Button } from "@/components/ui/Button"; import { Input } from "@/components/ui/Input"; import { Select } from "@/components/ui/Select"; import { Textarea } from "@/components/ui/Textarea"; import { fetchModelStreamModels, streamModelGeneration, type ModelStreamEvent, type ModelStreamModelsResponse, type ModelStreamRequest } from "@/lib/api"; import { cn } from "@/lib/utils"; interface ModelStreamPanelProps { open: boolean; onClose: () => void; onStreamStart: () => void; onStreamRawText: (rawText: string, done: boolean) => void; onStreamError: (message: string) => void; } const defaultPrompt = "If a store sells 3 notebooks for $12, how much would 8 notebooks cost at the same rate?"; const infiniteTokenCap = 32768; function numberValue(value: string, fallback: number) { const next = Number(value); return Number.isFinite(next) ? next : fallback; } function boundedIntegerValue(value: string, fallback: number, min: number, max: number) { return Math.min(max, Math.max(min, Math.round(numberValue(value, fallback)))); } function numericDiagnostic(value: unknown) { return typeof value === "number" && Number.isFinite(value) ? value : null; } function diagnosticValue(value: unknown) { if (value == null || value === "") return ""; if (Array.isArray(value)) return value.map(String).join(", "); if (typeof value === "object") { const entries = Object.entries(value as Record); if (entries.length === 0) return "none"; return entries.map(([key, count]) => `${key}:${String(count)}`).join(", "); } if (typeof value === "number") return Number.isInteger(value) ? value.toLocaleString() : value.toFixed(2); return String(value); } function generationDiagnosticItems(data: Record | null) { if (!data) return []; const tokensPerSecond = numericDiagnostic(data.tokensPerSecond); const generatedTokens = numericDiagnostic(data.generatedTokens); const adapterState = data.adapterMerged === true ? "merged" : data.adapterPath ? data.adapterMergeError === "quantized_adapter_wrapper" ? "wrapped" : "loaded" : "base"; const items = [ ["Device", diagnosticValue(data.device ?? data.parameterDevices)], ["Dtype", diagnosticValue(data.dtype ?? data.precision)], ["Attention", diagnosticValue(data.attention)], ["Prompt", data.promptTokens ? `${diagnosticValue(data.promptTokens)} tok` : ""], ["LoRA", adapterState], ["Offload", diagnosticValue(data.offloaded)] ]; if (generatedTokens != null) items.push(["Output", `${generatedTokens.toLocaleString()} tok`]); if (tokensPerSecond != null) items.push(["Speed", `${tokensPerSecond.toFixed(1)} tok/s`]); return items.filter(([, value]) => value); } export function ModelStreamPanel({ open, onClose, onStreamStart, onStreamRawText, onStreamError }: ModelStreamPanelProps) { const [prompt, setPrompt] = useState(defaultPrompt); const [modelCatalog, setModelCatalog] = useState(null); const [modelId, setModelId] = useState("Qwen/Qwen2.5-1.5B-Instruct"); const [adapterPath, setAdapterPath] = useState(""); const [systemPrompt, setSystemPrompt] = useState(""); const [systemPromptOpen, setSystemPromptOpen] = useState(false); const [precision, setPrecision] = useState("bf16"); const [tokenMode, setTokenMode] = useState<"limited" | "infinite">("limited"); const [maxNewTokens, setMaxNewTokens] = useState("2048"); const [temperature, setTemperature] = useState("0.7"); const [topP, setTopP] = useState("0.9"); const [topK, setTopK] = useState("50"); const [repetitionPenalty, setRepetitionPenalty] = useState("1.05"); const [doSample, setDoSample] = useState(true); const [rawGeneration, setRawGeneration] = useState(""); const [status, setStatus] = useState("idle"); const [error, setError] = useState(""); const [stage, setStage] = useState<"configure" | "streaming" | "done">("configure"); const [isStreaming, setIsStreaming] = useState(false); const [copiedRaw, setCopiedRaw] = useState(false); const [diagnostics, setDiagnostics] = useState | null>(null); const abortRef = useRef(null); const rawRef = useRef(""); const parsedLengthRef = useRef(0); const modelChoices = useMemo(() => modelCatalog?.models ?? [], [modelCatalog]); const selectedModel = useMemo( () => modelChoices.find((choice) => choice.modelId === modelId || choice.id === modelId) ?? null, [modelChoices, modelId] ); const checkpointOptions = selectedModel?.checkpoints ?? []; const diagnosticItems = useMemo(() => generationDiagnosticItems(diagnostics), [diagnostics]); useEffect(() => { if (!open || modelCatalog) return; let cancelled = false; setError(""); fetchModelStreamModels() .then((catalog) => { if (cancelled) return; const defaultModel = catalog.models.find((choice) => choice.modelId === catalog.defaultModelId) ?? catalog.models[0]; setModelCatalog(catalog); setModelId(defaultModel?.modelId ?? catalog.defaultModelId); setAdapterPath(defaultModel?.defaultCheckpointPath ?? catalog.defaultAdapterPath ?? ""); setSystemPrompt(defaultModel?.systemPrompt ?? ""); }) .catch((nextError) => { if (!cancelled) setError(nextError instanceof Error ? nextError.message : String(nextError)); }); return () => { cancelled = true; }; }, [modelCatalog, open]); const handleModelChange = (nextModelId: string) => { setModelId(nextModelId); const nextModel = modelChoices.find((choice) => choice.modelId === nextModelId || choice.id === nextModelId); setAdapterPath(nextModel?.defaultCheckpointPath ?? ""); setSystemPrompt(nextModel?.systemPrompt ?? ""); }; const handleEvent = (streamEvent: ModelStreamEvent) => { if (streamEvent.event === "status") { setStatus(String(streamEvent.data.message ?? "streaming")); setDiagnostics(streamEvent.data); return; } if (streamEvent.event === "token") { const text = typeof streamEvent.data.text === "string" ? streamEvent.data.text : ""; rawRef.current += text; setRawGeneration(rawRef.current); const completedIndex = Math.max( rawRef.current.lastIndexOf(""), rawRef.current.lastIndexOf(""), rawRef.current.lastIndexOf(""), rawRef.current.lastIndexOf("<|end_of_trajectory|>") ); if (completedIndex > parsedLengthRef.current) { parsedLengthRef.current = completedIndex; onStreamRawText(rawRef.current, false); } return; } if (streamEvent.event === "trajectory" || streamEvent.event === "trajectory_update") { parsedLengthRef.current = rawRef.current.length; onStreamRawText(rawRef.current, false); return; } if (streamEvent.event === "done") { const rawText = typeof streamEvent.data.rawText === "string" ? streamEvent.data.rawText : rawRef.current; rawRef.current = rawText; setRawGeneration(rawText); onStreamRawText(rawText, true); setStatus("done"); setDiagnostics((current) => ({ ...(current ?? {}), ...streamEvent.data, message: "done" })); setStage("done"); setIsStreaming(false); return; } if (streamEvent.event === "error") { const message = String(streamEvent.data.error ?? "Model stream failed."); setError(message); onStreamError(message); setStatus("error"); setStage("done"); setIsStreaming(false); } }; const startStream = async () => { if (!prompt.trim() || !modelId || isStreaming) return; const controller = new AbortController(); abortRef.current = controller; rawRef.current = ""; parsedLengthRef.current = 0; setRawGeneration(""); setCopiedRaw(false); setDiagnostics(null); setError(""); setStatus("starting"); setStage("streaming"); setIsStreaming(true); onStreamStart(); const defaultSystemPrompt = selectedModel?.systemPrompt?.trim() ?? ""; const nextSystemPrompt = systemPrompt.trim(); const systemPromptOverride = nextSystemPrompt && nextSystemPrompt !== defaultSystemPrompt ? nextSystemPrompt : null; try { await streamModelGeneration( { prompt, system_prompt: systemPromptOverride, model_id: modelId, adapter_path: adapterPath.trim() || null, precision, max_new_tokens: tokenMode === "limited" ? boundedIntegerValue(maxNewTokens, 2048, 1, infiniteTokenCap) : null, temperature: numberValue(temperature, 0.7), top_p: numberValue(topP, 0.9), top_k: boundedIntegerValue(topK, 50, 0, 1000), repetition_penalty: numberValue(repetitionPenalty, 1.05), do_sample: doSample }, handleEvent, controller.signal ); if (!controller.signal.aborted) { setStatus("done"); setStage("done"); } } catch (nextError) { if (controller.signal.aborted) { setStatus("stopped"); } else { const message = nextError instanceof Error ? nextError.message : String(nextError); setError(message); onStreamError(message); setStatus("error"); } setStage("done"); } finally { setIsStreaming(false); abortRef.current = null; } }; const stopStream = () => { abortRef.current?.abort(); setIsStreaming(false); setStatus("stopped"); setStage("done"); }; const copyRawGeneration = async () => { if (!rawGeneration) return; try { await navigator.clipboard.writeText(rawGeneration); setCopiedRaw(true); window.setTimeout(() => setCopiedRaw(false), 1400); } catch (nextError) { const message = nextError instanceof Error ? nextError.message : String(nextError); setError(`Copy failed: ${message}`); } }; const handleClose = () => { if (!isStreaming) setStage("configure"); onClose(); }; const openNewRun = () => { if (isStreaming) return; setError(""); setStatus("idle"); setRawGeneration(""); setCopiedRaw(false); setDiagnostics(null); rawRef.current = ""; parsedLengthRef.current = 0; setStage("configure"); }; const renderConfigureModal = () => (

Create tree by model

{status}