import {LlamaText} from "node-llama-cpp"; import path from "path"; import fs from "fs/promises"; /** * Output types for debugging */ const OutputTypes = { EXACT_PROMPT: 'exactPrompt', CONTEXT_STATE: 'contextState', STRUCTURED: 'structured' }; /** * Helper class for debugging and logging LLM prompts */ export class PromptDebugger { constructor(options = {}) { this.outputDir = options.outputDir || './'; this.filename = options.filename; this.includeTimestamp = options.includeTimestamp ?? false; this.appendMode = options.appendMode ?? false; // Configure which outputs to include this.outputTypes = options.outputTypes || [OutputTypes.EXACT_PROMPT]; // Ensure outputTypes is always an array if (!Array.isArray(this.outputTypes)) { this.outputTypes = [this.outputTypes]; } } /** * Captures only the exact prompt (user input + system + functions) * @param {Object} params * @param {Object} params.session - The chat session * @param {string} params.prompt - The user prompt * @param {string} params.systemPrompt - System prompt (optional) * @param {Object} params.functions - Available functions (optional) * @returns {Object} The exact prompt data */ captureExactPrompt(params) { const { session, prompt, systemPrompt, functions } = params; const chatWrapper = session.chatWrapper; // Build minimal history for exact prompt const history = [{ type: 'user', text: prompt }]; if (systemPrompt) { history.unshift({ type: 'system', text: systemPrompt }); } // Generate the context state with just the current prompt const state = chatWrapper.generateContextState({ chatHistory: history, availableFunctions: functions, systemPrompt: systemPrompt }); const formattedPrompt = state.contextText.toString(); return { exactPrompt: formattedPrompt, timestamp: new Date().toISOString(), prompt, systemPrompt, functions: functions ? Object.keys(functions) : [] }; } /** * Captures the full context state (includes assistant responses) * @param {Object} params * @param {Object} params.session - The chat session * @param {Object} params.model - The loaded model * @returns {Object} The context state data */ captureContextState(params) { const { session, model } = params; // Get the actual context from the session after responses const contextState = model.detokenize(session.sequence.contextTokens, true); return { contextState, timestamp: new Date().toISOString(), tokenCount: session.sequence.contextTokens.length }; } /** * Captures the structured token representation * @param {Object} params * @param {Object} params.session - The chat session * @param {Object} params.model - The loaded model * @returns {Object} The structured token data */ captureStructured(params) { const { session, model } = params; const structured = LlamaText.fromTokens(model.tokenizer, session.sequence.contextTokens); return { structured, timestamp: new Date().toISOString(), tokenCount: session.sequence.contextTokens.length }; } /** * Captures all configured output types * @param {Object} params - Contains all possible parameters * @returns {Object} Combined captured data based on configuration */ captureAll(params) { const result = { timestamp: new Date().toISOString() }; if (this.outputTypes.includes(OutputTypes.EXACT_PROMPT)) { const exactData = this.captureExactPrompt(params); result.exactPrompt = exactData.exactPrompt; result.prompt = exactData.prompt; result.systemPrompt = exactData.systemPrompt; result.functions = exactData.functions; } if (this.outputTypes.includes(OutputTypes.CONTEXT_STATE)) { const contextData = this.captureContextState(params); result.contextState = contextData.contextState; result.contextTokenCount = contextData.tokenCount; } if (this.outputTypes.includes(OutputTypes.STRUCTURED)) { const structuredData = this.captureStructured(params); result.structured = structuredData.structured; result.structuredTokenCount = structuredData.tokenCount; } return result; } /** * Formats the captured data based on configuration * @param {Object} capturedData - Data from capture methods * @returns {string} Formatted output */ formatOutput(capturedData) { let output = `\n========== PROMPT DEBUG OUTPUT ==========\n`; output += `Timestamp: ${capturedData.timestamp}\n`; if (capturedData.prompt) { output += `Original Prompt: ${capturedData.prompt}\n`; } if (capturedData.systemPrompt) { output += `System Prompt: ${capturedData.systemPrompt.substring(0, 50)}...\n`; } if (capturedData.functions && capturedData.functions.length > 0) { output += `Functions: ${capturedData.functions.join(', ')}\n`; } if (capturedData.exactPrompt) { output += `\n=== EXACT PROMPT ===\n`; output += capturedData.exactPrompt; output += `\n`; } if (capturedData.contextState) { output += `Token Count: ${capturedData.contextTokenCount || 'N/A'}\n`; output += `\n=== CONTEXT STATE ===\n`; output += capturedData.contextState; output += `\n`; } if (capturedData.structured) { output += `\n=== STRUCTURED ===\n`; output += `Token Count: ${capturedData.structuredTokenCount || 'N/A'}\n`; output += JSON.stringify(capturedData.structured, null, 2); output += `\n`; } output += `==========================================\n`; return output; } /** * Saves data to file * @param {Object} capturedData - Data to save * @param {null} customFilename - Optional custom filename */ async saveToFile(capturedData, customFilename = null) { const content = this.formatOutput(capturedData); let filename = customFilename || this.filename; if (this.includeTimestamp) { const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const ext = path.extname(filename); const base = path.basename(filename, ext); filename = `${base}_${timestamp}${ext}`; } const filepath = path.join(this.outputDir, filename); if (this.appendMode) { await fs.appendFile(filepath, content, 'utf8'); } else { await fs.writeFile(filepath, content, 'utf8'); } console.log(`Prompt debug output written to ${filepath}`); return filepath; } /** * Debug exact prompt only - minimal params needed * @param {Object} params - session, prompt, systemPrompt (optional), functions (optional) * @param customFilename */ async debugExactPrompt(params, customFilename = null) { const oldOutputTypes = this.outputTypes; this.outputTypes = [OutputTypes.EXACT_PROMPT]; const capturedData = this.captureAll(params); const filepath = await this.saveToFile(capturedData, customFilename); this.outputTypes = oldOutputTypes; return { capturedData, filepath }; } /** * Debug context state only - needs session and model * @param {Object} params - session, model * @param customFilename */ async debugContextState(params, customFilename = null) { const oldOutputTypes = this.outputTypes; this.outputTypes = [OutputTypes.CONTEXT_STATE]; const capturedData = this.captureAll(params); const filepath = await this.saveToFile(capturedData, customFilename); this.outputTypes = oldOutputTypes; return { capturedData, filepath }; } /** * Debug structured only - needs session and model * @param {Object} params - session, model * @param customFilename */ async debugStructured(params, customFilename = null) { const oldOutputTypes = this.outputTypes; this.outputTypes = [OutputTypes.STRUCTURED]; const capturedData = this.captureAll(params); const filepath = await this.saveToFile(capturedData, customFilename); this.outputTypes = oldOutputTypes; return { capturedData, filepath }; } /** * Debug with configured output types * @param {Object} params - All parameters (session, model, prompt, etc.) * @param customFilename */ async debug(params, customFilename = null) { const capturedData = this.captureAll(params); //const filepath = await this.saveToFile(capturedData, customFilename); return { capturedData }; } /** * Log to console only * @param {Object} params - Parameters based on configured output types */ logToConsole(params) { const capturedData = this.captureAll(params); console.log(this.formatOutput(capturedData)); return capturedData; } /** * Log exact prompt to console */ logExactPrompt(params) { const capturedData = this.captureExactPrompt(params); console.log(this.formatOutput(capturedData)); return capturedData; } /** * Log context state to console */ logContextState(params) { const capturedData = this.captureContextState(params); console.log(this.formatOutput(capturedData)); return capturedData; } /** * Log structured to console */ logStructured(params) { const capturedData = this.captureStructured(params); console.log(this.formatOutput(capturedData)); return capturedData; } } /** * Quick function to debug exact prompt only */ async function debugExactPrompt(params, options = {}) { const promptDebugger = new PromptDebugger({ ...options, outputTypes: [OutputTypes.EXACT_PROMPT] }); return await promptDebugger.debug(params); } /** * Quick function to debug context state only */ async function debugContextState(params, options = {}) { const promptDebugger = new PromptDebugger({ ...options, outputTypes: [OutputTypes.CONTEXT_STATE] }); return await promptDebugger.debug(params); } /** * Quick function to debug structured only */ async function debugStructured(params, options = {}) { const promptDebugger = new PromptDebugger({ ...options, outputTypes: [OutputTypes.STRUCTURED] }); return await promptDebugger.debug(params); } /** * Quick function to debug all outputs */ async function debugAll(params, options = {}) { const promptDebugger = new PromptDebugger({ ...options, outputTypes: [OutputTypes.EXACT_PROMPT, OutputTypes.CONTEXT_STATE, OutputTypes.STRUCTURED] }); return await promptDebugger.debug(params); }