/** * ZDR Metadata Tracker * * Always-on, privacy-first analytics for G0DM0D3. * Records EVERYTHING about requests — except the actual content. * * What IS tracked: * - Timestamps, endpoints, modes, tiers * - Models used, winner model, scores, latencies * - Pipeline flags (godmode, autotune, parseltongue, stm) * - AutoTune detected context type + confidence * - Per-model success/failure, response lengths, durations * - Error types (categorized, never raw error messages) * * What is NEVER tracked: * - Message content, system prompts, responses * - API keys, auth tokens, IP addresses * - Any PII whatsoever * * Storage: in-memory ring buffer, resets on restart. */ import { randomUUID } from 'crypto' // ── Types ──────────────────────────────────────────────────────────── export interface MetadataEvent { id: string timestamp: number // Request shape endpoint: string mode: 'standard' | 'ultraplinian' tier?: string // fast | standard | full (ultraplinian only) stream: boolean // Pipeline config (what was enabled, not what it produced) pipeline: { godmode: boolean autotune: boolean parseltongue: boolean stm_modules: string[] strategy?: string } // AutoTune metadata (context detection only, no content) autotune?: { detected_context: string confidence: number } // Model metadata model?: string // single-model mode models_queried?: number // ultraplinian models_succeeded?: number // Per-model results (no content, just numbers) model_results?: Array<{ model: string score: number duration_ms: number success: boolean content_length: number error_type?: string // categorized: 'timeout' | 'rate_limit' | 'auth' | 'model_error' | 'empty' | 'unknown' }> // Winner metadata winner?: { model: string score: number duration_ms: number content_length: number } // Response metadata total_duration_ms: number response_length: number // Liquid Response metadata (streaming upgrades) liquid?: { upgrades: number // how many leader changes happened first_response_ms: number // time to first leader } } // ── In-Memory Ring Buffer ──────────────────────────────────────────── let events: MetadataEvent[] = [] const MAX_EVENTS = 50000 // ~50k events before eviction // ── Recording ──────────────────────────────────────────────────────── export function recordEvent(event: Omit): string { const id = randomUUID() const record: MetadataEvent = { ...event, id, timestamp: Date.now(), } events.push(record) // Evict oldest if over cap if (events.length > MAX_EVENTS) { events = events.slice(events.length - MAX_EVENTS) } return id } // ── Error Categorization ───────────────────────────────────────────── export function categorizeError(error?: string): string | undefined { if (!error) return undefined const lower = error.toLowerCase() if (lower.includes('timeout') || lower.includes('abort')) return 'timeout' if (lower.includes('rate') || lower.includes('429')) return 'rate_limit' if (lower.includes('auth') || lower.includes('401') || lower.includes('403')) return 'auth' if (lower.includes('empty')) return 'empty' if (lower.includes('race ended')) return 'early_exit' if (lower.includes('api error') || lower.includes('http')) return 'model_error' return 'unknown' } // ── Query: Raw Events ──────────────────────────────────────────────── export function getEvents(opts?: { limit?: number offset?: number since?: number // unix ms timestamp mode?: 'standard' | 'ultraplinian' }): { events: MetadataEvent[]; total: number } { let filtered = events if (opts?.since) { filtered = filtered.filter(e => e.timestamp >= opts.since!) } if (opts?.mode) { filtered = filtered.filter(e => e.mode === opts.mode) } const total = filtered.length const offset = opts?.offset ?? 0 const limit = opts?.limit ?? 100 return { events: filtered.slice(offset, offset + limit), total, } } // ── Query: Aggregated Stats ────────────────────────────────────────── export interface MetadataStats { total_requests: number since: number | null // oldest event timestamp uptime_ms: number // Breakdown by mode by_mode: Record by_tier: Record by_endpoint: Record // Model usage models: { total_queries: number // sum of all individual model calls unique_models: number by_model: Record } // Pipeline usage pipeline: { godmode_rate: number // % of requests with godmode enabled autotune_rate: number parseltongue_rate: number stm_usage: Record // module -> count strategy_usage: Record } // Context detection (from autotune) contexts: Record // Latency latency: { avg_ms: number p50_ms: number p95_ms: number p99_ms: number } // Response sizes response_lengths: { avg: number p50: number p95: number } // Streaming streaming: { stream_rate: number // % of requests that used streaming avg_upgrades: number // avg leader changes per stream avg_first_response_ms: number // avg time to first leader } // Errors errors: { total_model_errors: number by_type: Record model_success_rate: number // across all individual model queries } } export function getStats(): MetadataStats { const total = events.length if (total === 0) { return { total_requests: 0, since: null, uptime_ms: 0, by_mode: {}, by_tier: {}, by_endpoint: {}, models: { total_queries: 0, unique_models: 0, by_model: {} }, pipeline: { godmode_rate: 0, autotune_rate: 0, parseltongue_rate: 0, stm_usage: {}, strategy_usage: {} }, contexts: {}, latency: { avg_ms: 0, p50_ms: 0, p95_ms: 0, p99_ms: 0 }, response_lengths: { avg: 0, p50: 0, p95: 0 }, streaming: { stream_rate: 0, avg_upgrades: 0, avg_first_response_ms: 0 }, errors: { total_model_errors: 0, by_type: {}, model_success_rate: 0 }, } } // Breakdowns const byMode: Record = {} const byTier: Record = {} const byEndpoint: Record = {} const contexts: Record = {} const stmUsage: Record = {} const strategyUsage: Record = {} const errorTypes: Record = {} // Model tracking const modelData: Record = {} // Accumulators let godmodeCount = 0 let autotuneCount = 0 let parseltongueCount = 0 let totalModelQueries = 0 let totalModelErrors = 0 let totalModelSuccesses = 0 const durations: number[] = [] const responseLengths: number[] = [] let streamCount = 0 let upgradeSum = 0 let firstResponseSum = 0 let streamWithLiquidCount = 0 for (const e of events) { byMode[e.mode] = (byMode[e.mode] || 0) + 1 if (e.tier) byTier[e.tier] = (byTier[e.tier] || 0) + 1 byEndpoint[e.endpoint] = (byEndpoint[e.endpoint] || 0) + 1 if (e.pipeline.godmode) godmodeCount++ if (e.pipeline.autotune) autotuneCount++ if (e.pipeline.parseltongue) parseltongueCount++ if (e.pipeline.strategy) { strategyUsage[e.pipeline.strategy] = (strategyUsage[e.pipeline.strategy] || 0) + 1 } for (const mod of e.pipeline.stm_modules) { stmUsage[mod] = (stmUsage[mod] || 0) + 1 } if (e.autotune?.detected_context) { const ctx = e.autotune.detected_context contexts[ctx] = (contexts[ctx] || 0) + 1 } durations.push(e.total_duration_ms) responseLengths.push(e.response_length) if (e.stream) streamCount++ if (e.liquid) { upgradeSum += e.liquid.upgrades firstResponseSum += e.liquid.first_response_ms streamWithLiquidCount++ } // Per-model results if (e.model_results) { for (const mr of e.model_results) { totalModelQueries++ if (!modelData[mr.model]) { modelData[mr.model] = { queries: 0, wins: 0, scores: [], durations: [], successes: 0 } } const md = modelData[mr.model] md.queries++ if (mr.success) { md.successes++ totalModelSuccesses++ md.scores.push(mr.score) md.durations.push(mr.duration_ms) } else { totalModelErrors++ if (mr.error_type) { errorTypes[mr.error_type] = (errorTypes[mr.error_type] || 0) + 1 } } } } // Winner tracking if (e.winner) { if (!modelData[e.winner.model]) { modelData[e.winner.model] = { queries: 0, wins: 0, scores: [], durations: [], successes: 0 } } modelData[e.winner.model].wins++ } } // Build per-model stats const byModel: Record = {} for (const [model, data] of Object.entries(modelData)) { byModel[model] = { queries: data.queries, wins: data.wins, avg_score: data.scores.length > 0 ? Math.round(data.scores.reduce((a, b) => a + b, 0) / data.scores.length) : 0, avg_duration_ms: data.durations.length > 0 ? Math.round(data.durations.reduce((a, b) => a + b, 0) / data.durations.length) : 0, success_rate: data.queries > 0 ? Math.round((data.successes / data.queries) * 100) / 100 : 0, } } return { total_requests: total, since: events[0].timestamp, uptime_ms: Date.now() - events[0].timestamp, by_mode: byMode, by_tier: byTier, by_endpoint: byEndpoint, models: { total_queries: totalModelQueries, unique_models: Object.keys(modelData).length, by_model: byModel, }, pipeline: { godmode_rate: Math.round((godmodeCount / total) * 100) / 100, autotune_rate: Math.round((autotuneCount / total) * 100) / 100, parseltongue_rate: Math.round((parseltongueCount / total) * 100) / 100, stm_usage: stmUsage, strategy_usage: strategyUsage, }, contexts, latency: { avg_ms: Math.round(durations.reduce((a, b) => a + b, 0) / durations.length), p50_ms: percentile(durations, 50), p95_ms: percentile(durations, 95), p99_ms: percentile(durations, 99), }, response_lengths: { avg: Math.round(responseLengths.reduce((a, b) => a + b, 0) / responseLengths.length), p50: percentile(responseLengths, 50), p95: percentile(responseLengths, 95), }, streaming: { stream_rate: Math.round((streamCount / total) * 100) / 100, avg_upgrades: streamWithLiquidCount > 0 ? Math.round((upgradeSum / streamWithLiquidCount) * 100) / 100 : 0, avg_first_response_ms: streamWithLiquidCount > 0 ? Math.round(firstResponseSum / streamWithLiquidCount) : 0, }, errors: { total_model_errors: totalModelErrors, by_type: errorTypes, model_success_rate: totalModelQueries > 0 ? Math.round((totalModelSuccesses / totalModelQueries) * 100) / 100 : 0, }, } } // ── Helpers ────────────────────────────────────────────────────────── function percentile(arr: number[], p: number): number { if (arr.length === 0) return 0 const sorted = [...arr].sort((a, b) => a - b) const idx = Math.ceil((p / 100) * sorted.length) - 1 return sorted[Math.max(0, idx)] }