/** * SuccessRateMonitor.ts * * Gap documentato: mancanza di Success Rate reale per funzione. * Questo modulo traccia successi e fallimenti per ogni TaskType in localStorage * (compatibile iPhone Safari) e fornisce API per leggere i tassi aggregati. * * Integrazione: * - recordOutcome(type, "success" | "failure") → chiamare dopo emitFinalText o su errore loop * - getSuccessRate(type) → tasso [0-1] per tipo, undefined se nessun dato * - getReport() → tabella completa (diagnostics / dev tools) * * Storage: localStorage key "agente_success_rates" → JSON { [type]: { s, f, last } } * Nessuna dipendenza esterna — puramente sincrono, non-blocking. */ import type { TaskType } from "../taskClassifier"; import { dispatchTask } from './AgentDispatcher'; // S681 import type { AgentTaskType } from './AgentDispatcher'; // S681 const _LS_KEY = "agente_success_rates"; const _MAX_HISTORY = 200; // max record totali per tipo (rolling window) interface TypeStats { s: number; // successes f: number; // failures last: number; // unix ms dell'ultimo evento } type RatesMap = Partial>; function _load(): RatesMap { try { const raw = localStorage.getItem(_LS_KEY); return raw ? (JSON.parse(raw) as RatesMap) : {}; } catch { return {}; } } function _save(map: RatesMap): void { try { localStorage.setItem(_LS_KEY, JSON.stringify(map)); } catch { /* non-blocking — Safari private mode */ } } /** * Registra l'esito di un task completato. * @param type TaskType classificato * @param outcome "success" = risposta emessa e accettata; "failure" = loop terminato senza risposta */ export function recordOutcome(type: TaskType, outcome: "success" | "failure"): void { const map = _load(); const entry = map[type] ?? { s: 0, f: 0, last: 0 }; // Rolling window: se s+f >= MAX_HISTORY, scala proporzionalmente. // S645: Math.round → Math.floor — con Math.round, 99.5 → 100 (arrotonda su) e il totale // non decresce mai (rimane >= _MAX_HISTORY per sempre). Math.floor garantisce che // scale * n <= n-1 per valori > 0, quindi la finestra continua a scorrere correttamente. if (entry.s + entry.f >= _MAX_HISTORY) { const scale = (_MAX_HISTORY - 1) / _MAX_HISTORY; entry.s = Math.floor(entry.s * scale); entry.f = Math.floor(entry.f * scale); } if (outcome === "success") entry.s++; else entry.f++; entry.last = Date.now(); map[type] = entry; _save(map); } /** * Restituisce il tasso di successo [0–1] per il TaskType dato. * undefined se non ci sono abbastanza dati (< 3 campioni). */ export function getSuccessRate(type: TaskType): number | undefined { const map = _load(); const entry = map[type]; if (!entry || entry.s + entry.f < 3) return undefined; return entry.s / (entry.s + entry.f); } /** * Tabella completa per diagnostics. * Restituisce righe ordinate per tasso crescente (prima i tipi più problematici). */ export function getReport(): Array<{ type: TaskType; rate: number; samples: number; last: Date }> { const map = _load(); return (Object.entries(map) as [TaskType, TypeStats][]) .filter(([, e]) => e.s + e.f >= 3) .map(([type, e]) => ({ type, rate: e.s / (e.s + e.f), samples: e.s + e.f, last: new Date(e.last), })) .sort((a, b) => a.rate - b.rate); // problematici prima } /** * Reset per testing / dev. Non chiamare in produzione. */ export function _resetMonitor(): void { try { localStorage.removeItem(_LS_KEY); } catch { /* noop */ } } // ─── S681: Routing Accuracy Tracking ───────────────────────────────────────── // Misura quante volte l'LLM sceglie uno dei tool suggeriti da AgentDispatcher. // Utile per misurare l'efficacia reale di S680 (dispatch hint nel system prompt). const _LS_ROUTING_KEY = "agente_routing_accuracy"; // S681 interface RoutingStats { matches: number; // tool scelto ∈ suggestedTools misses: number; // tool scelto ∉ suggestedTools last: number; // unix ms ultimo evento } type RoutingMap = Partial>; function _loadRouting(): RoutingMap { try { const raw = localStorage.getItem(_LS_ROUTING_KEY); return raw ? (JSON.parse(raw) as RoutingMap) : {}; } catch { return {}; } } function _saveRouting(map: RoutingMap): void { try { localStorage.setItem(_LS_ROUTING_KEY, JSON.stringify(map)); } catch { /**/ } } /** * S681: Registra se il tool scelto dall'LLM è in suggestedTools di AgentDispatcher. * Chiamare da toolExecutor dopo ogni tool call riuscita. Non-blocking — mai lancia. */ export function recordRoutingEvent(goal: string, chosenTool: string): void { if (!goal || !chosenTool) return; try { const result = dispatchTask(goal); if (result.agentType === 'general') return; const isMatch = result.suggestedTools.includes(chosenTool); const map = _loadRouting(); const entry = map[result.agentType] ?? { matches: 0, misses: 0, last: 0 }; if (isMatch) entry.matches++; else entry.misses++; entry.last = Date.now(); map[result.agentType] = entry; _saveRouting(map); } catch { /**/ } } /** * S681: Accuratezza di routing globale [0–1]. * undefined se meno di 5 campioni totali. */ export function getRoutingAccuracy(): number | undefined { const map = _loadRouting(); const totM = Object.values(map).reduce((a, e) => a + (e?.matches ?? 0), 0); const totF = Object.values(map).reduce((a, e) => a + (e?.misses ?? 0), 0); const tot = totM + totF; if (tot < 5) return undefined; return totM / tot; } /** * S681: Report accuratezza per tipo agente, ordinato per accuratezza crescente. */ export function getRoutingReport(): Array<{ type: AgentTaskType; accuracy: number; samples: number; last: Date; }> { const map = _loadRouting(); return (Object.entries(map) as [AgentTaskType, RoutingStats][]) .filter(([, e]) => e.matches + e.misses >= 3) .map(([type, e]) => ({ type, accuracy: e.matches / (e.matches + e.misses), samples: e.matches + e.misses, last: new Date(e.last), })) .sort((a, b) => a.accuracy - b.accuracy); } /** Reset routing stats — dev/test only. */ export function _resetRoutingMonitor(): void { try { localStorage.removeItem(_LS_ROUTING_KEY); } catch { /**/ } } // ─── ARL Signal Tracker ────────────────────────────────────────────────────── // Traccia coverage e round_to_converge per ogni tool call di ricerca. // Permette al planner di imparare nel tempo quali query traggono beneficio // da deep_research vs web_search. // // Storage: localStorage "agente_arl_signals" — rolling window 200 entries. // Pattern matching: top-3 token significativi (>4 chars, no stop-words). // Confidence: normalizzata su campioni (≥10 = confidence 1.0). const _LS_ARL_KEY = "agente_arl_signals"; const _ARL_MAX_HIST = 200; const _ARL_STOP = new Set([ "the","and","for","with","that","this","from","into","have","are","was","were", "che","del","della","per","con","una","uno","gli","dei","nel","nelle","alla", "sulle","degli","sono","come","quando","dove","anche","quindi","essere", ]); interface ArlSignalEntry { pattern: string; // sorted top-3 token, joined "_" tool: "deep_research" | "web_search"; // tool usato coverage: number; // goal coverage [0–1] rounds: number; // round ARL (1–3) ts: number; // unix ms } function _arlLoad(): ArlSignalEntry[] { try { const raw = localStorage.getItem(_LS_ARL_KEY); return raw ? (JSON.parse(raw) as ArlSignalEntry[]) : []; } catch { return []; } } function _arlSave(entries: ArlSignalEntry[]): void { try { localStorage.setItem(_LS_ARL_KEY, JSON.stringify(entries)); } catch { /**/ } } function _arlPattern(goal: string): string[] { return goal.toLowerCase() .split(/\W+/) .filter(t => t.length > 4 && !_ARL_STOP.has(t)) .slice(0, 3) .sort(); } /** * Registra il segnale ARL per una ricerca completata. * Chiamare subito dopo l'esecuzione di deep_research o web_search. * * @param goal Goal originale della ricerca * @param tool Tool usato: "deep_research" | "web_search" * @param coverage Goal coverage raggiunta [0–1] (0 se non disponibile) * @param roundsToConverge Round ARL necessari (1 per web_search singolo) */ export function recordArlSignal( goal: string, tool: "deep_research" | "web_search", coverage: number, roundsToConverge: number, ): void { if (!goal?.trim()) return; try { const pattern = _arlPattern(goal).join("_"); const entries = _arlLoad(); // Rolling window if (entries.length >= _ARL_MAX_HIST) { entries.splice(0, entries.length - _ARL_MAX_HIST + 1); } entries.push({ pattern, tool, coverage: Math.max(0, Math.min(1, coverage)), rounds: roundsToConverge, ts: Date.now() }); _arlSave(entries); } catch { /**/ } } /** * Restituisce un insight sul tool migliore per questo tipo di query, * basandosi sulla storia ARL accumulata. * * @returns { preferDeepResearch, confidence } oppure null se dati insufficienti */ export function getArlInsight(goal: string): { preferDeepResearch: boolean; confidence: number } | null { if (!goal?.trim()) return null; try { const goalPat = _arlPattern(goal); if (goalPat.length === 0) return null; const entries = _arlLoad(); // Match: almeno 2 token in comune con il pattern del goal const matching = entries.filter(e => { const ePat = e.pattern.split("_"); return goalPat.filter(t => ePat.includes(t)).length >= 2; }); if (matching.length < 3) return null; // troppo pochi campioni const byTool = (t: "deep_research" | "web_search") => matching.filter(e => e.tool === t); const drEntries = byTool("deep_research"); const wsEntries = byTool("web_search"); // Serve almeno un campione per entrambi i tool per confrontare if (drEntries.length === 0 && wsEntries.length === 0) return null; const avg = (arr: ArlSignalEntry[]) => arr.length === 0 ? 0 : arr.reduce((s, e) => s + e.coverage, 0) / arr.length; const drAvg = avg(drEntries); const wsAvg = avg(wsEntries); // Differenza minima significativa: 0.10 (evita rumore) if (Math.abs(drAvg - wsAvg) < 0.10 && drEntries.length > 0 && wsEntries.length > 0) { return null; // nessuna preferenza chiara } const samples = matching.length; const confidence = Math.min(1, samples / 10); // 10 campioni = confidence piena // Se un solo tool ha dati, preferisci quello con coverage > 0.5 if (drEntries.length === 0) return { preferDeepResearch: false, confidence }; if (wsEntries.length === 0) return { preferDeepResearch: drAvg >= 0.50, confidence }; return { preferDeepResearch: drAvg > wsAvg, confidence }; } catch { return null; } } /** * Report completo dei segnali ARL — per diagnostics / dev tools. * Raggruppa per pattern e calcola medie per tool. */ export function getArlReport(): Array<{ pattern: string; deepResearchAvgCov: number; webSearchAvgCov: number; preferredTool: "deep_research" | "web_search" | "tie"; samples: number; avgRoundsToConv: number; }> { try { const entries = _arlLoad(); const byPat = new Map(); for (const e of entries) { if (!byPat.has(e.pattern)) byPat.set(e.pattern, []); byPat.get(e.pattern)!.push(e); } const avg = (arr: ArlSignalEntry[], tool: "deep_research" | "web_search") => { const t = arr.filter(e => e.tool === tool); return t.length === 0 ? 0 : t.reduce((s, e) => s + e.coverage, 0) / t.length; }; return [...byPat.entries()] .map(([pattern, ents]) => { const drAvg = avg(ents, "deep_research"); const wsAvg = avg(ents, "web_search"); const preferred: "deep_research" | "web_search" | "tie" = Math.abs(drAvg - wsAvg) < 0.10 ? "tie" : drAvg > wsAvg ? "deep_research" : "web_search"; return { pattern, deepResearchAvgCov: drAvg, webSearchAvgCov: wsAvg, preferredTool: preferred, samples: ents.length, avgRoundsToConv: ents.reduce((s, e) => s + e.rounds, 0) / ents.length, }; }) .sort((a, b) => b.samples - a.samples); } catch { return []; } } /** Reset ARL signals — dev/test only. */ export function _resetArlSignals(): void { try { localStorage.removeItem(_LS_ARL_KEY); } catch { /**/ } }