AUDIT / src /lib /agent /SuccessRateMonitor.ts
Arypulka98's picture
feat(audit): deploy full backend cluster node (part 2)
cc11e77 verified
Raw
History Blame Contribute Delete
13 kB
/**
* SuccessRateMonitor.ts
*
* Gap documentato: mancanza di Success Rate reale per funzione.
* Questo modulo traccia successi e fallimenti per ogni TaskType in localStorage
* (compatibile iPhone Safari) e fornisce API per leggere i tassi aggregati.
*
* Integrazione:
* - recordOutcome(type, "success" | "failure") → chiamare dopo emitFinalText o su errore loop
* - getSuccessRate(type) → tasso [0-1] per tipo, undefined se nessun dato
* - getReport() → tabella completa (diagnostics / dev tools)
*
* Storage: localStorage key "agente_success_rates" → JSON { [type]: { s, f, last } }
* Nessuna dipendenza esterna — puramente sincrono, non-blocking.
*/
import type { TaskType } from "../taskClassifier";
import { dispatchTask } from './AgentDispatcher'; // S681
import type { AgentTaskType } from './AgentDispatcher'; // S681
const _LS_KEY = "agente_success_rates";
const _MAX_HISTORY = 200; // max record totali per tipo (rolling window)
interface TypeStats {
s: number; // successes
f: number; // failures
last: number; // unix ms dell'ultimo evento
}
type RatesMap = Partial<Record<TaskType, TypeStats>>;
function _load(): RatesMap {
try {
const raw = localStorage.getItem(_LS_KEY);
return raw ? (JSON.parse(raw) as RatesMap) : {};
} catch { return {}; }
}
function _save(map: RatesMap): void {
try { localStorage.setItem(_LS_KEY, JSON.stringify(map)); } catch { /* non-blocking — Safari private mode */ }
}
/**
* Registra l'esito di un task completato.
* @param type TaskType classificato
* @param outcome "success" = risposta emessa e accettata; "failure" = loop terminato senza risposta
*/
export function recordOutcome(type: TaskType, outcome: "success" | "failure"): void {
const map = _load();
const entry = map[type] ?? { s: 0, f: 0, last: 0 };
// Rolling window: se s+f >= MAX_HISTORY, scala proporzionalmente.
// S645: Math.round → Math.floor — con Math.round, 99.5 → 100 (arrotonda su) e il totale
// non decresce mai (rimane >= _MAX_HISTORY per sempre). Math.floor garantisce che
// scale * n <= n-1 per valori > 0, quindi la finestra continua a scorrere correttamente.
if (entry.s + entry.f >= _MAX_HISTORY) {
const scale = (_MAX_HISTORY - 1) / _MAX_HISTORY;
entry.s = Math.floor(entry.s * scale);
entry.f = Math.floor(entry.f * scale);
}
if (outcome === "success") entry.s++;
else entry.f++;
entry.last = Date.now();
map[type] = entry;
_save(map);
}
/**
* Restituisce il tasso di successo [0–1] per il TaskType dato.
* undefined se non ci sono abbastanza dati (< 3 campioni).
*/
export function getSuccessRate(type: TaskType): number | undefined {
const map = _load();
const entry = map[type];
if (!entry || entry.s + entry.f < 3) return undefined;
return entry.s / (entry.s + entry.f);
}
/**
* Tabella completa per diagnostics.
* Restituisce righe ordinate per tasso crescente (prima i tipi più problematici).
*/
export function getReport(): Array<{ type: TaskType; rate: number; samples: number; last: Date }> {
const map = _load();
return (Object.entries(map) as [TaskType, TypeStats][])
.filter(([, e]) => e.s + e.f >= 3)
.map(([type, e]) => ({
type,
rate: e.s / (e.s + e.f),
samples: e.s + e.f,
last: new Date(e.last),
}))
.sort((a, b) => a.rate - b.rate); // problematici prima
}
/**
* Reset per testing / dev. Non chiamare in produzione.
*/
export function _resetMonitor(): void {
try { localStorage.removeItem(_LS_KEY); } catch { /* noop */ }
}
// ─── S681: Routing Accuracy Tracking ─────────────────────────────────────────
// Misura quante volte l'LLM sceglie uno dei tool suggeriti da AgentDispatcher.
// Utile per misurare l'efficacia reale di S680 (dispatch hint nel system prompt).
const _LS_ROUTING_KEY = "agente_routing_accuracy"; // S681
interface RoutingStats {
matches: number; // tool scelto ∈ suggestedTools
misses: number; // tool scelto ∉ suggestedTools
last: number; // unix ms ultimo evento
}
type RoutingMap = Partial<Record<AgentTaskType, RoutingStats>>;
function _loadRouting(): RoutingMap {
try {
const raw = localStorage.getItem(_LS_ROUTING_KEY);
return raw ? (JSON.parse(raw) as RoutingMap) : {};
} catch { return {}; }
}
function _saveRouting(map: RoutingMap): void {
try { localStorage.setItem(_LS_ROUTING_KEY, JSON.stringify(map)); } catch { /**/ }
}
/**
* S681: Registra se il tool scelto dall'LLM è in suggestedTools di AgentDispatcher.
* Chiamare da toolExecutor dopo ogni tool call riuscita. Non-blocking — mai lancia.
*/
export function recordRoutingEvent(goal: string, chosenTool: string): void {
if (!goal || !chosenTool) return;
try {
const result = dispatchTask(goal);
if (result.agentType === 'general') return;
const isMatch = result.suggestedTools.includes(chosenTool);
const map = _loadRouting();
const entry = map[result.agentType] ?? { matches: 0, misses: 0, last: 0 };
if (isMatch) entry.matches++; else entry.misses++;
entry.last = Date.now();
map[result.agentType] = entry;
_saveRouting(map);
} catch { /**/ }
}
/**
* S681: Accuratezza di routing globale [0–1].
* undefined se meno di 5 campioni totali.
*/
export function getRoutingAccuracy(): number | undefined {
const map = _loadRouting();
const totM = Object.values(map).reduce((a, e) => a + (e?.matches ?? 0), 0);
const totF = Object.values(map).reduce((a, e) => a + (e?.misses ?? 0), 0);
const tot = totM + totF;
if (tot < 5) return undefined;
return totM / tot;
}
/**
* S681: Report accuratezza per tipo agente, ordinato per accuratezza crescente.
*/
export function getRoutingReport(): Array<{
type: AgentTaskType; accuracy: number; samples: number; last: Date;
}> {
const map = _loadRouting();
return (Object.entries(map) as [AgentTaskType, RoutingStats][])
.filter(([, e]) => e.matches + e.misses >= 3)
.map(([type, e]) => ({
type, accuracy: e.matches / (e.matches + e.misses),
samples: e.matches + e.misses, last: new Date(e.last),
}))
.sort((a, b) => a.accuracy - b.accuracy);
}
/** Reset routing stats — dev/test only. */
export function _resetRoutingMonitor(): void {
try { localStorage.removeItem(_LS_ROUTING_KEY); } catch { /**/ }
}
// ─── ARL Signal Tracker ──────────────────────────────────────────────────────
// Traccia coverage e round_to_converge per ogni tool call di ricerca.
// Permette al planner di imparare nel tempo quali query traggono beneficio
// da deep_research vs web_search.
//
// Storage: localStorage "agente_arl_signals" — rolling window 200 entries.
// Pattern matching: top-3 token significativi (>4 chars, no stop-words).
// Confidence: normalizzata su campioni (≥10 = confidence 1.0).
const _LS_ARL_KEY = "agente_arl_signals";
const _ARL_MAX_HIST = 200;
const _ARL_STOP = new Set([
"the","and","for","with","that","this","from","into","have","are","was","were",
"che","del","della","per","con","una","uno","gli","dei","nel","nelle","alla",
"sulle","degli","sono","come","quando","dove","anche","quindi","essere",
]);
interface ArlSignalEntry {
pattern: string; // sorted top-3 token, joined "_"
tool: "deep_research" | "web_search"; // tool usato
coverage: number; // goal coverage [0–1]
rounds: number; // round ARL (1–3)
ts: number; // unix ms
}
function _arlLoad(): ArlSignalEntry[] {
try {
const raw = localStorage.getItem(_LS_ARL_KEY);
return raw ? (JSON.parse(raw) as ArlSignalEntry[]) : [];
} catch { return []; }
}
function _arlSave(entries: ArlSignalEntry[]): void {
try { localStorage.setItem(_LS_ARL_KEY, JSON.stringify(entries)); } catch { /**/ }
}
function _arlPattern(goal: string): string[] {
return goal.toLowerCase()
.split(/\W+/)
.filter(t => t.length > 4 && !_ARL_STOP.has(t))
.slice(0, 3)
.sort();
}
/**
* Registra il segnale ARL per una ricerca completata.
* Chiamare subito dopo l'esecuzione di deep_research o web_search.
*
* @param goal Goal originale della ricerca
* @param tool Tool usato: "deep_research" | "web_search"
* @param coverage Goal coverage raggiunta [0–1] (0 se non disponibile)
* @param roundsToConverge Round ARL necessari (1 per web_search singolo)
*/
export function recordArlSignal(
goal: string,
tool: "deep_research" | "web_search",
coverage: number,
roundsToConverge: number,
): void {
if (!goal?.trim()) return;
try {
const pattern = _arlPattern(goal).join("_");
const entries = _arlLoad();
// Rolling window
if (entries.length >= _ARL_MAX_HIST) {
entries.splice(0, entries.length - _ARL_MAX_HIST + 1);
}
entries.push({ pattern, tool, coverage: Math.max(0, Math.min(1, coverage)), rounds: roundsToConverge, ts: Date.now() });
_arlSave(entries);
} catch { /**/ }
}
/**
* Restituisce un insight sul tool migliore per questo tipo di query,
* basandosi sulla storia ARL accumulata.
*
* @returns { preferDeepResearch, confidence } oppure null se dati insufficienti
*/
export function getArlInsight(goal: string): { preferDeepResearch: boolean; confidence: number } | null {
if (!goal?.trim()) return null;
try {
const goalPat = _arlPattern(goal);
if (goalPat.length === 0) return null;
const entries = _arlLoad();
// Match: almeno 2 token in comune con il pattern del goal
const matching = entries.filter(e => {
const ePat = e.pattern.split("_");
return goalPat.filter(t => ePat.includes(t)).length >= 2;
});
if (matching.length < 3) return null; // troppo pochi campioni
const byTool = (t: "deep_research" | "web_search") =>
matching.filter(e => e.tool === t);
const drEntries = byTool("deep_research");
const wsEntries = byTool("web_search");
// Serve almeno un campione per entrambi i tool per confrontare
if (drEntries.length === 0 && wsEntries.length === 0) return null;
const avg = (arr: ArlSignalEntry[]) =>
arr.length === 0 ? 0 : arr.reduce((s, e) => s + e.coverage, 0) / arr.length;
const drAvg = avg(drEntries);
const wsAvg = avg(wsEntries);
// Differenza minima significativa: 0.10 (evita rumore)
if (Math.abs(drAvg - wsAvg) < 0.10 && drEntries.length > 0 && wsEntries.length > 0) {
return null; // nessuna preferenza chiara
}
const samples = matching.length;
const confidence = Math.min(1, samples / 10); // 10 campioni = confidence piena
// Se un solo tool ha dati, preferisci quello con coverage > 0.5
if (drEntries.length === 0) return { preferDeepResearch: false, confidence };
if (wsEntries.length === 0) return { preferDeepResearch: drAvg >= 0.50, confidence };
return { preferDeepResearch: drAvg > wsAvg, confidence };
} catch { return null; }
}
/**
* Report completo dei segnali ARL — per diagnostics / dev tools.
* Raggruppa per pattern e calcola medie per tool.
*/
export function getArlReport(): Array<{
pattern: string;
deepResearchAvgCov: number;
webSearchAvgCov: number;
preferredTool: "deep_research" | "web_search" | "tie";
samples: number;
avgRoundsToConv: number;
}> {
try {
const entries = _arlLoad();
const byPat = new Map<string, ArlSignalEntry[]>();
for (const e of entries) {
if (!byPat.has(e.pattern)) byPat.set(e.pattern, []);
byPat.get(e.pattern)!.push(e);
}
const avg = (arr: ArlSignalEntry[], tool: "deep_research" | "web_search") => {
const t = arr.filter(e => e.tool === tool);
return t.length === 0 ? 0 : t.reduce((s, e) => s + e.coverage, 0) / t.length;
};
return [...byPat.entries()]
.map(([pattern, ents]) => {
const drAvg = avg(ents, "deep_research");
const wsAvg = avg(ents, "web_search");
const preferred: "deep_research" | "web_search" | "tie" =
Math.abs(drAvg - wsAvg) < 0.10 ? "tie"
: drAvg > wsAvg ? "deep_research" : "web_search";
return {
pattern,
deepResearchAvgCov: drAvg,
webSearchAvgCov: wsAvg,
preferredTool: preferred,
samples: ents.length,
avgRoundsToConv: ents.reduce((s, e) => s + e.rounds, 0) / ents.length,
};
})
.sort((a, b) => b.samples - a.samples);
} catch { return []; }
}
/** Reset ARL signals — dev/test only. */
export function _resetArlSignals(): void {
try { localStorage.removeItem(_LS_ARL_KEY); } catch { /**/ }
}