Spaces:
Sleeping
Sleeping
| /** | |
| * Agent Optimizer — token efficiency analysis and fleet benchmarking. | |
| * | |
| * Queries token_usage, tasks, mcp_call_log, and agent_trust_scores | |
| * to produce actionable recommendations for reducing agent cost and latency. | |
| */ | |
| import { getDatabase } from '@/lib/db' | |
| export interface TokenEfficiency { | |
| agentName: string | |
| totalInputTokens: number | |
| totalOutputTokens: number | |
| totalTokens: number | |
| totalCostUsd: number | |
| sessionsCount: number | |
| avgTokensPerSession: number | |
| avgCostPerSession: number | |
| } | |
| export interface ToolPatterns { | |
| agentName: string | |
| totalCalls: number | |
| uniqueTools: number | |
| topTools: Array<{ toolName: string; count: number; successRate: number }> | |
| failureRate: number | |
| avgDurationMs: number | |
| } | |
| export interface FleetBenchmark { | |
| agentName: string | |
| tokensPerTask: number | |
| costPerTask: number | |
| tasksCompleted: number | |
| trustScore: number | |
| toolCallsPerTask: number | |
| } | |
| export interface Recommendation { | |
| category: 'cost' | 'efficiency' | 'reliability' | 'trust' | |
| severity: 'info' | 'warning' | 'critical' | |
| message: string | |
| metric?: number | |
| } | |
| export function analyzeTokenEfficiency( | |
| agentName: string, | |
| hours: number = 24, | |
| workspaceId: number = 1, | |
| ): TokenEfficiency { | |
| const db = getDatabase() | |
| const since = Math.floor(Date.now() / 1000) - hours * 3600 | |
| const row = db.prepare(` | |
| SELECT | |
| COUNT(*) as sessions, | |
| COALESCE(SUM(input_tokens), 0) as input_tokens, | |
| COALESCE(SUM(output_tokens), 0) as output_tokens, | |
| COALESCE(SUM(cost_usd), 0) as total_cost | |
| FROM token_usage | |
| WHERE agent_name = ? AND created_at > ? | |
| `).get(agentName, since) as any | |
| const sessions = row?.sessions ?? 0 | |
| const inputTokens = row?.input_tokens ?? 0 | |
| const outputTokens = row?.output_tokens ?? 0 | |
| const totalTokens = inputTokens + outputTokens | |
| return { | |
| agentName, | |
| totalInputTokens: inputTokens, | |
| totalOutputTokens: outputTokens, | |
| totalTokens, | |
| totalCostUsd: Math.round((row?.total_cost ?? 0) * 10000) / 10000, | |
| sessionsCount: sessions, | |
| avgTokensPerSession: sessions > 0 ? Math.round(totalTokens / sessions) : 0, | |
| avgCostPerSession: sessions > 0 ? Math.round((row?.total_cost ?? 0) / sessions * 10000) / 10000 : 0, | |
| } | |
| } | |
| export function analyzeToolPatterns( | |
| agentName: string, | |
| hours: number = 24, | |
| workspaceId: number = 1, | |
| ): ToolPatterns { | |
| const db = getDatabase() | |
| const since = Math.floor(Date.now() / 1000) - hours * 3600 | |
| const totals = db.prepare(` | |
| SELECT | |
| COUNT(*) as total, | |
| COUNT(DISTINCT tool_name) as unique_tools, | |
| SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failures, | |
| AVG(duration_ms) as avg_duration | |
| FROM mcp_call_log | |
| WHERE agent_name = ? AND workspace_id = ? AND created_at > ? | |
| `).get(agentName, workspaceId, since) as any | |
| const topTools = db.prepare(` | |
| SELECT | |
| tool_name, | |
| COUNT(*) as count, | |
| SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) as success_rate | |
| FROM mcp_call_log | |
| WHERE agent_name = ? AND workspace_id = ? AND created_at > ? | |
| GROUP BY tool_name | |
| ORDER BY count DESC | |
| LIMIT 10 | |
| `).all(agentName, workspaceId, since) as any[] | |
| const total = totals?.total ?? 0 | |
| return { | |
| agentName, | |
| totalCalls: total, | |
| uniqueTools: totals?.unique_tools ?? 0, | |
| topTools: topTools.map((t: any) => ({ | |
| toolName: t.tool_name ?? 'unknown', | |
| count: t.count, | |
| successRate: Math.round(t.success_rate * 100) / 100, | |
| })), | |
| failureRate: total > 0 ? Math.round(((totals?.failures ?? 0) / total) * 10000) / 100 : 0, | |
| avgDurationMs: Math.round(totals?.avg_duration ?? 0), | |
| } | |
| } | |
| export function getFleetBenchmarks(workspaceId: number = 1): FleetBenchmark[] { | |
| const db = getDatabase() | |
| const rows = db.prepare(` | |
| SELECT | |
| a.agent_name, | |
| COALESCE(t.tokens_per_task, 0) as tokens_per_task, | |
| COALESCE(t.cost_per_task, 0) as cost_per_task, | |
| COALESCE(t.tasks_completed, 0) as tasks_completed, | |
| COALESCE(ats.trust_score, 1.0) as trust_score, | |
| COALESCE(m.tool_calls_per_task, 0) as tool_calls_per_task | |
| FROM (SELECT DISTINCT agent_name FROM agent_trust_scores WHERE workspace_id = ?) a | |
| LEFT JOIN ( | |
| SELECT | |
| agent_name, | |
| CASE WHEN COUNT(DISTINCT task_id) > 0 | |
| THEN SUM(input_tokens + output_tokens) * 1.0 / COUNT(DISTINCT task_id) | |
| ELSE 0 | |
| END as tokens_per_task, | |
| CASE WHEN COUNT(DISTINCT task_id) > 0 | |
| THEN SUM(COALESCE(cost_usd, 0)) * 1.0 / COUNT(DISTINCT task_id) | |
| ELSE 0 | |
| END as cost_per_task, | |
| COUNT(DISTINCT task_id) as tasks_completed | |
| FROM token_usage | |
| WHERE task_id IS NOT NULL | |
| GROUP BY agent_name | |
| ) t ON t.agent_name = a.agent_name | |
| LEFT JOIN agent_trust_scores ats ON ats.agent_name = a.agent_name AND ats.workspace_id = ? | |
| LEFT JOIN ( | |
| SELECT | |
| agent_name, | |
| COUNT(*) * 1.0 / NULLIF( | |
| (SELECT COUNT(DISTINCT task_id) FROM token_usage tu2 WHERE tu2.agent_name = mcl.agent_name AND tu2.task_id IS NOT NULL), | |
| 0 | |
| ) as tool_calls_per_task | |
| FROM mcp_call_log mcl | |
| WHERE workspace_id = ? | |
| GROUP BY agent_name | |
| ) m ON m.agent_name = a.agent_name | |
| `).all(workspaceId, workspaceId, workspaceId) as any[] | |
| return rows.map((r: any) => ({ | |
| agentName: r.agent_name, | |
| tokensPerTask: Math.round(r.tokens_per_task), | |
| costPerTask: Math.round(r.cost_per_task * 10000) / 10000, | |
| tasksCompleted: r.tasks_completed, | |
| trustScore: Math.round(r.trust_score * 100) / 100, | |
| toolCallsPerTask: Math.round(r.tool_calls_per_task * 10) / 10, | |
| })) | |
| } | |
| export function generateRecommendations( | |
| agentName: string, | |
| workspaceId: number = 1, | |
| ): Recommendation[] { | |
| const recommendations: Recommendation[] = [] | |
| const db = getDatabase() | |
| // Check trust score | |
| const trust = db.prepare(` | |
| SELECT * FROM agent_trust_scores WHERE agent_name = ? AND workspace_id = ? | |
| `).get(agentName, workspaceId) as any | |
| if (trust) { | |
| if (trust.trust_score < 0.5) { | |
| recommendations.push({ | |
| category: 'trust', | |
| severity: 'critical', | |
| message: `Trust score is critically low (${trust.trust_score.toFixed(2)}). Review security events.`, | |
| metric: trust.trust_score, | |
| }) | |
| } else if (trust.trust_score < 0.8) { | |
| recommendations.push({ | |
| category: 'trust', | |
| severity: 'warning', | |
| message: `Trust score is below threshold (${trust.trust_score.toFixed(2)}). Monitor for anomalies.`, | |
| metric: trust.trust_score, | |
| }) | |
| } | |
| if (trust.injection_attempts > 0) { | |
| recommendations.push({ | |
| category: 'trust', | |
| severity: 'critical', | |
| message: `${trust.injection_attempts} injection attempt(s) detected. Investigate immediately.`, | |
| metric: trust.injection_attempts, | |
| }) | |
| } | |
| } | |
| // Check tool failure rate | |
| const toolStats = db.prepare(` | |
| SELECT | |
| COUNT(*) as total, | |
| SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failures | |
| FROM mcp_call_log | |
| WHERE agent_name = ? AND workspace_id = ? AND created_at > ? | |
| `).get(agentName, workspaceId, Math.floor(Date.now() / 1000) - 86400) as any | |
| if (toolStats && toolStats.total > 10) { | |
| const failRate = toolStats.failures / toolStats.total | |
| if (failRate > 0.3) { | |
| recommendations.push({ | |
| category: 'reliability', | |
| severity: 'warning', | |
| message: `Tool failure rate is ${(failRate * 100).toFixed(1)}% in the last 24h. Check failing tools.`, | |
| metric: failRate, | |
| }) | |
| } | |
| } | |
| // Check token efficiency vs fleet average | |
| const agentCost = db.prepare(` | |
| SELECT COALESCE(SUM(cost_usd), 0) as cost, COUNT(DISTINCT task_id) as tasks | |
| FROM token_usage | |
| WHERE agent_name = ? AND task_id IS NOT NULL | |
| `).get(agentName) as any | |
| const fleetAvg = db.prepare(` | |
| SELECT AVG(cost_per_task) as avg_cost FROM ( | |
| SELECT SUM(COALESCE(cost_usd, 0)) * 1.0 / NULLIF(COUNT(DISTINCT task_id), 0) as cost_per_task | |
| FROM token_usage | |
| WHERE agent_name IS NOT NULL AND task_id IS NOT NULL | |
| GROUP BY agent_name | |
| ) | |
| `).get() as any | |
| if (agentCost?.tasks > 0 && fleetAvg?.avg_cost > 0) { | |
| const agentCostPerTask = agentCost.cost / agentCost.tasks | |
| if (agentCostPerTask > fleetAvg.avg_cost * 2) { | |
| recommendations.push({ | |
| category: 'cost', | |
| severity: 'warning', | |
| message: `Cost per task ($${agentCostPerTask.toFixed(4)}) is ${(agentCostPerTask / fleetAvg.avg_cost).toFixed(1)}x the fleet average.`, | |
| metric: agentCostPerTask, | |
| }) | |
| } | |
| } | |
| return recommendations | |
| } | |