| import { getCombos } from "@/lib/db/combos"; | |
| import { buildProviderHealthAutopilotReport } from "@/lib/monitoring/providerHealthAutopilot"; | |
| import { buildComboForecastResponse } from "@/lib/usage/comboForecast"; | |
| import { buildComboHealthResponse } from "@/lib/usage/comboHealth"; | |
| import type { | |
| ComboAutopilotAction, | |
| ComboAutopilotActionType, | |
| ComboAutopilotCombo, | |
| ComboAutopilotIssue, | |
| ComboAutopilotIssueKind, | |
| ComboAutopilotReport, | |
| ComboAutopilotSeverity, | |
| ComboAutopilotState, | |
| ComboAutopilotTargetRef, | |
| ComboForecastHorizon, | |
| ComboForecastMetrics, | |
| ComboForecastResponse, | |
| ComboForecastRiskLevel, | |
| ComboHealthMetrics, | |
| ComboHealthResponse, | |
| ComboRecord, | |
| UtilizationTimeRange, | |
| } from "@/shared/types/utilization"; | |
| type JsonRecord = Record<string, unknown>; | |
| export interface ComboHealthAutopilotOptions { | |
| range: UtilizationTimeRange; | |
| horizon: ComboForecastHorizon; | |
| comboId?: string; | |
| includeHealthy?: boolean; | |
| includeActions?: boolean; | |
| now?: number; | |
| combos?: ComboRecord[]; | |
| healthResponse?: ComboHealthResponse; | |
| forecastResponse?: ComboForecastResponse; | |
| } | |
| type ProviderIssueView = { | |
| severity?: ComboAutopilotSeverity; | |
| kind?: string; | |
| title?: string; | |
| recommendation?: string; | |
| target?: { | |
| provider?: string; | |
| connectionId?: string; | |
| }; | |
| evidence?: JsonRecord; | |
| }; | |
| function sanitizeId(parts: Array<string | null | undefined>): string { | |
| return parts | |
| .filter(Boolean) | |
| .join("_") | |
| .replace(/[^a-zA-Z0-9_.:-]/g, "_"); | |
| } | |
| function riskRank(risk: ComboForecastRiskLevel): number { | |
| return { unknown: 0, low: 1, medium: 2, high: 3, critical: 4 }[risk]; | |
| } | |
| function severityRank(severity: ComboAutopilotSeverity): number { | |
| return { info: 1, warning: 2, critical: 3 }[severity]; | |
| } | |
| function targetRef( | |
| combo: ComboHealthMetrics, | |
| target?: NonNullable<ComboHealthMetrics["targetHealth"]>[number] | |
| ): ComboAutopilotTargetRef { | |
| return { | |
| comboId: combo.comboId, | |
| comboName: combo.comboName, | |
| provider: target?.provider, | |
| connectionId: target?.connectionId, | |
| executionKey: target?.executionKey, | |
| model: target?.model, | |
| }; | |
| } | |
| function action( | |
| type: ComboAutopilotActionType, | |
| label: string, | |
| target: ComboAutopilotTargetRef, | |
| href?: string | |
| ): ComboAutopilotAction { | |
| return { | |
| type, | |
| mode: "manual", | |
| label, | |
| href, | |
| target, | |
| }; | |
| } | |
| function actionSet( | |
| target: ComboAutopilotTargetRef, | |
| includeActions: boolean, | |
| types: ComboAutopilotActionType[] | |
| ): ComboAutopilotAction[] { | |
| if (!includeActions) return []; | |
| return types.map((type) => { | |
| switch (type) { | |
| case "open_combo_editor": | |
| return action(type, "Open combo editor", target, "/dashboard/combos"); | |
| case "run_combo_test": | |
| return action(type, "Run combo test", target, "/dashboard/combos"); | |
| case "open_provider_health_autopilot": | |
| return action(type, "Open provider autopilot", target, "/dashboard/health"); | |
| case "review_quota_limits": | |
| return action(type, "Review quota limits", target, "/dashboard/providers"); | |
| case "review_pricing": | |
| return action(type, "Review pricing data", target, "/dashboard/settings"); | |
| } | |
| }); | |
| } | |
| function issue( | |
| combo: ComboHealthMetrics, | |
| kind: ComboAutopilotIssueKind, | |
| severity: ComboAutopilotSeverity, | |
| title: string, | |
| recommendation: string, | |
| evidence: JsonRecord, | |
| includeActions: boolean, | |
| actionTypes: ComboAutopilotActionType[], | |
| target?: NonNullable<ComboHealthMetrics["targetHealth"]>[number] | |
| ): ComboAutopilotIssue { | |
| const targetData = targetRef(combo, target); | |
| return { | |
| id: sanitizeId(["cha", kind, combo.comboId, target?.executionKey, target?.provider]), | |
| severity, | |
| kind, | |
| title, | |
| recommendation, | |
| evidence, | |
| target: targetData, | |
| actions: actionSet(targetData, includeActions, actionTypes), | |
| }; | |
| } | |
| function providerIssueMatchesTarget( | |
| providerIssue: ProviderIssueView, | |
| target: NonNullable<ComboHealthMetrics["targetHealth"]>[number] | |
| ): boolean { | |
| const provider = providerIssue.target?.provider; | |
| const connectionId = providerIssue.target?.connectionId; | |
| return Boolean( | |
| provider && | |
| provider === target.provider && | |
| (!connectionId || !target.connectionId || connectionId === target.connectionId) | |
| ); | |
| } | |
| function providerIssueEvidence(providerIssue: ProviderIssueView): JsonRecord { | |
| return { | |
| providerKind: providerIssue.kind ?? "provider_health_issue", | |
| providerTitle: providerIssue.title ?? "Provider health issue", | |
| providerRecommendation: providerIssue.recommendation ?? "Review provider health details.", | |
| ...(providerIssue.evidence ?? {}), | |
| }; | |
| } | |
| function buildProviderIssueIndex(providerIssues: ProviderIssueView[]): ProviderIssueView[] { | |
| return providerIssues.filter((entry) => Boolean(entry.target?.provider)); | |
| } | |
| function buildIssuesForCombo( | |
| combo: ComboHealthMetrics, | |
| forecast: ComboForecastMetrics | undefined, | |
| providerIssues: ProviderIssueView[], | |
| includeActions: boolean | |
| ): ComboAutopilotIssue[] { | |
| const issues: ComboAutopilotIssue[] = []; | |
| const targets = combo.targetHealth ?? []; | |
| if (targets.length === 0) { | |
| issues.push( | |
| issue( | |
| combo, | |
| "combo_no_targets", | |
| "critical", | |
| "Combo has no executable targets", | |
| "Open the combo editor and add at least one reachable provider/model target.", | |
| { targetCount: 0 }, | |
| includeActions, | |
| ["open_combo_editor"] | |
| ) | |
| ); | |
| } | |
| if (combo.performance.totalRequests === 0) { | |
| issues.push( | |
| issue( | |
| combo, | |
| "combo_no_recent_traffic", | |
| "info", | |
| "No recent combo traffic", | |
| "Run a combo test or send traffic before relying on health trends.", | |
| { totalRequests: 0 }, | |
| includeActions, | |
| ["run_combo_test"] | |
| ) | |
| ); | |
| } | |
| if (combo.performance.totalRequests >= 5 && combo.performance.successRate < 0.9) { | |
| const critical = combo.performance.successRate < 0.7; | |
| issues.push( | |
| issue( | |
| combo, | |
| "combo_low_success_rate", | |
| critical ? "critical" : "warning", | |
| "Combo success rate is below target", | |
| "Inspect failing targets and test fallback order before increasing traffic.", | |
| { | |
| successRate: combo.performance.successRate, | |
| totalRequests: combo.performance.totalRequests, | |
| }, | |
| includeActions, | |
| ["run_combo_test", "open_combo_editor"] | |
| ) | |
| ); | |
| } | |
| if (combo.usageSkew.giniCoefficient >= 0.65 && combo.performance.totalRequests > 0) { | |
| issues.push( | |
| issue( | |
| combo, | |
| "usage_skew_high", | |
| "warning", | |
| "Traffic is concentrated on few targets", | |
| "Review strategy weights or fallback order to avoid overloading one target.", | |
| { giniCoefficient: combo.usageSkew.giniCoefficient }, | |
| includeActions, | |
| ["open_combo_editor"] | |
| ) | |
| ); | |
| } | |
| for (const target of targets) { | |
| if (target.requests >= 3 && target.successRate < 80) { | |
| issues.push( | |
| issue( | |
| combo, | |
| "target_low_success_rate", | |
| target.successRate < 50 ? "critical" : "warning", | |
| "Target success rate is degraded", | |
| "Run a focused combo test and consider changing order, weight, or credentials.", | |
| { successRate: target.successRate, requests: target.requests }, | |
| includeActions, | |
| ["run_combo_test", "open_combo_editor"], | |
| target | |
| ) | |
| ); | |
| } | |
| if (target.lastStatus === "error") { | |
| issues.push( | |
| issue( | |
| combo, | |
| "target_last_error", | |
| "warning", | |
| "Target failed on its latest request", | |
| "Check provider/account health before sending more traffic to this target.", | |
| { lastStatus: target.lastStatus, lastUsedAt: target.lastUsedAt }, | |
| includeActions, | |
| ["open_provider_health_autopilot", "run_combo_test"], | |
| target | |
| ) | |
| ); | |
| } | |
| if (target.quotaIsExhausted) { | |
| issues.push( | |
| issue( | |
| combo, | |
| "target_quota_exhausted", | |
| "critical", | |
| "Target quota is exhausted", | |
| "Move traffic away from this target or rotate credentials until quota resets.", | |
| { quotaScope: target.quotaScope, quotaRemainingPct: target.quotaRemainingPct }, | |
| includeActions, | |
| ["review_quota_limits", "open_combo_editor"], | |
| target | |
| ) | |
| ); | |
| } else if (typeof target.quotaRemainingPct === "number" && target.quotaRemainingPct < 15) { | |
| issues.push( | |
| issue( | |
| combo, | |
| "target_low_quota", | |
| target.quotaRemainingPct < 5 ? "critical" : "warning", | |
| "Target quota is running low", | |
| "Review provider quota and rebalance traffic before the target is exhausted.", | |
| { quotaScope: target.quotaScope, quotaRemainingPct: target.quotaRemainingPct }, | |
| includeActions, | |
| ["review_quota_limits", "open_combo_editor"], | |
| target | |
| ) | |
| ); | |
| } | |
| for (const providerIssue of providerIssues) { | |
| if (!providerIssueMatchesTarget(providerIssue, target)) continue; | |
| issues.push( | |
| issue( | |
| combo, | |
| "provider_health_issue", | |
| providerIssue.severity === "critical" ? "critical" : "warning", | |
| providerIssue.title ?? "Provider health issue affects combo target", | |
| providerIssue.recommendation ?? "Review provider health autopilot details.", | |
| providerIssueEvidence(providerIssue), | |
| includeActions, | |
| ["open_provider_health_autopilot", "open_combo_editor"], | |
| target | |
| ) | |
| ); | |
| } | |
| } | |
| if (forecast && riskRank(forecast.quotaRisk.level) >= riskRank("medium")) { | |
| issues.push( | |
| issue( | |
| combo, | |
| "forecast_quota_risk", | |
| forecast.quotaRisk.level === "critical" ? "critical" : "warning", | |
| "Forecast predicts quota pressure", | |
| "Rebalance targets or review quotas before the forecast horizon is reached.", | |
| { | |
| risk: forecast.quotaRisk.level, | |
| projectedWorstRemainingPct: forecast.quotaRisk.projectedWorstRemainingPct, | |
| timeToExhaustDays: forecast.quotaRisk.timeToExhaustDays, | |
| }, | |
| includeActions, | |
| ["review_quota_limits", "open_combo_editor"] | |
| ) | |
| ); | |
| } | |
| if (forecast) { | |
| const hasDataQualityGap = | |
| forecast.confidence === "no_data" || | |
| forecast.confidence === "low" || | |
| forecast.dataQuality.pricingCoveragePct < 100 || | |
| forecast.dataQuality.quotaCoverage === "none" || | |
| forecast.dataQuality.quotaCoverage === "partial"; | |
| if (hasDataQualityGap) { | |
| issues.push( | |
| issue( | |
| combo, | |
| "data_quality_gap", | |
| forecast.dataQuality.pricingCoveragePct < 80 ? "warning" : "info", | |
| "Forecast data quality is incomplete", | |
| "Add pricing/quota data or generate more traffic to improve autopilot confidence.", | |
| { | |
| confidence: forecast.confidence, | |
| pricingCoveragePct: forecast.dataQuality.pricingCoveragePct, | |
| quotaCoverage: forecast.dataQuality.quotaCoverage, | |
| notes: forecast.dataQuality.notes, | |
| }, | |
| includeActions, | |
| ["review_pricing", "review_quota_limits"] | |
| ) | |
| ); | |
| } | |
| } | |
| return issues.sort((left, right) => severityRank(right.severity) - severityRank(left.severity)); | |
| } | |
| function stateForIssues(issues: ComboAutopilotIssue[]): ComboAutopilotState { | |
| if (issues.some((entry) => entry.severity === "critical")) return "down"; | |
| if (issues.some((entry) => entry.severity === "warning")) return "degraded"; | |
| return "healthy"; | |
| } | |
| function scoreForIssues(issues: ComboAutopilotIssue[]): number { | |
| const score = issues.reduce((current, entry) => { | |
| if (entry.severity === "critical") return current - 35; | |
| if (entry.severity === "warning") return current - 15; | |
| return current - 5; | |
| }, 100); | |
| return Math.max(0, Math.min(100, score)); | |
| } | |
| function buildAutopilotCombo( | |
| combo: ComboHealthMetrics, | |
| forecast: ComboForecastMetrics | undefined, | |
| providerIssues: ProviderIssueView[], | |
| includeActions: boolean | |
| ): ComboAutopilotCombo { | |
| const issues = buildIssuesForCombo(combo, forecast, providerIssues, includeActions); | |
| const state = stateForIssues(issues); | |
| const providerIssueCount = issues.filter( | |
| (entry) => entry.kind === "provider_health_issue" | |
| ).length; | |
| return { | |
| comboId: combo.comboId, | |
| comboName: combo.comboName, | |
| strategy: combo.strategy, | |
| state, | |
| score: scoreForIssues(issues), | |
| signals: { | |
| totalRequests: combo.performance.totalRequests, | |
| successRate: combo.performance.successRate, | |
| avgLatencyMs: combo.performance.avgLatencyMs, | |
| worstQuotaRemainingPct: | |
| combo.quotaHealth.providers.length > 0 ? combo.quotaHealth.worstRemainingPct : null, | |
| forecastRisk: forecast?.quotaRisk.level ?? "unknown", | |
| forecastConfidence: forecast?.confidence ?? "no_data", | |
| usageSkew: combo.usageSkew.giniCoefficient, | |
| targetCount: combo.targetHealth?.length ?? 0, | |
| providerIssueCount, | |
| dataQualityNotes: forecast?.dataQuality.notes ?? [], | |
| }, | |
| issues, | |
| }; | |
| } | |
| export async function buildComboHealthAutopilotReport( | |
| options: ComboHealthAutopilotOptions | |
| ): Promise<ComboAutopilotReport> { | |
| const includeHealthy = options.includeHealthy === true; | |
| const includeActions = options.includeActions !== false; | |
| const checkedAt = new Date(options.now ?? Date.now()).toISOString(); | |
| const combosSnapshot = | |
| options.combos ?? | |
| (options.healthResponse && options.forecastResponse | |
| ? undefined | |
| : ((await getCombos()) as ComboRecord[])); | |
| const [health, forecast, providerHealth] = await Promise.all([ | |
| options.healthResponse ?? | |
| buildComboHealthResponse({ | |
| range: options.range, | |
| comboId: options.comboId, | |
| now: options.now, | |
| combos: combosSnapshot, | |
| }), | |
| options.forecastResponse ?? | |
| buildComboForecastResponse({ | |
| range: options.range, | |
| horizon: options.horizon, | |
| comboId: options.comboId, | |
| now: options.now, | |
| combos: combosSnapshot, | |
| }), | |
| buildProviderHealthAutopilotReport({ includeHealthy: false, includeActions: false }), | |
| ]); | |
| const forecastsByComboId = new Map(forecast.combos.map((entry) => [entry.comboId, entry])); | |
| const providerIssues = buildProviderIssueIndex( | |
| providerHealth.providers.flatMap((provider) => provider.issues as ProviderIssueView[]) | |
| ); | |
| const allCombos = health.combos.map((combo) => | |
| buildAutopilotCombo( | |
| combo, | |
| forecastsByComboId.get(combo.comboId), | |
| providerIssues, | |
| includeActions | |
| ) | |
| ); | |
| const combos = includeHealthy | |
| ? allCombos | |
| : allCombos.filter((combo) => combo.state !== "healthy"); | |
| const downCount = allCombos.filter((combo) => combo.state === "down").length; | |
| const degradedCount = allCombos.filter((combo) => combo.state === "degraded").length; | |
| const healthyCount = allCombos.filter((combo) => combo.state === "healthy").length; | |
| const issueCount = allCombos.reduce((sum, combo) => sum + combo.issues.length, 0); | |
| const actionableCount = allCombos.reduce( | |
| (sum, combo) => | |
| sum + combo.issues.reduce((issueSum, issue) => issueSum + issue.actions.length, 0), | |
| 0 | |
| ); | |
| return { | |
| status: downCount > 0 ? "critical" : degradedCount > 0 ? "warning" : "healthy", | |
| checkedAt, | |
| timeRange: options.range, | |
| horizon: options.horizon, | |
| summary: { | |
| comboCount: allCombos.length, | |
| healthyCount, | |
| degradedCount, | |
| downCount, | |
| issueCount, | |
| actionableCount, | |
| }, | |
| combos, | |
| }; | |
| } | |