| import { getDb, getSetting, setSetting } from '../db/index.js'; |
| import { getProvider, resolveProvider } from '../providers/index.js'; |
| import { decrypt } from '../lib/crypto.js'; |
| import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider } from './ratelimit.js'; |
| import { |
| BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights, |
| reliabilityPosterior, expectedReliability, sampleBeta, |
| speedScore, intelligenceScore, headroomFactor, rateLimitFactor, combineScore, |
| } from './scoring.js'; |
| import { parseBudget } from '../lib/budget.js'; |
| import type { BaseProvider } from '../providers/base.js'; |
| import type { TursoDb } from '../db/client.js'; |
|
|
| interface KeyRow { |
| id: number; |
| platform: string; |
| encrypted_key: string; |
| iv: string; |
| auth_tag: string; |
| status: string; |
| enabled: number; |
| base_url: string | null; |
| } |
|
|
| |
| interface ChainRow { |
| model_db_id: number; |
| priority: number; |
| enabled: number; |
| platform: string; |
| model_id: string; |
| display_name: string; |
| intelligence_rank: number; |
| size_label: string; |
| monthly_token_budget: string; |
| rpm_limit: number | null; |
| rpd_limit: number | null; |
| tpm_limit: number | null; |
| tpd_limit: number | null; |
| supports_vision: number; |
| supports_tools: number; |
| context_window: number | null; |
| |
| |
| key_id: number | null; |
| } |
|
|
| export interface RouteResult { |
| provider: BaseProvider; |
| modelId: string; |
| modelDbId: number; |
| apiKey: string; |
| keyId: number; |
| platform: string; |
| displayName: string; |
| |
| |
| rpdLimit: number | null; |
| tpdLimit: number | null; |
| } |
|
|
| |
| const roundRobinIndex = new Map<string, number>(); |
|
|
| |
| |
| const rateLimitPenalties = new Map<number, { count: number; lastHit: number; penalty: number }>(); |
|
|
| |
| const PENALTY_PER_429 = 3; |
| const MAX_PENALTY = 10; |
| const DECAY_INTERVAL_MS = 2 * 60 * 1000; |
| const DECAY_AMOUNT = 1; |
|
|
| |
| |
| |
| export function recordRateLimitHit(modelDbId: number) { |
| const existing = rateLimitPenalties.get(modelDbId); |
| const now = Date.now(); |
| if (existing) { |
| existing.count++; |
| existing.lastHit = now; |
| existing.penalty = Math.min(existing.penalty + PENALTY_PER_429, MAX_PENALTY); |
| } else { |
| rateLimitPenalties.set(modelDbId, { count: 1, lastHit: now, penalty: PENALTY_PER_429 }); |
| } |
| } |
|
|
| |
| |
| |
| export function recordSuccess(modelDbId: number) { |
| const existing = rateLimitPenalties.get(modelDbId); |
| if (existing) { |
| existing.penalty = Math.max(0, existing.penalty - 1); |
| if (existing.penalty === 0) { |
| rateLimitPenalties.delete(modelDbId); |
| } |
| } |
| } |
|
|
| |
| |
| |
| function getPenalty(modelDbId: number): number { |
| const entry = rateLimitPenalties.get(modelDbId); |
| if (!entry) return 0; |
|
|
| |
| const now = Date.now(); |
| const elapsed = now - entry.lastHit; |
| const decaySteps = Math.floor(elapsed / DECAY_INTERVAL_MS); |
| if (decaySteps > 0) { |
| entry.penalty = Math.max(0, entry.penalty - (decaySteps * DECAY_AMOUNT)); |
| entry.lastHit = now; |
| if (entry.penalty === 0) { |
| rateLimitPenalties.delete(modelDbId); |
| return 0; |
| } |
| } |
|
|
| return entry.penalty; |
| } |
|
|
| |
| |
| |
| export function getAllPenalties(): Array<{ modelDbId: number; count: number; penalty: number }> { |
| const result: Array<{ modelDbId: number; count: number; penalty: number }> = []; |
| for (const [modelDbId, entry] of rateLimitPenalties) { |
| const penalty = getPenalty(modelDbId); |
| if (penalty > 0) { |
| result.push({ modelDbId, count: entry.count, penalty }); |
| } |
| } |
| return result.sort((a, b) => b.penalty - a.penalty); |
| } |
|
|
| |
| const STRATEGY_KEY = 'routing_strategy'; |
| const CUSTOM_WEIGHTS_KEY = 'routing_custom_weights'; |
| const VALID_STRATEGIES: RoutingStrategy[] = ['priority', 'balanced', 'smartest', 'fastest', 'reliable', 'custom']; |
|
|
| export async function getRoutingStrategy(): Promise<RoutingStrategy> { |
| const raw = await getSetting(STRATEGY_KEY); |
| return (raw && VALID_STRATEGIES.includes(raw as RoutingStrategy)) |
| ? (raw as RoutingStrategy) |
| : DEFAULT_STRATEGY; |
| } |
|
|
| export async function setRoutingStrategy(strategy: RoutingStrategy): Promise<void> { |
| if (!VALID_STRATEGIES.includes(strategy)) { |
| throw new Error(`Unknown routing strategy: ${strategy}`); |
| } |
| await setSetting(STRATEGY_KEY, strategy); |
| } |
|
|
| |
| |
| |
| |
| |
| export async function getCustomWeights(): Promise<RoutingWeights> { |
| const raw = await getSetting(CUSTOM_WEIGHTS_KEY); |
| if (raw) { |
| try { |
| const w = JSON.parse(raw) as RoutingWeights; |
| if ( |
| [w.reliability, w.speed, w.intelligence].every(v => Number.isFinite(v) && v >= 0) && |
| w.reliability + w.speed + w.intelligence > 0 |
| ) { |
| return { reliability: w.reliability, speed: w.speed, intelligence: w.intelligence }; |
| } |
| } catch { } |
| } |
| return { ...BANDIT_PRESETS.balanced }; |
| } |
|
|
| export async function setCustomWeights(weights: RoutingWeights): Promise<void> { |
| const { reliability, speed, intelligence } = weights; |
| if (![reliability, speed, intelligence].every(v => Number.isFinite(v) && v >= 0)) { |
| throw new Error('Custom weights must be non-negative numbers'); |
| } |
| const sum = reliability + speed + intelligence; |
| if (sum <= 0) { |
| throw new Error('Custom weights must not all be zero'); |
| } |
| await setSetting(CUSTOM_WEIGHTS_KEY, JSON.stringify({ |
| reliability: reliability / sum, |
| speed: speed / sum, |
| intelligence: intelligence / sum, |
| })); |
| } |
|
|
| async function weightsFor(strategy: RoutingStrategy): Promise<RoutingWeights | null> { |
| if (strategy === 'priority') return null; |
| if (strategy === 'custom') return await getCustomWeights(); |
| return BANDIT_PRESETS[strategy]; |
| } |
|
|
| |
| |
| |
| |
| |
| |
| const WINDOW_MS = 7 * 24 * 60 * 60 * 1000; |
| const HALF_LIFE_DAYS = 2; |
| const CACHE_TTL_MS = 60 * 1000; |
|
|
| interface ModelStats { |
| successes: number; |
| failures: number; |
| tokPerSec: number; |
| avgTtfbMs: number | null; |
| monthlyUsedTokens: number; |
| } |
|
|
| let statsCache: Map<string, ModelStats> | null = null; |
| let statsCacheTime = 0; |
|
|
| function decayWeight(ageDays: number): number { |
| return Math.pow(0.5, Math.max(0, ageDays) / HALF_LIFE_DAYS); |
| } |
|
|
| export async function refreshStatsCache(db: TursoDb, force = false): Promise<void> { |
| if (!force && statsCache && Date.now() - statsCacheTime < CACHE_TTL_MS) return; |
|
|
| const since = new Date(Date.now() - WINDOW_MS).toISOString(); |
| const buckets = await db.all<{ |
| platform: string; model_id: string; age_days: number; total: number; successes: number; |
| succ_out: number; succ_lat: number; succ_ttfb_sum: number; succ_ttfb_cnt: number; |
| }>(` |
| SELECT platform, model_id, |
| CAST((julianday('now') - julianday(created_at)) AS INTEGER) AS age_days, |
| COUNT(*) AS total, |
| SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS successes, |
| SUM(CASE WHEN status = 'success' THEN output_tokens ELSE 0 END) AS succ_out, |
| SUM(CASE WHEN status = 'success' THEN latency_ms ELSE 0 END) AS succ_lat, |
| SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN ttfb_ms ELSE 0 END) AS succ_ttfb_sum, |
| SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN 1 ELSE 0 END) AS succ_ttfb_cnt |
| FROM requests |
| WHERE created_at >= ? |
| GROUP BY platform, model_id, age_days |
| `, [since]); |
|
|
| |
| const acc = new Map<string, { |
| wSucc: number; wFail: number; wOut: number; wLat: number; wTtfbSum: number; wTtfbCnt: number; |
| }>(); |
| for (const b of buckets) { |
| const key = `${b.platform}:${b.model_id}`; |
| const w = decayWeight(b.age_days); |
| const a = acc.get(key) ?? { wSucc: 0, wFail: 0, wOut: 0, wLat: 0, wTtfbSum: 0, wTtfbCnt: 0 }; |
| a.wSucc += w * b.successes; |
| a.wFail += w * (b.total - b.successes); |
| a.wOut += w * b.succ_out; |
| a.wLat += w * b.succ_lat; |
| a.wTtfbSum += w * b.succ_ttfb_sum; |
| a.wTtfbCnt += w * b.succ_ttfb_cnt; |
| acc.set(key, a); |
| } |
|
|
| |
| const usageRows = await db.all<{ platform: string; model_id: string; used: number }>(` |
| SELECT platform, model_id, COALESCE(SUM(input_tokens + output_tokens), 0) AS used |
| FROM requests |
| WHERE created_at >= datetime('now', 'start of month') |
| AND request_type = 'chat' |
| GROUP BY platform, model_id |
| `); |
| const usageMap = new Map(usageRows.map(r => [`${r.platform}:${r.model_id}`, r.used])); |
|
|
| const next = new Map<string, ModelStats>(); |
| for (const [key, a] of acc) { |
| next.set(key, { |
| successes: a.wSucc, |
| failures: a.wFail, |
| tokPerSec: a.wLat > 0 ? (a.wOut * 1000) / a.wLat : 0, |
| avgTtfbMs: a.wTtfbCnt > 0 ? a.wTtfbSum / a.wTtfbCnt : null, |
| monthlyUsedTokens: usageMap.get(key) ?? 0, |
| }); |
| } |
| |
| for (const [key, used] of usageMap) { |
| if (!next.has(key)) { |
| next.set(key, { successes: 0, failures: 0, tokPerSec: 0, avgTtfbMs: null, monthlyUsedTokens: used }); |
| } |
| } |
|
|
| statsCache = next; |
| statsCacheTime = Date.now(); |
| } |
|
|
| |
| |
| |
| const TIER_VALUE: Record<string, number> = { Frontier: 4, Large: 3, Medium: 2, Small: 1 }; |
| function intelligenceComposite(sizeLabel: string, intelligenceRank: number): number { |
| const tier = TIER_VALUE[sizeLabel] ?? 0; |
| |
| return tier * 1000 - intelligenceRank; |
| } |
|
|
| |
| |
| interface ScoredEntry { |
| axes: { reliability: number; speed: number; intelligence: number }; |
| headroom: number; |
| rateLimit: number; |
| score: number; |
| } |
|
|
| function scoreChainEntry( |
| entry: ChainRow, |
| weights: RoutingWeights, |
| intelMin: number, |
| intelMax: number, |
| sampled: boolean, |
| ): ScoredEntry { |
| const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`); |
| const successes = stats?.successes ?? 0; |
| const failures = stats?.failures ?? 0; |
|
|
| let reliability: number; |
| if (sampled) { |
| const { alpha, beta } = reliabilityPosterior(successes, failures); |
| reliability = sampleBeta(alpha, beta); |
| } else { |
| reliability = expectedReliability(successes, failures); |
| } |
|
|
| const speed = speedScore(stats?.tokPerSec ?? 0, stats?.avgTtfbMs ?? null); |
| const intelligence = intelligenceScore( |
| intelligenceComposite(entry.size_label, entry.intelligence_rank), intelMin, intelMax, |
| ); |
|
|
| const budget = parseBudget(entry.monthly_token_budget); |
| const headroom = headroomFactor(stats?.monthlyUsedTokens ?? 0, budget); |
| const rl = rateLimitFactor(getPenalty(entry.model_db_id)); |
|
|
| const score = combineScore({ reliability, speed, intelligence, headroom, rateLimit: rl }, weights); |
| return { axes: { reliability, speed, intelligence }, headroom, rateLimit: rl, score }; |
| } |
|
|
| |
| |
| |
| |
| |
| |
| async function orderChain(chain: ChainRow[], strategy: RoutingStrategy): Promise<ChainRow[]> { |
| const weights = await weightsFor(strategy); |
| if (!weights) { |
| |
| return chain |
| .map(e => ({ e, eff: e.priority + getPenalty(e.model_db_id) })) |
| .sort((a, b) => a.eff - b.eff || a.e.priority - b.e.priority) |
| .map(x => x.e); |
| } |
|
|
| const composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank)); |
| const intelMin = composites.length ? Math.min(...composites) : 0; |
| const intelMax = composites.length ? Math.max(...composites) : 0; |
|
|
| return chain |
| .map(e => ({ e, s: scoreChainEntry(e, weights, intelMin, intelMax, true).score })) |
| |
| .sort((a, b) => b.s - a.s || a.e.priority - b.e.priority) |
| .map(x => x.e); |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| export async function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false): Promise<RouteResult> { |
| const db = getDb(); |
|
|
| const strategy = await getRoutingStrategy(); |
| if (strategy !== 'priority') await refreshStatsCache(db); |
|
|
| |
| const chain = await db.all<ChainRow>(` |
| SELECT fc.model_db_id, fc.priority, fc.enabled, |
| m.platform, m.model_id, m.display_name, m.intelligence_rank, |
| m.size_label, m.monthly_token_budget, |
| m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision, |
| m.supports_tools, m.context_window, m.key_id |
| FROM fallback_config fc |
| JOIN models m ON m.id = fc.model_db_id AND m.enabled = 1 |
| WHERE fc.enabled = 1 |
| `); |
|
|
| const sortedChain = await orderChain(chain, strategy); |
|
|
| |
| if (preferredModelDbId) { |
| const idx = sortedChain.findIndex(e => e.model_db_id === preferredModelDbId); |
| if (idx > 0) { |
| const [preferred] = sortedChain.splice(idx, 1); |
| sortedChain.unshift(preferred); |
| } |
| } |
|
|
| for (const entry of sortedChain) { |
| |
| |
| if (requireVision && !entry.supports_vision) continue; |
|
|
| |
| |
| |
| |
| |
| if (requireTools && !entry.supports_tools) continue; |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| if (entry.context_window != null && estimatedTokens > entry.context_window) continue; |
|
|
| |
| const provider = getProvider(entry.platform as any); |
| if (!provider) continue; |
|
|
| |
| const keys = await db.all<KeyRow>( |
| "SELECT * FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown')", |
| [entry.platform] |
| ); |
| if (keys.length === 0) continue; |
|
|
| |
| const limits = { |
| rpm: entry.rpm_limit, |
| rpd: entry.rpd_limit, |
| tpm: entry.tpm_limit, |
| tpd: entry.tpd_limit, |
| }; |
|
|
| |
| const rrKey = `${entry.platform}:${entry.model_id}`; |
| let idx = roundRobinIndex.get(rrKey) ?? 0; |
|
|
| for (let attempt = 0; attempt < keys.length; attempt++) { |
| const key = keys[idx % keys.length]; |
| idx++; |
|
|
| |
| |
| |
| |
| if (entry.platform === 'custom' && entry.key_id != null && key.id !== entry.key_id) continue; |
|
|
| const skipId = `${entry.platform}:${entry.model_id}:${key.id}`; |
| if (skipKeys?.has(skipId)) continue; |
|
|
| |
| if (await isOnCooldown(entry.platform, entry.model_id, key.id)) continue; |
|
|
| |
| |
| |
| if (!await canUseProvider(entry.platform, key.id)) continue; |
|
|
| if (!await canMakeRequest(entry.platform, entry.model_id, key.id, limits)) continue; |
| if (!await canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) continue; |
|
|
| let decryptedKey: string; |
| try { |
| decryptedKey = decrypt(key.encrypted_key, key.iv, key.auth_tag); |
| } catch { |
| await db.run("UPDATE api_keys SET status = 'error', last_checked_at = datetime('now') WHERE id = ?", [key.id]); |
| continue; |
| } |
|
|
| |
| |
| |
| const resolvedProvider = entry.platform === 'custom' |
| ? resolveProvider('custom', key.base_url) |
| : provider; |
| if (!resolvedProvider) continue; |
|
|
| |
| roundRobinIndex.set(rrKey, idx); |
| return { |
| provider: resolvedProvider, |
| modelId: entry.model_id, |
| modelDbId: entry.model_db_id, |
| apiKey: decryptedKey, |
| keyId: key.id, |
| platform: entry.platform, |
| displayName: entry.display_name, |
| rpdLimit: limits.rpd, |
| tpdLimit: limits.tpd, |
| }; |
| } |
|
|
| |
| |
| roundRobinIndex.set(rrKey, idx); |
|
|
| |
| |
| |
| } |
|
|
| const err = new Error('All models exhausted. Add more API keys or wait for rate limits to reset.') as any; |
| err.status = 429; |
| throw err; |
| } |
|
|
| |
| |
| |
| |
| |
| export interface RoutingScore { |
| modelDbId: number; |
| platform: string; |
| modelId: string; |
| displayName: string; |
| enabled: boolean; |
| reliability: number; |
| speed: number; |
| intelligence: number; |
| headroom: number; |
| rateLimit: number; |
| score: number; |
| totalRequests: number; |
| } |
|
|
| export async function getRoutingScores(): Promise<{ strategy: RoutingStrategy; weights: RoutingWeights | null; scores: RoutingScore[] }> { |
| const db = getDb(); |
| const strategy = await getRoutingStrategy(); |
| await refreshStatsCache(db); |
|
|
| const chain = await db.all<ChainRow>(` |
| SELECT fc.model_db_id, fc.priority, fc.enabled, |
| m.platform, m.model_id, m.display_name, m.intelligence_rank, |
| m.size_label, m.monthly_token_budget, |
| m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision, |
| m.supports_tools, m.context_window |
| FROM fallback_config fc |
| JOIN models m ON m.id = fc.model_db_id |
| WHERE m.enabled = 1 |
| `); |
|
|
| |
| |
| const weights = (await weightsFor(strategy)) ?? BANDIT_PRESETS.balanced; |
| const composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank)); |
| const intelMin = composites.length ? Math.min(...composites) : 0; |
| const intelMax = composites.length ? Math.max(...composites) : 0; |
|
|
| const scores: RoutingScore[] = chain.map(entry => { |
| const scored = scoreChainEntry(entry, weights, intelMin, intelMax, false); |
| const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`); |
| return { |
| modelDbId: entry.model_db_id, |
| platform: entry.platform, |
| modelId: entry.model_id, |
| displayName: entry.display_name, |
| enabled: entry.enabled === 1, |
| reliability: scored.axes.reliability, |
| speed: scored.axes.speed, |
| intelligence: scored.axes.intelligence, |
| headroom: scored.headroom, |
| rateLimit: scored.rateLimit, |
| score: scored.score, |
| totalRequests: Math.round((stats?.successes ?? 0) + (stats?.failures ?? 0)), |
| }; |
| }).sort((a, b) => b.score - a.score); |
|
|
| return { strategy, weights: await weightsFor(strategy), scores }; |
| } |
|
|
| |
| |
| |
| export async function hasEnabledVisionModel(): Promise<boolean> { |
| const db = getDb(); |
| const row = await db.get<{ cnt: number }>(` |
| SELECT COUNT(*) as cnt |
| FROM fallback_config fc |
| JOIN models m ON m.id = fc.model_db_id |
| WHERE fc.enabled = 1 AND m.enabled = 1 AND m.supports_vision = 1 |
| `); |
| return (row?.cnt ?? 0) > 0; |
| } |
|
|
| |
| |
| |
| export async function hasEnabledToolsModel(): Promise<boolean> { |
| const db = getDb(); |
| const row = await db.get<{ cnt: number }>(` |
| SELECT COUNT(*) as cnt |
| FROM fallback_config fc |
| JOIN models m ON m.id = fc.model_db_id |
| WHERE fc.enabled = 1 AND m.enabled = 1 AND m.supports_tools = 1 |
| `); |
| return (row?.cnt ?? 0) > 0; |
| } |
|
|