| import { getDb, getSetting, setSetting } from '../db/index.js';
|
| import { getProvider, resolveProvider } from '../providers/index.js';
|
| import { decrypt } from '../lib/crypto.js';
|
| import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider } from './ratelimit.js';
|
| import {
|
| BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,
|
| reliabilityPosterior, expectedReliability, sampleBeta,
|
| speedScore, intelligenceScore, headroomFactor, rateLimitFactor, combineScore,
|
| } from './scoring.js';
|
| import { parseBudget } from '../lib/budget.js';
|
| import type { BaseProvider } from '../providers/base.js';
|
| import type { Database } from 'better-sqlite3';
|
|
|
| interface KeyRow {
|
| id: number;
|
| platform: string;
|
| encrypted_key: string;
|
| iv: string;
|
| auth_tag: string;
|
| status: string;
|
| enabled: number;
|
| base_url: string | null;
|
| }
|
|
|
|
|
| interface ChainRow {
|
| model_db_id: number;
|
| priority: number;
|
| enabled: number;
|
| platform: string;
|
| model_id: string;
|
| display_name: string;
|
| intelligence_rank: number;
|
| size_label: string;
|
| monthly_token_budget: string;
|
| rpm_limit: number | null;
|
| rpd_limit: number | null;
|
| tpm_limit: number | null;
|
| tpd_limit: number | null;
|
| supports_vision: number;
|
| supports_tools: number;
|
| context_window: number | null;
|
|
|
|
|
| key_id: number | null;
|
| }
|
|
|
| export interface RouteResult {
|
| provider: BaseProvider;
|
| modelId: string;
|
| modelDbId: number;
|
| apiKey: string;
|
| keyId: number;
|
| platform: string;
|
| displayName: string;
|
|
|
|
|
| rpdLimit: number | null;
|
| tpdLimit: number | null;
|
| }
|
|
|
|
|
| const roundRobinIndex = new Map<string, number>();
|
|
|
|
|
|
|
| const rateLimitPenalties = new Map<number, { count: number; lastHit: number; penalty: number }>();
|
|
|
|
|
| const PENALTY_PER_429 = 3;
|
| const MAX_PENALTY = 10;
|
| const DECAY_INTERVAL_MS = 2 * 60 * 1000;
|
| const DECAY_AMOUNT = 1;
|
|
|
| |
| |
|
|
| export function recordRateLimitHit(modelDbId: number) {
|
| const existing = rateLimitPenalties.get(modelDbId);
|
| const now = Date.now();
|
| if (existing) {
|
| existing.count++;
|
| existing.lastHit = now;
|
| existing.penalty = Math.min(existing.penalty + PENALTY_PER_429, MAX_PENALTY);
|
| } else {
|
| rateLimitPenalties.set(modelDbId, { count: 1, lastHit: now, penalty: PENALTY_PER_429 });
|
| }
|
| }
|
|
|
| |
| |
|
|
| export function recordSuccess(modelDbId: number) {
|
| const existing = rateLimitPenalties.get(modelDbId);
|
| if (existing) {
|
| existing.penalty = Math.max(0, existing.penalty - 1);
|
| if (existing.penalty === 0) {
|
| rateLimitPenalties.delete(modelDbId);
|
| }
|
| }
|
| }
|
|
|
| |
| |
|
|
| function getPenalty(modelDbId: number): number {
|
| const entry = rateLimitPenalties.get(modelDbId);
|
| if (!entry) return 0;
|
|
|
|
|
| const now = Date.now();
|
| const elapsed = now - entry.lastHit;
|
| const decaySteps = Math.floor(elapsed / DECAY_INTERVAL_MS);
|
| if (decaySteps > 0) {
|
| entry.penalty = Math.max(0, entry.penalty - (decaySteps * DECAY_AMOUNT));
|
| entry.lastHit = now;
|
| if (entry.penalty === 0) {
|
| rateLimitPenalties.delete(modelDbId);
|
| return 0;
|
| }
|
| }
|
|
|
| return entry.penalty;
|
| }
|
|
|
| |
| |
|
|
| export function getAllPenalties(): Array<{ modelDbId: number; count: number; penalty: number }> {
|
| const result: Array<{ modelDbId: number; count: number; penalty: number }> = [];
|
| for (const [modelDbId, entry] of rateLimitPenalties) {
|
| const penalty = getPenalty(modelDbId);
|
| if (penalty > 0) {
|
| result.push({ modelDbId, count: entry.count, penalty });
|
| }
|
| }
|
| return result.sort((a, b) => b.penalty - a.penalty);
|
| }
|
|
|
|
|
| const STRATEGY_KEY = 'routing_strategy';
|
| const CUSTOM_WEIGHTS_KEY = 'routing_custom_weights';
|
| const VALID_STRATEGIES: RoutingStrategy[] = ['priority', 'balanced', 'smartest', 'fastest', 'reliable', 'custom'];
|
|
|
| export function getRoutingStrategy(): RoutingStrategy {
|
| const raw = getSetting(STRATEGY_KEY);
|
| return (raw && VALID_STRATEGIES.includes(raw as RoutingStrategy))
|
| ? (raw as RoutingStrategy)
|
| : DEFAULT_STRATEGY;
|
| }
|
|
|
| export function setRoutingStrategy(strategy: RoutingStrategy): void {
|
| if (!VALID_STRATEGIES.includes(strategy)) {
|
| throw new Error(`Unknown routing strategy: ${strategy}`);
|
| }
|
| setSetting(STRATEGY_KEY, strategy);
|
| }
|
|
|
|
|
|
|
|
|
|
|
|
|
| export function getCustomWeights(): RoutingWeights {
|
| const raw = getSetting(CUSTOM_WEIGHTS_KEY);
|
| if (raw) {
|
| try {
|
| const w = JSON.parse(raw) as RoutingWeights;
|
| if (
|
| [w.reliability, w.speed, w.intelligence].every(v => Number.isFinite(v) && v >= 0) &&
|
| w.reliability + w.speed + w.intelligence > 0
|
| ) {
|
| return { reliability: w.reliability, speed: w.speed, intelligence: w.intelligence };
|
| }
|
| } catch { }
|
| }
|
| return { ...BANDIT_PRESETS.balanced };
|
| }
|
|
|
| export function setCustomWeights(weights: RoutingWeights): void {
|
| const { reliability, speed, intelligence } = weights;
|
| if (![reliability, speed, intelligence].every(v => Number.isFinite(v) && v >= 0)) {
|
| throw new Error('Custom weights must be non-negative numbers');
|
| }
|
| const sum = reliability + speed + intelligence;
|
| if (sum <= 0) {
|
| throw new Error('Custom weights must not all be zero');
|
| }
|
| setSetting(CUSTOM_WEIGHTS_KEY, JSON.stringify({
|
| reliability: reliability / sum,
|
| speed: speed / sum,
|
| intelligence: intelligence / sum,
|
| }));
|
| }
|
|
|
| function weightsFor(strategy: RoutingStrategy): RoutingWeights | null {
|
| if (strategy === 'priority') return null;
|
| if (strategy === 'custom') return getCustomWeights();
|
| return BANDIT_PRESETS[strategy];
|
| }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| const WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
|
| const HALF_LIFE_DAYS = 2;
|
| const CACHE_TTL_MS = 60 * 1000;
|
|
|
| interface ModelStats {
|
| successes: number;
|
| failures: number;
|
| tokPerSec: number;
|
| avgTtfbMs: number | null;
|
| monthlyUsedTokens: number;
|
| }
|
|
|
| let statsCache: Map<string, ModelStats> | null = null;
|
| let statsCacheTime = 0;
|
|
|
| function decayWeight(ageDays: number): number {
|
| return Math.pow(0.5, Math.max(0, ageDays) / HALF_LIFE_DAYS);
|
| }
|
|
|
| export function refreshStatsCache(db: Database, force = false): void {
|
| if (!force && statsCache && Date.now() - statsCacheTime < CACHE_TTL_MS) return;
|
|
|
| const since = new Date(Date.now() - WINDOW_MS).toISOString();
|
| const buckets = db.prepare(`
|
| SELECT platform, model_id,
|
| CAST((julianday('now') - julianday(created_at)) AS INTEGER) AS age_days,
|
| COUNT(*) AS total,
|
| SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS successes,
|
| SUM(CASE WHEN status = 'success' THEN output_tokens ELSE 0 END) AS succ_out,
|
| SUM(CASE WHEN status = 'success' THEN latency_ms ELSE 0 END) AS succ_lat,
|
| SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN ttfb_ms ELSE 0 END) AS succ_ttfb_sum,
|
| SUM(CASE WHEN status = 'success' AND ttfb_ms IS NOT NULL THEN 1 ELSE 0 END) AS succ_ttfb_cnt
|
| FROM requests
|
| WHERE created_at >= ?
|
| GROUP BY platform, model_id, age_days
|
| `).all(since) as Array<{
|
| platform: string; model_id: string; age_days: number; total: number; successes: number;
|
| succ_out: number; succ_lat: number; succ_ttfb_sum: number; succ_ttfb_cnt: number;
|
| }>;
|
|
|
|
|
| const acc = new Map<string, {
|
| wSucc: number; wFail: number; wOut: number; wLat: number; wTtfbSum: number; wTtfbCnt: number;
|
| }>();
|
| for (const b of buckets) {
|
| const key = `${b.platform}:${b.model_id}`;
|
| const w = decayWeight(b.age_days);
|
| const a = acc.get(key) ?? { wSucc: 0, wFail: 0, wOut: 0, wLat: 0, wTtfbSum: 0, wTtfbCnt: 0 };
|
| a.wSucc += w * b.successes;
|
| a.wFail += w * (b.total - b.successes);
|
| a.wOut += w * b.succ_out;
|
| a.wLat += w * b.succ_lat;
|
| a.wTtfbSum += w * b.succ_ttfb_sum;
|
| a.wTtfbCnt += w * b.succ_ttfb_cnt;
|
| acc.set(key, a);
|
| }
|
|
|
|
|
| const usageRows = db.prepare(`
|
| SELECT platform, model_id, COALESCE(SUM(input_tokens + output_tokens), 0) AS used
|
| FROM requests
|
| WHERE created_at >= datetime('now', 'start of month')
|
| AND request_type = 'chat'
|
| GROUP BY platform, model_id
|
| `).all() as Array<{ platform: string; model_id: string; used: number }>;
|
| const usageMap = new Map(usageRows.map(r => [`${r.platform}:${r.model_id}`, r.used]));
|
|
|
| const next = new Map<string, ModelStats>();
|
| for (const [key, a] of acc) {
|
| next.set(key, {
|
| successes: a.wSucc,
|
| failures: a.wFail,
|
| tokPerSec: a.wLat > 0 ? (a.wOut * 1000) / a.wLat : 0,
|
| avgTtfbMs: a.wTtfbCnt > 0 ? a.wTtfbSum / a.wTtfbCnt : null,
|
| monthlyUsedTokens: usageMap.get(key) ?? 0,
|
| });
|
| }
|
|
|
| for (const [key, used] of usageMap) {
|
| if (!next.has(key)) {
|
| next.set(key, { successes: 0, failures: 0, tokPerSec: 0, avgTtfbMs: null, monthlyUsedTokens: used });
|
| }
|
| }
|
|
|
| statsCache = next;
|
| statsCacheTime = Date.now();
|
| }
|
|
|
|
|
|
|
|
|
| const TIER_VALUE: Record<string, number> = { Frontier: 4, Large: 3, Medium: 2, Small: 1 };
|
| function intelligenceComposite(sizeLabel: string, intelligenceRank: number): number {
|
| const tier = TIER_VALUE[sizeLabel] ?? 0;
|
|
|
| return tier * 1000 - intelligenceRank;
|
| }
|
|
|
|
|
|
|
| interface ScoredEntry {
|
| axes: { reliability: number; speed: number; intelligence: number };
|
| headroom: number;
|
| rateLimit: number;
|
| score: number;
|
| }
|
|
|
| function scoreChainEntry(
|
| entry: ChainRow,
|
| weights: RoutingWeights,
|
| intelMin: number,
|
| intelMax: number,
|
| sampled: boolean,
|
| ): ScoredEntry {
|
| const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`);
|
| const successes = stats?.successes ?? 0;
|
| const failures = stats?.failures ?? 0;
|
|
|
| let reliability: number;
|
| if (sampled) {
|
| const { alpha, beta } = reliabilityPosterior(successes, failures);
|
| reliability = sampleBeta(alpha, beta);
|
| } else {
|
| reliability = expectedReliability(successes, failures);
|
| }
|
|
|
| const speed = speedScore(stats?.tokPerSec ?? 0, stats?.avgTtfbMs ?? null);
|
| const intelligence = intelligenceScore(
|
| intelligenceComposite(entry.size_label, entry.intelligence_rank), intelMin, intelMax,
|
| );
|
|
|
| const budget = parseBudget(entry.monthly_token_budget);
|
| const headroom = headroomFactor(stats?.monthlyUsedTokens ?? 0, budget);
|
| const rl = rateLimitFactor(getPenalty(entry.model_db_id));
|
|
|
| const score = combineScore({ reliability, speed, intelligence, headroom, rateLimit: rl }, weights);
|
| return { axes: { reliability, speed, intelligence }, headroom, rateLimit: rl, score };
|
| }
|
|
|
| |
| |
| |
| |
| |
|
|
| function orderChain(chain: ChainRow[], strategy: RoutingStrategy): ChainRow[] {
|
| const weights = weightsFor(strategy);
|
| if (!weights) {
|
|
|
| return chain
|
| .map(e => ({ e, eff: e.priority + getPenalty(e.model_db_id) }))
|
| .sort((a, b) => a.eff - b.eff || a.e.priority - b.e.priority)
|
| .map(x => x.e);
|
| }
|
|
|
| const composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank));
|
| const intelMin = composites.length ? Math.min(...composites) : 0;
|
| const intelMax = composites.length ? Math.max(...composites) : 0;
|
|
|
| return chain
|
| .map(e => ({ e, s: scoreChainEntry(e, weights, intelMin, intelMax, true).score }))
|
|
|
| .sort((a, b) => b.s - a.s || a.e.priority - b.e.priority)
|
| .map(x => x.e);
|
| }
|
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false): RouteResult {
|
| const db = getDb();
|
|
|
| const strategy = getRoutingStrategy();
|
| if (strategy !== 'priority') refreshStatsCache(db);
|
|
|
|
|
| const chain = db.prepare(`
|
| SELECT fc.model_db_id, fc.priority, fc.enabled,
|
| m.platform, m.model_id, m.display_name, m.intelligence_rank,
|
| m.size_label, m.monthly_token_budget,
|
| m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision,
|
| m.supports_tools, m.context_window, m.key_id
|
| FROM fallback_config fc
|
| JOIN models m ON m.id = fc.model_db_id AND m.enabled = 1
|
| WHERE fc.enabled = 1
|
| `).all() as ChainRow[];
|
|
|
| const sortedChain = orderChain(chain, strategy);
|
|
|
|
|
| if (preferredModelDbId) {
|
| const idx = sortedChain.findIndex(e => e.model_db_id === preferredModelDbId);
|
| if (idx > 0) {
|
| const [preferred] = sortedChain.splice(idx, 1);
|
| sortedChain.unshift(preferred);
|
| }
|
| }
|
|
|
| for (const entry of sortedChain) {
|
|
|
|
|
| if (requireVision && !entry.supports_vision) continue;
|
|
|
|
|
|
|
|
|
|
|
|
|
| if (requireTools && !entry.supports_tools) continue;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| if (entry.context_window != null && estimatedTokens > entry.context_window) continue;
|
|
|
|
|
| const provider = getProvider(entry.platform as any);
|
| if (!provider) continue;
|
|
|
|
|
| const keys = db.prepare(
|
| "SELECT * FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown')"
|
| ).all(entry.platform) as KeyRow[];
|
|
|
| if (keys.length === 0) continue;
|
|
|
|
|
| const limits = {
|
| rpm: entry.rpm_limit,
|
| rpd: entry.rpd_limit,
|
| tpm: entry.tpm_limit,
|
| tpd: entry.tpd_limit,
|
| };
|
|
|
|
|
| const rrKey = `${entry.platform}:${entry.model_id}`;
|
| let idx = roundRobinIndex.get(rrKey) ?? 0;
|
|
|
| for (let attempt = 0; attempt < keys.length; attempt++) {
|
| const key = keys[idx % keys.length];
|
| idx++;
|
|
|
|
|
|
|
|
|
|
|
| if (entry.platform === 'custom' && entry.key_id != null && key.id !== entry.key_id) continue;
|
|
|
| const skipId = `${entry.platform}:${entry.model_id}:${key.id}`;
|
| if (skipKeys?.has(skipId)) continue;
|
|
|
|
|
| if (isOnCooldown(entry.platform, entry.model_id, key.id)) continue;
|
|
|
|
|
|
|
|
|
| if (!canUseProvider(entry.platform, key.id)) continue;
|
|
|
| if (!canMakeRequest(entry.platform, entry.model_id, key.id, limits)) continue;
|
| if (!canUseTokens(entry.platform, entry.model_id, key.id, estimatedTokens, limits)) continue;
|
|
|
| let decryptedKey: string;
|
| try {
|
| decryptedKey = decrypt(key.encrypted_key, key.iv, key.auth_tag);
|
| } catch {
|
| db.prepare("UPDATE api_keys SET status = 'error', last_checked_at = datetime('now') WHERE id = ?")
|
| .run(key.id);
|
| continue;
|
| }
|
|
|
|
|
|
|
|
|
| const resolvedProvider = entry.platform === 'custom'
|
| ? resolveProvider('custom', key.base_url)
|
| : provider;
|
| if (!resolvedProvider) continue;
|
|
|
|
|
| roundRobinIndex.set(rrKey, idx);
|
| return {
|
| provider: resolvedProvider,
|
| modelId: entry.model_id,
|
| modelDbId: entry.model_db_id,
|
| apiKey: decryptedKey,
|
| keyId: key.id,
|
| platform: entry.platform,
|
| displayName: entry.display_name,
|
| rpdLimit: limits.rpd,
|
| tpdLimit: limits.tpd,
|
| };
|
| }
|
|
|
|
|
|
|
| roundRobinIndex.set(rrKey, idx);
|
|
|
|
|
|
|
|
|
| }
|
|
|
| const err = new Error('All models exhausted. Add more API keys or wait for rate limits to reset.') as any;
|
| err.status = 429;
|
| throw err;
|
| }
|
|
|
| |
| |
| |
| |
|
|
| export interface RoutingScore {
|
| modelDbId: number;
|
| platform: string;
|
| modelId: string;
|
| displayName: string;
|
| enabled: boolean;
|
| reliability: number;
|
| speed: number;
|
| intelligence: number;
|
| headroom: number;
|
| rateLimit: number;
|
| score: number;
|
| totalRequests: number;
|
| }
|
|
|
| export function getRoutingScores(): { strategy: RoutingStrategy; weights: RoutingWeights | null; scores: RoutingScore[] } {
|
| const db = getDb();
|
| const strategy = getRoutingStrategy();
|
| refreshStatsCache(db);
|
|
|
| const chain = db.prepare(`
|
| SELECT fc.model_db_id, fc.priority, fc.enabled,
|
| m.platform, m.model_id, m.display_name, m.intelligence_rank,
|
| m.size_label, m.monthly_token_budget,
|
| m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision,
|
| m.supports_tools, m.context_window
|
| FROM fallback_config fc
|
| JOIN models m ON m.id = fc.model_db_id
|
| WHERE m.enabled = 1
|
| `).all() as ChainRow[];
|
|
|
|
|
|
|
| const weights = weightsFor(strategy) ?? BANDIT_PRESETS.balanced;
|
| const composites = chain.map(e => intelligenceComposite(e.size_label, e.intelligence_rank));
|
| const intelMin = composites.length ? Math.min(...composites) : 0;
|
| const intelMax = composites.length ? Math.max(...composites) : 0;
|
|
|
| const scores: RoutingScore[] = chain.map(entry => {
|
| const scored = scoreChainEntry(entry, weights, intelMin, intelMax, false);
|
| const stats = statsCache?.get(`${entry.platform}:${entry.model_id}`);
|
| return {
|
| modelDbId: entry.model_db_id,
|
| platform: entry.platform,
|
| modelId: entry.model_id,
|
| displayName: entry.display_name,
|
| enabled: entry.enabled === 1,
|
| reliability: scored.axes.reliability,
|
| speed: scored.axes.speed,
|
| intelligence: scored.axes.intelligence,
|
| headroom: scored.headroom,
|
| rateLimit: scored.rateLimit,
|
| score: scored.score,
|
| totalRequests: Math.round((stats?.successes ?? 0) + (stats?.failures ?? 0)),
|
| };
|
| }).sort((a, b) => b.score - a.score);
|
|
|
| return { strategy, weights: weightsFor(strategy), scores };
|
| }
|
|
|
|
|
|
|
|
|
| export function hasEnabledVisionModel(): boolean {
|
| const db = getDb();
|
| const row = db.prepare(`
|
| SELECT COUNT(*) as cnt
|
| FROM fallback_config fc
|
| JOIN models m ON m.id = fc.model_db_id
|
| WHERE fc.enabled = 1 AND m.enabled = 1 AND m.supports_vision = 1
|
| `).get() as { cnt: number };
|
| return row.cnt > 0;
|
| }
|
|
|
|
|
|
|
|
|
| export function hasEnabledToolsModel(): boolean {
|
| const db = getDb();
|
| const row = db.prepare(`
|
| SELECT COUNT(*) as cnt
|
| FROM fallback_config fc
|
| JOIN models m ON m.id = fc.model_db_id
|
| WHERE fc.enabled = 1 AND m.enabled = 1 AND m.supports_tools = 1
|
| `).get() as { cnt: number };
|
| return row.cnt > 0;
|
| }
|
|
|