freellmapi / server /src /routes /fallback.ts
Nryn215's picture
Deploy: Enable and enforce pure ELO-based Intelligence routing strategy
ed57015
Raw
History Blame Contribute Delete
12.3 kB
/**
* Express router handles model fallback configuration and token budget reporting.
* It integrates named profiles dynamically into the fallback routing logic and aggregates
* monthly token consumption and rate limits (RPM/RPD/TPM/TPD) across configured models.
*/
import { Router } from 'express';
import type { Request, Response } from 'express';
import { z } from 'zod';
import { getDb } from '../db/index.js';
import { getAllPenalties, getRoutingScores, getRoutingStrategy, setRoutingStrategy, setCustomWeights } from '../services/router.js';
import { BANDIT_PRESETS, type RoutingStrategy } from '../services/scoring.js';
import { parseBudget } from '../lib/budget.js';
import { getModelGroups } from '../services/model-groups.js';
export const fallbackRouter = Router();
// ── Bandit routing strategy ─────────────────────────────────────────────────
// GET /routing → active strategy, preset weights, and the per-model score
// breakdown (reliability / speed / intelligence + guardrails).
fallbackRouter.get('/routing', (_req: Request, res: Response) => {
res.json(getRoutingScores());
});
const routingSchema = z.object({
strategy: z.enum(['priority', 'balanced', 'smartest', 'fastest', 'reliable', 'intelligence', 'custom']),
// Only meaningful with strategy 'custom': the user's weight vector. Any
// non-negative vector is accepted; setCustomWeights renormalizes to sum 1.
weights: z.object({
reliability: z.number().nonnegative(),
speed: z.number().nonnegative(),
intelligence: z.number().nonnegative(),
}).optional(),
});
// PUT /routing → switch strategy. Presets are just weight vectors over the three
// axes; 'priority' falls back to the legacy manual chain order; 'custom' uses
// the user's saved weights (optionally updated in the same request).
fallbackRouter.put('/routing', (req: Request, res: Response) => {
const parsed = routingSchema.safeParse(req.body);
if (!parsed.success) {
res.status(400).json({ error: { message: parsed.error.errors.map(e => e.message).join(', ') } });
return;
}
// Persist the weights before flipping the strategy so the new mode reads the
// intended vector immediately. setCustomWeights throws on an all-zero vector.
if (parsed.data.weights) {
try {
setCustomWeights(parsed.data.weights);
} catch (err: any) {
res.status(400).json({ error: { message: err?.message ?? 'Invalid custom weights' } });
return;
}
}
setRoutingStrategy(parsed.data.strategy as RoutingStrategy);
res.json({ strategy: getRoutingStrategy(), presets: BANDIT_PRESETS });
});
// Get fallback chain (with dynamic penalties)
fallbackRouter.get('/', (_req: Request, res: Response) => {
const db = getDb();
const rows = db.prepare(`
SELECT fc.model_db_id, fc.priority, fc.enabled,
m.platform, m.model_id, m.display_name, m.intelligence_rank,
m.speed_rank, m.size_label, m.rpm_limit, m.rpd_limit,
m.tpm_limit, m.tpd_limit, m.context_window,
m.monthly_token_budget, m.supports_vision, m.supports_tools
FROM fallback_config fc
JOIN models m ON m.id = fc.model_db_id
WHERE m.enabled = 1
ORDER BY fc.priority ASC
`).all() as any[];
// Count enabled and working/unknown keys per platform
const keyCounts = db.prepare(`
SELECT platform, COUNT(*) as count
FROM api_keys WHERE enabled = 1 AND status IN ('healthy', 'unknown', 'error')
GROUP BY platform
`).all() as { platform: string; count: number }[];
const keyCountMap = new Map(keyCounts.map(k => [k.platform, k.count]));
// Get current dynamic penalties
const penalties = getAllPenalties();
const penaltyMap = new Map(penalties.map(p => [p.modelDbId, p]));
// Logical-model grouping per row, so the dashboard can collapse the same
// model served by several providers into one expandable group. Always sent
// (cheap); the client renders grouped only when its unify toggle is on.
const groupByDbId = new Map<number, { groupKey: string; canonicalId: string; groupLabel: string }>();
for (const g of getModelGroups()) {
for (const m of g.members) {
groupByDbId.set(m.model_db_id, { groupKey: g.groupKey, canonicalId: g.canonicalId, groupLabel: g.groupLabel });
}
}
res.json(rows.map(r => {
const penalty = penaltyMap.get(r.model_db_id);
const group = groupByDbId.get(r.model_db_id);
return {
modelDbId: r.model_db_id,
groupKey: group?.groupKey,
canonicalId: group?.canonicalId,
groupLabel: group?.groupLabel,
priority: r.priority,
effectivePriority: r.priority + (penalty?.penalty ?? 0),
penalty: penalty?.penalty ?? 0,
rateLimitHits: penalty?.count ?? 0,
enabled: r.enabled === 1,
platform: r.platform,
modelId: r.model_id,
displayName: r.display_name,
intelligenceRank: r.intelligence_rank,
speedRank: r.speed_rank,
sizeLabel: r.size_label,
rpmLimit: r.rpm_limit,
rpdLimit: r.rpd_limit,
tpmLimit: r.tpm_limit,
tpdLimit: r.tpd_limit,
// Max context length (tokens), used by the dashboard catalog filter. Null
// for models whose context window the catalog doesn't record.
contextWindow: r.context_window,
monthlyTokenBudget: r.monthly_token_budget,
// Parsed once here (single source of truth) so the dashboard never re-implements
// budget-label parsing; 0 for rate-limited/placeholder labels. See lib/budget.ts.
monthlyTokenBudgetTokens: parseBudget(r.monthly_token_budget),
supportsVision: r.supports_vision === 1,
supportsTools: r.supports_tools === 1,
keyCount: keyCountMap.get(r.platform) ?? 0,
};
}));
});
const updateSchema = z.array(z.object({
modelDbId: z.number(),
priority: z.number(),
enabled: z.boolean(),
}));
// Update fallback chain (full replace)
fallbackRouter.put('/', (req: Request, res: Response) => {
const parsed = updateSchema.safeParse(req.body);
if (!parsed.success) {
res.status(400).json({ error: { message: parsed.error.errors.map(e => e.message).join(', ') } });
return;
}
const db = getDb();
const update = db.prepare(`
UPDATE fallback_config SET priority = ?, enabled = ? WHERE model_db_id = ?
`);
const updateAll = db.transaction(() => {
for (const entry of parsed.data) {
update.run(entry.priority, entry.enabled ? 1 : 0, entry.modelDbId);
}
});
updateAll();
res.json({ success: true });
});
// `intelligence_rank` is scoped to each provider's own catalog — a provider's
// #1 model is not globally #1 (see issue #135: MiniMax's top model outranking
// Gemini Pro because both read "Intel #1"). `size_label` IS a cross-provider
// capability tier, so normalize on it first and use intelligence_rank only as
// an in-tier tiebreaker. Unknown labels sort last.
const INTELLIGENCE_TIER =
"CASE m.size_label WHEN 'Frontier' THEN 1 WHEN 'Large' THEN 2 WHEN 'Medium' THEN 3 WHEN 'Small' THEN 4 ELSE 5 END";
// Sort presets — `orderBy` is selected from a fixed whitelist, never from
// user input directly, so the interpolation below is safe.
const SORT_PRESETS: Record<string, string> = {
intelligence: `${INTELLIGENCE_TIER} ASC, m.intelligence_rank ASC`,
speed: 'm.speed_rank ASC',
};
function getBudgetScore(m: { monthly_token_budget: string; tpd_limit: number | null }): number {
if (m.tpd_limit != null) return m.tpd_limit * 30;
const str = m.monthly_token_budget;
if (!str) return 0;
if (str.toLowerCase().includes('unlimited') || str.includes('∞')) return Infinity;
const cleanStr = str.split('(')[0];
const matches = cleanStr.match(/[\d.]+/g);
let maxNum = 0;
if (matches) {
maxNum = Math.max(...matches.map(mStr => parseFloat(mStr)));
}
let mult = 1;
const upper = cleanStr.toUpperCase();
if (upper.includes('B')) mult = 1_000_000_000;
else if (upper.includes('M')) mult = 1_000_000;
else if (upper.includes('K')) mult = 1_000;
return maxNum * mult;
}
fallbackRouter.post('/sort/:preset', (req: Request, res: Response) => {
const preset = String(req.params.preset);
const db = getDb();
let models: { id: number }[] = [];
if (preset === 'budget') {
const allModels = db.prepare(`SELECT id, monthly_token_budget, tpd_limit FROM models`).all() as any[];
allModels.sort((a, b) => getBudgetScore(b) - getBudgetScore(a));
models = allModels.map(m => ({ id: m.id }));
} else {
const orderBy = SORT_PRESETS[preset];
if (!orderBy) {
res.status(400).json({ error: { message: `Unknown preset: ${preset}. Use: intelligence, speed, budget` } });
return;
}
models = db.prepare(`SELECT m.id FROM models m ORDER BY ${orderBy}`).all() as { id: number }[];
}
const update = db.prepare('UPDATE fallback_config SET priority = ? WHERE model_db_id = ?');
const reorder = db.transaction(() => {
for (let i = 0; i < models.length; i++) {
update.run(i + 1, models[i].id);
}
});
reorder();
res.json({ success: true, preset });
});
// Token usage per model for the stacked bar
fallbackRouter.get('/token-usage', (_req: Request, res: Response) => {
const db = getDb();
// Get platforms that have enabled keys
const platforms = db.prepare(`
SELECT DISTINCT ak.platform
FROM api_keys ak
WHERE ak.enabled = 1
`).all() as { platform: string }[];
const platformSet = new Set(platforms.map(p => p.platform));
// Check if there is an active profile
const settingRow = db.prepare(`SELECT value FROM settings WHERE key = 'active_profile_id'`).get() as { value: string } | undefined;
const activeProfileId = settingRow ? (parseInt(settingRow.value) || null) : null;
// Verify active profile still exists
const activeProfile = activeProfileId
? db.prepare('SELECT id FROM profiles WHERE id = ?').get(activeProfileId) as any
: null;
let rawModels: { model_db_id: number; platform: string; model_id: string; display_name: string; monthly_token_budget: string; priority: number; enabled: number; rpm_limit: number | null; rpd_limit: number | null; tpm_limit: number | null; tpd_limit: number | null }[];
if (activeProfile) {
// Profile mode: use profile_models chain (all models in profile, checked against enabled)
rawModels = db.prepare(`
SELECT m.id as model_db_id, m.platform, m.model_id, m.display_name, m.monthly_token_budget,
pm.priority, pm.enabled,
m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit
FROM profile_models pm
JOIN models m ON m.id = pm.model_db_id
WHERE pm.profile_id = ? AND m.enabled = 1
ORDER BY pm.priority ASC
`).all(activeProfileId) as any[];
} else {
// Default mode: use fallback_config (only include enabled models)
rawModels = db.prepare(`
SELECT m.id as model_db_id, m.platform, m.model_id, m.display_name, m.monthly_token_budget,
fc.priority, fc.enabled,
m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit
FROM fallback_config fc
JOIN models m ON m.id = fc.model_db_id
WHERE m.enabled = 1
ORDER BY fc.priority ASC
`).all() as any[];
}
// Build per-model breakdown (only platforms with keys), preserving enabled state
const modelBudgets = rawModels
.filter(m => platformSet.has(m.platform))
.map(m => ({
modelDbId: m.model_db_id,
displayName: m.display_name,
platform: m.platform,
budget: parseBudget(m.monthly_token_budget),
enabled: m.enabled === 1,
rpmLimit: m.rpm_limit,
rpdLimit: m.rpd_limit,
tpmLimit: m.tpm_limit,
tpdLimit: m.tpd_limit,
}));
// Total budget counts all models (both enabled and disabled — they contribute to the pool)
const totalBudget = modelBudgets.reduce((s, m) => s + m.budget, 0);
// Tokens used this month
const usage = db.prepare(`
SELECT
COALESCE(SUM(input_tokens + output_tokens), 0) as total_used
FROM requests
WHERE created_at >= datetime('now', 'start of month')
AND request_type = 'chat'
`).get() as { total_used: number };
res.json({
totalBudget,
totalUsed: usage.total_used,
models: modelBudgets,
});
});