Add support for configuring router multimodal model via env
Browse filesIntroduces the LLM_ROUTER_MULTIMODAL_MODEL environment variable to allow specifying a preferred multimodal model for the router. If set, the router will use the configured model for multimodal requests; otherwise, it falls back to the first available multimodal model. Updates are reflected in .env, prod.yaml, and endpoint.ts.
- .env +2 -0
- chart/env/prod.yaml +1 -0
- src/lib/server/router/endpoint.ts +21 -0
.env
CHANGED
|
@@ -65,6 +65,8 @@ LLM_ROUTER_ARCH_TIMEOUT_MS=10000
|
|
| 65 |
|
| 66 |
# Enable router multimodal fallback (set to true to allow image inputs via router)
|
| 67 |
LLM_ROUTER_ENABLE_MULTIMODAL=false
|
|
|
|
|
|
|
| 68 |
|
| 69 |
# Router UI overrides (client-visible)
|
| 70 |
# Public display name for the router entry in the model list. Defaults to "Omni".
|
|
|
|
| 65 |
|
| 66 |
# Enable router multimodal fallback (set to true to allow image inputs via router)
|
| 67 |
LLM_ROUTER_ENABLE_MULTIMODAL=false
|
| 68 |
+
# Optional: specific model to use for multimodal requests. If not set, uses first multimodal model
|
| 69 |
+
LLM_ROUTER_MULTIMODAL_MODEL=
|
| 70 |
|
| 71 |
# Router UI overrides (client-visible)
|
| 72 |
# Public display name for the router entry in the model list. Defaults to "Omni".
|
chart/env/prod.yaml
CHANGED
|
@@ -73,6 +73,7 @@ envVars:
|
|
| 73 |
LLM_ROUTER_OTHER_ROUTE: "casual_conversation"
|
| 74 |
LLM_ROUTER_ARCH_TIMEOUT_MS: "10000"
|
| 75 |
LLM_ROUTER_ENABLE_MULTIMODAL: "true"
|
|
|
|
| 76 |
PUBLIC_LLM_ROUTER_DISPLAY_NAME: "Omni"
|
| 77 |
PUBLIC_LLM_ROUTER_LOGO_URL: "https://cdn-uploads.huggingface.co/production/uploads/5f17f0a0925b9863e28ad517/C5V0v1xZXv6M7FXsdJH9b.png"
|
| 78 |
PUBLIC_LLM_ROUTER_ALIAS_ID: "omni"
|
|
|
|
| 73 |
LLM_ROUTER_OTHER_ROUTE: "casual_conversation"
|
| 74 |
LLM_ROUTER_ARCH_TIMEOUT_MS: "10000"
|
| 75 |
LLM_ROUTER_ENABLE_MULTIMODAL: "true"
|
| 76 |
+
LLM_ROUTER_MULTIMODAL_MODEL: "Qwen/Qwen3-VL-235B-A22B-Thinking"
|
| 77 |
PUBLIC_LLM_ROUTER_DISPLAY_NAME: "Omni"
|
| 78 |
PUBLIC_LLM_ROUTER_LOGO_URL: "https://cdn-uploads.huggingface.co/production/uploads/5f17f0a0925b9863e28ad517/C5V0v1xZXv6M7FXsdJH9b.png"
|
| 79 |
PUBLIC_LLM_ROUTER_ALIAS_ID: "omni"
|
src/lib/server/router/endpoint.ts
CHANGED
|
@@ -161,6 +161,27 @@ export async function makeRouterEndpoint(routerModel: ProcessedModel): Promise<E
|
|
| 161 |
try {
|
| 162 |
const mod = await import("../models");
|
| 163 |
const all = (mod as { models: ProcessedModel[] }).models;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
const first = all?.find((m) => !m.isRouter && m.multimodal);
|
| 165 |
return first?.id ?? first?.name;
|
| 166 |
} catch (e) {
|
|
|
|
| 161 |
try {
|
| 162 |
const mod = await import("../models");
|
| 163 |
const all = (mod as { models: ProcessedModel[] }).models;
|
| 164 |
+
|
| 165 |
+
// Check if a specific multimodal model is configured via env variable
|
| 166 |
+
const preferredModelId = config.LLM_ROUTER_MULTIMODAL_MODEL;
|
| 167 |
+
if (preferredModelId) {
|
| 168 |
+
const preferredModel = all?.find(
|
| 169 |
+
(m) => (m.id === preferredModelId || m.name === preferredModelId) && m.multimodal
|
| 170 |
+
);
|
| 171 |
+
if (preferredModel) {
|
| 172 |
+
logger.info(
|
| 173 |
+
{ model: preferredModel.id ?? preferredModel.name },
|
| 174 |
+
"[router] using configured multimodal model"
|
| 175 |
+
);
|
| 176 |
+
return preferredModel.id ?? preferredModel.name;
|
| 177 |
+
}
|
| 178 |
+
logger.warn(
|
| 179 |
+
{ configuredModel: preferredModelId },
|
| 180 |
+
"[router] configured multimodal model not found or not multimodal, falling back to first available"
|
| 181 |
+
);
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
// Fallback to first multimodal model
|
| 185 |
const first = all?.find((m) => !m.isRouter && m.multimodal);
|
| 186 |
return first?.id ?? first?.name;
|
| 187 |
} catch (e) {
|