victor HF Staff commited on
Commit
da4c9f6
·
1 Parent(s): cedc9cf

Add support for configuring router multimodal model via env

Browse files

Introduces the LLM_ROUTER_MULTIMODAL_MODEL environment variable to allow specifying a preferred multimodal model for the router. If set, the router will use the configured model for multimodal requests; otherwise, it falls back to the first available multimodal model. Updates are reflected in .env, prod.yaml, and endpoint.ts.

Files changed (3) hide show
  1. .env +2 -0
  2. chart/env/prod.yaml +1 -0
  3. src/lib/server/router/endpoint.ts +21 -0
.env CHANGED
@@ -65,6 +65,8 @@ LLM_ROUTER_ARCH_TIMEOUT_MS=10000
65
 
66
  # Enable router multimodal fallback (set to true to allow image inputs via router)
67
  LLM_ROUTER_ENABLE_MULTIMODAL=false
 
 
68
 
69
  # Router UI overrides (client-visible)
70
  # Public display name for the router entry in the model list. Defaults to "Omni".
 
65
 
66
  # Enable router multimodal fallback (set to true to allow image inputs via router)
67
  LLM_ROUTER_ENABLE_MULTIMODAL=false
68
+ # Optional: specific model to use for multimodal requests. If not set, uses first multimodal model
69
+ LLM_ROUTER_MULTIMODAL_MODEL=
70
 
71
  # Router UI overrides (client-visible)
72
  # Public display name for the router entry in the model list. Defaults to "Omni".
chart/env/prod.yaml CHANGED
@@ -73,6 +73,7 @@ envVars:
73
  LLM_ROUTER_OTHER_ROUTE: "casual_conversation"
74
  LLM_ROUTER_ARCH_TIMEOUT_MS: "10000"
75
  LLM_ROUTER_ENABLE_MULTIMODAL: "true"
 
76
  PUBLIC_LLM_ROUTER_DISPLAY_NAME: "Omni"
77
  PUBLIC_LLM_ROUTER_LOGO_URL: "https://cdn-uploads.huggingface.co/production/uploads/5f17f0a0925b9863e28ad517/C5V0v1xZXv6M7FXsdJH9b.png"
78
  PUBLIC_LLM_ROUTER_ALIAS_ID: "omni"
 
73
  LLM_ROUTER_OTHER_ROUTE: "casual_conversation"
74
  LLM_ROUTER_ARCH_TIMEOUT_MS: "10000"
75
  LLM_ROUTER_ENABLE_MULTIMODAL: "true"
76
+ LLM_ROUTER_MULTIMODAL_MODEL: "Qwen/Qwen3-VL-235B-A22B-Thinking"
77
  PUBLIC_LLM_ROUTER_DISPLAY_NAME: "Omni"
78
  PUBLIC_LLM_ROUTER_LOGO_URL: "https://cdn-uploads.huggingface.co/production/uploads/5f17f0a0925b9863e28ad517/C5V0v1xZXv6M7FXsdJH9b.png"
79
  PUBLIC_LLM_ROUTER_ALIAS_ID: "omni"
src/lib/server/router/endpoint.ts CHANGED
@@ -161,6 +161,27 @@ export async function makeRouterEndpoint(routerModel: ProcessedModel): Promise<E
161
  try {
162
  const mod = await import("../models");
163
  const all = (mod as { models: ProcessedModel[] }).models;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164
  const first = all?.find((m) => !m.isRouter && m.multimodal);
165
  return first?.id ?? first?.name;
166
  } catch (e) {
 
161
  try {
162
  const mod = await import("../models");
163
  const all = (mod as { models: ProcessedModel[] }).models;
164
+
165
+ // Check if a specific multimodal model is configured via env variable
166
+ const preferredModelId = config.LLM_ROUTER_MULTIMODAL_MODEL;
167
+ if (preferredModelId) {
168
+ const preferredModel = all?.find(
169
+ (m) => (m.id === preferredModelId || m.name === preferredModelId) && m.multimodal
170
+ );
171
+ if (preferredModel) {
172
+ logger.info(
173
+ { model: preferredModel.id ?? preferredModel.name },
174
+ "[router] using configured multimodal model"
175
+ );
176
+ return preferredModel.id ?? preferredModel.name;
177
+ }
178
+ logger.warn(
179
+ { configuredModel: preferredModelId },
180
+ "[router] configured multimodal model not found or not multimodal, falling back to first available"
181
+ );
182
+ }
183
+
184
+ // Fallback to first multimodal model
185
  const first = all?.find((m) => !m.isRouter && m.multimodal);
186
  return first?.id ?? first?.name;
187
  } catch (e) {