Refresh models (#1961)

* Add model refresh API and refactor model loading

Introduces a POST /models/refresh endpoint for admins to trigger a model refresh and receive a summary of changes. Refactors model loading logic in models.ts to support dynamic refresh, exposes refreshModels and lastModelRefreshSummary, and adds a test script (test-refresh.mjs) to verify refresh functionality. Minor code cleanup and improved override handling included.

* Delete test-refresh.mjs

src/lib/components/chat/ChatMessage.svelte

@@ -68,7 +68,7 @@
 	let editFormEl: HTMLFormElement | undefined = $state();
 
 	// Zero-config reasoning autodetection: detect <think> blocks in content
-		const THINK_BLOCK_REGEX = /(<think>[\s\S]*?(?:<\/think>|$))/gi;
+	const THINK_BLOCK_REGEX = /(<think>[\s\S]*?(?:<\/think>|$))/gi;
 	let hasClientThink = $derived(message.content.split(THINK_BLOCK_REGEX).length > 1);
 
 	// Strip think blocks for clipboard copy (always, regardless of detection)

src/lib/server/api/routes/groups/models.ts

@@ -1,4 +1,5 @@
-import { Elysia } from "elysia";
+import { Elysia, status } from "elysia";
+import { refreshModels, lastModelRefreshSummary } from "$lib/server/models";
 import type { BackendModel } from "$lib/server/models";
 import { authPlugin } from "../../authPlugin";
 import { authCondition } from "$lib/server/auth";
@@ -70,6 +71,42 @@ export const modelGroup = new Elysia().group("/models", (app) =>
 		.get("/old", async () => {
 			return [] as GETOldModelsResponse;
 		})
+		.group("/refresh", (app) =>
+			app.use(authPlugin).post("", async ({ locals }) => {
+				if (!locals.user && !locals.sessionId) {
+					throw status(401, "Unauthorized");
+				}
+				if (!locals.isAdmin) {
+					throw status(403, "Admin privileges required");
+				}
+
+				const previous = lastModelRefreshSummary;
+
+				try {
+					const summary = await refreshModels();
+
+					return {
+						refreshedAt: summary.refreshedAt.toISOString(),
+						durationMs: summary.durationMs,
+						added: summary.added,
+						removed: summary.removed,
+						changed: summary.changed,
+						total: summary.total,
+						hadChanges:
+							summary.added.length > 0 || summary.removed.length > 0 || summary.changed.length > 0,
+						previous:
+							previous.refreshedAt.getTime() > 0
+								? {
+										refreshedAt: previous.refreshedAt.toISOString(),
+										total: previous.total,
+									}
+								: null,
+					};
+				} catch (err) {
+					throw status(502, "Model refresh failed");
+				}
+			})
+		)
 		.group("/:namespace/:model?", (app) =>
 			app
 				.derive(async ({ params, error }) => {

src/lib/server/models.ts

@@ -76,18 +76,229 @@ const overrideEntrySchema = modelConfig
 
 type ModelOverride = z.infer<typeof overrideEntrySchema>;
 
-// ggufModelsConfig unused in this build
-
-// Source models exclusively from an OpenAI-compatible endpoint.
-let modelsRaw: ModelConfig[] = [];
-
-// Require explicit base URL; no implicit default here
 const openaiBaseUrl = config.OPENAI_BASE_URL
 	? config.OPENAI_BASE_URL.replace(/\/$/, "")
 	: undefined;
 const isHFRouter = openaiBaseUrl === "https://router.huggingface.co/v1";
 
-if (openaiBaseUrl) {
+const listSchema = z
+	.object({
+		data: z.array(
+			z.object({
+				id: z.string(),
+				description: z.string().optional(),
+				providers: z
+					.array(z.object({ supports_tools: z.boolean().optional() }).passthrough())
+					.optional(),
+				architecture: z
+					.object({
+						input_modalities: z.array(z.string()).optional(),
+					})
+					.passthrough()
+					.optional(),
+			})
+		),
+	})
+	.passthrough();
+
+function getChatPromptRender(_m: ModelConfig): (inputs: ChatTemplateInput) => string {
+	// Minimal template to support legacy "completions" flow if ever used.
+	// We avoid any tokenizer/Jinja usage in this build.
+	return ({ messages, preprompt }) => {
+		const parts: string[] = [];
+		if (preprompt) parts.push(`[SYSTEM]\n${preprompt}`);
+		for (const msg of messages) {
+			const role = msg.from === "assistant" ? "ASSISTANT" : msg.from.toUpperCase();
+			parts.push(`[${role}]\n${msg.content}`);
+		}
+		parts.push(`[ASSISTANT]`);
+		return parts.join("\n\n");
+	};
+}
+
+const processModel = async (m: ModelConfig) => ({
+	...m,
+	chatPromptRender: await getChatPromptRender(m),
+	id: m.id || m.name,
+	displayName: m.displayName || m.name,
+	preprompt: m.prepromptUrl ? await fetch(m.prepromptUrl).then((r) => r.text()) : m.preprompt,
+	parameters: { ...m.parameters, stop_sequences: m.parameters?.stop },
+	unlisted: m.unlisted ?? false,
+});
+
+const addEndpoint = (m: Awaited<ReturnType<typeof processModel>>) => ({
+	...m,
+	getEndpoint: async (): Promise<Endpoint> => {
+		if (!m.endpoints || m.endpoints.length === 0) {
+			throw new Error("No endpoints configured. This build requires OpenAI-compatible endpoints.");
+		}
+		// Only support OpenAI-compatible endpoints in this build
+		const endpoint = m.endpoints[0];
+		if (endpoint.type !== "openai") {
+			throw new Error("Only 'openai' endpoint type is supported in this build");
+		}
+		return await endpoints.openai({ ...endpoint, model: m });
+	},
+});
+
+type InternalProcessedModel = Awaited<ReturnType<typeof addEndpoint>> & {
+	isRouter: boolean;
+	hasInferenceAPI: boolean;
+};
+
+const inferenceApiIds: string[] = [];
+
+const getModelOverrides = (): ModelOverride[] => {
+	const overridesEnv = (Reflect.get(config, "MODELS") as string | undefined) ?? "";
+
+	if (!overridesEnv.trim()) {
+		return [];
+	}
+
+	try {
+		return z.array(overrideEntrySchema).parse(JSON5.parse(sanitizeJSONEnv(overridesEnv, "[]")));
+	} catch (error) {
+		logger.error(error, "[models] Failed to parse MODELS overrides");
+		return [];
+	}
+};
+
+export type ModelsRefreshSummary = {
+	refreshedAt: Date;
+	durationMs: number;
+	added: string[];
+	removed: string[];
+	changed: string[];
+	total: number;
+};
+
+export type ProcessedModel = InternalProcessedModel;
+
+export let models: ProcessedModel[] = [];
+export let defaultModel!: ProcessedModel;
+export let taskModel!: ProcessedModel;
+export let validModelIdSchema: z.ZodType<string> = z.string();
+export let lastModelRefresh = new Date(0);
+export let lastModelRefreshDurationMs = 0;
+export let lastModelRefreshSummary: ModelsRefreshSummary = {
+	refreshedAt: new Date(0),
+	durationMs: 0,
+	added: [],
+	removed: [],
+	changed: [],
+	total: 0,
+};
+
+let inflightRefresh: Promise<ModelsRefreshSummary> | null = null;
+
+const createValidModelIdSchema = (modelList: ProcessedModel[]): z.ZodType<string> => {
+	if (modelList.length === 0) {
+		throw new Error("No models available to build validation schema");
+	}
+	const ids = new Set(modelList.map((m) => m.id));
+	return z.string().refine((value) => ids.has(value), "Invalid model id");
+};
+
+const resolveTaskModel = (modelList: ProcessedModel[]) => {
+	if (modelList.length === 0) {
+		throw new Error("No models available to select task model");
+	}
+
+	if (config.TASK_MODEL) {
+		const preferred = modelList.find(
+			(m) => m.name === config.TASK_MODEL || m.id === config.TASK_MODEL
+		);
+		if (preferred) {
+			return preferred;
+		}
+	}
+
+	return modelList[0];
+};
+
+const signatureForModel = (model: ProcessedModel) =>
+	JSON.stringify({
+		description: model.description,
+		displayName: model.displayName,
+		providers: model.providers,
+		parameters: model.parameters,
+		preprompt: model.preprompt,
+		prepromptUrl: model.prepromptUrl,
+		endpoints:
+			model.endpoints?.map((endpoint) => {
+				if (endpoint.type === "openai") {
+					const { type, baseURL } = endpoint;
+					return { type, baseURL };
+				}
+				return { type: endpoint.type };
+			}) ?? null,
+		multimodal: model.multimodal,
+		multimodalAcceptedMimetypes: model.multimodalAcceptedMimetypes,
+		isRouter: model.isRouter,
+		hasInferenceAPI: model.hasInferenceAPI,
+	});
+
+const applyModelState = (newModels: ProcessedModel[], startedAt: number): ModelsRefreshSummary => {
+	if (newModels.length === 0) {
+		throw new Error("Failed to load any models from upstream");
+	}
+
+	const previousIds = new Set(models.map((m) => m.id));
+	const previousSignatures = new Map(models.map((m) => [m.id, signatureForModel(m)]));
+	const refreshedAt = new Date();
+	const durationMs = Date.now() - startedAt;
+
+	models = newModels;
+	defaultModel = models[0];
+	taskModel = resolveTaskModel(models);
+	validModelIdSchema = createValidModelIdSchema(models);
+	lastModelRefresh = refreshedAt;
+	lastModelRefreshDurationMs = durationMs;
+
+	const added = newModels.map((m) => m.id).filter((id) => !previousIds.has(id));
+	const removed = Array.from(previousIds).filter(
+		(id) => !newModels.some((model) => model.id === id)
+	);
+	const changed = newModels
+		.filter((model) => {
+			const previousSignature = previousSignatures.get(model.id);
+			return previousSignature !== undefined && previousSignature !== signatureForModel(model);
+		})
+		.map((model) => model.id);
+
+	const summary: ModelsRefreshSummary = {
+		refreshedAt,
+		durationMs,
+		added,
+		removed,
+		changed,
+		total: models.length,
+	};
+
+	lastModelRefreshSummary = summary;
+
+	logger.info(
+		{
+			total: summary.total,
+			added: summary.added,
+			removed: summary.removed,
+			changed: summary.changed,
+			durationMs: summary.durationMs,
+		},
+		"[models] Model cache refreshed"
+	);
+
+	return summary;
+};
+
+const buildModels = async (): Promise<ProcessedModel[]> => {
+	if (!openaiBaseUrl) {
+		logger.error(
+			"OPENAI_BASE_URL is required. Set it to an OpenAI-compatible base (e.g., https://router.huggingface.co/v1)."
+		);
+		throw new Error("OPENAI_BASE_URL not set");
+	}
+
 	try {
 		const baseURL = openaiBaseUrl;
 		logger.info({ baseURL }, "[models] Using OpenAI-compatible base URL");
@@ -113,30 +324,10 @@ if (openaiBaseUrl) {
 		const json = await response.json();
 		logger.info({ keys: Object.keys(json || {}) }, "[models] Response keys");
 
-		const listSchema = z
-			.object({
-				data: z.array(
-					z.object({
-						id: z.string(),
-						description: z.string().optional(),
-						providers: z
-							.array(z.object({ supports_tools: z.boolean().optional() }).passthrough())
-							.optional(),
-						architecture: z
-							.object({
-								input_modalities: z.array(z.string()).optional(),
-							})
-							.passthrough()
-							.optional(),
-					})
-				),
-			})
-			.passthrough();
-
 		const parsed = listSchema.parse(json);
 		logger.info({ count: parsed.data.length }, "[models] Parsed models count");
 
-		modelsRaw = parsed.data.map((m) => {
+		let modelsRaw = parsed.data.map((m) => {
 			let logoUrl: string | undefined = undefined;
 			if (isHFRouter && m.id.includes("/")) {
 				const org = m.id.split("/")[0];
@@ -166,163 +357,118 @@ if (openaiBaseUrl) {
 				],
 			} as ModelConfig;
 		}) as ModelConfig[];
-	} catch (e) {
-		logger.error(e, "Failed to load models from OpenAI base URL");
-		throw e;
-	}
-} else {
-	logger.error(
-		"OPENAI_BASE_URL is required. Set it to an OpenAI-compatible base (e.g., https://router.huggingface.co/v1)."
-	);
-	throw new Error("OPENAI_BASE_URL not set");
-}
-
-let modelOverrides: ModelOverride[] = [];
-const overridesEnv = (Reflect.get(config, "MODELS") as string | undefined) ?? "";
-
-if (overridesEnv.trim()) {
-	try {
-		modelOverrides = z
-			.array(overrideEntrySchema)
-			.parse(JSON5.parse(sanitizeJSONEnv(overridesEnv, "[]")));
-	} catch (error) {
-		logger.error(error, "[models] Failed to parse MODELS overrides");
-	}
-}
 
-if (modelOverrides.length) {
-	const overrideMap = new Map<string, ModelOverride>();
-	for (const override of modelOverrides) {
-		for (const key of [override.id, override.name]) {
-			const trimmed = key?.trim();
-			if (trimmed) overrideMap.set(trimmed, override);
-		}
-	}
+		const overrides = getModelOverrides();
 
-	modelsRaw = modelsRaw.map((model) => {
-		const override = overrideMap.get(model.id ?? "") ?? overrideMap.get(model.name ?? "");
-		if (!override) return model;
+		if (overrides.length) {
+			const overrideMap = new Map<string, ModelOverride>();
+			for (const override of overrides) {
+				for (const key of [override.id, override.name]) {
+					const trimmed = key?.trim();
+					if (trimmed) overrideMap.set(trimmed, override);
+				}
+			}
 
-		const { id, name, ...rest } = override;
-		void id;
-		void name;
+			modelsRaw = modelsRaw.map((model) => {
+				const override = overrideMap.get(model.id ?? "") ?? overrideMap.get(model.name ?? "");
+				if (!override) return model;
 
-		return {
-			...model,
-			...rest,
-		};
-	});
-}
+				const { id, name, ...rest } = override;
+				void id;
+				void name;
 
-function getChatPromptRender(_m: ModelConfig): (inputs: ChatTemplateInput) => string {
-	// Minimal template to support legacy "completions" flow if ever used.
-	// We avoid any tokenizer/Jinja usage in this build.
-	return ({ messages, preprompt }) => {
-		const parts: string[] = [];
-		if (preprompt) parts.push(`[SYSTEM]\n${preprompt}`);
-		for (const msg of messages) {
-			const role = msg.from === "assistant" ? "ASSISTANT" : msg.from.toUpperCase();
-			parts.push(`[${role}]\n${msg.content}`);
+				return {
+					...model,
+					...rest,
+				};
+			});
 		}
-		parts.push(`[ASSISTANT]`);
-		return parts.join("\n\n");
-	};
-}
 
-const processModel = async (m: ModelConfig) => ({
-	...m,
-	chatPromptRender: await getChatPromptRender(m),
-	id: m.id || m.name,
-	displayName: m.displayName || m.name,
-	preprompt: m.prepromptUrl ? await fetch(m.prepromptUrl).then((r) => r.text()) : m.preprompt,
-	parameters: { ...m.parameters, stop_sequences: m.parameters?.stop },
-	unlisted: m.unlisted ?? false,
-});
+		const builtModels = await Promise.all(
+			modelsRaw.map((e) =>
+				processModel(e)
+					.then(addEndpoint)
+					.then(async (m) => ({
+						...m,
+						hasInferenceAPI: inferenceApiIds.includes(m.id ?? m.name),
+						// router decoration added later
+						isRouter: false as boolean,
+					}))
+			)
+		);
+
+		const archBase = (config.LLM_ROUTER_ARCH_BASE_URL || "").trim();
+		const routerLabel = (config.PUBLIC_LLM_ROUTER_DISPLAY_NAME || "Omni").trim() || "Omni";
+		const routerLogo = (config.PUBLIC_LLM_ROUTER_LOGO_URL || "").trim();
+		const routerAliasId = (config.PUBLIC_LLM_ROUTER_ALIAS_ID || "omni").trim() || "omni";
+		const routerMultimodalEnabled =
+			(config.LLM_ROUTER_ENABLE_MULTIMODAL || "").toLowerCase() === "true";
+
+		let decorated = builtModels as ProcessedModel[];
+
+		if (archBase) {
+			// Build a minimal model config for the alias
+			const aliasRaw = {
+				id: routerAliasId,
+				name: routerAliasId,
+				displayName: routerLabel,
+				logoUrl: routerLogo || undefined,
+				preprompt: "",
+				endpoints: [
+					{
+						type: "openai" as const,
+						baseURL: openaiBaseUrl,
+					},
+				],
+				// Keep the alias visible
+				unlisted: false,
+			} as ModelConfig;
 
-const addEndpoint = (m: Awaited<ReturnType<typeof processModel>>) => ({
-	...m,
-	getEndpoint: async (): Promise<Endpoint> => {
-		if (!m.endpoints || m.endpoints.length === 0) {
-			throw new Error("No endpoints configured. This build requires OpenAI-compatible endpoints.");
-		}
-		// Only support OpenAI-compatible endpoints in this build
-		const endpoint = m.endpoints[0];
-		if (endpoint.type !== "openai") {
-			throw new Error("Only 'openai' endpoint type is supported in this build");
-		}
-		return await endpoints.openai({ ...endpoint, model: m });
-	},
-});
+			if (routerMultimodalEnabled) {
+				aliasRaw.multimodal = true;
+				aliasRaw.multimodalAcceptedMimetypes = ["image/*"];
+			}
 
-const inferenceApiIds: string[] = [];
+			const aliasBase = await processModel(aliasRaw);
+			// Create a self-referential ProcessedModel for the router endpoint
+			const aliasModel: ProcessedModel = {
+				...aliasBase,
+				isRouter: true,
+				hasInferenceAPI: false,
+				// getEndpoint uses the router wrapper regardless of the endpoints array
+				getEndpoint: async (): Promise<Endpoint> => makeRouterEndpoint(aliasModel),
+			} as ProcessedModel;
+
+			// Put alias first
+			decorated = [aliasModel, ...decorated];
+		}
 
-const builtModels = await Promise.all(
-	modelsRaw.map((e) =>
-		processModel(e)
-			.then(addEndpoint)
-			.then(async (m) => ({
-				...m,
-				hasInferenceAPI: inferenceApiIds.includes(m.id ?? m.name),
-				// router decoration added later
-				isRouter: false as boolean,
-			}))
-	)
-);
-
-// Inject a synthetic router alias ("Omni") if Arch router is configured
-const archBase = (config.LLM_ROUTER_ARCH_BASE_URL || "").trim();
-const routerLabel = (config.PUBLIC_LLM_ROUTER_DISPLAY_NAME || "Omni").trim() || "Omni";
-const routerLogo = (config.PUBLIC_LLM_ROUTER_LOGO_URL || "").trim();
-const routerAliasId = (config.PUBLIC_LLM_ROUTER_ALIAS_ID || "omni").trim() || "omni";
-const routerMultimodalEnabled =
-	(config.LLM_ROUTER_ENABLE_MULTIMODAL || "").toLowerCase() === "true";
-
-let decorated = builtModels as ProcessedModel[];
-
-if (archBase) {
-	// Build a minimal model config for the alias
-	const aliasRaw: ModelConfig = {
-		id: routerAliasId,
-		name: routerAliasId,
-		displayName: routerLabel,
-		logoUrl: routerLogo || undefined,
-		preprompt: "",
-		endpoints: [
-			{
-				type: "openai" as const,
-				baseURL: openaiBaseUrl,
-			},
-		],
-		// Keep the alias visible
-		unlisted: false,
-	} as ProcessedModel;
-
-	if (routerMultimodalEnabled) {
-		aliasRaw.multimodal = true;
-		aliasRaw.multimodalAcceptedMimetypes = ["image/*"];
+		return decorated;
+	} catch (e) {
+		logger.error(e, "Failed to load models from OpenAI base URL");
+		throw e;
 	}
+};
 
-	const aliasBase = await processModel(aliasRaw);
-	// Create a self-referential ProcessedModel for the router endpoint
-	const aliasModel: ProcessedModel = {
-		...aliasBase,
-		isRouter: true,
-		// getEndpoint uses the router wrapper regardless of the endpoints array
-		getEndpoint: async (): Promise<Endpoint> => makeRouterEndpoint(aliasModel),
-	} as ProcessedModel;
-
-	// Put alias first
-	decorated = [aliasModel, ...decorated];
-}
+const rebuildModels = async (): Promise<ModelsRefreshSummary> => {
+	const startedAt = Date.now();
+	const newModels = await buildModels();
+	return applyModelState(newModels, startedAt);
+};
 
-export const models = decorated as typeof builtModels;
+await rebuildModels();
 
-export type ProcessedModel = (typeof models)[number] & { isRouter?: boolean };
+export const refreshModels = async (): Promise<ModelsRefreshSummary> => {
+	if (inflightRefresh) {
+		return inflightRefresh;
+	}
 
-// super ugly but not sure how to make typescript happier
-export const validModelIdSchema = z.enum(models.map((m) => m.id) as [string, ...string[]]);
+	inflightRefresh = rebuildModels().finally(() => {
+		inflightRefresh = null;
+	});
 
-export const defaultModel = models[0];
+	return inflightRefresh;
+};
 
 export const validateModel = (_models: BackendModel[]) => {
 	// Zod enum function requires 2 parameters
@@ -331,13 +477,6 @@ export const validateModel = (_models: BackendModel[]) => {
 
 // if `TASK_MODEL` is string & name of a model in `MODELS`, then we use `MODELS[TASK_MODEL]`, else we try to parse `TASK_MODEL` as a model config itself
 
-export const taskModel = addEndpoint(
-	config.TASK_MODEL
-		? (models.find((m) => m.name === config.TASK_MODEL || m.id === config.TASK_MODEL) ??
-				defaultModel)
-		: defaultModel
-);
-
 export type BackendModel = Optional<
 	typeof defaultModel,
 	"preprompt" | "parameters" | "multimodal" | "unlisted" | "hasInferenceAPI"