Add Prometheus metrics support with prom-client (#1934)

.env

@@ -137,6 +137,8 @@ WEBHOOK_URL_REPORT_ASSISTANT=#provide slack webhook url to get notified for repo
 
 
 ### Metrics ###
+METRICS_ENABLED=false
+METRICS_PORT=5565
 LOG_LEVEL=info
 
 

chart/env/prod.yaml

@@ -58,7 +58,7 @@ envVars:
   COOKIE_SAMESITE: "lax"
   COOKIE_SECURE: "true"
   EXPOSE_API: "true"
-  METRICS_PORT: "5565"
+  METRICS_ENABLED: "true"
   LOG_LEVEL: "debug"
 
   OPENAI_BASE_URL: "https://router.huggingface.co/v1"

chart/templates/deployment.yaml

@@ -53,6 +53,11 @@ spec:
             - containerPort: {{ $.Values.envVars.APP_PORT | default 3000 | int }}
               name: http
               protocol: TCP
+            {{- if eq "true" $.Values.envVars.METRICS_ENABLED }}
+            - containerPort: {{ $.Values.envVars.METRICS_PORT | default 5565 | int }}
+              name: metrics
+              protocol: TCP
+            {{- end }}
           resources: {{ toYaml .Values.resources | nindent 12 }}
           {{- with $.Values.extraEnv }}
           env:

chart/templates/service-monitor.yaml

@@ -0,0 +1,15 @@
+{{- if eq "true" $.Values.envVars.METRICS_ENABLED }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels: {{ include "labels.standard" . | nindent 4 }}
+  name: {{ include "name" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels: {{ include "labels.standard" . | nindent 6 }}
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 15s
+{{- end }}

chart/templates/service.yaml

@@ -11,5 +11,11 @@ spec:
     port: 80
     protocol: TCP
     targetPort: http
+  {{- if eq "true" $.Values.envVars.METRICS_ENABLED }}
+  - name: metrics
+    port: {{ $.Values.envVars.METRICS_PORT | default 5565 | int }}
+    protocol: TCP
+    targetPort: {{ $.Values.envVars.METRICS_PORT | default 5565 | quote }}
+  {{- end }}
   selector: {{ include "labels.standard" . | nindent 4 }}
   type: {{.Values.service.type}}

package-lock.json

@@ -35,6 +35,7 @@
 				"pino": "^9.0.0",
 				"pino-pretty": "^11.0.0",
 				"postcss": "^8.4.31",
+				"prom-client": "^15.1.3",
 				"satori": "^0.10.11",
 				"satori-html": "^0.3.2",
 				"sharp": "^0.33.4",
@@ -1748,6 +1749,15 @@
 				"node": ">= 8"
 			}
 		},
+		"node_modules/@opentelemetry/api": {
+			"version": "1.9.0",
+			"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
+			"integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
+			"license": "Apache-2.0",
+			"engines": {
+				"node": ">=8.0.0"
+			}
+		},
 		"node_modules/@pkgjs/parseargs": {
 			"version": "0.11.0",
 			"resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
@@ -3515,6 +3525,12 @@
 			"license": "MIT",
 			"optional": true
 		},
+		"node_modules/bintrees": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz",
+			"integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==",
+			"license": "MIT"
+		},
 		"node_modules/brace-expansion": {
 			"version": "2.0.2",
 			"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
@@ -8066,6 +8082,19 @@
 			],
 			"license": "MIT"
 		},
+		"node_modules/prom-client": {
+			"version": "15.1.3",
+			"resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz",
+			"integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==",
+			"license": "Apache-2.0",
+			"dependencies": {
+				"@opentelemetry/api": "^1.4.0",
+				"tdigest": "^0.1.1"
+			},
+			"engines": {
+				"node": "^16 || ^18 || >=20"
+			}
+		},
 		"node_modules/psl": {
 			"version": "1.15.0",
 			"resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz",
@@ -9303,6 +9332,15 @@
 				"streamx": "^2.15.0"
 			}
 		},
+		"node_modules/tdigest": {
+			"version": "0.1.2",
+			"resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz",
+			"integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==",
+			"license": "MIT",
+			"dependencies": {
+				"bintrees": "1.0.2"
+			}
+		},
 		"node_modules/text-decoder": {
 			"version": "1.2.3",
 			"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz",

package.json

@@ -93,6 +93,7 @@
 		"pino": "^9.0.0",
 		"pino-pretty": "^11.0.0",
 		"postcss": "^8.4.31",
+		"prom-client": "^15.1.3",
 		"satori": "^0.10.11",
 		"satori-html": "^0.3.2",
 		"sharp": "^0.33.4",

src/hooks.server.ts

@@ -18,6 +18,7 @@ import { initExitHandler } from "$lib/server/exitHandler";
 import { refreshConversationStats } from "$lib/jobs/refresh-conversation-stats";
 import { adminTokenManager } from "$lib/server/adminToken";
 import { isHostLocalhost } from "$lib/server/isURLLocal";
+import { MetricsServer } from "$lib/server/metrics";
 
 export const init: ServerInit = async () => {
 	// Wait for config to be fully loaded
@@ -41,6 +42,10 @@ export const init: ServerInit = async () => {
 		logger.info("Starting server...");
 		initExitHandler();
 
+		if (config.METRICS_ENABLED === "true") {
+			MetricsServer.getInstance();
+		}
+
 		checkAndRunMigrations();
 		refreshConversationStats();
 

src/lib/server/config.ts

@@ -151,7 +151,12 @@ export const ready = (async () => {
 	}
 })();
 
-type ExtraConfigKeys = "HF_TOKEN" | "ENABLE_ASSISTANTS";
+type ExtraConfigKeys =
+	| "HF_TOKEN"
+	| "OLD_MODELS"
+	| "ENABLE_ASSISTANTS"
+	| "METRICS_ENABLED"
+	| "METRICS_PORT";
 
 type ConfigProxy = ConfigManager & { [K in ConfigKey | ExtraConfigKeys]: string };
 

src/lib/server/metrics.ts

@@ -0,0 +1,255 @@
+import { collectDefaultMetrics, Counter, Registry, Summary } from "prom-client";
+import { logger } from "$lib/server/logger";
+import { config } from "$lib/server/config";
+import { createServer, type Server as HttpServer } from "http";
+import { onExit } from "./exitHandler";
+
+type ModelLabel = "model";
+type ToolLabel = "tool";
+
+interface Metrics {
+	model: {
+		conversationsTotal: Counter<ModelLabel>;
+		messagesTotal: Counter<ModelLabel>;
+		tokenCountTotal: Counter<ModelLabel>;
+		timePerOutputToken: Summary<ModelLabel>;
+		timeToFirstToken: Summary<ModelLabel>;
+		latency: Summary<ModelLabel>;
+		votesPositive: Counter<ModelLabel>;
+		votesNegative: Counter<ModelLabel>;
+	};
+	webSearch: {
+		requestCount: Counter;
+		pageFetchCount: Counter;
+		pageFetchCountError: Counter;
+		pageFetchDuration: Summary;
+		embeddingDuration: Summary;
+	};
+	tool: {
+		toolUseCount: Counter<ToolLabel>;
+		toolUseCountError: Counter<ToolLabel>;
+		toolUseDuration: Summary<ToolLabel>;
+		timeToChooseTools: Summary<ModelLabel>;
+	};
+}
+
+export class MetricsServer {
+	private static instance: MetricsServer | undefined;
+	private readonly enabled: boolean;
+	private readonly register: Registry;
+	private readonly metrics: Metrics;
+	private httpServer: HttpServer | undefined;
+
+	private constructor() {
+		this.enabled = config.METRICS_ENABLED === "true";
+		this.register = new Registry();
+
+		if (this.enabled) {
+			collectDefaultMetrics({ register: this.register });
+		}
+
+		this.metrics = this.createMetrics();
+
+		if (this.enabled) {
+			this.startStandaloneServer();
+		}
+	}
+
+	public static getInstance(): MetricsServer {
+		if (!MetricsServer.instance) {
+			MetricsServer.instance = new MetricsServer();
+		}
+		return MetricsServer.instance;
+	}
+
+	public static getMetrics(): Metrics {
+		return MetricsServer.getInstance().metrics;
+	}
+
+	public static isEnabled(): boolean {
+		return config.METRICS_ENABLED === "true";
+	}
+
+	public async render(): Promise<string> {
+		if (!this.enabled) {
+			return "";
+		}
+
+		return this.register.metrics();
+	}
+
+	private createMetrics(): Metrics {
+		const labelNames: ModelLabel[] = ["model"];
+		const toolLabelNames: ToolLabel[] = ["tool"];
+
+		const noopRegistry = new Registry();
+
+		const registry = this.enabled ? this.register : noopRegistry;
+
+		return {
+			model: {
+				conversationsTotal: new Counter<ModelLabel>({
+					name: "model_conversations_total",
+					help: "Total number of conversations",
+					labelNames,
+					registers: [registry],
+				}),
+				messagesTotal: new Counter<ModelLabel>({
+					name: "model_messages_total",
+					help: "Total number of messages",
+					labelNames,
+					registers: [registry],
+				}),
+				tokenCountTotal: new Counter<ModelLabel>({
+					name: "model_token_count_total",
+					help: "Total number of tokens emitted by the model",
+					labelNames,
+					registers: [registry],
+				}),
+				timePerOutputToken: new Summary<ModelLabel>({
+					name: "model_time_per_output_token_ms",
+					help: "Per-token latency in milliseconds",
+					labelNames,
+					registers: [registry],
+					maxAgeSeconds: 5 * 60,
+					ageBuckets: 5,
+				}),
+				timeToFirstToken: new Summary<ModelLabel>({
+					name: "model_time_to_first_token_ms",
+					help: "Time to first token in milliseconds",
+					labelNames,
+					registers: [registry],
+					maxAgeSeconds: 5 * 60,
+					ageBuckets: 5,
+				}),
+				latency: new Summary<ModelLabel>({
+					name: "model_latency_ms",
+					help: "Total time to complete a response in milliseconds",
+					labelNames,
+					registers: [registry],
+					maxAgeSeconds: 5 * 60,
+					ageBuckets: 5,
+				}),
+				votesPositive: new Counter<ModelLabel>({
+					name: "model_votes_positive_total",
+					help: "Total number of positive votes on model messages",
+					labelNames,
+					registers: [registry],
+				}),
+				votesNegative: new Counter<ModelLabel>({
+					name: "model_votes_negative_total",
+					help: "Total number of negative votes on model messages",
+					labelNames,
+					registers: [registry],
+				}),
+			},
+			webSearch: {
+				requestCount: new Counter({
+					name: "web_search_request_count",
+					help: "Total number of web search requests",
+					registers: [registry],
+				}),
+				pageFetchCount: new Counter({
+					name: "web_search_page_fetch_count",
+					help: "Total number of web search page fetches",
+					registers: [registry],
+				}),
+				pageFetchCountError: new Counter({
+					name: "web_search_page_fetch_count_error",
+					help: "Total number of web search page fetch errors",
+					registers: [registry],
+				}),
+				pageFetchDuration: new Summary({
+					name: "web_search_page_fetch_duration_ms",
+					help: "Duration of web search page fetches in milliseconds",
+					registers: [registry],
+					maxAgeSeconds: 5 * 60,
+					ageBuckets: 5,
+				}),
+				embeddingDuration: new Summary({
+					name: "web_search_embedding_duration_ms",
+					help: "Duration of web search embeddings in milliseconds",
+					registers: [registry],
+					maxAgeSeconds: 5 * 60,
+					ageBuckets: 5,
+				}),
+			},
+			tool: {
+				toolUseCount: new Counter<ToolLabel>({
+					name: "tool_use_count",
+					help: "Total number of tool invocations",
+					labelNames: toolLabelNames,
+					registers: [registry],
+				}),
+				toolUseCountError: new Counter<ToolLabel>({
+					name: "tool_use_count_error",
+					help: "Total number of tool invocation errors",
+					labelNames: toolLabelNames,
+					registers: [registry],
+				}),
+				toolUseDuration: new Summary<ToolLabel>({
+					name: "tool_use_duration_ms",
+					help: "Duration of tool invocations in milliseconds",
+					labelNames: toolLabelNames,
+					registers: [registry],
+					maxAgeSeconds: 30 * 60,
+					ageBuckets: 5,
+				}),
+				timeToChooseTools: new Summary<ModelLabel>({
+					name: "time_to_choose_tools_ms",
+					help: "Time spent selecting tools in milliseconds",
+					labelNames,
+					registers: [registry],
+					maxAgeSeconds: 5 * 60,
+					ageBuckets: 5,
+				}),
+			},
+		};
+	}
+
+	private startStandaloneServer() {
+		const port = Number(config.METRICS_PORT || "5565");
+
+		if (!Number.isInteger(port) || port < 0 || port > 65535) {
+			logger.warn(`Invalid METRICS_PORT value: ${config.METRICS_PORT}`);
+			return;
+		}
+
+		this.httpServer = createServer(async (req, res) => {
+			if (req.method !== "GET") {
+				res.statusCode = 405;
+				res.end("Method Not Allowed");
+				return;
+			}
+
+			try {
+				const payload = await this.render();
+				res.setHeader("Content-Type", "text/plain; version=0.0.4");
+				res.end(payload);
+			} catch (error) {
+				logger.error(error, "Failed to render metrics");
+				res.statusCode = 500;
+				res.end("Failed to render metrics");
+			}
+		});
+
+		this.httpServer.listen(port, () => {
+			logger.info(`Metrics server listening on port ${port}`);
+		});
+
+		onExit(async () => {
+			if (!this.httpServer) return;
+			logger.info("Shutting down metrics server...");
+			await new Promise<void>((resolve, reject) => {
+				this.httpServer?.close((err) => {
+					if (err) {
+						reject(err);
+						return;
+					}
+					resolve();
+				});
+			}).catch((error) => logger.error(error, "Failed to close metrics server"));
+			this.httpServer = undefined;
+		});
+	}
+}

src/routes/conversation/+server.ts

@@ -9,6 +9,7 @@ import { models, validateModel } from "$lib/server/models";
 import { v4 } from "uuid";
 import { authCondition } from "$lib/server/auth";
 import { usageLimits } from "$lib/server/usageLimits";
+import { MetricsServer } from "$lib/server/metrics";
 
 export const POST: RequestHandler = async ({ locals, request }) => {
 	const body = await request.text();
@@ -97,6 +98,10 @@ export const POST: RequestHandler = async ({ locals, request }) => {
 		...(values.fromShare ? { meta: { fromShareId: values.fromShare } } : {}),
 	});
 
+	if (MetricsServer.isEnabled()) {
+		MetricsServer.getMetrics().model.conversationsTotal.inc({ model: values.model });
+	}
+
 	return new Response(
 		JSON.stringify({
 			conversationId: res.insertedId.toString(),

src/routes/conversation/[id]/+server.ts

@@ -23,6 +23,7 @@ import { textGeneration } from "$lib/server/textGeneration";
 import type { TextGenerationContext } from "$lib/server/textGeneration/types";
 import { logger } from "$lib/server/logger.js";
 import { AbortRegistry } from "$lib/server/abortRegistry";
+import { MetricsServer } from "$lib/server/metrics";
 
 export async function POST({ request, locals, params, getClientAddress }) {
 	const id = z.string().parse(params.id);
@@ -313,6 +314,11 @@ export async function POST({ request, locals, params, getClientAddress }) {
 	let clientDetached = false;
 
 	let lastTokenTimestamp: undefined | Date = undefined;
+	let firstTokenObserved = false;
+	const metricsEnabled = MetricsServer.isEnabled();
+	const metrics = metricsEnabled ? MetricsServer.getMetrics() : undefined;
+	const metricsModelId = model.id ?? model.name ?? conv.model;
+	const metricsLabels = { model: metricsModelId };
 
 	const persistConversation = async () => {
 		await collections.conversations.updateOne(
@@ -344,9 +350,24 @@ export async function POST({ request, locals, params, getClientAddress }) {
 					if (event.token === "") return;
 					messageToWriteTo.content += event.token;
 
-					if (!lastTokenTimestamp) {
-						lastTokenTimestamp = new Date();
+					if (metricsEnabled && metrics) {
+						const now = Date.now();
+						metrics.model.tokenCountTotal.inc(metricsLabels);
+
+						if (!firstTokenObserved) {
+							metrics.model.timeToFirstToken.observe(
+								metricsLabels,
+								now - promptedAt.getTime()
+							);
+							firstTokenObserved = true;
+						}
+
+						const previousTimestamp = lastTokenTimestamp
+							? lastTokenTimestamp.getTime()
+							: promptedAt.getTime();
+						metrics.model.timePerOutputToken.observe(metricsLabels, now - previousTimestamp);
 					}
+
 					lastTokenTimestamp = new Date();
 				}
 
@@ -366,6 +387,10 @@ export async function POST({ request, locals, params, getClientAddress }) {
 					messageToWriteTo.interrupted = event.interrupted;
 					messageToWriteTo.content = initialMessageContent + event.text;
 					finalAnswerReceived = true;
+
+					if (metricsEnabled && metrics) {
+						metrics.model.latency.observe(metricsLabels, Date.now() - promptedAt.getTime());
+					}
 				}
 
 				// Add file
@@ -533,6 +558,10 @@ export async function POST({ request, locals, params, getClientAddress }) {
 		},
 	});
 
+	if (metricsEnabled && metrics) {
+		metrics.model.messagesTotal.inc(metricsLabels);
+	}
+
 	// Todo: maybe we should wait for the message to be saved before ending the response - in case of errors
 	return new Response(stream, {
 		headers: {

src/routes/metrics/+server.ts

@@ -0,0 +1,18 @@
+import { config } from "$lib/server/config";
+import { MetricsServer } from "$lib/server/metrics";
+
+export async function GET() {
+	if (config.METRICS_ENABLED !== "true") {
+		return new Response("Not Found", { status: 404 });
+	}
+
+	const payload = await MetricsServer.getInstance().render();
+
+	return new Response(payload, {
+		status: 200,
+		headers: {
+			"Content-Type": "text/plain; version=0.0.4",
+			"Cache-Control": "no-store",
+		},
+	});
+}