Refactor token parameters to standardize naming across models and generation settings

src/lib/server/endpoints/openai/endpointOai.ts

@@ -119,11 +119,11 @@ export async function endpointOai(
 				model: model.id ?? model.name,
 				prompt,
 				stream: true,
-				max_tokens: parameters?.max_new_tokens,
+				max_tokens: parameters?.max_tokens,
 				stop: parameters?.stop,
 				temperature: parameters?.temperature,
 				top_p: parameters?.top_p,
-				frequency_penalty: parameters?.repetition_penalty,
+					frequency_penalty: parameters?.frequency_penalty,
 				presence_penalty: parameters?.presence_penalty,
 			};
 
@@ -168,12 +168,12 @@ export async function endpointOai(
 				stream: streamingSupported,
 				// Support two different ways of specifying token limits depending on the model
 				...(useCompletionTokens
-					? { max_completion_tokens: parameters?.max_new_tokens }
-					: { max_tokens: parameters?.max_new_tokens }),
+					? { max_completion_tokens: parameters?.max_tokens }
+					: { max_tokens: parameters?.max_tokens }),
 				stop: parameters?.stop,
 				temperature: parameters?.temperature,
 				top_p: parameters?.top_p,
-				frequency_penalty: parameters?.repetition_penalty,
+					frequency_penalty: parameters?.frequency_penalty,
 				presence_penalty: parameters?.presence_penalty,
 			};
 

src/lib/server/models.ts

@@ -60,11 +60,11 @@ const modelConfig = z.object({
 		.object({
 			temperature: z.number().min(0).max(2).optional(),
 			truncate: z.number().int().positive().optional(),
-			max_new_tokens: z.number().int().positive().optional(),
+			max_tokens: z.number().int().positive().optional(),
 			stop: z.array(z.string()).optional(),
 			top_p: z.number().positive().optional(),
 			top_k: z.number().positive().optional(),
-			repetition_penalty: z.number().min(-2).max(2).optional(),
+				frequency_penalty: z.number().min(-2).max(2).optional(),
 			presence_penalty: z.number().min(-2).max(2).optional(),
 		})
 		.passthrough()

src/lib/server/textGeneration/generate.ts

@@ -111,7 +111,7 @@ If the user is just having a casual conversation that doesn't require explanatio
 
 Do not use prefixes such as Response: or Answer: when answering to the user.`,
 						generateSettings: {
-							max_new_tokens: 1024,
+							max_tokens: 1024,
 						},
 						modelId: model.id,
 					});

src/lib/server/textGeneration/reasoning.ts

@@ -21,9 +21,9 @@ export async function generateSummaryOfReasoning(
 				preprompt: `You are tasked with summarizing the latest reasoning steps. Never describe results of the reasoning, only the process. Remain vague in your summary.
             The text might be incomplete, try your best to summarize it in one very short sentence, starting with a gerund and ending with three points. 
             Example: "Thinking about life...", "Summarizing the results...", "Processing the input..."`,
-				generateSettings: {
-					max_new_tokens: 50,
-				},
+			generateSettings: {
+				max_tokens: 50,
+			},
 				modelId,
 			})
 		);

src/lib/server/textGeneration/title.ts

@@ -44,9 +44,9 @@ Do not answer the question.
 Do not include the word prompt into your response.
 Do not include quotes, emojis, hashtags or trailing punctuation.
 Return ONLY the title text.`,
-			generateSettings: {
-				max_new_tokens: 30,
-			},
+		generateSettings: {
+			max_tokens: 30,
+		},
 			modelId,
 		})
 	)

src/lib/types/Assistant.ts

@@ -19,7 +19,7 @@ export interface Assistant extends Timestamps {
 	generateSettings?: {
 		temperature?: number;
 		top_p?: number;
-		repetition_penalty?: number;
+		frequency_penalty?: number;
 		top_k?: number;
 	};
 	dynamicPrompt?: boolean;