nsarrazin commited on
Commit
f887995
·
unverified ·
1 Parent(s): 4f27ac2

feat: allow strings in `MODELS` env var (#1791)

Browse files

* feat: allow strings in `MODELS` env var

This feature lets you simplify your `MODELS` env var by just specifying a bunch of model names on the hub and letting chat-ui deal with the rest.

For example

```
MODELS=['hf:HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF']
```

would automatically download the relevant file and serve it locally with `node-llama-cpp`

* fix: make it simpler to use

Files changed (1) hide show
  1. src/lib/server/models.ts +25 -1
src/lib/server/models.ts CHANGED
@@ -113,7 +113,31 @@ const ggufModelsConfig = await Promise.all(
113
  })
114
  );
115
 
116
- let modelsRaw = z.array(modelConfig).parse(JSON5.parse(env.MODELS ?? "[]"));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
117
 
118
  if (env.LOAD_GGUF_MODELS === "true" || modelsRaw.length === 0) {
119
  const parsedGgufModels = z.array(modelConfig).parse(ggufModelsConfig);
 
113
  })
114
  );
115
 
116
+ const turnStringIntoLocalModel = z.preprocess((obj: unknown) => {
117
+ if (typeof obj !== "string") return obj;
118
+
119
+ const name = obj.startsWith("hf:") ? obj.split(":")[1] : obj;
120
+ const displayName = obj.startsWith("hf:")
121
+ ? obj.split(":")[1].split("/").slice(0, 2).join("/")
122
+ : obj.endsWith(".gguf")
123
+ ? obj.replace(".gguf", "")
124
+ : obj;
125
+
126
+ const modelPath = obj.includes("/") && !obj.startsWith("hf:") ? `hf:${obj}` : obj;
127
+
128
+ return {
129
+ name,
130
+ displayName,
131
+ endpoints: [
132
+ {
133
+ type: "local",
134
+ modelPath,
135
+ },
136
+ ],
137
+ } satisfies z.input<typeof modelConfig>;
138
+ }, modelConfig);
139
+
140
+ let modelsRaw = z.array(turnStringIntoLocalModel).parse(JSON5.parse(env.MODELS ?? "[]"));
141
 
142
  if (env.LOAD_GGUF_MODELS === "true" || modelsRaw.length === 0) {
143
  const parsedGgufModels = z.array(modelConfig).parse(ggufModelsConfig);