Spaces:
Sleeping
Sleeping
feat: allow strings in `MODELS` env var (#1791)
Browse files* feat: allow strings in `MODELS` env var
This feature lets you simplify your `MODELS` env var by just specifying a bunch of model names on the hub and letting chat-ui deal with the rest.
For example
```
MODELS=['hf:HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF']
```
would automatically download the relevant file and serve it locally with `node-llama-cpp`
* fix: make it simpler to use
- src/lib/server/models.ts +25 -1
src/lib/server/models.ts
CHANGED
|
@@ -113,7 +113,31 @@ const ggufModelsConfig = await Promise.all(
|
|
| 113 |
})
|
| 114 |
);
|
| 115 |
|
| 116 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
|
| 118 |
if (env.LOAD_GGUF_MODELS === "true" || modelsRaw.length === 0) {
|
| 119 |
const parsedGgufModels = z.array(modelConfig).parse(ggufModelsConfig);
|
|
|
|
| 113 |
})
|
| 114 |
);
|
| 115 |
|
| 116 |
+
const turnStringIntoLocalModel = z.preprocess((obj: unknown) => {
|
| 117 |
+
if (typeof obj !== "string") return obj;
|
| 118 |
+
|
| 119 |
+
const name = obj.startsWith("hf:") ? obj.split(":")[1] : obj;
|
| 120 |
+
const displayName = obj.startsWith("hf:")
|
| 121 |
+
? obj.split(":")[1].split("/").slice(0, 2).join("/")
|
| 122 |
+
: obj.endsWith(".gguf")
|
| 123 |
+
? obj.replace(".gguf", "")
|
| 124 |
+
: obj;
|
| 125 |
+
|
| 126 |
+
const modelPath = obj.includes("/") && !obj.startsWith("hf:") ? `hf:${obj}` : obj;
|
| 127 |
+
|
| 128 |
+
return {
|
| 129 |
+
name,
|
| 130 |
+
displayName,
|
| 131 |
+
endpoints: [
|
| 132 |
+
{
|
| 133 |
+
type: "local",
|
| 134 |
+
modelPath,
|
| 135 |
+
},
|
| 136 |
+
],
|
| 137 |
+
} satisfies z.input<typeof modelConfig>;
|
| 138 |
+
}, modelConfig);
|
| 139 |
+
|
| 140 |
+
let modelsRaw = z.array(turnStringIntoLocalModel).parse(JSON5.parse(env.MODELS ?? "[]"));
|
| 141 |
|
| 142 |
if (env.LOAD_GGUF_MODELS === "true" || modelsRaw.length === 0) {
|
| 143 |
const parsedGgufModels = z.array(modelConfig).parse(ggufModelsConfig);
|