Revert "Generic Multimodal Support" (#1146)

Revert "Generic Multimodal Support (#1021)"

This reverts commit 57f89346cc6d953246d0f1705e342e2e66fce386.

README.md

@@ -230,7 +230,7 @@ The following is the default `chatPromptTemplate`, although newlines and indenti
 
 #### Multi modal model
 
-We currently support [IDEFICS](https://huggingface.co/blog/idefics) (hosted on TGI), OpenAI and Claude 3 as multimodal models. You can enable it by setting `multimodal: true` in your `MODELS` configuration. For IDEFICS, you must have a [PRO HF Api token](https://huggingface.co/settings/tokens). For OpenAI, see the [OpenAI section](#OpenAI). For Anthropic, see the [Anthropic section](#Anthropic).
+We currently only support IDEFICS as a multimodal model, hosted on TGI. You can enable it by using the following config (if you have a PRO HF Api token):
 
 ```env
     {
@@ -465,34 +465,14 @@ MODELS=`[
 
 #### Anthropic
 
-We also support Anthropic models (including multimodal ones via `multmodal: true`) through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example.
+We also support Anthropic models through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example.
 
 ```
 MODELS=`[
-  {
-      "name": "claude-3-haiku-20240307",
-      "displayName": "Claude 3 Haiku",
-      "description": "Fastest and most compact model for near-instant responsiveness",
-      "multimodal": true,
-      "parameters": {
-        "max_new_tokens": 4096,
-      },
-      "endpoints": [
-        {
-          "type": "anthropic",
-          // optionals
-          "apiKey": "sk-ant-...",
-          "baseURL": "https://api.anthropic.com",
-          "defaultHeaders": {},
-          "defaultQuery": {}
-        }
-      ]
-  },
   {
       "name": "claude-3-sonnet-20240229",
       "displayName": "Claude 3 Sonnet",
       "description": "Ideal balance of intelligence and speed",
-      "multimodal": true,
       "parameters": {
         "max_new_tokens": 4096,
       },
@@ -511,7 +491,6 @@ MODELS=`[
       "name": "claude-3-opus-20240229",
       "displayName": "Claude 3 Opus",
       "description": "Most powerful model for highly complex tasks",
-      "multimodal": true,
       "parameters": {
          "max_new_tokens": 4096
       },
@@ -537,7 +516,6 @@ MODELS=`[
       "name": "claude-3-sonnet@20240229",
       "displayName": "Claude 3 Sonnet",
       "description": "Ideal balance of intelligence and speed",
-      "multimodal": true,
       "parameters": {
         "max_new_tokens": 4096,
       },
@@ -556,7 +534,6 @@ MODELS=`[
       "name": "claude-3-haiku@20240307",
       "displayName": "Claude 3 Haiku",
       "description": "Fastest, most compact model for near-instant responsiveness",
-      "multimodal": true,
       "parameters": {
          "max_new_tokens": 4096
       },

package-lock.json

@@ -13,14 +13,13 @@
 				"@huggingface/inference": "^2.6.3",
 				"@iconify-json/bi": "^1.1.21",
 				"@playwright/browser-chromium": "^1.43.1",
-				"@resvg/resvg-js": "^2.6.2",
+				"@resvg/resvg-js": "^2.6.0",
 				"@xenova/transformers": "^2.16.1",
 				"autoprefixer": "^10.4.14",
 				"browser-image-resizer": "^2.4.1",
 				"date-fns": "^2.29.3",
 				"dotenv": "^16.0.3",
 				"express": "^4.19.2",
-				"file-type": "^19.0.0",
 				"handlebars": "^4.7.8",
 				"highlight.js": "^11.7.0",
 				"image-size": "^1.0.2",
@@ -42,7 +41,7 @@
 				"satori-html": "^0.3.2",
 				"sbd": "^1.0.19",
 				"serpapi": "^1.1.1",
-				"sharp": "^0.33.3",
+				"sharp": "^0.33.2",
 				"tailwind-scrollbar": "^3.0.0",
 				"tailwindcss": "^3.4.0",
 				"uuid": "^9.0.1",
@@ -89,7 +88,7 @@
 				"@google-cloud/vertexai": "^1.1.0",
 				"aws4fetch": "^1.0.17",
 				"cohere-ai": "^7.9.0",
-				"openai": "^4.44.0"
+				"openai": "^4.14.2"
 			}
 		},
 		"node_modules/@alloc/quick-lru": {
@@ -238,9 +237,9 @@
 			}
 		},
 		"node_modules/@emnapi/runtime": {
-			"version": "1.1.1",
-			"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.1.1.tgz",
-			"integrity": "sha512-3bfqkzuR1KLx57nZfjr2NLnFOobvyS0aTszaEGCGqmYMVDRaGvgIZbjGSV/MHSSmLgQ/b9JFHQ5xm5WRZYd+XQ==",
+			"version": "0.45.0",
+			"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-0.45.0.tgz",
+			"integrity": "sha512-Txumi3td7J4A/xTTwlssKieHKTGl3j4A1tglBx72auZ49YK7ePY6XZricgIg9mnZT4xPfA+UPCUdnhRuEFDL+w==",
 			"optional": true,
 			"dependencies": {
 				"tslib": "^2.4.0"
@@ -812,9 +811,9 @@
 			}
 		},
 		"node_modules/@img/sharp-darwin-arm64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.3.tgz",
-			"integrity": "sha512-FaNiGX1MrOuJ3hxuNzWgsT/mg5OHG/Izh59WW2mk1UwYHUwtfbhk5QNKYZgxf0pLOhx9ctGiGa2OykD71vOnSw==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.2.tgz",
+			"integrity": "sha512-itHBs1rPmsmGF9p4qRe++CzCgd+kFYktnsoR1sbIAfsRMrJZau0Tt1AH9KVnufc2/tU02Gf6Ibujx+15qRE03w==",
 			"cpu": [
 				"arm64"
 			],
@@ -833,13 +832,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-darwin-arm64": "1.0.2"
+				"@img/sharp-libvips-darwin-arm64": "1.0.1"
 			}
 		},
 		"node_modules/@img/sharp-darwin-x64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.3.tgz",
-			"integrity": "sha512-2QeSl7QDK9ru//YBT4sQkoq7L0EAJZA3rtV+v9p8xTKl4U1bUqTIaCnoC7Ctx2kCjQgwFXDasOtPTCT8eCTXvw==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.2.tgz",
+			"integrity": "sha512-/rK/69Rrp9x5kaWBjVN07KixZanRr+W1OiyKdXcbjQD6KbW+obaTeBBtLUAtbBsnlTTmWthw99xqoOS7SsySDg==",
 			"cpu": [
 				"x64"
 			],
@@ -858,13 +857,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-darwin-x64": "1.0.2"
+				"@img/sharp-libvips-darwin-x64": "1.0.1"
 			}
 		},
 		"node_modules/@img/sharp-libvips-darwin-arm64": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.2.tgz",
-			"integrity": "sha512-tcK/41Rq8IKlSaKRCCAuuY3lDJjQnYIW1UXU1kxcEKrfL8WR7N6+rzNoOxoQRJWTAECuKwgAHnPvqXGN8XfkHA==",
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.1.tgz",
+			"integrity": "sha512-kQyrSNd6lmBV7O0BUiyu/OEw9yeNGFbQhbxswS1i6rMDwBBSX+e+rPzu3S+MwAiGU3HdLze3PanQ4Xkfemgzcw==",
 			"cpu": [
 				"arm64"
 			],
@@ -883,9 +882,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-darwin-x64": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.2.tgz",
-			"integrity": "sha512-Ofw+7oaWa0HiiMiKWqqaZbaYV3/UGL2wAPeLuJTx+9cXpCRdvQhCLG0IH8YGwM0yGWGLpsF4Su9vM1o6aer+Fw==",
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.1.tgz",
+			"integrity": "sha512-eVU/JYLPVjhhrd8Tk6gosl5pVlvsqiFlt50wotCvdkFGf+mDNBJxMh+bvav+Wt3EBnNZWq8Sp2I7XfSjm8siog==",
 			"cpu": [
 				"x64"
 			],
@@ -904,9 +903,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linux-arm": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.2.tgz",
-			"integrity": "sha512-iLWCvrKgeFoglQxdEwzu1eQV04o8YeYGFXtfWU26Zr2wWT3q3MTzC+QTCO3ZQfWd3doKHT4Pm2kRmLbupT+sZw==",
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.1.tgz",
+			"integrity": "sha512-FtdMvR4R99FTsD53IA3LxYGghQ82t3yt0ZQ93WMZ2xV3dqrb0E8zq4VHaTOuLEAuA83oDawHV3fd+BsAPadHIQ==",
 			"cpu": [
 				"arm"
 			],
@@ -925,9 +924,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linux-arm64": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.2.tgz",
-			"integrity": "sha512-x7kCt3N00ofFmmkkdshwj3vGPCnmiDh7Gwnd4nUwZln2YjqPxV1NlTyZOvoDWdKQVDL911487HOueBvrpflagw==",
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.1.tgz",
+			"integrity": "sha512-bnGG+MJjdX70mAQcSLxgeJco11G+MxTz+ebxlz8Y3dxyeb3Nkl7LgLI0mXupoO+u1wRNx/iRj5yHtzA4sde1yA==",
 			"cpu": [
 				"arm64"
 			],
@@ -946,9 +945,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linux-s390x": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.2.tgz",
-			"integrity": "sha512-cmhQ1J4qVhfmS6szYW7RT+gLJq9dH2i4maq+qyXayUSn9/3iY2ZeWpbAgSpSVbV2E1JUL2Gg7pwnYQ1h8rQIog==",
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.1.tgz",
+			"integrity": "sha512-3+rzfAR1YpMOeA2zZNp+aYEzGNWK4zF3+sdMxuCS3ey9HhDbJ66w6hDSHDMoap32DueFwhhs3vwooAB2MaK4XQ==",
 			"cpu": [
 				"s390x"
 			],
@@ -967,9 +966,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linux-x64": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.2.tgz",
-			"integrity": "sha512-E441q4Qdb+7yuyiADVi5J+44x8ctlrqn8XgkDTwr4qPJzWkaHwD489iZ4nGDgcuya4iMN3ULV6NwbhRZJ9Z7SQ==",
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.1.tgz",
+			"integrity": "sha512-3NR1mxFsaSgMMzz1bAnnKbSAI+lHXVTqAHgc1bgzjHuXjo4hlscpUxc0vFSAPKI3yuzdzcZOkq7nDPrP2F8Jgw==",
 			"cpu": [
 				"x64"
 			],
@@ -988,9 +987,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linuxmusl-arm64": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.2.tgz",
-			"integrity": "sha512-3CAkndNpYUrlDqkCM5qhksfE+qSIREVpyoeHIU6jd48SJZViAmznoQQLAv4hVXF7xyUB9zf+G++e2v1ABjCbEQ==",
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.1.tgz",
+			"integrity": "sha512-5aBRcjHDG/T6jwC3Edl3lP8nl9U2Yo8+oTl5drd1dh9Z1EBfzUKAJFUDTDisDjUwc7N4AjnPGfCA3jl3hY8uDg==",
 			"cpu": [
 				"arm64"
 			],
@@ -1009,9 +1008,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linuxmusl-x64": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.2.tgz",
-			"integrity": "sha512-VI94Q6khIHqHWNOh6LLdm9s2Ry4zdjWJwH56WoiJU7NTeDwyApdZZ8c+SADC8OH98KWNQXnE01UdJ9CSfZvwZw==",
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.1.tgz",
+			"integrity": "sha512-dcT7inI9DBFK6ovfeWRe3hG30h51cBAP5JXlZfx6pzc/Mnf9HFCQDLtYf4MCBjxaaTfjCCjkBxcy3XzOAo5txw==",
 			"cpu": [
 				"x64"
 			],
@@ -1030,9 +1029,9 @@
 			}
 		},
 		"node_modules/@img/sharp-linux-arm": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.3.tgz",
-			"integrity": "sha512-Q7Ee3fFSC9P7vUSqVEF0zccJsZ8GiiCJYGWDdhEjdlOeS9/jdkyJ6sUSPj+bL8VuOYFSbofrW0t/86ceVhx32w==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.2.tgz",
+			"integrity": "sha512-Fndk/4Zq3vAc4G/qyfXASbS3HBZbKrlnKZLEJzPLrXoJuipFNNwTes71+Ki1hwYW5lch26niRYoZFAtZVf3EGA==",
 			"cpu": [
 				"arm"
 			],
@@ -1051,13 +1050,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linux-arm": "1.0.2"
+				"@img/sharp-libvips-linux-arm": "1.0.1"
 			}
 		},
 		"node_modules/@img/sharp-linux-arm64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.3.tgz",
-			"integrity": "sha512-Zf+sF1jHZJKA6Gor9hoYG2ljr4wo9cY4twaxgFDvlG0Xz9V7sinsPp8pFd1XtlhTzYo0IhDbl3rK7P6MzHpnYA==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.2.tgz",
+			"integrity": "sha512-pz0NNo882vVfqJ0yNInuG9YH71smP4gRSdeL09ukC2YLE6ZyZePAlWKEHgAzJGTiOh8Qkaov6mMIMlEhmLdKew==",
 			"cpu": [
 				"arm64"
 			],
@@ -1076,13 +1075,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linux-arm64": "1.0.2"
+				"@img/sharp-libvips-linux-arm64": "1.0.1"
 			}
 		},
 		"node_modules/@img/sharp-linux-s390x": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.3.tgz",
-			"integrity": "sha512-vFk441DKRFepjhTEH20oBlFrHcLjPfI8B0pMIxGm3+yilKyYeHEVvrZhYFdqIseSclIqbQ3SnZMwEMWonY5XFA==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.2.tgz",
+			"integrity": "sha512-MBoInDXDppMfhSzbMmOQtGfloVAflS2rP1qPcUIiITMi36Mm5YR7r0ASND99razjQUpHTzjrU1flO76hKvP5RA==",
 			"cpu": [
 				"s390x"
 			],
@@ -1101,13 +1100,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linux-s390x": "1.0.2"
+				"@img/sharp-libvips-linux-s390x": "1.0.1"
 			}
 		},
 		"node_modules/@img/sharp-linux-x64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.3.tgz",
-			"integrity": "sha512-Q4I++herIJxJi+qmbySd072oDPRkCg/SClLEIDh5IL9h1zjhqjv82H0Seupd+q2m0yOfD+/fJnjSoDFtKiHu2g==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.2.tgz",
+			"integrity": "sha512-xUT82H5IbXewKkeF5aiooajoO1tQV4PnKfS/OZtb5DDdxS/FCI/uXTVZ35GQ97RZXsycojz/AJ0asoz6p2/H/A==",
 			"cpu": [
 				"x64"
 			],
@@ -1126,13 +1125,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linux-x64": "1.0.2"
+				"@img/sharp-libvips-linux-x64": "1.0.1"
 			}
 		},
 		"node_modules/@img/sharp-linuxmusl-arm64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.3.tgz",
-			"integrity": "sha512-qnDccehRDXadhM9PM5hLvcPRYqyFCBN31kq+ErBSZtZlsAc1U4Z85xf/RXv1qolkdu+ibw64fUDaRdktxTNP9A==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.2.tgz",
+			"integrity": "sha512-F+0z8JCu/UnMzg8IYW1TMeiViIWBVg7IWP6nE0p5S5EPQxlLd76c8jYemG21X99UzFwgkRo5yz2DS+zbrnxZeA==",
 			"cpu": [
 				"arm64"
 			],
@@ -1151,13 +1150,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linuxmusl-arm64": "1.0.2"
+				"@img/sharp-libvips-linuxmusl-arm64": "1.0.1"
 			}
 		},
 		"node_modules/@img/sharp-linuxmusl-x64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.3.tgz",
-			"integrity": "sha512-Jhchim8kHWIU/GZ+9poHMWRcefeaxFIs9EBqf9KtcC14Ojk6qua7ghKiPs0sbeLbLj/2IGBtDcxHyjCdYWkk2w==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.2.tgz",
+			"integrity": "sha512-+ZLE3SQmSL+Fn1gmSaM8uFusW5Y3J9VOf+wMGNnTtJUMUxFhv+P4UPaYEYT8tqnyYVaOVGgMN/zsOxn9pSsO2A==",
 			"cpu": [
 				"x64"
 			],
@@ -1176,19 +1175,19 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linuxmusl-x64": "1.0.2"
+				"@img/sharp-libvips-linuxmusl-x64": "1.0.1"
 			}
 		},
 		"node_modules/@img/sharp-wasm32": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.3.tgz",
-			"integrity": "sha512-68zivsdJ0koE96stdUfM+gmyaK/NcoSZK5dV5CAjES0FUXS9lchYt8LAB5rTbM7nlWtxaU/2GON0HVN6/ZYJAQ==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.2.tgz",
+			"integrity": "sha512-fLbTaESVKuQcpm8ffgBD7jLb/CQLcATju/jxtTXR1XCLwbOQt+OL5zPHSDMmp2JZIeq82e18yE0Vv7zh6+6BfQ==",
 			"cpu": [
 				"wasm32"
 			],
 			"optional": true,
 			"dependencies": {
-				"@emnapi/runtime": "^1.1.0"
+				"@emnapi/runtime": "^0.45.0"
 			},
 			"engines": {
 				"node": "^18.17.0 || ^20.3.0 || >=21.0.0",
@@ -1201,9 +1200,9 @@
 			}
 		},
 		"node_modules/@img/sharp-win32-ia32": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.3.tgz",
-			"integrity": "sha512-CyimAduT2whQD8ER4Ux7exKrtfoaUiVr7HG0zZvO0XTFn2idUWljjxv58GxNTkFb8/J9Ub9AqITGkJD6ZginxQ==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.2.tgz",
+			"integrity": "sha512-okBpql96hIGuZ4lN3+nsAjGeggxKm7hIRu9zyec0lnfB8E7Z6p95BuRZzDDXZOl2e8UmR4RhYt631i7mfmKU8g==",
 			"cpu": [
 				"ia32"
 			],
@@ -1222,9 +1221,9 @@
 			}
 		},
 		"node_modules/@img/sharp-win32-x64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.3.tgz",
-			"integrity": "sha512-viT4fUIDKnli3IfOephGnolMzhz5VaTvDRkYqtZxOMIoMQ4MrAziO7pT1nVnOt2FAm7qW5aa+CCc13aEY6Le0g==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.2.tgz",
+			"integrity": "sha512-E4magOks77DK47FwHUIGH0RYWSgRBfGdK56kIHSVeB9uIS4pPFr4N2kIVsXdQQo4LzOsENKV5KAhRlRL7eMAdg==",
 			"cpu": [
 				"x64"
 			],
@@ -1445,31 +1444,31 @@
 			"integrity": "sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q=="
 		},
 		"node_modules/@resvg/resvg-js": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.2.tgz",
-			"integrity": "sha512-xBaJish5OeGmniDj9cW5PRa/PtmuVU3ziqrbr5xJj901ZDN4TosrVaNZpEiLZAxdfnhAe7uQ7QFWfjPe9d9K2Q==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.0.tgz",
+			"integrity": "sha512-Tf3YpbBKcQn991KKcw/vg7vZf98v01seSv6CVxZBbRkL/xyjnoYB6KgrFL6zskT1A4dWC/vg77KyNOW+ePaNlA==",
 			"engines": {
 				"node": ">= 10"
 			},
 			"optionalDependencies": {
-				"@resvg/resvg-js-android-arm-eabi": "2.6.2",
-				"@resvg/resvg-js-android-arm64": "2.6.2",
-				"@resvg/resvg-js-darwin-arm64": "2.6.2",
-				"@resvg/resvg-js-darwin-x64": "2.6.2",
-				"@resvg/resvg-js-linux-arm-gnueabihf": "2.6.2",
-				"@resvg/resvg-js-linux-arm64-gnu": "2.6.2",
-				"@resvg/resvg-js-linux-arm64-musl": "2.6.2",
-				"@resvg/resvg-js-linux-x64-gnu": "2.6.2",
-				"@resvg/resvg-js-linux-x64-musl": "2.6.2",
-				"@resvg/resvg-js-win32-arm64-msvc": "2.6.2",
-				"@resvg/resvg-js-win32-ia32-msvc": "2.6.2",
-				"@resvg/resvg-js-win32-x64-msvc": "2.6.2"
+				"@resvg/resvg-js-android-arm-eabi": "2.6.0",
+				"@resvg/resvg-js-android-arm64": "2.6.0",
+				"@resvg/resvg-js-darwin-arm64": "2.6.0",
+				"@resvg/resvg-js-darwin-x64": "2.6.0",
+				"@resvg/resvg-js-linux-arm-gnueabihf": "2.6.0",
+				"@resvg/resvg-js-linux-arm64-gnu": "2.6.0",
+				"@resvg/resvg-js-linux-arm64-musl": "2.6.0",
+				"@resvg/resvg-js-linux-x64-gnu": "2.6.0",
+				"@resvg/resvg-js-linux-x64-musl": "2.6.0",
+				"@resvg/resvg-js-win32-arm64-msvc": "2.6.0",
+				"@resvg/resvg-js-win32-ia32-msvc": "2.6.0",
+				"@resvg/resvg-js-win32-x64-msvc": "2.6.0"
 			}
 		},
 		"node_modules/@resvg/resvg-js-android-arm-eabi": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm-eabi/-/resvg-js-android-arm-eabi-2.6.2.tgz",
-			"integrity": "sha512-FrJibrAk6v29eabIPgcTUMPXiEz8ssrAk7TXxsiZzww9UTQ1Z5KAbFJs+Z0Ez+VZTYgnE5IQJqBcoSiMebtPHA==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm-eabi/-/resvg-js-android-arm-eabi-2.6.0.tgz",
+			"integrity": "sha512-lJnZ/2P5aMocrFMW7HWhVne5gH82I8xH6zsfH75MYr4+/JOaVcGCTEQ06XFohGMdYRP3v05SSPLPvTM/RHjxfA==",
 			"cpu": [
 				"arm"
 			],
@@ -1482,9 +1481,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-android-arm64": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm64/-/resvg-js-android-arm64-2.6.2.tgz",
-			"integrity": "sha512-VcOKezEhm2VqzXpcIJoITuvUS/fcjIw5NA/w3tjzWyzmvoCdd+QXIqy3FBGulWdClvp4g+IfUemigrkLThSjAQ==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm64/-/resvg-js-android-arm64-2.6.0.tgz",
+			"integrity": "sha512-N527f529bjMwYWShZYfBD60dXA4Fux+D695QsHQ93BDYZSHUoOh1CUGUyICevnTxs7VgEl98XpArmUWBZQVMfQ==",
 			"cpu": [
 				"arm64"
 			],
@@ -1497,9 +1496,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-darwin-arm64": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-arm64/-/resvg-js-darwin-arm64-2.6.2.tgz",
-			"integrity": "sha512-nmok2LnAd6nLUKI16aEB9ydMC6Lidiiq2m1nEBDR1LaaP7FGs4AJ90qDraxX+CWlVuRlvNjyYJTNv8qFjtL9+A==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-arm64/-/resvg-js-darwin-arm64-2.6.0.tgz",
+			"integrity": "sha512-MabUKLVayEwlPo0mIqAmMt+qESN8LltCvv5+GLgVga1avpUrkxj/fkU1TKm8kQegutUjbP/B0QuMuUr0uhF8ew==",
 			"cpu": [
 				"arm64"
 			],
@@ -1512,9 +1511,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-darwin-x64": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-x64/-/resvg-js-darwin-x64-2.6.2.tgz",
-			"integrity": "sha512-GInyZLjgWDfsVT6+SHxQVRwNzV0AuA1uqGsOAW+0th56J7Nh6bHHKXHBWzUrihxMetcFDmQMAX1tZ1fZDYSRsw==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-x64/-/resvg-js-darwin-x64-2.6.0.tgz",
+			"integrity": "sha512-zrFetdnSw/suXjmyxSjfDV7i61hahv6DDG6kM7BYN2yJ3Es5+BZtqYZTcIWogPJedYKmzN1YTMWGd/3f0ubFiA==",
 			"cpu": [
 				"x64"
 			],
@@ -1527,9 +1526,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-arm-gnueabihf": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm-gnueabihf/-/resvg-js-linux-arm-gnueabihf-2.6.2.tgz",
-			"integrity": "sha512-YIV3u/R9zJbpqTTNwTZM5/ocWetDKGsro0SWp70eGEM9eV2MerWyBRZnQIgzU3YBnSBQ1RcxRZvY/UxwESfZIw==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm-gnueabihf/-/resvg-js-linux-arm-gnueabihf-2.6.0.tgz",
+			"integrity": "sha512-sH4gxXt7v7dGwjGyzLwn7SFGvwZG6DQqLaZ11MmzbCwd9Zosy1TnmrMJfn6TJ7RHezmQMgBPi18bl55FZ1AT4A==",
 			"cpu": [
 				"arm"
 			],
@@ -1542,9 +1541,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-arm64-gnu": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-gnu/-/resvg-js-linux-arm64-gnu-2.6.2.tgz",
-			"integrity": "sha512-zc2BlJSim7YR4FZDQ8OUoJg5holYzdiYMeobb9pJuGDidGL9KZUv7SbiD4E8oZogtYY42UZEap7dqkkYuA91pg==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-gnu/-/resvg-js-linux-arm64-gnu-2.6.0.tgz",
+			"integrity": "sha512-fCyMncqCJtrlANADIduYF4IfnWQ295UKib7DAxFXQhBsM9PLDTpizr0qemZcCNadcwSVHnAIzL4tliZhCM8P6A==",
 			"cpu": [
 				"arm64"
 			],
@@ -1557,9 +1556,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-arm64-musl": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-musl/-/resvg-js-linux-arm64-musl-2.6.2.tgz",
-			"integrity": "sha512-3h3dLPWNgSsD4lQBJPb4f+kvdOSJHa5PjTYVsWHxLUzH4IFTJUAnmuWpw4KqyQ3NA5QCyhw4TWgxk3jRkQxEKg==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-musl/-/resvg-js-linux-arm64-musl-2.6.0.tgz",
+			"integrity": "sha512-ouLjTgBQHQyxLht4FdMPTvuY8xzJigM9EM2Tlu0llWkN1mKyTQrvYWi6TA6XnKdzDJHy7ZLpWpjZi7F5+Pg+Vg==",
 			"cpu": [
 				"arm64"
 			],
@@ -1572,9 +1571,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-x64-gnu": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.2.tgz",
-			"integrity": "sha512-IVUe+ckIerA7xMZ50duAZzwf1U7khQe2E0QpUxu5MBJNao5RqC0zwV/Zm965vw6D3gGFUl7j4m+oJjubBVoftw==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.0.tgz",
+			"integrity": "sha512-n3zC8DWsvxC1AwxpKFclIPapDFibs5XdIRoV/mcIlxlh0vseW1F49b97F33BtJQRmlntsqqN6GMMqx8byB7B+Q==",
 			"cpu": [
 				"x64"
 			],
@@ -1587,9 +1586,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-x64-musl": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-musl/-/resvg-js-linux-x64-musl-2.6.2.tgz",
-			"integrity": "sha512-UOf83vqTzoYQO9SZ0fPl2ZIFtNIz/Rr/y+7X8XRX1ZnBYsQ/tTb+cj9TE+KHOdmlTFBxhYzVkP2lRByCzqi4jQ==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-musl/-/resvg-js-linux-x64-musl-2.6.0.tgz",
+			"integrity": "sha512-n4tasK1HOlAxdTEROgYA1aCfsEKk0UOFDNd/AQTTZlTmCbHKXPq+O8npaaKlwXquxlVK8vrkcWbksbiGqbCAcw==",
 			"cpu": [
 				"x64"
 			],
@@ -1602,9 +1601,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-win32-arm64-msvc": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-arm64-msvc/-/resvg-js-win32-arm64-msvc-2.6.2.tgz",
-			"integrity": "sha512-7C/RSgCa+7vqZ7qAbItfiaAWhyRSoD4l4BQAbVDqRRsRgY+S+hgS3in0Rxr7IorKUpGE69X48q6/nOAuTJQxeQ==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-arm64-msvc/-/resvg-js-win32-arm64-msvc-2.6.0.tgz",
+			"integrity": "sha512-X2+EoBJFwDI5LDVb51Sk7ldnVLitMGr9WwU/i21i3fAeAXZb3hM16k67DeTy16OYkT2dk/RfU1tP1wG+rWbz2Q==",
 			"cpu": [
 				"arm64"
 			],
@@ -1617,9 +1616,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-win32-ia32-msvc": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-ia32-msvc/-/resvg-js-win32-ia32-msvc-2.6.2.tgz",
-			"integrity": "sha512-har4aPAlvjnLcil40AC77YDIk6loMawuJwFINEM7n0pZviwMkMvjb2W5ZirsNOZY4aDbo5tLx0wNMREp5Brk+w==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-ia32-msvc/-/resvg-js-win32-ia32-msvc-2.6.0.tgz",
+			"integrity": "sha512-L7oevWjQoUgK5W1fCKn0euSVemhDXVhrjtwqpc7MwBKKimYeiOshO1Li1pa8bBt5PESahenhWgdB6lav9O0fEg==",
 			"cpu": [
 				"ia32"
 			],
@@ -1632,9 +1631,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-win32-x64-msvc": {
-			"version": "2.6.2",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-x64-msvc/-/resvg-js-win32-x64-msvc-2.6.2.tgz",
-			"integrity": "sha512-ZXtYhtUr5SSaBrUDq7DiyjOFJqBVL/dOBN7N/qmi/pO0IgiWW/f/ue3nbvu9joWE5aAKDoIzy/CxsY0suwGosQ==",
+			"version": "2.6.0",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-x64-msvc/-/resvg-js-win32-x64-msvc-2.6.0.tgz",
+			"integrity": "sha512-8lJlghb+Unki5AyKgsnFbRJwkEj9r1NpwyuBG8yEJiG1W9eEGl03R3I7bsVa3haof/3J1NlWf0rzSa1G++A2iw==",
 			"cpu": [
 				"x64"
 			],
@@ -2076,11 +2075,6 @@
 				"node": ">=4"
 			}
 		},
-		"node_modules/@tokenizer/token": {
-			"version": "0.3.0",
-			"resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz",
-			"integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="
-		},
 		"node_modules/@tootallnate/once": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
@@ -3837,9 +3831,9 @@
 			}
 		},
 		"node_modules/detect-libc": {
-			"version": "2.0.3",
-			"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz",
-			"integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==",
+			"version": "2.0.2",
+			"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz",
+			"integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==",
 			"engines": {
 				"node": ">=8"
 			}
@@ -4562,22 +4556,6 @@
 				"node": "^10.12.0 || >=12.0.0"
 			}
 		},
-		"node_modules/file-type": {
-			"version": "19.0.0",
-			"resolved": "https://registry.npmjs.org/file-type/-/file-type-19.0.0.tgz",
-			"integrity": "sha512-s7cxa7/leUWLiXO78DVVfBVse+milos9FitauDLG1pI7lNaJ2+5lzPnr2N24ym+84HVwJL6hVuGfgVE+ALvU8Q==",
-			"dependencies": {
-				"readable-web-to-node-stream": "^3.0.2",
-				"strtok3": "^7.0.0",
-				"token-types": "^5.0.1"
-			},
-			"engines": {
-				"node": ">=18"
-			},
-			"funding": {
-				"url": "https://github.com/sindresorhus/file-type?sponsor=1"
-			}
-		},
 		"node_modules/fill-range": {
 			"version": "7.0.1",
 			"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -6407,15 +6385,16 @@
 			}
 		},
 		"node_modules/openai": {
-			"version": "4.47.1",
-			"resolved": "https://registry.npmjs.org/openai/-/openai-4.47.1.tgz",
-			"integrity": "sha512-WWSxhC/69ZhYWxH/OBsLEirIjUcfpQ5+ihkXKp06hmeYXgBBIUCa9IptMzYx6NdkiOCsSGYCnTIsxaic3AjRCQ==",
+			"version": "4.14.2",
+			"resolved": "https://registry.npmjs.org/openai/-/openai-4.14.2.tgz",
+			"integrity": "sha512-JGlm7mMC7J+cyQZnQMOH7daD9cBqqWqLtlBsejElEkgoehPrYfdyxSxIGICz5xk4YimbwI5FlLATSVojLtCKXQ==",
 			"optional": true,
 			"dependencies": {
 				"@types/node": "^18.11.18",
 				"@types/node-fetch": "^2.6.4",
 				"abort-controller": "^3.0.0",
 				"agentkeepalive": "^4.2.1",
+				"digest-fetch": "^1.3.0",
 				"form-data-encoder": "1.7.2",
 				"formdata-node": "^4.3.2",
 				"node-fetch": "^2.6.7",
@@ -6641,18 +6620,6 @@
 				"node": "*"
 			}
 		},
-		"node_modules/peek-readable": {
-			"version": "5.0.0",
-			"resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.0.0.tgz",
-			"integrity": "sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==",
-			"engines": {
-				"node": ">=14.16"
-			},
-			"funding": {
-				"type": "github",
-				"url": "https://github.com/sponsors/Borewit"
-			}
-		},
 		"node_modules/periscopic": {
 			"version": "3.1.0",
 			"resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.1.0.tgz",
@@ -7508,21 +7475,6 @@
 				"node": ">= 6"
 			}
 		},
-		"node_modules/readable-web-to-node-stream": {
-			"version": "3.0.2",
-			"resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz",
-			"integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==",
-			"dependencies": {
-				"readable-stream": "^3.6.0"
-			},
-			"engines": {
-				"node": ">=8"
-			},
-			"funding": {
-				"type": "github",
-				"url": "https://github.com/sponsors/Borewit"
-			}
-		},
 		"node_modules/readdirp": {
 			"version": "3.6.0",
 			"resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
@@ -7784,9 +7736,12 @@
 			"integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
 		},
 		"node_modules/semver": {
-			"version": "7.6.2",
-			"resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz",
-			"integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==",
+			"version": "7.5.4",
+			"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+			"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+			"dependencies": {
+				"lru-cache": "^6.0.0"
+			},
 			"bin": {
 				"semver": "bin/semver.js"
 			},
@@ -7885,42 +7840,42 @@
 			"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
 		},
 		"node_modules/sharp": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.3.tgz",
-			"integrity": "sha512-vHUeXJU1UvlO/BNwTpT0x/r53WkLUVxrmb5JTgW92fdFCFk0ispLMAeu/jPO2vjkXM1fYUi3K7/qcLF47pwM1A==",
+			"version": "0.33.2",
+			"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.2.tgz",
+			"integrity": "sha512-WlYOPyyPDiiM07j/UO+E720ju6gtNtHjEGg5vovUk1Lgxyjm2LFO+37Nt/UI3MMh2l6hxTWQWi7qk3cXJTutcQ==",
 			"hasInstallScript": true,
 			"dependencies": {
 				"color": "^4.2.3",
-				"detect-libc": "^2.0.3",
-				"semver": "^7.6.0"
+				"detect-libc": "^2.0.2",
+				"semver": "^7.5.4"
 			},
 			"engines": {
-				"libvips": ">=8.15.2",
+				"libvips": ">=8.15.1",
 				"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
 			},
 			"funding": {
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-darwin-arm64": "0.33.3",
-				"@img/sharp-darwin-x64": "0.33.3",
-				"@img/sharp-libvips-darwin-arm64": "1.0.2",
-				"@img/sharp-libvips-darwin-x64": "1.0.2",
-				"@img/sharp-libvips-linux-arm": "1.0.2",
-				"@img/sharp-libvips-linux-arm64": "1.0.2",
-				"@img/sharp-libvips-linux-s390x": "1.0.2",
-				"@img/sharp-libvips-linux-x64": "1.0.2",
-				"@img/sharp-libvips-linuxmusl-arm64": "1.0.2",
-				"@img/sharp-libvips-linuxmusl-x64": "1.0.2",
-				"@img/sharp-linux-arm": "0.33.3",
-				"@img/sharp-linux-arm64": "0.33.3",
-				"@img/sharp-linux-s390x": "0.33.3",
-				"@img/sharp-linux-x64": "0.33.3",
-				"@img/sharp-linuxmusl-arm64": "0.33.3",
-				"@img/sharp-linuxmusl-x64": "0.33.3",
-				"@img/sharp-wasm32": "0.33.3",
-				"@img/sharp-win32-ia32": "0.33.3",
-				"@img/sharp-win32-x64": "0.33.3"
+				"@img/sharp-darwin-arm64": "0.33.2",
+				"@img/sharp-darwin-x64": "0.33.2",
+				"@img/sharp-libvips-darwin-arm64": "1.0.1",
+				"@img/sharp-libvips-darwin-x64": "1.0.1",
+				"@img/sharp-libvips-linux-arm": "1.0.1",
+				"@img/sharp-libvips-linux-arm64": "1.0.1",
+				"@img/sharp-libvips-linux-s390x": "1.0.1",
+				"@img/sharp-libvips-linux-x64": "1.0.1",
+				"@img/sharp-libvips-linuxmusl-arm64": "1.0.1",
+				"@img/sharp-libvips-linuxmusl-x64": "1.0.1",
+				"@img/sharp-linux-arm": "0.33.2",
+				"@img/sharp-linux-arm64": "0.33.2",
+				"@img/sharp-linux-s390x": "0.33.2",
+				"@img/sharp-linux-x64": "0.33.2",
+				"@img/sharp-linuxmusl-arm64": "0.33.2",
+				"@img/sharp-linuxmusl-x64": "0.33.2",
+				"@img/sharp-wasm32": "0.33.2",
+				"@img/sharp-win32-ia32": "0.33.2",
+				"@img/sharp-win32-x64": "0.33.2"
 			}
 		},
 		"node_modules/shebang-command": {
@@ -8232,22 +8187,6 @@
 				"url": "https://github.com/sponsors/antfu"
 			}
 		},
-		"node_modules/strtok3": {
-			"version": "7.0.0",
-			"resolved": "https://registry.npmjs.org/strtok3/-/strtok3-7.0.0.tgz",
-			"integrity": "sha512-pQ+V+nYQdC5H3Q7qBZAz/MO6lwGhoC2gOAjuouGf/VO0m7vQRh8QNMl2Uf6SwAtzZ9bOw3UIeBukEGNJl5dtXQ==",
-			"dependencies": {
-				"@tokenizer/token": "^0.3.0",
-				"peek-readable": "^5.0.0"
-			},
-			"engines": {
-				"node": ">=14.16"
-			},
-			"funding": {
-				"type": "github",
-				"url": "https://github.com/sponsors/Borewit"
-			}
-		},
 		"node_modules/sucrase": {
 			"version": "3.32.0",
 			"resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.32.0.tgz",
@@ -8770,22 +8709,6 @@
 				"node": ">=0.6"
 			}
 		},
-		"node_modules/token-types": {
-			"version": "5.0.1",
-			"resolved": "https://registry.npmjs.org/token-types/-/token-types-5.0.1.tgz",
-			"integrity": "sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==",
-			"dependencies": {
-				"@tokenizer/token": "^0.3.0",
-				"ieee754": "^1.2.1"
-			},
-			"engines": {
-				"node": ">=14.16"
-			},
-			"funding": {
-				"type": "github",
-				"url": "https://github.com/sponsors/Borewit"
-			}
-		},
 		"node_modules/totalist": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.0.tgz",

package.json

@@ -57,15 +57,14 @@
 		"@huggingface/hub": "^0.5.1",
 		"@huggingface/inference": "^2.6.3",
 		"@iconify-json/bi": "^1.1.21",
-		"@resvg/resvg-js": "^2.6.2",
 		"@playwright/browser-chromium": "^1.43.1",
+		"@resvg/resvg-js": "^2.6.0",
 		"@xenova/transformers": "^2.16.1",
 		"autoprefixer": "^10.4.14",
 		"browser-image-resizer": "^2.4.1",
 		"date-fns": "^2.29.3",
 		"dotenv": "^16.0.3",
 		"express": "^4.19.2",
-		"file-type": "^19.0.0",
 		"handlebars": "^4.7.8",
 		"highlight.js": "^11.7.0",
 		"image-size": "^1.0.2",
@@ -87,7 +86,7 @@
 		"satori-html": "^0.3.2",
 		"sbd": "^1.0.19",
 		"serpapi": "^1.1.1",
-		"sharp": "^0.33.3",
+		"sharp": "^0.33.2",
 		"tailwind-scrollbar": "^3.0.0",
 		"tailwindcss": "^3.4.0",
 		"uuid": "^9.0.1",
@@ -99,6 +98,6 @@
 		"@google-cloud/vertexai": "^1.1.0",
 		"aws4fetch": "^1.0.17",
 		"cohere-ai": "^7.9.0",
-		"openai": "^4.44.0"
+		"openai": "^4.14.2"
 	}
 }

src/lib/components/chat/ChatMessage.svelte

@@ -308,17 +308,17 @@
 			{#if message.files && message.files.length > 0}
 				<div class="mx-auto grid w-fit grid-cols-2 gap-5 px-5">
 					{#each message.files as file}
-						<!-- handle the case where this is a hash that points to an image in the db -->
-						{#if file.type === "hash"}
+						<!-- handle the case where this is a hash that points to an image in the db, hash is always 64 char long -->
+						{#if file.length === 64}
 							<img
-								src={$page.url.pathname + "/output/" + file.value}
+								src={$page.url.pathname + "/output/" + file}
 								alt="input from user"
 								class="my-2 aspect-auto max-h-48 rounded-lg shadow-lg"
 							/>
 						{:else}
 							<!-- handle the case where this is a base64 encoded image -->
 							<img
-								src={`data:${file.mime};base64,${file.value}`}
+								src={"data:image/*;base64," + file}
 								alt="input from user"
 								class="my-2 aspect-auto max-h-48 rounded-lg shadow-lg"
 							/>

src/lib/components/chat/ChatWindow.svelte

@@ -92,9 +92,7 @@
 		(lastMessage.from === "user" ||
 			lastMessage.updates?.findIndex((u) => u.type === "status" && u.status === "error") !== -1);
 
-	$: sources = files?.map((file) =>
-		file2base64(file).then((value) => ({ type: "base64", value, mime: file.type }))
-	);
+	$: sources = files.map((file) => file2base64(file));
 
 	function onShare() {
 		dispatch("share");
@@ -231,13 +229,13 @@
 	<div
 		class="dark:via-gray-80 pointer-events-none absolute inset-x-0 bottom-0 z-0 mx-auto flex w-full max-w-3xl flex-col items-center justify-center bg-gradient-to-t from-white via-white/80 to-white/0 px-3.5 py-4 max-md:border-t max-md:bg-white sm:px-5 md:py-8 xl:max-w-4xl dark:border-gray-800 dark:from-gray-900 dark:to-gray-900/0 max-md:dark:bg-gray-900 [&>*]:pointer-events-auto"
 	>
-		{#if sources?.length}
+		{#if sources.length}
 			<div class="flex flex-row flex-wrap justify-center gap-2.5 max-md:pb-3">
 				{#each sources as source, index}
 					{#await source then src}
 						<div class="relative h-16 w-16 overflow-hidden rounded-lg shadow-lg">
 							<img
-								src={`data:${src.mime};base64,${src.value}`}
+								src={`data:image/*;base64,${src}`}
 								alt="input content"
 								class="h-full w-full rounded-lg bg-gray-400 object-cover dark:bg-gray-900"
 							/>

src/lib/server/endpoints/anthropic/endpointAnthropic.ts

@@ -1,9 +1,7 @@
 import { z } from "zod";
-import type { Endpoint } from "../endpoints";
 import { env } from "$env/dynamic/private";
+import type { Endpoint } from "../endpoints";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
-import { createImageProcessorOptionsValidator } from "../images";
-import { endpointMessagesToAnthropicMessages } from "./utils";
 
 export const endpointAnthropicParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -13,24 +11,12 @@ export const endpointAnthropicParametersSchema = z.object({
 	apiKey: z.string().default(env.ANTHROPIC_API_KEY ?? "sk-"),
 	defaultHeaders: z.record(z.string()).optional(),
 	defaultQuery: z.record(z.string()).optional(),
-	multimodal: z
-		.object({
-			image: createImageProcessorOptionsValidator({
-				supportedMimeTypes: ["image/png", "image/jpeg", "image/webp"],
-				preferredMimeType: "image/webp",
-				// The 4 / 3 compensates for the 33% increase in size when converting to base64
-				maxSizeInMB: (5 / 4) * 3,
-				maxWidth: 4096,
-				maxHeight: 4096,
-			}),
-		})
-		.default({}),
 });
 
 export async function endpointAnthropic(
 	input: z.input<typeof endpointAnthropicParametersSchema>
 ): Promise<Endpoint> {
-	const { baseURL, apiKey, model, defaultHeaders, defaultQuery, multimodal } =
+	const { baseURL, apiKey, model, defaultHeaders, defaultQuery } =
 		endpointAnthropicParametersSchema.parse(input);
 	let Anthropic;
 	try {
@@ -52,6 +38,16 @@ export async function endpointAnthropic(
 			system = messages[0].content;
 		}
 
+		const messagesFormatted = messages
+			.filter((message) => message.from !== "system")
+			.map((message) => ({
+				role: message.from,
+				content: message.content,
+			})) as unknown as {
+			role: "user" | "assistant";
+			content: string;
+		}[];
+
 		let tokenId = 0;
 
 		const parameters = { ...model.parameters, ...generateSettings };
@@ -59,7 +55,7 @@ export async function endpointAnthropic(
 		return (async function* () {
 			const stream = anthropic.messages.stream({
 				model: model.id ?? model.name,
-				messages: await endpointMessagesToAnthropicMessages(messages, multimodal),
+				messages: messagesFormatted,
 				max_tokens: parameters?.max_new_tokens,
 				temperature: parameters?.temperature,
 				top_p: parameters?.top_p,

src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts

@@ -1,8 +1,6 @@
 import { z } from "zod";
 import type { Endpoint } from "../endpoints";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
-import { createImageProcessorOptionsValidator } from "../images";
-import { endpointMessagesToAnthropicMessages } from "./utils";
 
 export const endpointAnthropicVertexParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -12,24 +10,12 @@ export const endpointAnthropicVertexParametersSchema = z.object({
 	projectId: z.string(),
 	defaultHeaders: z.record(z.string()).optional(),
 	defaultQuery: z.record(z.string()).optional(),
-	multimodal: z
-		.object({
-			image: createImageProcessorOptionsValidator({
-				supportedMimeTypes: ["image/png", "image/jpeg", "image/webp"],
-				preferredMimeType: "image/webp",
-				// The 4 / 3 compensates for the 33% increase in size when converting to base64
-				maxSizeInMB: (5 / 4) * 3,
-				maxWidth: 4096,
-				maxHeight: 4096,
-			}),
-		})
-		.default({}),
 });
 
 export async function endpointAnthropicVertex(
 	input: z.input<typeof endpointAnthropicVertexParametersSchema>
 ): Promise<Endpoint> {
-	const { region, projectId, model, defaultHeaders, defaultQuery, multimodal } =
+	const { region, projectId, model, defaultHeaders, defaultQuery } =
 		endpointAnthropicVertexParametersSchema.parse(input);
 	let AnthropicVertex;
 	try {
@@ -52,11 +38,21 @@ export async function endpointAnthropicVertex(
 			system = messages[0].content;
 		}
 
+		const messagesFormatted = messages
+			.filter((message) => message.from !== "system")
+			.map((message) => ({
+				role: message.from,
+				content: message.content,
+			})) as unknown as {
+			role: "user" | "assistant";
+			content: string;
+		}[];
+
 		let tokenId = 0;
 		return (async function* () {
 			const stream = anthropic.messages.stream({
 				model: model.id ?? model.name,
-				messages: await endpointMessagesToAnthropicMessages(messages, multimodal),
+				messages: messagesFormatted,
 				max_tokens: model.parameters?.max_new_tokens,
 				temperature: model.parameters?.temperature,
 				top_p: model.parameters?.top_p,

src/lib/server/endpoints/anthropic/utils.ts

@@ -1,44 +0,0 @@
-import type { ImageBlockParam, MessageParam } from "@anthropic-ai/sdk/resources";
-import { makeImageProcessor, type ImageProcessorOptions } from "../images";
-import type { EndpointMessage } from "../endpoints";
-import type { MessageFile } from "$lib/types/Message";
-
-export async function fileToImageBlock(
-	file: MessageFile,
-	opts: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp">
-): Promise<ImageBlockParam> {
-	const processor = makeImageProcessor(opts);
-	const { image, mime } = await processor(file);
-
-	return {
-		type: "image",
-		source: {
-			type: "base64",
-			media_type: mime,
-			data: image.toString("base64"),
-		},
-	};
-}
-
-type NonSystemMessage = EndpointMessage & { from: "user" | "assistant" };
-
-export async function endpointMessagesToAnthropicMessages(
-	messages: EndpointMessage[],
-	multimodal: { image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> }
-): Promise<MessageParam[]> {
-	return await Promise.all(
-		messages
-			.filter((message): message is NonSystemMessage => message.from !== "system")
-			.map<Promise<MessageParam>>(async (message) => {
-				return {
-					role: message.from,
-					content: [
-						...(await Promise.all(
-							(message.files ?? []).map((file) => fileToImageBlock(file, multimodal.image))
-						)),
-						{ type: "text", text: message.content },
-					],
-				};
-			})
-	);
-}

src/lib/server/endpoints/endpoints.ts

@@ -1,5 +1,4 @@
 import type { Conversation } from "$lib/types/Conversation";
-import type { Message } from "$lib/types/Message";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
 import { endpointTgi, endpointTgiParametersSchema } from "./tgi/endpointTgi";
 import { z } from "zod";
@@ -26,10 +25,9 @@ import endpointLangserve, {
 	endpointLangserveParametersSchema,
 } from "./langserve/endpointLangserve";
 
-export type EndpointMessage = Omit<Message, "id">;
 // parameters passed when generating text
 export interface EndpointParameters {
-	messages: EndpointMessage[];
+	messages: Omit<Conversation["messages"][0], "id">[];
 	preprompt?: Conversation["preprompt"];
 	continueMessage?: boolean; // used to signal that the last message will be extended
 	generateSettings?: Partial<Model["parameters"]>;

src/lib/server/endpoints/images.ts

@@ -1,211 +0,0 @@
-import type { Sharp } from "sharp";
-import sharp from "sharp";
-import type { MessageFile } from "$lib/types/Message";
-import { z, type util } from "zod";
-
-export interface ImageProcessorOptions<TMimeType extends string = string> {
-	supportedMimeTypes: TMimeType[];
-	preferredMimeType: TMimeType;
-	maxSizeInMB: number;
-	maxWidth: number;
-	maxHeight: number;
-}
-export type ImageProcessor<TMimeType extends string = string> = (file: MessageFile) => Promise<{
-	image: Buffer;
-	mime: TMimeType;
-}>;
-
-export function createImageProcessorOptionsValidator<TMimeType extends string = string>(
-	defaults: ImageProcessorOptions<TMimeType>
-) {
-	return z
-		.object({
-			supportedMimeTypes: z
-				.array(
-					z.enum<string, [TMimeType, ...TMimeType[]]>([
-						defaults.supportedMimeTypes[0],
-						...defaults.supportedMimeTypes.slice(1),
-					])
-				)
-				.default(defaults.supportedMimeTypes),
-			preferredMimeType: z
-				.enum([defaults.supportedMimeTypes[0], ...defaults.supportedMimeTypes.slice(1)])
-				.default(defaults.preferredMimeType as util.noUndefined<TMimeType>),
-			maxSizeInMB: z.number().positive().default(defaults.maxSizeInMB),
-			maxWidth: z.number().int().positive().default(defaults.maxWidth),
-			maxHeight: z.number().int().positive().default(defaults.maxHeight),
-		})
-		.default(defaults);
-}
-
-export function makeImageProcessor<TMimeType extends string = string>(
-	options: ImageProcessorOptions<TMimeType>
-): ImageProcessor<TMimeType> {
-	return async (file) => {
-		const { supportedMimeTypes, preferredMimeType, maxSizeInMB, maxWidth, maxHeight } = options;
-		const { mime, value } = file;
-
-		const buffer = Buffer.from(value, "base64");
-		let sharpInst = sharp(buffer);
-
-		const metadata = await sharpInst.metadata();
-		if (!metadata) throw Error("Failed to read image metadata");
-		const { width, height } = metadata;
-		if (width === undefined || height === undefined) throw Error("Failed to read image size");
-
-		const tooLargeInSize = width > maxWidth || height > maxHeight;
-		const tooLargeInBytes = buffer.byteLength > maxSizeInMB * 1000 * 1000;
-
-		const outputMime = chooseMimeType(supportedMimeTypes, preferredMimeType, mime, {
-			preferSizeReduction: tooLargeInBytes,
-		});
-
-		// Resize if necessary
-		if (tooLargeInSize || tooLargeInBytes) {
-			const size = chooseImageSize({
-				mime: outputMime,
-				width,
-				height,
-				maxWidth,
-				maxHeight,
-				maxSizeInMB,
-			});
-			if (size.width !== width || size.height !== height) {
-				sharpInst = resizeImage(sharpInst, size.width, size.height);
-			}
-		}
-
-		// Convert format if necessary
-		// We always want to convert the image when the file was too large in bytes
-		// so we can guarantee that ideal options are used, which are expected when
-		// choosing the image size
-		if (outputMime !== mime || tooLargeInBytes) {
-			sharpInst = convertImage(sharpInst, outputMime);
-		}
-
-		const processedImage = await sharpInst.toBuffer();
-		return { image: processedImage, mime: outputMime };
-	};
-}
-
-const outputFormats = ["png", "jpeg", "webp", "avif", "tiff", "gif"] as const;
-type OutputImgFormat = (typeof outputFormats)[number];
-const isOutputFormat = (format: string): format is (typeof outputFormats)[number] =>
-	outputFormats.includes(format as OutputImgFormat);
-
-export function convertImage(sharpInst: Sharp, outputMime: string): Sharp {
-	const [type, format] = outputMime.split("/");
-	if (type !== "image") throw Error(`Requested non-image mime type: ${outputMime}`);
-	if (!isOutputFormat(format)) {
-		throw Error(`Requested to convert to an unsupported format: ${format}`);
-	}
-
-	return sharpInst[format]();
-}
-
-// heic/heif requires proprietary license
-// TODO: blocking heif may be incorrect considering it also supports av1, so we should instead
-// detect the compression method used via sharp().metadata().compression
-// TODO: consider what to do about animated formats: apng, gif, animated webp, ...
-const blocklistedMimes = ["image/heic", "image/heif"];
-
-/** Sorted from largest to smallest */
-const mimesBySizeDesc = [
-	"image/png",
-	"image/tiff",
-	"image/gif",
-	"image/jpeg",
-	"image/webp",
-	"image/avif",
-];
-
-/**
- * Defaults to preferred format or uses existing mime if supported
- * When preferSizeReduction is true, it will choose the smallest format that is supported
- **/
-function chooseMimeType<T extends readonly string[]>(
-	supportedMimes: T,
-	preferredMime: string,
-	mime: string,
-	{ preferSizeReduction }: { preferSizeReduction: boolean }
-): T[number] {
-	if (!supportedMimes.includes(preferredMime)) {
-		const supportedMimesStr = supportedMimes.join(", ");
-		throw Error(
-			`Preferred format "${preferredMime}" not found in supported mimes: ${supportedMimesStr}`
-		);
-	}
-
-	const [type] = mime.split("/");
-	if (type !== "image") throw Error(`Received non-image mime type: ${mime}`);
-
-	if (supportedMimes.includes(mime) && !preferSizeReduction) return mime;
-
-	if (blocklistedMimes.includes(mime)) throw Error(`Received blocklisted mime type: ${mime}`);
-
-	const smallestMime = mimesBySizeDesc.findLast((m) => supportedMimes.includes(m));
-	return smallestMime ?? preferredMime;
-}
-
-interface ImageSizeOptions {
-	mime: string;
-	width: number;
-	height: number;
-	maxWidth: number;
-	maxHeight: number;
-	maxSizeInMB: number;
-}
-
-/** Resizes the image to fit within the specified size in MB by guessing the output size */
-export function chooseImageSize({
-	mime,
-	width,
-	height,
-	maxWidth,
-	maxHeight,
-	maxSizeInMB,
-}: ImageSizeOptions): { width: number; height: number } {
-	const biggestDiscrepency = Math.max(1, width / maxWidth, height / maxHeight);
-
-	let selectedWidth = Math.ceil(width / biggestDiscrepency);
-	let selectedHeight = Math.ceil(height / biggestDiscrepency);
-
-	do {
-		const estimatedSize = estimateImageSizeInBytes(mime, selectedWidth, selectedHeight);
-		if (estimatedSize < maxSizeInMB * 1024 * 1024) {
-			return { width: selectedWidth, height: selectedHeight };
-		}
-		selectedWidth = Math.floor(selectedWidth / 1.1);
-		selectedHeight = Math.floor(selectedHeight / 1.1);
-	} while (selectedWidth > 1 && selectedHeight > 1);
-
-	throw Error(`Failed to resize image to fit within ${maxSizeInMB}MB`);
-}
-
-const mimeToCompressionRatio: Record<string, number> = {
-	"image/png": 1 / 2,
-	"image/jpeg": 1 / 10,
-	"image/webp": 1 / 4,
-	"image/avif": 1 / 5,
-	"image/tiff": 1,
-	"image/gif": 1 / 5,
-};
-
-/**
- * Guesses the side of an image in MB based on its format and dimensions
- * Should guess the worst case
- **/
-function estimateImageSizeInBytes(mime: string, width: number, height: number): number {
-	const compressionRatio = mimeToCompressionRatio[mime];
-	if (!compressionRatio) throw Error(`Unsupported image format: ${mime}`);
-
-	const bitsPerPixel = 32; // Assuming 32-bit color depth for 8-bit R G B A
-	const bytesPerPixel = bitsPerPixel / 8;
-	const uncompressedSize = width * height * bytesPerPixel;
-
-	return uncompressedSize * compressionRatio;
-}
-
-export function resizeImage(sharpInst: Sharp, maxWidth: number, maxHeight: number): Sharp {
-	return sharpInst.resize({ width: maxWidth, height: maxHeight, fit: "inside" });
-}

src/lib/server/endpoints/openai/endpointOai.ts

@@ -6,10 +6,6 @@ import type { ChatCompletionCreateParamsStreaming } from "openai/resources/chat/
 import { buildPrompt } from "$lib/buildPrompt";
 import { env } from "$env/dynamic/private";
 import type { Endpoint } from "../endpoints";
-import type OpenAI from "openai";
-import { createImageProcessorOptionsValidator, makeImageProcessor } from "../images";
-import type { MessageFile } from "$lib/types/Message";
-import type { EndpointMessage } from "../endpoints";
 
 export const endpointOAIParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -23,41 +19,13 @@ export const endpointOAIParametersSchema = z.object({
 	defaultHeaders: z.record(z.string()).optional(),
 	defaultQuery: z.record(z.string()).optional(),
 	extraBody: z.record(z.any()).optional(),
-	multimodal: z
-		.object({
-			image: createImageProcessorOptionsValidator({
-				supportedMimeTypes: [
-					"image/png",
-					"image/jpeg",
-					"image/webp",
-					"image/avif",
-					"image/tiff",
-					"image/gif",
-				],
-				preferredMimeType: "image/webp",
-				maxSizeInMB: Infinity,
-				maxWidth: 4096,
-				maxHeight: 4096,
-			}),
-		})
-		.default({}),
 });
 
 export async function endpointOai(
 	input: z.input<typeof endpointOAIParametersSchema>
 ): Promise<Endpoint> {
-	const {
-		baseURL,
-		apiKey,
-		completion,
-		model,
-		defaultHeaders,
-		defaultQuery,
-		multimodal,
-		extraBody,
-	} = endpointOAIParametersSchema.parse(input);
-
-	/* eslint-disable-next-line no-shadow */
+	const { baseURL, apiKey, completion, model, defaultHeaders, defaultQuery, extraBody } =
+		endpointOAIParametersSchema.parse(input);
 	let OpenAI;
 	try {
 		OpenAI = (await import("openai")).OpenAI;
@@ -72,8 +40,6 @@ export async function endpointOai(
 		defaultQuery,
 	});
 
-	const imageProcessor = makeImageProcessor(multimodal.image);
-
 	if (completion === "completions") {
 		return async ({ messages, preprompt, continueMessage, generateSettings }) => {
 			const prompt = await buildPrompt({
@@ -103,8 +69,10 @@ export async function endpointOai(
 		};
 	} else if (completion === "chat_completions") {
 		return async ({ messages, preprompt, generateSettings }) => {
-			let messagesOpenAI: OpenAI.Chat.Completions.ChatCompletionMessageParam[] =
-				await prepareMessages(messages, imageProcessor);
+			let messagesOpenAI = messages.map((message) => ({
+				role: message.from,
+				content: message.content,
+			}));
 
 			if (messagesOpenAI?.[0]?.role !== "system") {
 				messagesOpenAI = [{ role: "system", content: "" }, ...messagesOpenAI];
@@ -136,39 +104,3 @@ export async function endpointOai(
 		throw new Error("Invalid completion type");
 	}
 }
-
-async function prepareMessages(
-	messages: EndpointMessage[],
-	imageProcessor: ReturnType<typeof makeImageProcessor>
-): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]> {
-	return Promise.all(
-		messages.map(async (message) => {
-			if (message.from === "user") {
-				return {
-					role: message.from,
-					content: [
-						...(await prepareFiles(imageProcessor, message.files ?? [])),
-						{ type: "text", text: message.content },
-					],
-				};
-			}
-			return {
-				role: message.from,
-				content: message.content,
-			};
-		})
-	);
-}
-
-async function prepareFiles(
-	imageProcessor: ReturnType<typeof makeImageProcessor>,
-	files: MessageFile[]
-): Promise<OpenAI.Chat.Completions.ChatCompletionContentPartImage[]> {
-	const processedFiles = await Promise.all(files.map(imageProcessor));
-	return processedFiles.map((file) => ({
-		type: "image_url" as const,
-		image_url: {
-			url: `data:${file.mime};base64,${file.image.toString("base64")}`,
-		},
-	}));
-}

src/lib/server/endpoints/preprocessMessages.ts

@@ -1,56 +0,0 @@
-import type { Message } from "$lib/types/Message";
-import { format } from "date-fns";
-import type { EndpointMessage } from "./endpoints";
-import { downloadFile } from "../files/downloadFile";
-import type { ObjectId } from "mongodb";
-
-export async function preprocessMessages(
-	messages: Message[],
-	webSearch: Message["webSearch"],
-	convId: ObjectId
-): Promise<EndpointMessage[]> {
-	return Promise.resolve(messages)
-		.then((msgs) => addWebSearchContext(msgs, webSearch))
-		.then((msgs) => downloadFiles(msgs, convId));
-}
-
-function addWebSearchContext(messages: Message[], webSearch: Message["webSearch"]) {
-	const webSearchContext = webSearch?.contextSources
-		.map(({ context }) => context.trim())
-		.join("\n\n----------\n\n");
-
-	// No web search context available, skip
-	if (!webSearch || !webSearchContext?.trim()) return messages;
-	// No messages available, skip
-	if (messages.length === 0) return messages;
-
-	const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? "";
-	const previousQuestions = messages
-		.filter((el) => el.from === "user")
-		.slice(0, -1)
-		.map((el) => el.content);
-	const currentDate = format(new Date(), "MMMM d, yyyy");
-
-	const finalMessage = {
-		...messages[messages.length - 1],
-		content: `I searched the web using the query: ${webSearch.searchQuery}.
-Today is ${currentDate} and here are the results:
-=====================
-${webSearchContext}
-=====================
-${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""}
-Answer the question: ${lastQuestion}`,
-	};
-
-	return [...messages.slice(0, -1), finalMessage];
-}
-
-async function downloadFiles(messages: Message[], convId: ObjectId): Promise<EndpointMessage[]> {
-	return Promise.all(
-		messages.map<Promise<EndpointMessage>>((message) =>
-			Promise.all((message.files ?? []).map((file) => downloadFile(file.value, convId))).then(
-				(files) => ({ ...message, files })
-			)
-		)
-	);
-}

src/lib/server/endpoints/tgi/endpointTgi.ts

@@ -1,13 +1,8 @@
 import { env } from "$env/dynamic/private";
 import { buildPrompt } from "$lib/buildPrompt";
 import { textGenerationStream } from "@huggingface/inference";
-import type { Endpoint, EndpointMessage } from "../endpoints";
+import type { Endpoint } from "../endpoints";
 import { z } from "zod";
-import {
-	createImageProcessorOptionsValidator,
-	makeImageProcessor,
-	type ImageProcessor,
-} from "../images";
 
 export const endpointTgiParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -16,32 +11,14 @@ export const endpointTgiParametersSchema = z.object({
 	url: z.string().url(),
 	accessToken: z.string().default(env.HF_TOKEN ?? env.HF_ACCESS_TOKEN),
 	authorization: z.string().optional(),
-	multimodal: z
-		.object({
-			// Assumes IDEFICS
-			image: createImageProcessorOptionsValidator({
-				supportedMimeTypes: ["image/jpeg", "image/webp"],
-				preferredMimeType: "image/webp",
-				maxSizeInMB: 5,
-				maxWidth: 224,
-				maxHeight: 224,
-			}),
-		})
-		.default({}),
 });
 
 export function endpointTgi(input: z.input<typeof endpointTgiParametersSchema>): Endpoint {
-	const { url, accessToken, model, authorization, multimodal } =
-		endpointTgiParametersSchema.parse(input);
-	const imageProcessor = makeImageProcessor(multimodal.image);
+	const { url, accessToken, model, authorization } = endpointTgiParametersSchema.parse(input);
 
 	return async ({ messages, preprompt, continueMessage, generateSettings }) => {
-		const messagesWithResizedFiles = await Promise.all(
-			messages.map((message) => prepareMessage(message, imageProcessor))
-		);
-
 		const prompt = await buildPrompt({
-			messages: messagesWithResizedFiles,
+			messages,
 			preprompt,
 			model,
 			continueMessage,
@@ -71,23 +48,4 @@ export function endpointTgi(input: z.input<typeof endpointTgiParametersSchema>):
 	};
 }
 
-const whiteImage = {
-	mime: "image/png",
-	image: Buffer.from(
-		"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAQABADAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD+/igAoAKACgD/2Q==",
-		"base64"
-	),
-};
-
-async function prepareMessage(
-	message: EndpointMessage,
-	imageProcessor: ImageProcessor
-): Promise<EndpointMessage> {
-	const files = await Promise.all(message.files?.map(imageProcessor) ?? [whiteImage]);
-	const markdowns = files.map(
-		(file) => `![](data:${file.mime};base64,${file.image.toString("base64")})`
-	);
-	const content = message.content + "\n" + markdowns.join("\n ");
-
-	return { ...message, content };
-}
+export default endpointTgi;

src/lib/server/files/downloadFile.ts

@@ -2,16 +2,15 @@ import { error } from "@sveltejs/kit";
 import { collections } from "$lib/server/database";
 import type { Conversation } from "$lib/types/Conversation";
 import type { SharedConversation } from "$lib/types/SharedConversation";
-import type { MessageFile } from "$lib/types/Message";
 
 export async function downloadFile(
 	sha256: string,
 	convId: Conversation["_id"] | SharedConversation["_id"]
-): Promise<MessageFile & { type: "base64" }> {
+) {
 	const fileId = collections.bucket.find({ filename: `${convId.toString()}-${sha256}` });
 	let mime = "";
 
-	const buffer = await fileId.next().then(async (file) => {
+	const content = await fileId.next().then(async (file) => {
 		if (!file) {
 			throw error(404, "File not found");
 		}
@@ -33,5 +32,5 @@ export async function downloadFile(
 		return fileBuffer;
 	});
 
-	return { type: "base64", value: buffer.toString("base64"), mime };
+	return { content, mime };
 }

src/lib/server/files/uploadFile.ts

@@ -1,27 +1,21 @@
 import type { Conversation } from "$lib/types/Conversation";
-import type { MessageFile } from "$lib/types/Message";
 import { sha256 } from "$lib/utils/sha256";
-import { fileTypeFromBuffer } from "file-type";
 import { collections } from "$lib/server/database";
 
-export async function uploadFile(file: File, conv: Conversation): Promise<MessageFile> {
+export async function uploadFile(file: Blob, conv: Conversation): Promise<string> {
 	const sha = await sha256(await file.text());
-	const buffer = await file.arrayBuffer();
-
-	// Attempt to detect the mime type of the file, fallback to the uploaded mime
-	const mime = await fileTypeFromBuffer(buffer).then((fileType) => fileType?.mime ?? file.type);
 
 	const upload = collections.bucket.openUploadStream(`${conv._id}-${sha}`, {
-		metadata: { conversation: conv._id.toString(), mime },
+		metadata: { conversation: conv._id.toString(), mime: "image/jpeg" },
 	});
 
 	upload.write((await file.arrayBuffer()) as unknown as Buffer);
 	upload.end();
 
-	// only return the filename when upload throws a finish event or a 20s time out occurs
+	// only return the filename when upload throws a finish event or a 10s time out occurs
 	return new Promise((resolve, reject) => {
-		upload.once("finish", () => resolve({ type: "hash", value: sha, mime: file.type }));
+		upload.once("finish", () => resolve(sha));
 		upload.once("error", reject);
-		setTimeout(() => reject(new Error("Upload timed out")), 20_000);
+		setTimeout(() => reject(new Error("Upload timed out")), 10000);
 	});
 }

src/lib/server/generateFromDefaultEndpoint.ts

@@ -1,12 +1,12 @@
 import { smallModel } from "$lib/server/models";
-import type { EndpointMessage } from "./endpoints/endpoints";
+import type { Conversation } from "$lib/types/Conversation";
 
 export async function generateFromDefaultEndpoint({
 	messages,
 	preprompt,
 	generateSettings,
 }: {
-	messages: EndpointMessage[];
+	messages: Omit<Conversation["messages"][0], "id">[];
 	preprompt?: string;
 	generateSettings?: Record<string, unknown>;
 }): Promise<string> {

src/lib/server/models.ts

@@ -3,7 +3,7 @@ import type { ChatTemplateInput } from "$lib/types/Template";
 import { compileTemplate } from "$lib/utils/template";
 import { z } from "zod";
 import endpoints, { endpointSchema, type Endpoint } from "./endpoints/endpoints";
-import { endpointTgi } from "./endpoints/tgi/endpointTgi";
+import endpointTgi from "./endpoints/tgi/endpointTgi";
 import { sum } from "$lib/utils/sum";
 import { embeddingModels, validateEmbeddingModelByName } from "./embeddingModels";
 

src/lib/server/preprocessMessages.ts

@@ -0,0 +1,61 @@
+import type { Conversation } from "$lib/types/Conversation";
+import type { Message } from "$lib/types/Message";
+import { format } from "date-fns";
+import { downloadFile } from "./files/downloadFile";
+import { logger } from "$lib/server/logger";
+
+export async function preprocessMessages(
+	messages: Message[],
+	webSearch: Message["webSearch"],
+	multimodal: boolean,
+	id: Conversation["_id"]
+): Promise<Message[]> {
+	return await Promise.all(
+		structuredClone(messages).map(async (message, idx) => {
+			const webSearchContext = webSearch?.contextSources
+				.map(({ context }) => context.trim())
+				.join("\n\n----------\n\n");
+
+			// start by adding websearch to the last message
+			if (idx === messages.length - 1 && webSearch && webSearchContext?.trim()) {
+				const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? "";
+				const previousQuestions = messages
+					.filter((el) => el.from === "user")
+					.slice(0, -1)
+					.map((el) => el.content);
+				const currentDate = format(new Date(), "MMMM d, yyyy");
+
+				message.content = `I searched the web using the query: ${webSearch.searchQuery}.
+Today is ${currentDate} and here are the results:
+=====================
+${webSearchContext}
+=====================
+${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""}
+Answer the question: ${lastQuestion}`;
+			}
+			// handle files if model is multimodal
+			if (multimodal) {
+				if (message.files && message.files.length > 0) {
+					const markdowns = await Promise.all(
+						message.files.map(async (hash) => {
+							try {
+								const { content: image, mime } = await downloadFile(hash, id);
+								const b64 = image.toString("base64");
+								return `![](data:${mime};base64,${b64})})`;
+							} catch (e) {
+								logger.error(e);
+							}
+						})
+					);
+					message.content += markdowns.join("\n ");
+				} else {
+					// if no image, append an empty white image
+					message.content +=
+						"\n![](data:image/png;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAQABADAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD+/igAoAKACgD/2Q==)";
+				}
+			}
+
+			return message;
+		})
+	);
+}

src/lib/server/summarize.ts

@@ -1,6 +1,6 @@
 import { env } from "$env/dynamic/private";
 import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint";
-import type { EndpointMessage } from "./endpoints/endpoints";
+import type { Message } from "$lib/types/Message";
 import { logger } from "$lib/server/logger";
 
 export async function summarize(prompt: string) {
@@ -8,7 +8,7 @@ export async function summarize(prompt: string) {
 		return prompt.split(/\s+/g).slice(0, 5).join(" ");
 	}
 
-	const messages: Array<EndpointMessage> = [
+	const messages: Array<Omit<Message, "id">> = [
 		{ from: "user", content: "Who is the president of Gabon?" },
 		{ from: "assistant", content: "🇬🇦 President of Gabon" },
 		{ from: "user", content: "Who is Julien Chaumond?" },

src/lib/server/websearch/search/generateQuery.ts

@@ -1,6 +1,5 @@
 import type { Message } from "$lib/types/Message";
 import { format } from "date-fns";
-import type { EndpointMessage } from "../../endpoints/endpoints";
 import { generateFromDefaultEndpoint } from "../../generateFromDefaultEndpoint";
 
 export async function generateQuery(messages: Message[]) {
@@ -10,7 +9,7 @@ export async function generateQuery(messages: Message[]) {
 
 	const lastMessage = userMessages.slice(-1)[0];
 
-	const convQuery: Array<EndpointMessage> = [
+	const convQuery: Array<Omit<Message, "id">> = [
 		{
 			from: "user",
 			content: `Previous Questions:

src/lib/types/Message.ts

@@ -11,11 +11,7 @@ export type Message = Partial<Timestamps> & {
 	webSearchId?: WebSearch["_id"]; // legacy version
 	webSearch?: WebSearch;
 	score?: -1 | 0 | 1;
-	/**
-	 * Either contains the base64 encoded image data
-	 * or the hash of the file stored on the server
-	 **/
-	files?: MessageFile[];
+	files?: string[]; // can contain either the hash of the file or the b64 encoded image data on the client side when uploading
 	interrupted?: boolean;
 
 	// needed for conversation trees
@@ -24,9 +20,3 @@ export type Message = Partial<Timestamps> & {
 	// goes one level deep
 	children?: Message["id"][];
 };
-
-export type MessageFile = {
-	type: "hash" | "base64";
-	value: string;
-	mime: string;
-};

src/lib/utils/messageUpdates.ts

@@ -1,4 +1,3 @@
-import type { MessageFile } from "$lib/types/Message";
 import type { MessageUpdate, TextStreamUpdate } from "$lib/types/MessageUpdate";
 
 type MessageUpdateRequestOptions = {
@@ -8,7 +7,7 @@ type MessageUpdateRequestOptions = {
 	isRetry: boolean;
 	isContinue: boolean;
 	webSearch: boolean;
-	files?: MessageFile[];
+	files?: string[];
 };
 export async function fetchMessageUpdates(
 	conversationId: string,

src/routes/conversation/[id]/+page.svelte

@@ -75,10 +75,20 @@
 			loading = true;
 			pending = true;
 
-			const base64Files = await Promise.all(
-				(files ?? []).map((file) =>
-					file2base64(file).then((value) => ({ type: "base64" as const, value, mime: file.type }))
-				)
+			const module = await import("browser-image-resizer");
+			// currently, only IDEFICS is supported by TGI
+			// the size of images is hardcoded to 224x224 in TGI
+			// this will need to be configurable when support for more models is added
+			const resizedImages = await Promise.all(
+				files.map(async (file) => {
+					return await module
+						.readAndCompressImage(file, {
+							maxHeight: 224,
+							maxWidth: 224,
+							quality: 1,
+						})
+						.then(async (el) => await file2base64(el as File));
+				})
 			);
 
 			let messageToWriteToId: Message["id"] | undefined = undefined;
@@ -110,11 +120,7 @@
 							messages,
 							rootMessageId: data.rootMessageId,
 						},
-						{
-							from: "user",
-							content: prompt,
-							files: messageToRetry.files,
-						},
+						{ from: "user", content: prompt },
 						messageId
 					);
 					messageToWriteToId = addChildren(
@@ -122,7 +128,7 @@
 							messages,
 							rootMessageId: data.rootMessageId,
 						},
-						{ from: "assistant", content: "" },
+						{ from: "assistant", content: "", files: resizedImages },
 						newUserMessageId
 					);
 				} else if (messageToRetry?.from === "assistant") {
@@ -148,7 +154,7 @@
 					{
 						from: "user",
 						content: prompt ?? "",
-						files: base64Files,
+						files: resizedImages,
 						createdAt: new Date(),
 						updatedAt: new Date(),
 					},
@@ -175,7 +181,6 @@
 			}
 
 			messages = [...messages];
-			const userMessage = messages.find((message) => message.id === messageId);
 			const messageToWriteTo = messages.find((message) => message.id === messageToWriteToId);
 			if (!messageToWriteTo) {
 				throw new Error("Message to write to not found");
@@ -193,7 +198,7 @@
 					isRetry,
 					isContinue,
 					webSearch: !hasAssistant && $webSearchParameters.useSearch,
-					files: isRetry ? userMessage?.files : base64Files,
+					files: isRetry ? undefined : resizedImages,
 				},
 				messageUpdatesAbortController.signal
 			).catch((err) => {

src/routes/conversation/[id]/+server.ts

@@ -13,13 +13,14 @@ import { runWebSearch } from "$lib/server/websearch/runWebSearch";
 import { AbortedGenerations } from "$lib/server/abortedGenerations";
 import { summarize } from "$lib/server/summarize";
 import { uploadFile } from "$lib/server/files/uploadFile";
+import sizeof from "image-size";
 import type { Assistant } from "$lib/types/Assistant";
 import { convertLegacyConversation } from "$lib/utils/tree/convertLegacyConversation";
 import { isMessageId } from "$lib/utils/tree/isMessageId";
 import { buildSubtree } from "$lib/utils/tree/buildSubtree.js";
 import { addChildren } from "$lib/utils/tree/addChildren.js";
 import { addSibling } from "$lib/utils/tree/addSibling.js";
-import { preprocessMessages } from "$lib/server/endpoints/preprocessMessages.js";
+import { preprocessMessages } from "$lib/server/preprocessMessages.js";
 import { usageLimits } from "$lib/server/usageLimits";
 import { isURLLocal } from "$lib/server/isURLLocal.js";
 import { logger } from "$lib/server/logger.js";
@@ -133,7 +134,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 		is_retry: isRetry,
 		is_continue: isContinue,
 		web_search: webSearch,
-		files: inputFiles,
+		files: b64files,
 	} = z
 		.object({
 			id: z.string().uuid().refine(isMessageId).optional(), // parent message id to append to for a normal message, or the message id for a retry/continue
@@ -146,43 +147,44 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			is_retry: z.optional(z.boolean()),
 			is_continue: z.optional(z.boolean()),
 			web_search: z.optional(z.boolean()),
-			files: z.optional(
-				z.array(
-					z.object({
-						type: z.literal("base64").or(z.literal("hash")),
-						value: z.string(),
-						mime: z.string(),
-					})
-				)
-			),
+			files: z.optional(z.array(z.string())),
 		})
 		.parse(json);
 
 	if (usageLimits?.messageLength && (newPrompt?.length ?? 0) > usageLimits.messageLength) {
 		throw error(400, "Message too long.");
 	}
+	// files is an array of base64 strings encoding Blob objects
+	// we need to convert this array to an array of File objects
 
-	// each file is either:
-	// base64 string requiring upload to the server
-	// hash pointing to an existing file
-	const hashFiles = inputFiles?.filter((file) => file.type === "hash") ?? [];
-	const b64Files =
-		inputFiles
-			?.filter((file) => file.type !== "hash")
-			.map((file) => {
-				const blob = Buffer.from(file.value, "base64");
-				return new File([blob], "file", { type: file.mime });
-			}) ?? [];
+	const files = b64files?.map((file) => {
+		const blob = Buffer.from(file, "base64");
+		return new File([blob], "image.png");
+	});
 
 	// check sizes
-	// todo: make configurable
-	if (b64Files.some((file) => file.size > 10 * 1024 * 1024)) {
-		throw error(413, "File too large, should be <10MB");
+	if (files) {
+		const filechecks = await Promise.all(
+			files.map(async (file) => {
+				const dimensions = sizeof(Buffer.from(await file.arrayBuffer()));
+				return (
+					file.size > 2 * 1024 * 1024 ||
+					(dimensions.width ?? 0) > 224 ||
+					(dimensions.height ?? 0) > 224
+				);
+			})
+		);
+
+		if (filechecks.some((check) => check)) {
+			throw error(413, "File too large, should be <2MB and 224x224 max.");
+		}
 	}
 
-	const uploadedFiles = await Promise.all(b64Files.map((file) => uploadFile(file, conv))).then(
-		(files) => [...files, ...hashFiles]
-	);
+	let hashes: undefined | string[];
+
+	if (files) {
+		hashes = await Promise.all(files.map(async (file) => await uploadFile(file, conv)));
+	}
 
 	// we will append tokens to the content of this message
 	let messageToWriteToId: Message["id"] | undefined = undefined;
@@ -214,13 +216,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			// add a children to that sibling, where we can write to
 			const newUserMessageId = addSibling(
 				conv,
-				{
-					from: "user",
-					content: newPrompt,
-					files: uploadedFiles,
-					createdAt: new Date(),
-					updatedAt: new Date(),
-				},
+				{ from: "user", content: newPrompt, createdAt: new Date(), updatedAt: new Date() },
 				messageId
 			);
 			messageToWriteToId = addChildren(
@@ -228,6 +224,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 				{
 					from: "assistant",
 					content: "",
+					files: hashes,
 					createdAt: new Date(),
 					updatedAt: new Date(),
 				},
@@ -253,7 +250,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			{
 				from: "user",
 				content: newPrompt ?? "",
-				files: uploadedFiles,
+				files: hashes,
 				createdAt: new Date(),
 				updatedAt: new Date(),
 			},
@@ -414,9 +411,10 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			}
 
 			// inject websearch result & optionally images into the messages
-			const processedMessages = preprocessMessages(
+			const processedMessages = await preprocessMessages(
 				messagesForPrompt,
 				messageToWriteTo.webSearch,
+				model.multimodal,
 				convId
 			);
 
@@ -431,7 +429,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			try {
 				const endpoint = await model.getEndpoint();
 				for await (const output of await endpoint({
-					messages: await processedMessages,
+					messages: processedMessages,
 					preprompt,
 					continueMessage: isContinue,
 					generateSettings: assistant?.generateSettings,

src/routes/conversation/[id]/output/[sha256]/+server.ts

@@ -39,9 +39,9 @@ export const GET: RequestHandler = async ({ locals, params }) => {
 		}
 	}
 
-	const { value, mime } = await downloadFile(sha256, params.id);
+	const { content, mime } = await downloadFile(sha256, params.id);
 
-	return new Response(Buffer.from(value, "base64"), {
+	return new Response(content, {
 		headers: {
 			"Content-Type": mime ?? "application/octet-stream",
 		},