remove token stuff

src/pages/api/llm.js

@@ -1,20 +1,10 @@
 import { GoogleCustomSearch } from "openai-function-calling-tools";
 import { LLMError, LLMStream } from './stream';
 
-// @ts-expect-error
-import wasm from '@dqbd/tiktoken/lite/tiktoken_bg.wasm?module';
-
-import tiktokenModel from '@dqbd/tiktoken/encoders/cl100k_base.json';
-import { Tiktoken, init } from '@dqbd/tiktoken/lite/init';
-
-export const config = {};
-
 const handler = async (req) => {
   try {
     const { question } = (await req.json());
 
-    await init((imports) => WebAssembly.instantiate(wasm, imports));
-
     const googleCustomSearch = new GoogleCustomSearch({
         apiKey: process.env.API_KEY,
         googleCSEId: process.env.CONTEXT_KEY,
@@ -30,11 +20,6 @@ const handler = async (req) => {
     const functions = {
         googleCustomSearch,
     };
-    const encoding = new Tiktoken(
-      tiktokenModel.bpe_ranks,
-      tiktokenModel.special_tokens,
-      tiktokenModel.pat_str,
-    );
 
     let promptToSend = question;
     if (!promptToSend) {
@@ -46,25 +31,7 @@ const handler = async (req) => {
       temperatureToUse = 0.8;
     }
 
-    const prompt_tokens = encoding.encode(promptToSend);
-
-    let tokenCount = prompt_tokens.length;
-    let messagesToSend = [];
-
-    for (let i = messages.length - 1; i >= 0; i--) {
-      const message = messages[i];
-      const tokens = encoding.encode(message.content);
-
-      if (tokenCount + tokens.length + 1000 > model.tokenLimit) {
-        break;
-      }
-      tokenCount += tokens.length;
-      messagesToSend = [message, ...messagesToSend];
-    }
-
-    encoding.free();
-
-    const stream = await LLMStream(model, promptToSend, temperatureToUse, key, messagesToSend, functions);
+    const stream = await LLMStream(model, promptToSend, temperatureToUse, key, messages, functions);
 
     return new Response(stream);
   } catch (error) {