a

app.py

@@ -93,14 +93,17 @@ def respond(
             messages.append({"role": "assistant", "content": val[1]})
     messages.append({"role": "user", "content": message})
     
-    # Tokenize the prompt
-    inputs = tokenizer(messages, return_tensors="pt", truncation=True)
-    input_ids = inputs.input_ids.to("cpu")  # Ensure input is on the CPU
-
+    # Tokenize the messages
+    inputs = tokenizer.apply_chat_template(
+        messages,
+        tokenize = True,
+        add_generation_prompt = True, # Must add for generation
+        return_tensors = "pt",
+    )
     # Generate tokens incrementally
     streamer = TextStreamer(tokenizer, skip_prompt=True)
     generation_kwargs = {
-        "input_ids": input_ids,
+        "input_ids": inputs,
         "max_new_tokens": max_tokens,
         "temperature": temperature,
         "top_p": top_p,