test

app.py

@@ -59,10 +59,13 @@ def respond(
         add_generation_prompt = True, # Must add for generation
         return_tensors = "pt",
     )
-    text_streamer = TextStreamer(tokenizer, skip_prompt = True)
-    for response in model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = max_tokens, use_cache = True,
-                            temperature = temperature, min_p = top_p):
-        yield response
+    # text_streamer = TextStreamer(tokenizer, skip_prompt = True)
+    # for response in model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = max_tokens, use_cache = True,
+    #                         temperature = temperature, min_p = top_p):
+    #     yield response
+    outputs = model.generate(input_ids = inputs, max_new_tokens = max_tokens, use_cache = True,
+                         temperature = temperature, min_p = top_p)
+    return tokenizer.batch_decode(outputs)
 
 
 """