Update app.py

app.py

@@ -82,8 +82,8 @@ def respond(
         if llama == None:
             llama = Llama("models/t5-query-reformulation-RL-q8_0.gguf",flash_attn=False,
                         n_gpu_layers=0,
-                        n_batch=64,
-                        n_ctx=256,
+                        n_batch=32,
+                        n_ctx=512,
                         n_threads=2,
                         n_threads_batch=2)
         
@@ -91,11 +91,14 @@ def respond(
         llama.encode(tokens)
         tokens = [llama.decoder_start_token()]
         outputs =""
-        for token in llama.generate(tokens, top_k=top_k, top_p=top_p, temp=temperature, repeat_penalty=repeat_penalty):
-            outputs+= llama.detokenize([token]).decode()
-            yield outputs
-            if token == llama.token_eos():
-                break
+        iteration = 5
+        for i in range(iteration):
+            for token in llama.generate(tokens, top_k=top_k, top_p=top_p, temp=temperature, repeat_penalty=repeat_penalty):
+                outputs+= llama.detokenize([token]).decode()
+                yield outputs
+                if token == llama.token_eos():
+                    break
+            outputs+="\n"
         return outputs
     except Exception as e:
         # Custom exception handling