after bufsize=1 change, exploring word or token level stream

app.py

@@ -3,6 +3,7 @@ import random
 import subprocess
 import time
 
+"""
 def generate_response(user_message): #Figure Out the parameters later and find a way to get the ram usage
     cmd = [
         "/app/llama.cpp/main",  # Path to the executable
@@ -32,9 +33,9 @@ def generate_response(user_message): #Figure Out the parameters later and find a
     if process.returncode != 0:
         error_message = process.stderr.read()
         print(f"Error: {error_message}")
+"""
 
-
-def generate_response_token_by_token(user_message):
+def generate_response(user_message): #generate_response_token_by_token
     cmd = [
         "/app/llama.cpp/main",  # Path to the executable
         "-m", "/app/llama.cpp/models/stablelm-2-zephyr-1_6b-Q4_0.gguf",
@@ -58,13 +59,14 @@ def generate_response_token_by_token(user_message):
             if char == ' ' or char == '\n':  # Token delimiters
                 elapsed_time = time.time() - start_time  # Calculate elapsed time
                 alltokens += token_buffer
-                yield f"{alltokens} [Inference time: {elapsed_time:.2f} seconds]"
+                yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds]"
                 token_buffer = ''  # Reset token buffer
     
     # Yield the last token if there is any
     if token_buffer:
         elapsed_time = time.time() - start_time  # Calculate elapsed time
-        yield f"{token_buffer} [Inference time: {elapsed_time:.2f} seconds]"
+        alltokens += token_buffer
+        yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds]"
 
     # Wait for the subprocess to finish if it hasn't already
     process.wait()
@@ -93,7 +95,7 @@ with gr.Blocks() as iface:
         description="No Message History for now - Enter your message and get a response. (One sentence every 20s)",
         flagging_dir="/usr/src/app/flagged",
     )
-    gr.Interface(fn=generate_response_token_by_token, inputs=gr.Textbox(lines=2, placeholder='Type prompt here...'), outputs="text", description="More Responsive streaming test")
+    #gr.Interface(fn=generate_response_token_by_token, inputs=gr.Textbox(lines=2, placeholder='Type prompt here...'), outputs="text", description="More Responsive streaming test")
     with gr.Group():
         gr.HTML("Test for wrapping generator (20 seconds a piece of the response)")
         MainOutput = gr.TextArea(placeholder='Output will show here')