Trying streaming again

app.py

@@ -1,5 +1,6 @@
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForCausalLM, TextGenerationPipeline
+import asyncio
 
 # Load the model and tokenizer
 model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
@@ -10,11 +11,13 @@ model = AutoModelForCausalLM.from_pretrained(model_name)
 pipeline = TextGenerationPipeline(model=model, tokenizer=tokenizer)
 
 # Define the inference function with streaming
-def generate_text(prompt):
+async def generate_text(prompt):
+    loop = asyncio.get_event_loop()
     output = ""
     for token in pipeline(prompt, max_length=100, num_return_sequences=1, do_sample=True, top_k=50, top_p=0.95):
         output += token['generated_text']
         yield output  # Stream the output
+        await loop.run_in_executor(None, lambda: None)  # Yield control to the event loop
 
 # Create a Gradio interface
 iface = gr.Interface(