tried streaming

app.py

@@ -83,13 +83,18 @@ def get_top_chunks(query, chunk_embeddings, text_chunks):
 client = InferenceClient("Qwen/Qwen2.5-7B-Instruct-1M")
 
 def respond(message, history):
+    response = ""
     information = get_top_chunks(message,chunk_embeddings,cleaned_chunks)
-    messages = [{"role":"system", "content": f"You are a friendly chatbot. You base your response on the following information: {information}"}]
+    messages = [{"role":"system", "content": f"You are a friendly and informative chatbot. You answer in full sentences. You base your response on the following information: {information}"}]
     if history:
         messages.extend(history)
     messages.append({"role": "user", "content": message})
-    response = client.chat_completion(messages,max_tokens=100)
-    return response["choices"][0]["message"]["content"].strip()
+    complete = client.chat_completion(messages,max_tokens=1000,temperature = 0.2, stream = True)
+    for i in complete:
+        token = message.choices[0].delta.content
+        response += token
+        yield response["choices"][0]["message"]["content"].strip()
+    #return response["choices"][0]["message"]["content"].strip()
 
 chatBot = gr.ChatInterface(respond, type = "messages")
 chatBot.launch()