stream

.gitignore

@@ -172,3 +172,5 @@ cython_debug/
 
 # PyPI configuration file
 .pypirc
+/.github/
+/.github/

chat_app.py

@@ -3,6 +3,7 @@ import os
 import gradio as gr
 from typing import List
 
+import langchain_core.callbacks
 from langchain_huggingface import HuggingFaceEndpoint
 
 from langchain.schema import BaseMessage
@@ -45,10 +46,13 @@ prompt = ChatPromptTemplate.from_messages([
 ])
 
 model_id="mistralai/Mistral-7B-Instruct-v0.3"
+callbacks = [langchain_core.callbacks.StreamingStdOutCallbackHandler()]
 llm = HuggingFaceEndpoint(
     repo_id=model_id,
     max_new_tokens=1024,
     temperature=0.1,
+    callbacks=callbacks,
+    streaming=True,
     huggingfacehub_api_token=os.getenv('HF_TOKEN'),
 )