Update app.py

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-from gradio.components import Textbox, Chat
+from gradio.components import Textbox, Chat, Slider
 
 load_dotenv()
 
@@ -14,47 +14,45 @@ def format_chat_prompt(message, chat_history, instruction):
     prompt = f"{prompt}\nUser: {message}\nAssistant:"
     return prompt
 
-def query(payload):
-    response = requests.post(API_URL, headers=headers, json=payload)
-    return response.json()
-
-def respond(message, chat_history=[], instruction="A conversation between a user and an AI assistant. The assistant gives helpful and honest answers."):
-    chat_history.append((message, ""))
+def respond(message, chat_history, instruction, temperature=0.7):
     prompt = format_chat_prompt(message, chat_history, instruction)
-    response = query({"inputs": prompt})
-    assistant_message = response['generated_text'].split("Assistant:")[-1].strip()
-    chat_history[-1] = (message, assistant_message)
-    return "", chat_history
-
-def clear_inputs(input1, input2, input3):
-    input1.value = ""
-    input2.value = []
-    input3.value = "A conversation between a user and an AI assistant. The assistant gives helpful and honest answers."
-    return input1, input2, input3
-
-iface = gr.Interface(
-    fn=respond, 
-    inputs=[
-        Textbox(label="Prompt"),
-        Chat(label="Chat History", height=240),
-        Textbox(label="System message", lines=2, default="A conversation between a user and an AI assistant. The assistant gives helpful and honest answers.")
-    ],
-    outputs=[
-        Textbox(label="Prompt"),
-        Chat(label="Chat History", height=240),
-        Textbox(label="System message")
-    ],
-    server_name="0.0.0.0",
-    server_port=7860,
-    allow_flagging=False,
-    title="Chat with AI",
-    description="This is a chat interface that connects to a language model.",
-    analytics_enabled=False,
-)
-
-iface.add_button("Clear", clear_inputs)
-iface.add_button("Submit", respond)
-
-iface.launch()
+    chat_history = chat_history + [[message, ""]]
+    stream = client.generate_stream(prompt,
+                                      max_new_tokens=1024,
+                                      stop_sequences=["\nUser:", "<|endoftext|>"],
+                                      temperature=temperature)
+                                      #stop_sequences to not generate the user answer
+    acc_text = ""
+    #Streaming the tokens
+    for idx, response in enumerate(stream):
+            text_token = response.token.text
+
+            if response.details:
+                return
+
+            if idx == 0 and text_token.startswith(" "):
+                text_token = text_token[1:]
+
+            acc_text += text_token
+            last_turn = list(chat_history.pop(-1))
+            last_turn[-1] += acc_text
+            chat_history = chat_history + [last_turn]
+            yield "", chat_history
+            acc_text = ""
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot(height=240) #just to fit the notebook
+    msg = gr.Textbox(label="Prompt")
+    with gr.Accordion(label="Advanced options",open=False):
+        system = gr.Textbox(label="System message", lines=2, value="A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.")
+        temperature = gr.Slider(label="temperature", minimum=0.1, maximum=1, value=0.7, step=0.1)
+    btn = gr.Button("Submit")
+    clear = gr.ClearButton(components=[msg, chatbot], value="Clear console")
+
+    btn.click(respond, inputs=[msg, chatbot, system], outputs=[msg, chatbot])
+    msg.submit(respond, inputs=[msg, chatbot, system], outputs=[msg, chatbot]) #Press enter to submit
+gr.close_all()
+demo.queue().launch(share=True, server_port=int(os.environ['PORT4']))
+