Create app.py

app.py

@@ -0,0 +1,67 @@
+import gradio as gr
+import random
+import time
+import requests
+import json
+
+
+def http_yield(prompt):
+    print(prompt)
+    bot_message = random.choice(["How are you?", "I love you", "I'm very hungry"])
+    for character in bot_message:
+        yield character
+
+def http_bot_yield(prompt):
+    headers = {"User-Agent": "vLLM Client"}
+    pload = {
+        "prompt": f"<s>[INST] {prompt} [/INST] ",
+        "stream": True,
+        "max_tokens": 256,
+        "temperature": 0.1
+    }
+    response = requests.post(
+        'https://ca21-34-126-145-53.ngrok-free.app/generate',
+        headers=headers,
+        json=pload,
+        stream=True
+    )
+
+    for chunk in response.iter_lines(chunk_size=8192,
+                                     decode_unicode=False,
+                                     delimiter=b"\0"):
+        if chunk:
+            data = json.loads(chunk.decode("utf-8"))
+            output = data["text"][0].split('[/INST] ')[-1]
+            # print("output --->", output)
+            yield output
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot()
+    msg = gr.Textbox()
+    clear = gr.Button("Clear")
+
+    def user(user_message, history):
+        print("", history + [[user_message, None]])
+        return "", history + [[user_message, None]]
+
+    def bot(history):
+        print(history[-1][0])
+        prompt = history[-1][0]
+        history[-1][1] = ""
+        # b_text = ""
+        print(history)
+        for character in http_bot_yield(prompt):
+            # print("yld --- > ", history[-1][1])
+            history[-1][1] = character
+            # b_text += character 
+            # print("yield --- > ", b_text)
+            time.sleep(0.05)
+            yield history
+
+    msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
+        bot, chatbot, chatbot
+    )
+    clear.click(lambda: None, None, chatbot, queue=False)
+    
+demo.queue()
+demo.launch()