app.py

import gradio as gr
import random as rand
from huggingface_hub import InferenceClient

client = InferenceClient("Qwen/Qwen2.5-7B-Instruct") #todo: change model?

def respond(message, history):
    messages = [{"role": "system",
                "content": "You are a friendly chatbot."
                }]
    if history:
        messages.extend(history)
    messages.append({"role": "user",
                    "content": message
                    })
    response = ""
    for msg in client.chat_completion(messages, max_tokens=200, temperature=1, top_p=0.5, stream = True):
        token = msg.choices[0].delta.content
        response += msg
        yield response

chatbot = gr.ChatInterface(fn=respond, title="Magic 8 Ball", description="Discover your fortune...", editable=True)
chatbot.launch()