Updated the model

app.py

@@ -1,16 +1,58 @@
 import gradio as gr
-
-# Load model and tokenizer
 from transformers import AutoTokenizer, AutoModelForCausalLM
 
-tokenizer = AutoTokenizer.from_pretrained("SebastianSchramm/UniNER-7B-type-GPTQ-4bit-128g-actorder_True")
-model = AutoModelForCausalLM.from_pretrained("SebastianSchramm/UniNER-7B-type-GPTQ-4bit-128g-actorder_True")
+# Load the model and tokenizer
+tokenizer = AutoTokenizer.from_pretrained(
+    "SebastianSchramm/UniNER-7B-type-GPTQ-4bit-128g-actorder_True", 
+    legacy=False
+)
+model = AutoModelForCausalLM.from_pretrained(
+    "SebastianSchramm/UniNER-7B-type-GPTQ-4bit-128g-actorder_True",
+    device_map="auto"
+)
 
-def chat(prompt):
+# Define a function to handle user input and generate a response
+def chatbot_response(user_input, chat_history=[]):
+    # Append user input to chat history
+    chat_history.append(("User", user_input))
+    
+    # Prepare the prompt based on the chat history
+    prompt = " ".join([f"{speaker}: {text}" for speaker, text in chat_history])
+    
+    # Generate a response
     inputs = tokenizer(prompt, return_tensors="pt")
     outputs = model.generate(inputs.input_ids, max_length=200, num_return_sequences=1)
-    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+    bot_response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    
+    # Append bot response to chat history
+    chat_history.append(("Bot", bot_response))
+    
+    # Return updated chat history
+    return chat_history, chat_history
+
+# Gradio interface
+with gr.Blocks() as interface:
+    gr.Markdown("### Chat with Your LLM")
+
+    chat_history = gr.State([])  # To store the conversation history
+    chat_display = gr.Chatbot()  # Chat display for the conversation
+    user_input = gr.Textbox(
+        show_label=False, 
+        placeholder="Type your message and press Enter"
+    )
+    send_button = gr.Button("Send")
+    
+    # Define the interaction
+    send_button.click(
+        chatbot_response, 
+        inputs=[user_input, chat_history], 
+        outputs=[chat_display, chat_history]
+    )
+    user_input.submit(
+        chatbot_response, 
+        inputs=[user_input, chat_history], 
+        outputs=[chat_display, chat_history]
+    )
 
-# Create Gradio interface
-iface = gr.Interface(fn=chat, inputs="text", outputs="text", title="Open-Insurance LLM Chat")
-iface.launch()
+# Launch the Gradio app
+interface.launch()