Trying to use model output

app.py

@@ -1,48 +1,59 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+from unsloth import FastLanguageModel
+from transformers import TextStreamer
+import torch
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("DominusDeorum/llama-3.2-lora_model")
+# Initialize the model and tokenizer
+def initialize_model(model_name, max_seq_length, dtype, load_in_4bit):
+    model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name=model_name,  # Your Lora model name
+        max_seq_length=max_seq_length,
+        dtype=dtype,
+        load_in_4bit=load_in_4bit,
+    )
+    FastLanguageModel.for_inference(model)  # Enable 2x faster inference
+    return model, tokenizer
 
+# Load model and tokenizer
+model_name = "DominusDeorum/llama-3.2-lora_model"  # Replace with your model
+max_seq_length = 2048  # Adjust as needed
+dtype = torch.float16  # Set dtype (can also use torch.bfloat16, etc.)
+load_in_4bit = True  # Set to True if using 4-bit inference
 
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
+model, tokenizer = initialize_model(model_name, max_seq_length, dtype, load_in_4bit)
 
+def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p):
+    # Prepare the chat history and system message
+    messages = [{"role": "system", "content": system_message}]
+    
     for val in history:
         if val[0]:
             messages.append({"role": "user", "content": val[0]})
         if val[1]:
             messages.append({"role": "assistant", "content": val[1]})
-
+    
+    # Add the user's new message
     messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
+    
+    # Prepare inputs for the model
+    inputs = tokenizer.apply_chat_template(
         messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
+        tokenize=True,
+        add_generation_prompt=True,
+        return_tensors="pt",
+    ).to("cuda")
+    
+    # Generate response with streaming
+    text_streamer = TextStreamer(tokenizer, skip_prompt=True)
+    response = ""
+    
+    for output in model.generate(input_ids=inputs, streamer=text_streamer, max_new_tokens=max_tokens,
+                                 use_cache=True, temperature=temperature, top_p=top_p):
+        token = tokenizer.decode(output, skip_special_tokens=True)
         response += token
         yield response
 
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
+# Set up Gradio interface
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
@@ -59,6 +70,5 @@ demo = gr.ChatInterface(
     ],
 )
 
-
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()