Update app.py

app.py

@@ -1,17 +1,21 @@
 import gradio as gr
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from peft import PeftModel
 from threading import Thread
 import torch
 
-# Load your model once at startup
-print("Loading model...")
-tokenizer = AutoTokenizer.from_pretrained("drumwell/autotrain-2duhi-5mmyz")
+# Load base model + your adapter
+print("Loading base model...")
+tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct")
 model = AutoModelForCausalLM.from_pretrained(
-    "drumwell/autotrain-2duhi-5mmyz",
+    "meta-llama/Llama-3.1-8B-Instruct",
     device_map="auto",
     torch_dtype=torch.float16,
     load_in_8bit=True,
 )
+
+print("Loading your fine-tuned adapter...")
+model = PeftModel.from_pretrained(model, "drumwell/autotrain-2duhi-5mmyz")
 print("Model loaded!")
 
 def respond(
@@ -23,18 +27,13 @@ def respond(
     top_p,
     hf_token: gr.OAuthToken,
 ):
-    """
-    For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-    """
     messages = [{"role": "system", "content": system_message}]
     messages.extend(history)
     messages.append({"role": "user", "content": message})
     
-    # Apply chat template
     text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
     inputs = tokenizer(text, return_tensors="pt").to(model.device)
     
-    # Setup streaming
     streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
     generation_kwargs = dict(
         inputs,
@@ -46,7 +45,6 @@ def respond(
         repetition_penalty=1.1,
     )
     
-    # Generate in a separate thread for streaming
     thread = Thread(target=model.generate, kwargs=generation_kwargs)
     thread.start()
     
@@ -55,9 +53,6 @@ def respond(
         response += token
         yield response
 
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
 chatbot = gr.ChatInterface(
     respond,
     type="messages",