mistral load

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
-from unsloth import FastLanguageModel
+from peft import PeftModel, PeftConfig
 
 # Model and tokenizer initialization
 MODEL_NAME = "satishpednekar/sbxcertqueryhelper"
@@ -21,32 +21,31 @@ def load_model_org():
 
 
 def load_model():
-    model, tokenizer = FastLanguageModel.from_pretrained(
-        model_name="satishpednekar/sbxcertqueryhelper",  # Use the path where you saved the model
-        max_seq_length=4096,  # Use the same as during training
-        dtype=torch.float16,
-        load_in_4bit=False,
-        token="ff"
+    # Load base model first
+    base_model = AutoModelForCausalLM.from_pretrained(
+        "unsloth/mistral-7b-v0.3",  # Use your base model name
+        torch_dtype=torch.float16,
+        device_map="auto",
+        trust_remote_code=True
+    )
+    
+    # Load the PEFT adapter weights
+    model = PeftModel.from_pretrained(
+        base_model,
+        "satishpednekar/sbx-qhelper-mistral-loraWeights",  # Path to your trained LoRA weights
+        torch_dtype=torch.float16,
+        device_map="auto"
     )
     
-    # Configure PEFT settings exactly as during training
-    model = FastLanguageModel.get_peft_model(
-        model,
-        r=16,
-        target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
-                       "gate_proj", "up_proj", "down_proj"],
-        lora_alpha=16,
-        lora_dropout=0,
-        bias="none",
-        use_gradient_checkpointing="unsloth",
-        random_state=3407,
-        use_rslora=False,
-        loftq_config=None
+    tokenizer = AutoTokenizer.from_pretrained(
+        "unsloth/mistral-7b-v0.3",  # Use your base model name
+        trust_remote_code=True
     )
     
     return model, tokenizer
 
 
+
 # Initialize model and tokenizer
 print("Loading model...")
 model, tokenizer = load_model()
@@ -89,15 +88,15 @@ def generate_response(prompt, max_length=512, temperature=0.7, top_p=0.95):
 def main():
     with gr.Blocks(title="Salesforce Certification Query Helper") as demo:
         gr.Markdown("""
-        # Salesforce Certification Query Helper
-        Ask questions about Salesforce certifications and get detailed answers!
+        # SBX Certification Query Helper
+        Ask questions about SBX certifications and get detailed answers!
         """)
         
         with gr.Row():
             with gr.Column():
                 input_text = gr.Textbox(
                     label="Your Question",
-                    placeholder="Enter your question about Salesforce certifications...",
+                    placeholder="Enter your question about SBX certifications...",
                     lines=3
                 )