try

app.py

@@ -1,23 +1,34 @@
+import torch
 from unsloth import FastLanguageModel
-from peft import PeftModel
-from transformers import AutoTokenizer
-import gradio as gr
 
+# Check if CUDA is available
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# Load the base model
 base_model_name = "unsloth/Llama-3.2-3B-Instruct"
 base_model, tokenizer = FastLanguageModel.from_pretrained(
     model_name=base_model_name,
     max_seq_length=2048,
-    dtype=None,
-    load_in_4bit=True 
+    dtype=None,  # Auto-detect data type
+    load_in_4bit=False,  # Disable 4-bit quantization for CPU
 )
+base_model.to(device)
+
+# Apply LoRA adapters
+from peft import PeftModel
 
-# lora adapters from my Hugging Face model
-lora_model_name = "oskaralf/lora_model"  # Hugging Face repository for LoRA adapters
+lora_model_name = "oskaralf/lora_model"  # Replace with your LoRA model path
 model = PeftModel.from_pretrained(base_model, lora_model_name)
+model.to(device)
+
+# Prepare for inference
 FastLanguageModel.for_inference(model)
 
+# Gradio interface
+import gradio as gr
+
 def chatbot(input_text):
-    inputs = tokenizer(input_text, return_tensors="pt").to("cuda")  
+    inputs = tokenizer(input_text, return_tensors="pt").to(device)
     outputs = model.generate(input_ids=inputs["input_ids"], max_new_tokens=64)
     response = tokenizer.decode(outputs[0], skip_special_tokens=True)
     return response