8bit

app/embedding_setup.py

@@ -53,9 +53,7 @@ model = AutoModelForCausalLM.from_pretrained(
 
 rec_adapter = PeftModel.from_pretrained(
             model,
-            lora_weights_rec,
-            torch_dtype=torch.float16,
-            device_map={'': 0}
+            lora_weights_rec
         )
       
 
@@ -148,9 +146,7 @@ def compare_docs_with_context(doc_a, doc_b, target_occupation_name, target_occup
 #-----------------------------------------explanation-------------------------------------
 exp_adapter = PeftModel.from_pretrained(
             model,
-            lora_weights_exp,
-            torch_dtype=torch.float16,
-            device_map={'': 0}
+            lora_weights_exp
         )
 exp_adapter.eval()