4bit

app/embedding_setup.py

@@ -45,7 +45,7 @@ first_id = tokenizer.convert_tokens_to_ids(first_token)
 second_id = tokenizer.convert_tokens_to_ids(second_token)
 model = AutoModelForCausalLM.from_pretrained(
             LLM_MODEL,
-            load_in_8bit=True,
+            load_in_4bit=True,
             torch_dtype=torch.float16,
             token=hf_auth,
         )