quantization

app/embedding_setup.py

@@ -40,7 +40,7 @@ quantization_config = BitsAndBytesConfig(
     bnb_4bit_quant_type="nf4"
 )
 
-tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL, quantization_config=quantization_config, token=hf_auth)
+tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL, token=hf_auth)
 
 
 first_token = 'First'
@@ -50,6 +50,7 @@ first_id = tokenizer.convert_tokens_to_ids(first_token)
 second_id = tokenizer.convert_tokens_to_ids(second_token)
 model = AutoModelForCausalLM.from_pretrained(
             LLM_MODEL,
+             quantization_config=quantization_config,
             torch_dtype=torch.float16,
             device_map="auto",
             token=hf_auth,