Qconfig

app.py

@@ -5,7 +5,7 @@ import open_clip
 
 from huggingface_hub import hf_hub_download
 
-quantizer_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=True)
+quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=True)
 # Use when running on a CPU
 device_map = {
     "transformer.word_embeddings": 0,
@@ -17,7 +17,12 @@ device_map = {
 
 # Load the Blip2 model
 preprocessor_blip2_8_bit = AutoProcessor.from_pretrained("Salesforce/blip2-opt-2.7b")
-model_blip2_8_bit = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-2.7b", device_map="auto", load_in_8bit=True)
+model_blip2_8_bit = Blip2ForConditionalGeneration.from_pretrained(
+	"Salesforce/blip2-opt-2.7b",
+	device_map=device_map,
+    quantization_config=quantization_config,
+	load_in_8bit=True
+)
 
 # Load the Blip base model
 # preprocessor_blip_base = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base")