Updated Blip device map

app.py

@@ -5,10 +5,14 @@ import open_clip
 
 from huggingface_hub import hf_hub_download
 
+# Use when running on a CPU
+device_map = {
+    "embedding": "cpu",
+}
 
 # Load the Blip2 model
 preprocessor_blip2_8_bit = AutoProcessor.from_pretrained("Salesforce/blip2-opt-6.7b")
-model_blip2_8_bit = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-6.7b", device_map="auto", load_in_8bit=True)
+model_blip2_8_bit = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-6.7b", device_map=device_map, load_in_8bit_fp32_cpu_offload=True)
 
 # Load the Blip base model
 preprocessor_blip_base = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base")