Add 8-bit quantization to reduce memory usage

- Load model with load_in_8bit=True
- Add bitsandbytes dependency
- Should reduce memory footprint by ~50%
- Fixes memory limit exceeded error on free tier

app.py

@@ -13,18 +13,19 @@ import gradio as gr
 MODEL_ID = "convaiinnovations/ECG-Instruct-Llama-3.2-11B-Vision"
 
 print(f"Loading model: {MODEL_ID}")
+print("Loading in 8-bit mode to reduce memory usage...")
 print("This may take a few minutes on first load...")
 
-# Load model and processor
+# Load model and processor with 8-bit quantization to reduce memory
 model = MllamaForConditionalGeneration.from_pretrained(
     MODEL_ID,
-    dtype=torch.bfloat16,
+    load_in_8bit=True,
     device_map="auto",
 )
 
 processor = AutoProcessor.from_pretrained(MODEL_ID)
 
-print("Model loaded successfully!")
+print("Model loaded successfully in 8-bit mode!")
 
 
 # Helper functions

requirements.txt

@@ -1,6 +1,7 @@
 torch>=2.0.0
 transformers>=4.45.0
 accelerate>=0.20.0
+bitsandbytes>=0.41.0
 Pillow>=10.0.0
 sentencepiece>=0.1.99
 protobuf>=3.20.0