quantization not working?

app.py

@@ -51,33 +51,19 @@ def load_model():
         processor = AutoProcessor.from_pretrained(MODEL_ID)
         logger.info("Processor loaded successfully")
         
-        # Try loading model with quantization first
-        try:
-            logger.info(f"Attempting to load model with quantization from {MODEL_ID}")
-            from transformers import BitsAndBytesConfig
-            
-            # Configure BitsAndBytes for 4-bit quantization
-            bnb_config = BitsAndBytesConfig(
-                load_in_4bit=True,
-                bnb_4bit_use_double_quant=True,
-                bnb_4bit_quant_type="nf4",
-                bnb_4bit_compute_dtype=torch.float16
-            )
-            
-            model = Qwen2AudioForConditionalGeneration.from_pretrained(
-                MODEL_ID,
-                quantization_config=bnb_config,
-                device_map="auto",
-                low_cpu_mem_usage=True
-            )
-            logger.info("Model loaded successfully with quantization")
-        except Exception as quant_error:
-            # If quantization fails, fall back to basic loading
-            logger.warning(f"Quantization failed: {quant_error}. Falling back to standard loading.")
-            
-            # Try FP16 if GPU available
-            if torch.cuda.is_available():
-                try:
+        # Skip quantization attempts since we know it's problematic with CUDA 12.4
+        logger.info(f"Loading model with optimized settings for your environment")
+        
+        # Check if GPU is available and has enough memory
+        if torch.cuda.is_available():
+            try:
+                # Get GPU memory info
+                gpu_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3
+                logger.info(f"GPU memory: {gpu_memory:.2f} GB")
+                
+                # If GPU has enough memory, try loading directly without quantization
+                if gpu_memory > 16:  # For GPUs with >16GB memory
+                    logger.info("Using FP16 precision on GPU")
                     model = Qwen2AudioForConditionalGeneration.from_pretrained(
                         MODEL_ID,
                         torch_dtype=torch.float16,
@@ -85,22 +71,33 @@ def load_model():
                         low_cpu_mem_usage=True
                     )
                     logger.info("Model loaded successfully with FP16")
-                except Exception as fp16_error:
-                    logger.warning(f"FP16 loading failed: {fp16_error}. Falling back to CPU.")
+                else:
+                    # For smaller GPUs, use CPU offloading
+                    logger.info("Using CPU offloading for model components")
                     model = Qwen2AudioForConditionalGeneration.from_pretrained(
-                        MODEL_ID, 
-                        device_map="cpu",
+                        MODEL_ID,
+                        torch_dtype=torch.float16,
+                        device_map="auto",
+                        offload_folder="offload",
                         low_cpu_mem_usage=True
                     )
-                    logger.info("Model loaded successfully on CPU")
-            else:
-                # Load on CPU if no GPU
+                    logger.info("Model loaded successfully with CPU offloading")
+            except Exception as gpu_error:
+                logger.warning(f"GPU loading failed: {gpu_error}. Falling back to CPU.")
                 model = Qwen2AudioForConditionalGeneration.from_pretrained(
                     MODEL_ID, 
                     device_map="cpu",
                     low_cpu_mem_usage=True
                 )
                 logger.info("Model loaded successfully on CPU")
+        else:
+            # Load on CPU if no GPU
+            model = Qwen2AudioForConditionalGeneration.from_pretrained(
+                MODEL_ID, 
+                device_map="cpu",
+                low_cpu_mem_usage=True
+            )
+            logger.info("Model loaded successfully on CPU")
         
         model.eval()
         log_gpu_memory("After model loading")

requirements.txt

@@ -8,5 +8,4 @@ soundfile>=0.12.1
 requests>=2.28.0
 pillow>=9.5.0
 huggingface_hub>=0.16.0
-bitsandbytes>=0.41.0
 scikit-learn>=1.0.2