help

app.py

@@ -51,53 +51,38 @@ def load_model():
         processor = AutoProcessor.from_pretrained(MODEL_ID)
         logger.info("Processor loaded successfully")
         
-        # Skip quantization attempts since we know it's problematic with CUDA 12.4
-        logger.info(f"Loading model with optimized settings for your environment")
+        # Check if Accelerate is available
+        try:
+            import accelerate
+            logger.info(f"Accelerate version: {accelerate.__version__}")
+            has_accelerate = True
+        except ImportError:
+            logger.warning("Accelerate not found. Will load model without device mapping.")
+            has_accelerate = False
         
-        # Check if GPU is available and has enough memory
+        # Check if GPU is available
         if torch.cuda.is_available():
-            try:
-                # Get GPU memory info
-                gpu_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3
-                logger.info(f"GPU memory: {gpu_memory:.2f} GB")
-                
-                # If GPU has enough memory, try loading directly without quantization
-                if gpu_memory > 16:  # For GPUs with >16GB memory
-                    logger.info("Using FP16 precision on GPU")
-                    model = Qwen2AudioForConditionalGeneration.from_pretrained(
-                        MODEL_ID,
-                        torch_dtype=torch.float16,
-                        device_map="auto",
-                        low_cpu_mem_usage=True
-                    )
-                    logger.info("Model loaded successfully with FP16")
-                else:
-                    # For smaller GPUs, use CPU offloading
-                    logger.info("Using CPU offloading for model components")
-                    model = Qwen2AudioForConditionalGeneration.from_pretrained(
-                        MODEL_ID,
-                        torch_dtype=torch.float16,
-                        device_map="auto",
-                        offload_folder="offload",
-                        low_cpu_mem_usage=True
-                    )
-                    logger.info("Model loaded successfully with CPU offloading")
-            except Exception as gpu_error:
-                logger.warning(f"GPU loading failed: {gpu_error}. Falling back to CPU.")
+            gpu_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3
+            logger.info(f"GPU memory: {gpu_memory:.2f} GB")
+            
+            if has_accelerate:
+                # With Accelerate, use device mapping
+                logger.info("Loading model with FP16 precision on GPU")
                 model = Qwen2AudioForConditionalGeneration.from_pretrained(
-                    MODEL_ID, 
-                    device_map="cpu",
+                    MODEL_ID,
+                    torch_dtype=torch.float16,
+                    device_map="auto",
                     low_cpu_mem_usage=True
                 )
-                logger.info("Model loaded successfully on CPU")
+            else:
+                # Without Accelerate, load directly to GPU
+                logger.info("Loading model directly to GPU")
+                model = Qwen2AudioForConditionalGeneration.from_pretrained(MODEL_ID)
+                model = model.to("cuda").half()  # Move to GPU and convert to FP16
         else:
             # Load on CPU if no GPU
-            model = Qwen2AudioForConditionalGeneration.from_pretrained(
-                MODEL_ID, 
-                device_map="cpu",
-                low_cpu_mem_usage=True
-            )
-            logger.info("Model loaded successfully on CPU")
+            logger.info("Loading model on CPU")
+            model = Qwen2AudioForConditionalGeneration.from_pretrained(MODEL_ID)
         
         model.eval()
         log_gpu_memory("After model loading")
@@ -105,7 +90,7 @@ def load_model():
     except Exception as e:
         logger.error(f"Error loading model or processor: {e}")
         raise
-
+    
 def process_audio(audio_url):
     """Process audio from URL"""
     logger.info(f"Processing audio: {audio_url}")

requirements.txt

@@ -4,7 +4,7 @@ transformers
 datasets
 peft
 bitsandbytes==0.41.1
-accelerate==0.25.0
+accelerate>=0.26.0
 hf_transfer
 tensorboard
 requests