Update handler.py

handler.py

@@ -21,18 +21,12 @@ class EndpointHandler:
         self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
         print(f"Using device: {self.device}")
         
-        # Load model WITHOUT flash attention
+       # Load model in float32 to avoid dtype conflicts
         model_kwargs = {
             'trust_remote_code': True,
+            'torch_dtype': torch.float32  # Use float32 instead of float16
         }
         
-        # Use appropriate dtype based on GPU capability
-        if self.device == 'cuda':
-            # T4 and L4 work better with float16
-            model_kwargs['torch_dtype'] = torch.float16
-        else:
-            model_kwargs['torch_dtype'] = torch.float32
-        
         # Explicitly disable flash attention
         model_kwargs['_attn_implementation'] = 'eager'