Upload model

config.json

@@ -20,9 +20,8 @@
     "dropout": 0.0,
     "layer_norm_eps": 1e-06,
     "nhead": 12,
-    "norm_first": true
+    "norm_first": false
   },
-  "dtype": "float32",
   "encoder_cfg": {
     "enable_nested_tensor": false,
     "mask_check": true,
@@ -37,7 +36,7 @@
     "dropout": 0.0,
     "layer_norm_eps": 1e-06,
     "nhead": 12,
-    "norm_first": true
+    "norm_first": false
   },
   "extractor_config": {
     "conv_bias": false,
@@ -49,5 +48,6 @@
     "share_weights_over_channels": false
   },
   "model_type": "wavjepa-nat-base",
-  "transformers_version": "4.57.1"
+  "torch_dtype": "float32",
+  "transformers_version": "4.53.2"
 }

configuration_wavjepa_nat.py

@@ -18,7 +18,7 @@ class WavJEPANatConfig(PretrainedConfig):
         encoder_d_model: int = 768,
         encoder_nhead : int = 12,
         encoder_batch_first = True,
-        encoder_norm_first = True, 
+        encoder_norm_first = False, 
         encoder_bias = True,
         encoder_mlp_ratio = 4.0,
         encoder_dropout = 0.0,
@@ -29,7 +29,7 @@ class WavJEPANatConfig(PretrainedConfig):
         decoder_d_model: int = 384,
         decoder_nhead : int = 12,
         decoder_batch_first = True,
-        decoder_norm_first = True, 
+        decoder_norm_first = False, 
         decoder_bias = True,
         decoder_mlp_ratio = 4.0,
         decoder_dropout = 0.0,