End of epoch 1 | AVG Contr_Loss: 5846.9614 | AVG Diver_Loss: 440.8876 | PPL: 454.6

config.json

@@ -13,7 +13,7 @@
   "classifier_proj_size": 256,
   "codebook_negatives": 0,
   "codevector_dim": 768,
-  "codevector_entropy_weight": 0.2,
+  "codevector_entropy_weight": 2.0,
   "contrastive_logits_temperature": 0.1,
   "conv_bias": true,
   "conv_dim": [
@@ -47,7 +47,7 @@
   "ctc_zero_infinity": false,
   "diversity_loss_weight": 0.1,
   "do_stable_layer_norm": true,
-  "dtype": "bfloat16",
+  "dtype": "float32",
   "eos_token_id": 2,
   "feat_extract_activation": "gelu",
   "feat_extract_dropout": 0.0,
@@ -56,7 +56,7 @@
   "feat_quantizer_dropout": 0.0,
   "final_dropout": 0.0,
   "gradient_checkpointing": false,
-  "gumbel_temperature": 1.9999400007499946,
+  "gumbel_temperature": 1.999890002749958,
   "hidden_act": "gelu",
   "hidden_dropout": 0.1,
   "hidden_size": 1024,
@@ -73,7 +73,7 @@
   "mask_feature_length": 10,
   "mask_feature_min_masks": 0,
   "mask_feature_prob": 0.0,
-  "mask_time_length": 10,
+  "mask_time_length": 8,
   "mask_time_min_masks": 2,
   "mask_time_min_space": 1,
   "mask_time_other": 0.0,
@@ -88,7 +88,7 @@
   "num_conv_pos_embeddings": 128,
   "num_feat_extract_layers": 7,
   "num_hidden_layers": 24,
-  "num_negatives": 100,
+  "num_negatives": 50,
   "output_hidden_size": 1024,
   "pad_token_id": 0,
   "proj_codevector_dim": 768,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9297df4345779cc7b6391c662347fc9224cd1ce1e7d6bfc6ea2b54123f07904f
-size 634834360
+oid sha256:478c22da2e42062ae01980e30e72f906d7f410dba9c9a9226096077acb7e5078
+size 1269615400