End of epoch 1 | AVG Contr_Loss: 7309.7822 | AVG Diver_Loss: 954.4370 | PPL: 268.6

config.json

@@ -56,7 +56,7 @@
   "feat_quantizer_dropout": 0.0,
   "final_dropout": 0.0,
   "gradient_checkpointing": false,
-  "gumbel_temperature": 1.99999,
+  "gumbel_temperature": 1.9999400007499946,
   "hidden_act": "gelu",
   "hidden_dropout": 0.1,
   "hidden_size": 1024,
@@ -88,7 +88,7 @@
   "num_conv_pos_embeddings": 128,
   "num_feat_extract_layers": 7,
   "num_hidden_layers": 24,
-  "num_negatives": 20,
+  "num_negatives": 100,
   "output_hidden_size": 1024,
   "pad_token_id": 0,
   "proj_codevector_dim": 768,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fd02ca27cd933f549f90956c9816e90cbaa84c5f567a99ea57f248e83be12da
+oid sha256:9297df4345779cc7b6391c662347fc9224cd1ce1e7d6bfc6ea2b54123f07904f
 size 634834360