FilipT/babylm-ft-final-test

README.md

@@ -41,7 +41,7 @@ The following hyperparameters were used during training:
 - total_train_batch_size: 128
 - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: linear
-- training_steps: 1
+- training_steps: 30
 
 ### Training results
 

config.json

@@ -2,14 +2,14 @@
   "architectures": [
     "LtgBertForMaskedLM"
   ],
-  "attention_probs_dropout_prob": 0.1,
+  "attention_probs_dropout_prob": 0.15,
   "auto_map": {
     "AutoConfig": "configuration_ltgbert.LtgBertConfig",
     "AutoModelForMaskedLM": "modeling_ltgbert.LtgBertForMaskedLM",
     "AutoModelForSequenceClassification": "modeling_ltgbert.LtgBertForSequenceClassification"
   },
   "classifier_dropout": 0.2,
-  "hidden_dropout_prob": 0.1,
+  "hidden_dropout_prob": 0.2,
   "hidden_size": 768,
   "intermediate_size": 2048,
   "layer_norm_eps": 1e-07,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b078b9d79767451ce1f73158f08eea9e86d8ac818b52236ac585f39207fefee0
+oid sha256:6fce0a039deaef3a926b3dad8c938f459c0578865b621d38791909ac858ad0d9
 size 443265688

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ee5618843a16723d0c62107898d6500334a49fbb2822a78b3487cac464245fe
+oid sha256:c18800c8cfd89a8eb5069f9f6792e713167cca90bbf5621997a64024f858e23b
 size 5304