Training in progress, step 10000

config.json

@@ -91,7 +91,7 @@
   },
   "decoder_start_token_id": 0,
   "encoder": {
-    "_name_or_path": "facebook/esm2_t12_35M_UR50D",
+    "_name_or_path": "facebook/esm2_t33_650M_UR50D",
     "add_cross_attention": false,
     "architectures": [
       "EsmForMaskedLM"
@@ -117,13 +117,13 @@
     "forced_eos_token_id": null,
     "hidden_act": "gelu",
     "hidden_dropout_prob": 0.15,
-    "hidden_size": 480,
+    "hidden_size": 1280,
     "id2label": {
       "0": "LABEL_0",
       "1": "LABEL_1"
     },
     "initializer_range": 0.02,
-    "intermediate_size": 1920,
+    "intermediate_size": 5120,
     "is_decoder": false,
     "is_encoder_decoder": false,
     "is_folding_model": false,
@@ -142,7 +142,7 @@
     "num_attention_heads": 20,
     "num_beam_groups": 1,
     "num_beams": 1,
-    "num_hidden_layers": 12,
+    "num_hidden_layers": 33,
     "num_return_sequences": 1,
     "output_attentions": false,
     "output_hidden_states": false,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:373861c1055362604f812be6963895cc1dded4b0ccb402022d58276fd3a385b4
-size 949216292
+oid sha256:d847ec7213205c197037d10c42b2e13d24ce2a180ea36176e2eff1ef4ee3ea5d
+size 3425985780

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8ccb6e2a9963d08ae07ae6b260b9e04779507b2ff21e19c425493335328cfa3
-size 5304
+oid sha256:bffe7f7c92365465b71d2103f925bc5b750100e31c3046ba4f986011de01a856
+size 5240