Training in progress, epoch 1

config.json

@@ -1,15 +1,32 @@
 {
-  "_name_or_path": "nlpaueb/legal-bert-small-uncased",
+  "_name_or_path": "/content/drive/MyDrive/bregman_mimic_40ksteps/40000",
   "architectures": [
-    "BertForSequenceClassification"
+    "LongformerForSequenceClassification"
   ],
+  "attention_mode": "longformer",
   "attention_probs_dropout_prob": 0.1,
-  "bos_token_id": 0,
+  "attention_window": [
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128
+  ],
+  "bos_token_id": null,
   "classifier_dropout": null,
-  "eos_token_ids": 0,
+  "cls_token_id": 101,
+  "eos_token_id": null,
+  "gradient_checkpointing": false,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 512,
+  "hidden_size": 768,
   "id2label": {
     "0": "LABEL_0",
     "1": "LABEL_1",
@@ -20,13 +37,32 @@
     "6": "LABEL_6",
     "7": "LABEL_7",
     "8": "LABEL_8",
-    "9": "LABEL_9"
+    "9": "LABEL_9",
+    "10": "LABEL_10",
+    "11": "LABEL_11",
+    "12": "LABEL_12",
+    "13": "LABEL_13",
+    "14": "LABEL_14",
+    "15": "LABEL_15",
+    "16": "LABEL_16",
+    "17": "LABEL_17",
+    "18": "LABEL_18"
   },
+  "ignore_attention_mask": false,
   "initializer_range": 0.02,
-  "intermediate_size": 2048,
+  "intermediate_size": 3072,
   "label2id": {
     "LABEL_0": 0,
     "LABEL_1": 1,
+    "LABEL_10": 10,
+    "LABEL_11": 11,
+    "LABEL_12": 12,
+    "LABEL_13": 13,
+    "LABEL_14": 14,
+    "LABEL_15": 15,
+    "LABEL_16": 16,
+    "LABEL_17": 17,
+    "LABEL_18": 18,
     "LABEL_2": 2,
     "LABEL_3": 3,
     "LABEL_4": 4,
@@ -36,15 +72,17 @@
     "LABEL_8": 8,
     "LABEL_9": 9
   },
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
-  "num_attention_heads": 8,
-  "num_hidden_layers": 6,
-  "output_past": true,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 4098,
+  "model_max_length": 4096,
+  "model_type": "longformer",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "onnx_export": false,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "problem_type": "multi_label_classification",
+  "sep_token_id": 102,
   "torch_dtype": "float32",
   "transformers_version": "4.25.1",
   "type_vocab_size": 2,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4856891ee67c3e24b802f84aa683b6bdb2842c575911df58d034103cd880f7c
-size 140334005
+oid sha256:ca163dfd7352c0d7a72223cf73e0dd091a8d737b31fb01eb946ad090ee2c920b
+size 536507161

tokenizer_config.json

@@ -3,8 +3,8 @@
   "do_basic_tokenize": true,
   "do_lower_case": true,
   "mask_token": "[MASK]",
-  "model_max_length": 512,
-  "name_or_path": "nlpaueb/legal-bert-small-uncased",
+  "model_max_length": 4096,
+  "name_or_path": "/content/drive/MyDrive/bregman_mimic_40ksteps/40000",
   "never_split": null,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce7afc4e786f73c06e083fd2171b30cf6e23df894efb7f258d7f54a68b10cf67
+oid sha256:e703b12fd68fe2c571ef89b6bf1c1dbc86cb30bb6cf0298b0e32b7ee8b31ec66
 size 3451