Model save

README.md

@@ -17,8 +17,8 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [Mardiyyah/biomedbert_model_extended_untrained](https://huggingface.co/Mardiyyah/biomedbert_model_extended_untrained) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 6.6288
-- Accuracy: 0.2237
+- Loss: 4.9618
+- Accuracy: 0.3431
 
 ## Model description
 
@@ -45,23 +45,31 @@ The following hyperparameters were used during training:
 - total_train_batch_size: 32
 - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-06 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: linear
-- lr_scheduler_warmup_ratio: 0.01
-- num_epochs: 10
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 20
 - mixed_precision_training: Native AMP
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss | Accuracy |
-|:-------------:|:------:|:----:|:---------------:|:--------:|
-| 10.133        | 1.0    | 6    | 10.3504         | 0.0      |
-| 8.7383        | 2.0    | 12   | 8.7064          | 0.0113   |
-| 7.6303        | 3.0    | 18   | 8.0542          | 0.0578   |
-| 7.06          | 4.0    | 24   | 7.5514          | 0.0821   |
-| 6.6524        | 5.0    | 30   | 7.0707          | 0.1521   |
-| 6.2891        | 6.0    | 36   | 6.9006          | 0.1806   |
-| 6.0336        | 7.0    | 42   | 6.6465          | 0.2103   |
-| 5.9085        | 8.0    | 48   | 6.6159          | 0.2208   |
-| 6.375         | 8.3636 | 50   | 6.6288          | 0.2237   |
+| Training Loss | Epoch   | Step | Validation Loss | Accuracy |
+|:-------------:|:-------:|:----:|:---------------:|:--------:|
+| 10.1666       | 1.0     | 6    | 11.0374         | 0.0      |
+| 9.6209        | 2.0     | 12   | 9.6987          | 0.0      |
+| 8.2654        | 3.0     | 18   | 8.4261          | 0.0281   |
+| 7.3452        | 4.0     | 24   | 7.7401          | 0.0637   |
+| 6.7638        | 5.0     | 30   | 7.0736          | 0.1537   |
+| 6.2231        | 6.0     | 36   | 6.6987          | 0.1956   |
+| 5.7561        | 7.0     | 42   | 6.2060          | 0.2480   |
+| 5.4166        | 8.0     | 48   | 5.9637          | 0.2752   |
+| 5.1312        | 9.0     | 54   | 5.7060          | 0.2832   |
+| 4.9182        | 10.0    | 60   | 5.5990          | 0.2995   |
+| 4.6794        | 11.0    | 66   | 5.3070          | 0.3117   |
+| 4.5745        | 12.0    | 72   | 5.2026          | 0.3180   |
+| 4.4439        | 13.0    | 78   | 5.0539          | 0.3331   |
+| 4.412         | 14.0    | 84   | 5.0538          | 0.3469   |
+| 4.2761        | 15.0    | 90   | 5.2681          | 0.3167   |
+| 4.3235        | 16.0    | 96   | 4.9841          | 0.3259   |
+| 4.5983        | 16.7273 | 100  | 4.9618          | 0.3431   |
 
 
 ### Framework versions

all_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 8.363636363636363,
+    "eval_accuracy": 0.22999581064097194,
+    "eval_loss": 6.545156478881836,
+    "eval_perplexity": 695.8655586952032,
+    "eval_runtime": 0.6885,
+    "eval_samples": 31,
+    "eval_samples_per_second": 45.027,
+    "eval_steps_per_second": 2.905,
+    "total_flos": 368493460070400.0,
+    "train_loss": 7.268417072296143,
+    "train_perplexity": 1434.278296330838,
+    "train_runtime": 82.8209,
+    "train_samples": 167,
+    "train_samples_per_second": 20.164,
+    "train_steps_per_second": 0.604
+}

eval_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 8.363636363636363,
+    "eval_accuracy": 0.22999581064097194,
+    "eval_loss": 6.545156478881836,
+    "eval_perplexity": 695.8655586952032,
+    "eval_runtime": 0.6885,
+    "eval_samples": 31,
+    "eval_samples_per_second": 45.027,
+    "eval_steps_per_second": 2.905,
+    "total_flos": 368493460070400.0,
+    "train_loss": 7.268417072296143,
+    "train_perplexity": 1434.278296330838,
+    "train_runtime": 82.8209,
+    "train_samples": 167,
+    "train_samples_per_second": 20.164,
+    "train_steps_per_second": 0.604
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96c33829b56a66d56df6e2f80f3b94129750cc65e84302ced1f553a0c2d94c0c
+oid sha256:09a76b651c039967573b93171023623ee39eafe2c2ece87833d40daa3e4bad60
 size 442888692

train_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 8.363636363636363,
+    "total_flos": 368493460070400.0,
+    "train_loss": 7.268417072296143,
+    "train_perplexity": 1434.278296330838,
+    "train_runtime": 82.8209,
+    "train_samples": 167,
+    "train_samples_per_second": 20.164,
+    "train_steps_per_second": 0.604
+}

trainer_state.json

@@ -0,0 +1,186 @@
+{
+  "best_metric": 0.22371177209886886,
+  "best_model_checkpoint": "/nfs/production/literature/amina-mardiyyah/new_data/OT-Entity-Extraction-Pipeline/model_outputs/Continued_pretraining/TAPT/Mardiyyah/biomedbert_model_extended_untrained/cellate-tapt_base-LR_2e-05/checkpoint-50",
+  "epoch": 8.363636363636363,
+  "eval_steps": 500,
+  "global_step": 50,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "grad_norm": 22.1080379486084,
+      "learning_rate": 1.9183673469387756e-05,
+      "loss": 10.133,
+      "step": 6
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.0,
+      "eval_loss": 10.35041332244873,
+      "eval_runtime": 0.7052,
+      "eval_samples_per_second": 43.957,
+      "eval_steps_per_second": 2.836,
+      "step": 6
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 8.525097846984863,
+      "learning_rate": 1.673469387755102e-05,
+      "loss": 8.7383,
+      "step": 12
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.011311269375785506,
+      "eval_loss": 8.706360816955566,
+      "eval_runtime": 0.6712,
+      "eval_samples_per_second": 46.188,
+      "eval_steps_per_second": 2.98,
+      "step": 12
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 4.951538562774658,
+      "learning_rate": 1.4285714285714287e-05,
+      "loss": 7.6303,
+      "step": 18
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.057813154587348134,
+      "eval_loss": 8.054218292236328,
+      "eval_runtime": 0.6857,
+      "eval_samples_per_second": 45.211,
+      "eval_steps_per_second": 2.917,
+      "step": 18
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 4.658953666687012,
+      "learning_rate": 1.1836734693877552e-05,
+      "loss": 7.06,
+      "step": 24
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.08211143695014662,
+      "eval_loss": 7.551440238952637,
+      "eval_runtime": 0.6769,
+      "eval_samples_per_second": 45.798,
+      "eval_steps_per_second": 2.955,
+      "step": 24
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 4.83123779296875,
+      "learning_rate": 9.387755102040818e-06,
+      "loss": 6.6524,
+      "step": 30
+    },
+    {
+      "epoch": 5.0,
+      "eval_accuracy": 0.15207373271889402,
+      "eval_loss": 7.070725440979004,
+      "eval_runtime": 0.6777,
+      "eval_samples_per_second": 45.742,
+      "eval_steps_per_second": 2.951,
+      "step": 30
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 4.068853378295898,
+      "learning_rate": 6.938775510204082e-06,
+      "loss": 6.2891,
+      "step": 36
+    },
+    {
+      "epoch": 6.0,
+      "eval_accuracy": 0.1805613741097612,
+      "eval_loss": 6.90059232711792,
+      "eval_runtime": 0.6808,
+      "eval_samples_per_second": 45.536,
+      "eval_steps_per_second": 2.938,
+      "step": 36
+    },
+    {
+      "epoch": 7.0,
+      "grad_norm": 6.190578460693359,
+      "learning_rate": 4.489795918367348e-06,
+      "loss": 6.0336,
+      "step": 42
+    },
+    {
+      "epoch": 7.0,
+      "eval_accuracy": 0.210305823209049,
+      "eval_loss": 6.646459579467773,
+      "eval_runtime": 0.7705,
+      "eval_samples_per_second": 40.235,
+      "eval_steps_per_second": 2.596,
+      "step": 42
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 4.320862293243408,
+      "learning_rate": 2.0408163265306125e-06,
+      "loss": 5.9085,
+      "step": 48
+    },
+    {
+      "epoch": 8.0,
+      "eval_accuracy": 0.22077922077922077,
+      "eval_loss": 6.615872383117676,
+      "eval_runtime": 0.679,
+      "eval_samples_per_second": 45.655,
+      "eval_steps_per_second": 2.945,
+      "step": 48
+    },
+    {
+      "epoch": 8.363636363636363,
+      "grad_norm": 6.226965427398682,
+      "learning_rate": 1.2244897959183673e-06,
+      "loss": 6.375,
+      "step": 50
+    },
+    {
+      "epoch": 8.363636363636363,
+      "eval_accuracy": 0.22371177209886886,
+      "eval_loss": 6.628802299499512,
+      "eval_runtime": 0.7625,
+      "eval_samples_per_second": 40.656,
+      "eval_steps_per_second": 2.623,
+      "step": 50
+    },
+    {
+      "epoch": 8.363636363636363,
+      "step": 50,
+      "total_flos": 368493460070400.0,
+      "train_loss": 7.268417072296143,
+      "train_runtime": 82.8209,
+      "train_samples_per_second": 20.164,
+      "train_steps_per_second": 0.604
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 50,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 368493460070400.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83b6393facefc005a011b15243f23b8e9f0e6b5a1638c16fe70da1dab87e7587
+oid sha256:dfeee50a49d24056ce9152272d9d86ffeb390bbfec10314b48c0effbe4e019f4
 size 5624