ILC-CNR/gs-greBERTa

README.md

@@ -16,13 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [bowphs/GreBerta](https://huggingface.co/bowphs/GreBerta) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.4879
-- Top1 Acc: 0.3563
-- Top5 Acc: 0.6438
-- Top10 Acc: 0.775
-- Top15 Acc: 0.8688
-- Top20 Acc: 0.9125
-- Top25 Acc: 0.9187
+- Loss: 0.5878
 
 ## Model description
 
@@ -41,7 +35,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-05
+- learning_rate: 4e-05
 - train_batch_size: 16
 - eval_batch_size: 8
 - seed: 42
@@ -52,23 +46,23 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step  | Validation Loss | Top1 Acc | Top5 Acc | Top10 Acc | Top15 Acc | Top20 Acc | Top25 Acc |
-|:-------------:|:-----:|:-----:|:---------------:|:--------:|:--------:|:---------:|:---------:|:---------:|:---------:|
-| 1.2617        | 1.0   | 5710  | 0.9286          | 0.8304   | 0.8947   | 0.9064    | 0.9415    | 0.9532    | 0.9532    |
-| 0.938         | 2.0   | 11420 | 0.8058          | 0.8071   | 0.9357   | 0.9643    | 0.9643    | 0.9714    | 0.9786    |
-| 0.825         | 3.0   | 17130 | 0.7296          | 0.8375   | 0.95     | 0.9625    | 0.9812    | 0.9875    | 0.9875    |
-| 0.765         | 4.0   | 22840 | 0.6903          | 0.775    | 0.8875   | 0.9437    | 0.95      | 0.95      | 0.9563    |
-| 0.7226        | 5.0   | 28550 | 0.6579          | 0.8026   | 0.9342   | 0.9539    | 0.9605    | 0.9605    | 0.9605    |
-| 0.6998        | 6.0   | 34260 | 0.6540          | 0.8088   | 0.9265   | 0.9412    | 0.9485    | 0.9559    | 0.9632    |
-| 1.1697        | 7.0   | 39970 | 2.6136          | 0.4178   | 0.6301   | 0.7397    | 0.8288    | 0.8562    | 0.8699    |
-| 2.3523        | 8.0   | 45680 | 2.4223          | 0.3764   | 0.5843   | 0.7191    | 0.7753    | 0.8146    | 0.8483    |
-| 2.5091        | 9.0   | 51390 | 2.3962          | 0.4286   | 0.6599   | 0.7619    | 0.8299    | 0.8776    | 0.9116    |
-| 2.5581        | 10.0  | 57100 | 2.4889          | 0.34     | 0.6      | 0.7667    | 0.8267    | 0.8533    | 0.9       |
+| Training Loss | Epoch | Step  | Validation Loss |
+|:-------------:|:-----:|:-----:|:---------------:|
+| 1.2756        | 1.0   | 5710  | 0.9367          |
+| 0.9522        | 2.0   | 11420 | 0.8217          |
+| 0.8493        | 3.0   | 17130 | 0.7453          |
+| 0.781         | 4.0   | 22840 | 0.6875          |
+| 0.7316        | 5.0   | 28550 | 0.6624          |
+| 0.6985        | 6.0   | 34260 | 0.6378          |
+| 0.6736        | 7.0   | 39970 | 0.6155          |
+| 0.6502        | 8.0   | 45680 | 0.6063          |
+| 0.6345        | 9.0   | 51390 | 0.5895          |
+| 0.6232        | 10.0  | 57100 | 0.5866          |
 
 
 ### Framework versions
 
 - Transformers 4.51.3
 - Pytorch 2.7.0+cu126
-- Datasets 3.5.1
+- Datasets 3.6.0
 - Tokenizers 0.21.1

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 10.0,
-    "eval_loss": 2.4879400730133057,
-    "eval_runtime": 4368.2627,
-    "eval_samples_per_second": 2.341,
-    "eval_steps_per_second": 0.293,
+    "eval_loss": 0.5878283381462097,
+    "eval_runtime": 36.4992,
+    "eval_samples_per_second": 280.116,
+    "eval_steps_per_second": 35.014,
     "eval_top10_acc": 0.775,
     "eval_top15_acc": 0.86875,
     "eval_top1_acc": 0.35625,
@@ -12,8 +12,8 @@
     "eval_top5_acc": 0.64375,
     "step": 57100,
     "total_flos": 6.01310491705344e+16,
-    "train_loss": 1.3801261920561514,
-    "train_runtime": 49255.6771,
-    "train_samples_per_second": 18.55,
-    "train_steps_per_second": 1.159
+    "train_loss": 0.7869737379939251,
+    "train_runtime": 10124.3653,
+    "train_samples_per_second": 90.247,
+    "train_steps_per_second": 5.64
 }

eval_results.json

@@ -1,13 +1,7 @@
 {
     "epoch": 10.0,
-    "eval_loss": 2.4879400730133057,
-    "eval_runtime": 4368.2627,
-    "eval_samples_per_second": 2.341,
-    "eval_steps_per_second": 0.293,
-    "eval_top10_acc": 0.775,
-    "eval_top15_acc": 0.86875,
-    "eval_top1_acc": 0.35625,
-    "eval_top20_acc": 0.9125,
-    "eval_top25_acc": 0.91875,
-    "eval_top5_acc": 0.64375
+    "eval_loss": 0.5878283381462097,
+    "eval_runtime": 36.4992,
+    "eval_samples_per_second": 280.116,
+    "eval_steps_per_second": 35.014
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0e941c7589f36b7e6d63a780831db82f151251110cc805d2b46fa864d38bc23
+oid sha256:643e2522fe0e5dca911e3232fa230abe289b64cadad8156ac1d31580e5549d40
 size 504150808

train_results.json

@@ -2,8 +2,8 @@
     "epoch": 10.0,
     "step": 57100,
     "total_flos": 6.01310491705344e+16,
-    "train_loss": 1.3801261920561514,
-    "train_runtime": 49255.6771,
-    "train_samples_per_second": 18.55,
-    "train_steps_per_second": 1.159
+    "train_loss": 0.7869737379939251,
+    "train_runtime": 10124.3653,
+    "train_samples_per_second": 90.247,
+    "train_steps_per_second": 5.64
 }

trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": null,
-  "best_metric": null,
+  "best_metric": 0.5865987539291382,
   "best_model_checkpoint": null,
   "epoch": 10.0,
   "eval_steps": 500,
@@ -11,235 +11,169 @@
   "log_history": [
     {
       "epoch": 1.0,
-      "grad_norm": 4.705949783325195,
-      "learning_rate": 4.500700525394046e-05,
-      "loss": 1.2617,
+      "grad_norm": 5.180218696594238,
+      "learning_rate": 3.600420315236428e-05,
+      "loss": 1.2756,
       "step": 5710
     },
     {
       "epoch": 1.0,
-      "eval_loss": 0.928647518157959,
-      "eval_runtime": 4234.8005,
-      "eval_samples_per_second": 2.414,
-      "eval_steps_per_second": 0.302,
-      "eval_top10_acc": 0.9064327485380117,
-      "eval_top15_acc": 0.9415204678362573,
-      "eval_top1_acc": 0.8304093567251462,
-      "eval_top20_acc": 0.9532163742690059,
-      "eval_top25_acc": 0.9532163742690059,
-      "eval_top5_acc": 0.8947368421052632,
+      "eval_loss": 0.9367296695709229,
+      "eval_runtime": 76.8888,
+      "eval_samples_per_second": 132.971,
+      "eval_steps_per_second": 16.621,
       "step": 5710
     },
     {
       "epoch": 2.0,
-      "grad_norm": 5.407649040222168,
-      "learning_rate": 4.0007005253940456e-05,
-      "loss": 0.938,
+      "grad_norm": 5.356524467468262,
+      "learning_rate": 3.200630472854641e-05,
+      "loss": 0.9522,
       "step": 11420
     },
     {
       "epoch": 2.0,
-      "eval_loss": 0.8057610392570496,
-      "eval_runtime": 4258.0614,
-      "eval_samples_per_second": 2.401,
-      "eval_steps_per_second": 0.3,
-      "eval_top10_acc": 0.9642857142857143,
-      "eval_top15_acc": 0.9642857142857143,
-      "eval_top1_acc": 0.8071428571428572,
-      "eval_top20_acc": 0.9714285714285714,
-      "eval_top25_acc": 0.9785714285714285,
-      "eval_top5_acc": 0.9357142857142857,
+      "eval_loss": 0.8216637969017029,
+      "eval_runtime": 73.2993,
+      "eval_samples_per_second": 139.483,
+      "eval_steps_per_second": 17.435,
       "step": 11420
     },
     {
       "epoch": 3.0,
-      "grad_norm": 4.782800674438477,
-      "learning_rate": 3.500875656742557e-05,
-      "loss": 0.825,
+      "grad_norm": 4.990707874298096,
+      "learning_rate": 2.800700525394046e-05,
+      "loss": 0.8493,
       "step": 17130
     },
     {
       "epoch": 3.0,
-      "eval_loss": 0.7296221256256104,
-      "eval_runtime": 4280.2,
-      "eval_samples_per_second": 2.389,
-      "eval_steps_per_second": 0.299,
-      "eval_top10_acc": 0.9625,
-      "eval_top15_acc": 0.98125,
-      "eval_top1_acc": 0.8375,
-      "eval_top20_acc": 0.9875,
-      "eval_top25_acc": 0.9875,
-      "eval_top5_acc": 0.95,
+      "eval_loss": 0.7453346848487854,
+      "eval_runtime": 76.8549,
+      "eval_samples_per_second": 133.03,
+      "eval_steps_per_second": 16.629,
       "step": 17130
     },
     {
       "epoch": 4.0,
-      "grad_norm": 4.224056243896484,
-      "learning_rate": 3.0011383537653244e-05,
-      "loss": 0.765,
+      "grad_norm": 4.565978527069092,
+      "learning_rate": 2.4009106830122595e-05,
+      "loss": 0.781,
       "step": 22840
     },
     {
       "epoch": 4.0,
-      "eval_loss": 0.6902998089790344,
-      "eval_runtime": 4258.1988,
-      "eval_samples_per_second": 2.401,
-      "eval_steps_per_second": 0.3,
-      "eval_top10_acc": 0.94375,
-      "eval_top15_acc": 0.95,
-      "eval_top1_acc": 0.775,
-      "eval_top20_acc": 0.95,
-      "eval_top25_acc": 0.95625,
-      "eval_top5_acc": 0.8875,
+      "eval_loss": 0.6874940991401672,
+      "eval_runtime": 38.7504,
+      "eval_samples_per_second": 263.842,
+      "eval_steps_per_second": 32.98,
       "step": 22840
     },
     {
       "epoch": 5.0,
-      "grad_norm": 4.620619773864746,
-      "learning_rate": 2.5013134851138352e-05,
-      "loss": 0.7226,
+      "grad_norm": 4.659147262573242,
+      "learning_rate": 2.0010507880910684e-05,
+      "loss": 0.7316,
       "step": 28550
     },
     {
       "epoch": 5.0,
-      "eval_loss": 0.6579228639602661,
-      "eval_runtime": 4227.7384,
-      "eval_samples_per_second": 2.418,
-      "eval_steps_per_second": 0.302,
-      "eval_top10_acc": 0.9539473684210527,
-      "eval_top15_acc": 0.9605263157894737,
-      "eval_top1_acc": 0.8026315789473685,
-      "eval_top20_acc": 0.9605263157894737,
-      "eval_top25_acc": 0.9605263157894737,
-      "eval_top5_acc": 0.9342105263157895,
+      "eval_loss": 0.6624111533164978,
+      "eval_runtime": 36.5212,
+      "eval_samples_per_second": 279.947,
+      "eval_steps_per_second": 34.993,
       "step": 28550
     },
     {
       "epoch": 6.0,
-      "grad_norm": 4.630845546722412,
-      "learning_rate": 2.001751313485114e-05,
-      "loss": 0.6998,
+      "grad_norm": 4.955957412719727,
+      "learning_rate": 1.6011908931698776e-05,
+      "loss": 0.6985,
       "step": 34260
     },
     {
       "epoch": 6.0,
-      "eval_loss": 0.6539566516876221,
-      "eval_runtime": 4239.6205,
-      "eval_samples_per_second": 2.412,
-      "eval_steps_per_second": 0.301,
-      "eval_top10_acc": 0.9411764705882353,
-      "eval_top15_acc": 0.9485294117647058,
-      "eval_top1_acc": 0.8088235294117647,
-      "eval_top20_acc": 0.9558823529411765,
-      "eval_top25_acc": 0.9632352941176471,
-      "eval_top5_acc": 0.9264705882352942,
+      "eval_loss": 0.6377598643302917,
+      "eval_runtime": 35.9349,
+      "eval_samples_per_second": 284.514,
+      "eval_steps_per_second": 35.564,
       "step": 34260
     },
     {
       "epoch": 7.0,
-      "grad_norm": 5.699009895324707,
-      "learning_rate": 1.5021891418563924e-05,
-      "loss": 1.1697,
+      "grad_norm": 4.549160003662109,
+      "learning_rate": 1.2012609457092821e-05,
+      "loss": 0.6736,
       "step": 39970
     },
     {
       "epoch": 7.0,
-      "eval_loss": 2.613582134246826,
-      "eval_runtime": 4560.5825,
-      "eval_samples_per_second": 2.242,
-      "eval_steps_per_second": 0.28,
-      "eval_top10_acc": 0.7397260273972602,
-      "eval_top15_acc": 0.8287671232876712,
-      "eval_top1_acc": 0.4178082191780822,
-      "eval_top20_acc": 0.8561643835616438,
-      "eval_top25_acc": 0.8698630136986302,
-      "eval_top5_acc": 0.6301369863013698,
+      "eval_loss": 0.6155329346656799,
+      "eval_runtime": 37.8645,
+      "eval_samples_per_second": 270.015,
+      "eval_steps_per_second": 33.752,
       "step": 39970
     },
     {
       "epoch": 8.0,
-      "grad_norm": 5.335746765136719,
-      "learning_rate": 1.002276707530648e-05,
-      "loss": 2.3523,
+      "grad_norm": 4.006523609161377,
+      "learning_rate": 8.01260945709282e-06,
+      "loss": 0.6502,
       "step": 45680
     },
     {
       "epoch": 8.0,
-      "eval_loss": 2.4223251342773438,
-      "eval_runtime": 4532.3712,
-      "eval_samples_per_second": 2.256,
-      "eval_steps_per_second": 0.282,
-      "eval_top10_acc": 0.7191011235955056,
-      "eval_top15_acc": 0.7752808988764045,
-      "eval_top1_acc": 0.37640449438202245,
-      "eval_top20_acc": 0.8146067415730337,
-      "eval_top25_acc": 0.848314606741573,
-      "eval_top5_acc": 0.5842696629213483,
+      "eval_loss": 0.6062974333763123,
+      "eval_runtime": 36.5102,
+      "eval_samples_per_second": 280.031,
+      "eval_steps_per_second": 35.004,
       "step": 45680
     },
     {
       "epoch": 9.0,
-      "grad_norm": 6.699865818023682,
-      "learning_rate": 5.024518388791594e-06,
-      "loss": 2.5091,
+      "grad_norm": 3.7265231609344482,
+      "learning_rate": 4.013309982486865e-06,
+      "loss": 0.6345,
       "step": 51390
     },
     {
       "epoch": 9.0,
-      "eval_loss": 2.396181344985962,
-      "eval_runtime": 4441.5711,
-      "eval_samples_per_second": 2.302,
-      "eval_steps_per_second": 0.288,
-      "eval_top10_acc": 0.7619047619047619,
-      "eval_top15_acc": 0.8299319727891157,
-      "eval_top1_acc": 0.42857142857142855,
-      "eval_top20_acc": 0.8775510204081632,
-      "eval_top25_acc": 0.9115646258503401,
-      "eval_top5_acc": 0.6598639455782312,
+      "eval_loss": 0.5894550681114197,
+      "eval_runtime": 43.2376,
+      "eval_samples_per_second": 236.461,
+      "eval_steps_per_second": 29.558,
       "step": 51390
     },
     {
       "epoch": 10.0,
-      "grad_norm": 4.715834140777588,
-      "learning_rate": 2.6269702276707533e-08,
-      "loss": 2.5581,
+      "grad_norm": 3.897273302078247,
+      "learning_rate": 1.611208406304729e-08,
+      "loss": 0.6232,
       "step": 57100
     },
     {
       "epoch": 10.0,
-      "eval_loss": 2.488941192626953,
-      "eval_runtime": 4366.5233,
-      "eval_samples_per_second": 2.341,
-      "eval_steps_per_second": 0.293,
-      "eval_top10_acc": 0.7666666666666667,
-      "eval_top15_acc": 0.8266666666666667,
-      "eval_top1_acc": 0.34,
-      "eval_top20_acc": 0.8533333333333334,
-      "eval_top25_acc": 0.9,
-      "eval_top5_acc": 0.6,
+      "eval_loss": 0.5865987539291382,
+      "eval_runtime": 36.5729,
+      "eval_samples_per_second": 279.551,
+      "eval_steps_per_second": 34.944,
       "step": 57100
     },
     {
       "epoch": 10.0,
       "step": 57100,
       "total_flos": 6.01310491705344e+16,
-      "train_loss": 1.3801261920561514,
-      "train_runtime": 49255.6771,
-      "train_samples_per_second": 18.55,
-      "train_steps_per_second": 1.159
+      "train_loss": 0.7869737379939251,
+      "train_runtime": 10124.3653,
+      "train_samples_per_second": 90.247,
+      "train_steps_per_second": 5.64
     },
     {
       "epoch": 10.0,
-      "eval_loss": 2.4879400730133057,
-      "eval_runtime": 4368.2627,
-      "eval_samples_per_second": 2.341,
-      "eval_steps_per_second": 0.293,
-      "eval_top10_acc": 0.775,
-      "eval_top15_acc": 0.86875,
-      "eval_top1_acc": 0.35625,
-      "eval_top20_acc": 0.9125,
-      "eval_top25_acc": 0.91875,
-      "eval_top5_acc": 0.64375,
+      "eval_loss": 0.5878283381462097,
+      "eval_runtime": 36.4992,
+      "eval_samples_per_second": 280.116,
+      "eval_steps_per_second": 35.014,
       "step": 57100
     }
   ],
@@ -249,6 +183,15 @@
   "num_train_epochs": 10,
   "save_steps": 500,
   "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 2,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
     "TrainerControl": {
       "args": {
         "should_epoch_stop": false,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d86f31b8645edb5a8bc695e48ee26e49fb8c60618206647233a2cba8872a191c
+oid sha256:2f8e8baf6eeb9717a999831a644b81e1618482d42a4c4d6f0efb269f9d424652
 size 5649