ILC-CNR/gs-greBERTa

README.md

@@ -1,21 +1,26 @@
 ---
 library_name: transformers
-base_model: Jacobo/aristoBERTo
+license: apache-2.0
+base_model: bowphs/GreBerta
 tags:
 - generated_from_trainer
 model-index:
-- name: model-checkpoints
+- name: gs-greBERTa
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# model-checkpoints
+# gs-greBERTa
 
-This model is a fine-tuned version of [Jacobo/aristoBERTo](https://huggingface.co/Jacobo/aristoBERTo) on an unknown dataset.
+This model is a fine-tuned version of [bowphs/GreBerta](https://huggingface.co/bowphs/GreBerta) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.0007
+- Loss: 0.5786
+- Top1 Acc: 0.8589
+- Top5 Acc: 0.9202
+- Top10 Acc: 0.9448
+- Top20 Acc: 0.9632
 
 ## Model description
 
@@ -36,7 +41,7 @@ More information needed
 The following hyperparameters were used during training:
 - learning_rate: 5e-05
 - train_batch_size: 16
-- eval_batch_size: 16
+- eval_batch_size: 8
 - seed: 42
 - optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: linear
@@ -45,23 +50,23 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step  | Validation Loss |
-|:-------------:|:-----:|:-----:|:---------------:|
-| 2.9726        | 1.0   | 5652  | 2.5978          |
-| 2.5639        | 2.0   | 11304 | 2.3816          |
-| 2.4014        | 3.0   | 16956 | 2.2645          |
-| 2.2988        | 4.0   | 22608 | 2.2166          |
-| 2.2089        | 5.0   | 28260 | 2.1206          |
-| 2.1371        | 6.0   | 33912 | 2.0921          |
-| 2.0808        | 7.0   | 39564 | 2.0355          |
-| 2.0361        | 8.0   | 45216 | 2.0193          |
-| 2.001         | 9.0   | 50868 | 1.9988          |
-| 1.977         | 10.0  | 56520 | 1.9922          |
+| Training Loss | Epoch | Step  | Validation Loss | Top1 Acc | Top5 Acc | Top10 Acc | Top20 Acc |
+|:-------------:|:-----:|:-----:|:---------------:|:--------:|:--------:|:---------:|:---------:|
+| 1.2561        | 1.0   | 5634  | 0.9441          | 0.7965   | 0.8837   | 0.9302    | 0.9593    |
+| 0.969         | 2.0   | 11268 | 0.8007          | 0.8028   | 0.9296   | 0.9366    | 0.9648    |
+| 0.8407        | 3.0   | 16902 | 0.7505          | 0.8092   | 0.9249   | 0.9480    | 0.9769    |
+| 0.7791        | 4.0   | 22536 | 0.6900          | 0.825    | 0.9313   | 0.95      | 0.975     |
+| 0.7264        | 5.0   | 28170 | 0.6541          | 0.8824   | 0.9471   | 0.9706    | 0.9765    |
+| 0.6872        | 6.0   | 33804 | 0.6343          | 0.8344   | 0.9264   | 0.9571    | 0.9877    |
+| 0.6553        | 7.0   | 39438 | 0.6069          | 0.8705   | 0.9568   | 0.9712    | 0.9784    |
+| 0.6479        | 8.0   | 45072 | 0.5924          | 0.8905   | 0.9708   | 0.9854    | 0.9927    |
+| 0.6181        | 9.0   | 50706 | 0.5827          | 0.8834   | 0.9571   | 0.9693    | 0.9816    |
+| 0.6051        | 10.0  | 56340 | 0.5851          | 0.8922   | 0.9701   | 1.0       | 1.0       |
 
 
 ### Framework versions
 
 - Transformers 4.51.3
 - Pytorch 2.7.0+cu126
-- Datasets 3.5.0
+- Datasets 3.5.1
 - Tokenizers 0.21.1

all_results.json

@@ -1,13 +1,17 @@
 {
     "epoch": 10.0,
-    "eval_loss": 2.0006515979766846,
-    "eval_runtime": 11.5932,
-    "eval_samples_per_second": 839.288,
-    "eval_steps_per_second": 52.531,
-    "step": 56520,
-    "total_flos": 3.105015735189504e+16,
-    "train_loss": 2.067556116325038,
-    "train_runtime": 7586.095,
-    "train_samples_per_second": 124.388,
-    "train_steps_per_second": 7.773
+    "eval_loss": 0.5786494612693787,
+    "eval_runtime": 736.6563,
+    "eval_samples_per_second": 3.851,
+    "eval_steps_per_second": 0.482,
+    "eval_top10_acc": 0.9447852760736196,
+    "eval_top1_acc": 0.8588957055214724,
+    "eval_top20_acc": 0.9631901840490797,
+    "eval_top5_acc": 0.9202453987730062,
+    "step": 56340,
+    "total_flos": 5.933070595915776e+16,
+    "train_loss": 0.7784920386062855,
+    "train_runtime": 14293.974,
+    "train_samples_per_second": 63.066,
+    "train_steps_per_second": 3.942
 }

config.json

@@ -1,25 +1,26 @@
 {
   "architectures": [
-    "BertForMaskedLM"
+    "RobertaForMaskedLM"
   ],
   "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
   "classifier_dropout": null,
+  "eos_token_id": 2,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
   "initializer_range": 0.02,
   "intermediate_size": 3072,
   "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
-  "output_past": true,
-  "pad_token_id": 0,
+  "pad_token_id": 1,
   "position_embedding_type": "absolute",
   "torch_dtype": "float32",
   "transformers_version": "4.51.3",
-  "type_vocab_size": 2,
+  "type_vocab_size": 1,
   "use_cache": true,
-  "vocab_size": 35000
+  "vocab_size": 52000
 }

eval_results.json

@@ -1,7 +1,11 @@
 {
     "epoch": 10.0,
-    "eval_loss": 2.0006515979766846,
-    "eval_runtime": 11.5932,
-    "eval_samples_per_second": 839.288,
-    "eval_steps_per_second": 52.531
+    "eval_loss": 0.5786494612693787,
+    "eval_runtime": 736.6563,
+    "eval_samples_per_second": 3.851,
+    "eval_steps_per_second": 0.482,
+    "eval_top10_acc": 0.9447852760736196,
+    "eval_top1_acc": 0.8588957055214724,
+    "eval_top20_acc": 0.9631901840490797,
+    "eval_top5_acc": 0.9202453987730062
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ac48d52adb26e2fddddb2ca2062b638e638322e6dcaa395cb7f33fd5e6f5338
-size 451855232
+oid sha256:dbc282e85cce75ef4bb3e2e8e5e0fe234ef492beebc040cd69cd3f3746b1457c
+size 504150808

special_tokens_map.json

@@ -1,7 +1,51 @@
 {
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -1,58 +1,58 @@
 {
+  "add_prefix_space": false,
   "added_tokens_decoder": {
     "0": {
-      "content": "[PAD]",
+      "content": "<s>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "100": {
-      "content": "[UNK]",
+    "1": {
+      "content": "<pad>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "101": {
-      "content": "[CLS]",
+    "2": {
+      "content": "</s>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "102": {
-      "content": "[SEP]",
+    "3": {
+      "content": "<unk>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "103": {
-      "content": "[MASK]",
-      "lstrip": false,
-      "normalized": false,
+    "4": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
-  "clean_up_tokenization_spaces": true,
-  "cls_token": "[CLS]",
-  "do_basic_tokenize": true,
-  "do_lower_case": true,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
   "extra_special_tokens": {},
-  "mask_token": "[MASK]",
+  "mask_token": "<mask>",
   "model_max_length": 512,
-  "never_split": null,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "strip_accents": null,
-  "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizer",
-  "unk_token": "[UNK]"
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
 }

train_results.json

@@ -1,8 +1,9 @@
 {
     "epoch": 10.0,
-    "eval_loss": 1.980637788772583,
-    "eval_runtime": 11.5702,
-    "eval_samples_per_second": 840.952,
-    "eval_steps_per_second": 52.635,
-    "step": 56520
+    "step": 56340,
+    "total_flos": 5.933070595915776e+16,
+    "train_loss": 0.7784920386062855,
+    "train_runtime": 14293.974,
+    "train_samples_per_second": 63.066,
+    "train_steps_per_second": 3.942
 }

trainer_state.json

@@ -4,197 +4,225 @@
   "best_model_checkpoint": null,
   "epoch": 10.0,
   "eval_steps": 500,
-  "global_step": 56520,
+  "global_step": 56340,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "grad_norm": 12.995927810668945,
-      "learning_rate": 4.50070771408351e-05,
-      "loss": 2.9726,
-      "step": 5652
+      "grad_norm": 5.6999335289001465,
+      "learning_rate": 4.5005324813631524e-05,
+      "loss": 1.2561,
+      "step": 5634
     },
     {
       "epoch": 1.0,
-      "eval_loss": 2.5977725982666016,
-      "eval_runtime": 19.8579,
-      "eval_samples_per_second": 489.981,
-      "eval_steps_per_second": 30.668,
-      "step": 5652
+      "eval_loss": 0.9440904259681702,
+      "eval_runtime": 727.0892,
+      "eval_samples_per_second": 3.902,
+      "eval_steps_per_second": 0.488,
+      "eval_top10_acc": 0.9302325581395349,
+      "eval_top1_acc": 0.7965116279069767,
+      "eval_top20_acc": 0.9593023255813954,
+      "eval_top5_acc": 0.8837209302325582,
+      "step": 5634
     },
     {
       "epoch": 2.0,
-      "grad_norm": 10.8749361038208,
-      "learning_rate": 4.000796178343949e-05,
-      "loss": 2.5639,
-      "step": 11304
+      "grad_norm": 4.9532036781311035,
+      "learning_rate": 4.000887468938587e-05,
+      "loss": 0.969,
+      "step": 11268
     },
     {
       "epoch": 2.0,
-      "eval_loss": 2.381584405899048,
-      "eval_runtime": 19.7379,
-      "eval_samples_per_second": 492.959,
-      "eval_steps_per_second": 30.854,
-      "step": 11304
+      "eval_loss": 0.8007155656814575,
+      "eval_runtime": 732.2211,
+      "eval_samples_per_second": 3.875,
+      "eval_steps_per_second": 0.485,
+      "eval_top10_acc": 0.9366197183098591,
+      "eval_top1_acc": 0.8028169014084507,
+      "eval_top20_acc": 0.9647887323943662,
+      "eval_top5_acc": 0.9295774647887324,
+      "step": 11268
     },
     {
       "epoch": 3.0,
-      "grad_norm": 11.009535789489746,
-      "learning_rate": 3.500973106864827e-05,
-      "loss": 2.4014,
-      "step": 16956
+      "grad_norm": 4.767992973327637,
+      "learning_rate": 3.5011537096201635e-05,
+      "loss": 0.8407,
+      "step": 16902
     },
     {
       "epoch": 3.0,
-      "eval_loss": 2.2645437717437744,
-      "eval_runtime": 19.6521,
-      "eval_samples_per_second": 495.112,
-      "eval_steps_per_second": 30.989,
-      "step": 16956
+      "eval_loss": 0.750490665435791,
+      "eval_runtime": 746.6021,
+      "eval_samples_per_second": 3.8,
+      "eval_steps_per_second": 0.475,
+      "eval_top10_acc": 0.9479768786127167,
+      "eval_top1_acc": 0.8092485549132948,
+      "eval_top20_acc": 0.976878612716763,
+      "eval_top5_acc": 0.9248554913294798,
+      "step": 16902
     },
     {
       "epoch": 4.0,
-      "grad_norm": 11.495499610900879,
-      "learning_rate": 3.0011500353857042e-05,
-      "loss": 2.2988,
-      "step": 22608
+      "grad_norm": 4.196132659912109,
+      "learning_rate": 3.0011537096201635e-05,
+      "loss": 0.7791,
+      "step": 22536
     },
     {
       "epoch": 4.0,
-      "eval_loss": 2.2165863513946533,
-      "eval_runtime": 19.8532,
-      "eval_samples_per_second": 490.097,
-      "eval_steps_per_second": 30.675,
-      "step": 22608
+      "eval_loss": 0.6900169849395752,
+      "eval_runtime": 744.1442,
+      "eval_samples_per_second": 3.812,
+      "eval_steps_per_second": 0.477,
+      "eval_top10_acc": 0.95,
+      "eval_top1_acc": 0.825,
+      "eval_top20_acc": 0.975,
+      "eval_top5_acc": 0.93125,
+      "step": 22536
     },
     {
       "epoch": 5.0,
-      "grad_norm": 11.591689109802246,
-      "learning_rate": 2.501238499646143e-05,
-      "loss": 2.2089,
-      "step": 28260
+      "grad_norm": 4.124821186065674,
+      "learning_rate": 2.5014199503017393e-05,
+      "loss": 0.7264,
+      "step": 28170
     },
     {
       "epoch": 5.0,
-      "eval_loss": 2.120560884475708,
-      "eval_runtime": 19.806,
-      "eval_samples_per_second": 491.265,
-      "eval_steps_per_second": 30.748,
-      "step": 28260
+      "eval_loss": 0.6541450619697571,
+      "eval_runtime": 732.9508,
+      "eval_samples_per_second": 3.871,
+      "eval_steps_per_second": 0.484,
+      "eval_top10_acc": 0.9705882352941176,
+      "eval_top1_acc": 0.8823529411764706,
+      "eval_top20_acc": 0.9764705882352941,
+      "eval_top5_acc": 0.9470588235294117,
+      "step": 28170
     },
     {
       "epoch": 6.0,
-      "grad_norm": 10.091324806213379,
-      "learning_rate": 2.0015923566878984e-05,
-      "loss": 2.1371,
-      "step": 33912
+      "grad_norm": 4.150660991668701,
+      "learning_rate": 2.001508697195598e-05,
+      "loss": 0.6872,
+      "step": 33804
     },
     {
       "epoch": 6.0,
-      "eval_loss": 2.0921123027801514,
-      "eval_runtime": 19.6342,
-      "eval_samples_per_second": 495.564,
-      "eval_steps_per_second": 31.017,
-      "step": 33912
+      "eval_loss": 0.6342827081680298,
+      "eval_runtime": 735.093,
+      "eval_samples_per_second": 3.859,
+      "eval_steps_per_second": 0.483,
+      "eval_top10_acc": 0.9570552147239264,
+      "eval_top1_acc": 0.8343558282208589,
+      "eval_top20_acc": 0.9877300613496932,
+      "eval_top5_acc": 0.9263803680981595,
+      "step": 33804
     },
     {
       "epoch": 7.0,
-      "grad_norm": 9.590017318725586,
-      "learning_rate": 1.5016808209483369e-05,
-      "loss": 2.0808,
-      "step": 39564
+      "grad_norm": 4.235877990722656,
+      "learning_rate": 1.501597444089457e-05,
+      "loss": 0.6553,
+      "step": 39438
     },
     {
       "epoch": 7.0,
-      "eval_loss": 2.035478115081787,
-      "eval_runtime": 19.7938,
-      "eval_samples_per_second": 491.568,
-      "eval_steps_per_second": 30.767,
-      "step": 39564
+      "eval_loss": 0.6069093942642212,
+      "eval_runtime": 729.8603,
+      "eval_samples_per_second": 3.887,
+      "eval_steps_per_second": 0.486,
+      "eval_top10_acc": 0.9712230215827338,
+      "eval_top1_acc": 0.8705035971223022,
+      "eval_top20_acc": 0.9784172661870504,
+      "eval_top5_acc": 0.9568345323741008,
+      "step": 39438
     },
     {
       "epoch": 8.0,
-      "grad_norm": 11.4172945022583,
-      "learning_rate": 1.0017692852087757e-05,
-      "loss": 2.0361,
-      "step": 45216
+      "grad_norm": 4.277960777282715,
+      "learning_rate": 1.0021299254526093e-05,
+      "loss": 0.6479,
+      "step": 45072
     },
     {
       "epoch": 8.0,
-      "eval_loss": 2.019296169281006,
-      "eval_runtime": 19.7587,
-      "eval_samples_per_second": 492.442,
-      "eval_steps_per_second": 30.822,
-      "step": 45216
+      "eval_loss": 0.5924458503723145,
+      "eval_runtime": 735.4415,
+      "eval_samples_per_second": 3.858,
+      "eval_steps_per_second": 0.483,
+      "eval_top10_acc": 0.9854014598540146,
+      "eval_top1_acc": 0.8905109489051095,
+      "eval_top20_acc": 0.9927007299270073,
+      "eval_top5_acc": 0.9708029197080292,
+      "step": 45072
     },
     {
       "epoch": 9.0,
-      "grad_norm": 13.36776351928711,
-      "learning_rate": 5.019462137296532e-06,
-      "loss": 2.001,
-      "step": 50868
+      "grad_norm": 3.431784152984619,
+      "learning_rate": 5.023074192403266e-06,
+      "loss": 0.6181,
+      "step": 50706
     },
     {
       "epoch": 9.0,
-      "eval_loss": 1.9987555742263794,
-      "eval_runtime": 19.4188,
-      "eval_samples_per_second": 501.06,
-      "eval_steps_per_second": 31.361,
-      "step": 50868
+      "eval_loss": 0.5826597213745117,
+      "eval_runtime": 740.5141,
+      "eval_samples_per_second": 3.831,
+      "eval_steps_per_second": 0.479,
+      "eval_top10_acc": 0.9693251533742331,
+      "eval_top1_acc": 0.8834355828220859,
+      "eval_top20_acc": 0.9815950920245399,
+      "eval_top5_acc": 0.9570552147239264,
+      "step": 50706
     },
     {
       "epoch": 10.0,
-      "grad_norm": 11.329434394836426,
-      "learning_rate": 2.1231422505307857e-08,
-      "loss": 1.977,
-      "step": 56520
+      "grad_norm": 4.740328788757324,
+      "learning_rate": 2.5736599219027333e-08,
+      "loss": 0.6051,
+      "step": 56340
     },
     {
       "epoch": 10.0,
-      "eval_loss": 1.992159366607666,
-      "eval_runtime": 19.7519,
-      "eval_samples_per_second": 492.612,
-      "eval_steps_per_second": 30.833,
-      "step": 56520
+      "eval_loss": 0.5851157307624817,
+      "eval_runtime": 739.7027,
+      "eval_samples_per_second": 3.835,
+      "eval_steps_per_second": 0.48,
+      "eval_top10_acc": 1.0,
+      "eval_top1_acc": 0.8922155688622755,
+      "eval_top20_acc": 1.0,
+      "eval_top5_acc": 0.9700598802395209,
+      "step": 56340
     },
     {
       "epoch": 10.0,
-      "step": 56520,
-      "total_flos": 2.975422706417664e+16,
-      "train_loss": 2.2677627250309627,
-      "train_runtime": 6971.4542,
-      "train_samples_per_second": 129.738,
-      "train_steps_per_second": 8.107
+      "step": 56340,
+      "total_flos": 5.933070595915776e+16,
+      "train_loss": 0.7784920386062855,
+      "train_runtime": 14293.974,
+      "train_samples_per_second": 63.066,
+      "train_steps_per_second": 3.942
     },
     {
       "epoch": 10.0,
-      "eval_loss": 1.9793492555618286,
-      "eval_runtime": 19.7795,
-      "eval_samples_per_second": 491.923,
-      "eval_steps_per_second": 30.789,
-      "step": 56520
-    },
-    {
-      "epoch": 10.0,
-      "eval_loss": 1.980637788772583,
-      "eval_runtime": 11.5702,
-      "eval_samples_per_second": 840.952,
-      "eval_steps_per_second": 52.635,
-      "step": 56520
-    },
-    {
-      "epoch": 10.0,
-      "eval_loss": 2.0006515979766846,
-      "eval_runtime": 11.5932,
-      "eval_samples_per_second": 839.288,
-      "eval_steps_per_second": 52.531,
-      "step": 56520
+      "eval_loss": 0.5786494612693787,
+      "eval_runtime": 736.6563,
+      "eval_samples_per_second": 3.851,
+      "eval_steps_per_second": 0.482,
+      "eval_top10_acc": 0.9447852760736196,
+      "eval_top1_acc": 0.8588957055214724,
+      "eval_top20_acc": 0.9631901840490797,
+      "eval_top5_acc": 0.9202453987730062,
+      "step": 56340
     }
   ],
   "logging_steps": 500,
-  "max_steps": 56520,
+  "max_steps": 56340,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
   "save_steps": 500,
@@ -210,7 +238,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.975422706417664e+16,
+  "total_flos": 5.933070595915776e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b67c2a3013a187a18e50cba2537a82dc555169447cfcc62169c9520a9e77c19
-size 5713
+oid sha256:d86f31b8645edb5a8bc695e48ee26e49fb8c60618206647233a2cba8872a191c
+size 5649