abdeljalilELmajjodi/test-bert

README.md

@@ -1,7 +1,7 @@
 ---
 library_name: transformers
 license: mit
-base_model: FacebookAI/xlm-roberta-base
+base_model: atlasia/XLM-RoBERTa-Morocco
 tags:
 - generated_from_trainer
 model-index:
@@ -14,9 +14,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # model
 
-This model is a fine-tuned version of [FacebookAI/xlm-roberta-base](https://huggingface.co/FacebookAI/xlm-roberta-base) on an unknown dataset.
-It achieves the following results on the evaluation set:
-- Loss: 3.2858
+This model is a fine-tuned version of [atlasia/XLM-RoBERTa-Morocco](https://huggingface.co/atlasia/XLM-RoBERTa-Morocco) on an unknown dataset.
 
 ## Model description
 
@@ -37,43 +35,19 @@ More information needed
 The following hyperparameters were used during training:
 - learning_rate: 2e-05
 - train_batch_size: 4
-- eval_batch_size: 16
+- eval_batch_size: 8
 - seed: 42
 - gradient_accumulation_steps: 4
 - total_train_batch_size: 16
-- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.03
 - num_epochs: 1
 - mixed_precision_training: Native AMP
 
-### Training results
-
-| Training Loss | Epoch  | Step  | Validation Loss |
-|:-------------:|:------:|:-----:|:---------------:|
-| 3.2251        | 0.0533 | 1000  | 4.1491          |
-| 2.7413        | 0.1067 | 2000  | 3.7912          |
-| 2.5416        | 0.16   | 3000  | 3.6801          |
-| 2.371         | 0.2133 | 4000  | 3.6439          |
-| 2.2968        | 0.2667 | 5000  | 3.5301          |
-| 2.1989        | 0.32   | 6000  | 3.3905          |
-| 2.0841        | 0.3733 | 7000  | 3.5244          |
-| 2.0032        | 0.4267 | 8000  | 3.3268          |
-| 1.9618        | 0.48   | 9000  | 3.3207          |
-| 1.9114        | 0.5333 | 10000 | 3.4544          |
-| 1.8472        | 0.5867 | 11000 | 3.2520          |
-| 1.8068        | 0.64   | 12000 | 3.3389          |
-| 1.7692        | 0.6933 | 13000 | 3.2428          |
-| 1.7236        | 0.7467 | 14000 | 3.3926          |
-| 1.7219        | 0.8    | 15000 | 3.2721          |
-| 1.6838        | 0.8533 | 16000 | 3.2671          |
-| 1.6771        | 0.9067 | 17000 | 3.2732          |
-| 1.6531        | 0.96   | 18000 | 3.2858          |
-
-
 ### Framework versions
 
-- Transformers 4.48.1
-- Pytorch 2.1.1+cu121
-- Datasets 3.2.0
-- Tokenizers 0.21.0
+- Transformers 4.51.3
+- Pytorch 2.6.0+cu124
+- Datasets 3.5.1
+- Tokenizers 0.21.1

config.json

@@ -1,5 +1,4 @@
 {
-  "_name_or_path": "FacebookAI/xlm-roberta-base",
   "architectures": [
     "XLMRobertaForMaskedLM"
   ],
@@ -9,20 +8,21 @@
   "eos_token_id": 2,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
+  "hidden_size": 1024,
   "initializer_range": 0.02,
-  "intermediate_size": 3072,
+  "intermediate_size": 4096,
   "layer_norm_eps": 1e-05,
   "max_position_embeddings": 514,
   "model_type": "xlm-roberta",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
   "output_past": true,
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
   "torch_dtype": "float32",
-  "transformers_version": "4.48.1",
+  "transformers_version": "4.51.3",
   "type_vocab_size": 1,
   "use_cache": true,
+  "use_flash_attention_2": true,
   "vocab_size": 250002
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4ce63c40c9137609281d850473d880d4688929df4da249fd60ad23cbc668a50
-size 1113205088
+oid sha256:c90fe26fd285a5881e806e6b4626cf982897354f8453b972e43f5ff44cc37c06
+size 2240618752

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -1,7 +1,25 @@
 {
-  "bos_token": "<s>",
-  "cls_token": "<s>",
-  "eos_token": "</s>",
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
@@ -9,7 +27,25 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "unk_token": "<unk>"
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6094422212ef0ee74526132298f9c3679a759d99ca9ddb95fa929af9ad8589ae
+oid sha256:dd330ca9891f488a7c79ac078633a6bea1f643f674bc03901cb4c732575d9663
 size 5304