binary-38

README.md

@@ -23,13 +23,13 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6784388150837528
+      value: 0.6928497604059006
     - name: Precision
       type: precision
-      value: 0.613033104682374
+      value: 0.7049713193116635
     - name: Recall
       type: recall
-      value: 0.7594679475337152
+      value: 0.6811380011084427
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -39,10 +39,10 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [thejosango/nuha-mlm](https://huggingface.co/thejosango/nuha-mlm) on the nuha-dataset dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6316
-- F1: 0.6784
-- Precision: 0.6130
-- Recall: 0.7595
+- Loss: 0.4966
+- F1: 0.6928
+- Precision: 0.7050
+- Recall: 0.6811
 - Support: None
 
 ## Model description
@@ -62,28 +62,32 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-05
-- train_batch_size: 64
-- eval_batch_size: 64
+- learning_rate: 1e-05
+- train_batch_size: 32
+- eval_batch_size: 32
 - seed: 42
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: linear
-- num_epochs: 30
+- lr_scheduler_type: constant
+- num_epochs: 5
 - label_smoothing_factor: 0.1
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss | F1     | Precision | Recall | Support |
 |:-------------:|:-----:|:----:|:---------------:|:------:|:---------:|:------:|:-------:|
-| 0.5588        | 0.37  | 500  | 0.5412          | 0.6800 | 0.6016    | 0.7818 | None    |
-| 0.5387        | 0.74  | 1000 | 0.5741          | 0.6859 | 0.5719    | 0.8568 | None    |
-| 0.5189        | 1.11  | 1500 | 0.5581          | 0.6892 | 0.5867    | 0.8352 | None    |
-| 0.4758        | 1.48  | 2000 | 0.5360          | 0.6972 | 0.6339    | 0.7744 | None    |
-| 0.4744        | 1.85  | 2500 | 0.5382          | 0.6983 | 0.6151    | 0.8077 | None    |
-| 0.3943        | 2.22  | 3000 | 0.5652          | 0.6917 | 0.6418    | 0.7500 | None    |
-| 0.4018        | 2.58  | 3500 | 0.5676          | 0.6856 | 0.6370    | 0.7423 | None    |
-| 0.4071        | 2.95  | 4000 | 0.5517          | 0.6903 | 0.6285    | 0.7656 | None    |
-| 0.3339        | 3.32  | 4500 | 0.6316          | 0.6784 | 0.6130    | 0.7595 | None    |
+| 0.5537        | 0.25  | 500  | 0.5298          | 0.6135 | 0.6830    | 0.5568 | None    |
+| 0.5294        | 0.5   | 1000 | 0.5174          | 0.6260 | 0.7106    | 0.5594 | None    |
+| 0.5236        | 0.75  | 1500 | 0.5129          | 0.6751 | 0.6752    | 0.6750 | None    |
+| 0.5211        | 1.0   | 2000 | 0.5038          | 0.6656 | 0.7084    | 0.6277 | None    |
+| 0.4972        | 1.26  | 2500 | 0.5034          | 0.6486 | 0.7288    | 0.5843 | None    |
+| 0.4926        | 1.51  | 3000 | 0.5058          | 0.6944 | 0.6755    | 0.7144 | None    |
+| 0.4914        | 1.76  | 3500 | 0.5006          | 0.6936 | 0.6901    | 0.6970 | None    |
+| 0.4899        | 2.01  | 4000 | 0.5019          | 0.6881 | 0.7036    | 0.6732 | None    |
+| 0.4715        | 2.26  | 4500 | 0.4987          | 0.6949 | 0.6977    | 0.6920 | None    |
+| 0.474         | 2.51  | 5000 | 0.4990          | 0.6998 | 0.6922    | 0.7076 | None    |
+| 0.4729        | 2.76  | 5500 | 0.4966          | 0.6928 | 0.7050    | 0.6811 | None    |
 
 
 ### Framework versions

config.json

@@ -23,7 +23,7 @@
   "max_position_embeddings": 512,
   "model_type": "bert",
   "num_attention_heads": 12,
-  "num_hidden_layers": 4,
+  "num_hidden_layers": 12,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "problem_type": "single_label_classification",

config.toml

@@ -1,29 +1,29 @@
 [experiment]
-name = "binary-37"
+name = "binary-38"
 type = "binary"
 
 
 [dataset]
 path = "thejosango/nuha-dataset"
 dataset_revision = "main"
-augment_ratio = 0.25
-undersampling_strategy = "majority"
+augment_ratio = 0.0
+undersampling_strategy = false
 
 
 [model]
 pretrained_model_name_or_path = "thejosango/nuha-mlm"
-revision = "main"
-num_hidden_layers = 4
+revision = "ce20f497544665775129f9ff5b3cd2a3e350dce8"
+
 
 [training]
-num_train_epochs = 30 
+num_train_epochs = 5
 warmup_steps = 0
-lr_scheduler_type = "linear"
-learning_rate = 5e-5
-per_device_train_batch_size = 64 
-per_device_eval_batch_size = 64
-gradient_accumulation_steps = 1 
-weight_decay = 0.01
+lr_scheduler_type = "constant"
+learning_rate = 1e-5
+per_device_train_batch_size = 32
+per_device_eval_batch_size = 32
+gradient_accumulation_steps = 2
+weight_decay = 0.00
 label_smoothing_factor = 0.1
 weighted_loss = false
 early_stopping_patience = 5

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36cd0e680db3b2e7f4b4bc051d8c4f81bf3ad8719e204e2964a9a13173b79913
-size 313992076
+oid sha256:c3eceddd77e5a00819ab30f4f9f90180c97198b12fefd284d24ecf49c2b80ce5
+size 540847921

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {

tokenizer_config.json

@@ -5,17 +5,24 @@
   "do_lower_case": false,
   "mask_token": "[MASK]",
   "max_len": 512,
+  "max_length": 512,
   "model_max_length": 512,
   "never_split": [
     "[بريد]",
     "[مستخدم]",
     "[رابط]"
   ],
+  "pad_to_multiple_of": null,
   "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "[SEP]",
+  "stride": 0,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "[UNK]",
   "use_fast": true
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b51c8aa63e710c62301328222b5db461f379c1ac8c6027a7b352d4fe74a10c0
+oid sha256:08daaa9929d5ac6d9774c9e26f13cc06d212873de4e3172ec8b81ea97d9d5d6b
 size 4091