binary-34

README.md

@@ -23,13 +23,13 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6829268292682927
+      value: 0.6843479680104456
     - name: Precision
       type: precision
-      value: 0.6481911715897777
+      value: 0.6129220874141208
     - name: Recall
       type: recall
-      value: 0.7215961573988546
+      value: 0.7746166635876593
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -39,10 +39,10 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [thejosango/nuha-mlm](https://huggingface.co/thejosango/nuha-mlm) on the nuha-dataset dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5679
-- F1: 0.6829
-- Precision: 0.6482
-- Recall: 0.7216
+- Loss: 0.6703
+- F1: 0.6843
+- Precision: 0.6129
+- Recall: 0.7746
 - Support: None
 
 ## Model description
@@ -63,11 +63,9 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-05
-- train_batch_size: 32
-- eval_batch_size: 32
+- train_batch_size: 64
+- eval_batch_size: 64
 - seed: 42
-- gradient_accumulation_steps: 2
-- total_train_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - num_epochs: 30
@@ -77,13 +75,16 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | F1     | Precision | Recall | Support |
 |:-------------:|:-----:|:----:|:---------------:|:------:|:---------:|:------:|:-------:|
-| 0.5606        | 0.37  | 500  | 0.5393          | 0.6786 | 0.6092    | 0.7659 | None    |
-| 0.5414        | 0.74  | 1000 | 0.5770          | 0.6845 | 0.5747    | 0.8463 | None    |
-| 0.5178        | 1.11  | 1500 | 0.5384          | 0.6888 | 0.6069    | 0.7962 | None    |
-| 0.4777        | 1.48  | 2000 | 0.5407          | 0.6926 | 0.6266    | 0.7742 | None    |
-| 0.4752        | 1.85  | 2500 | 0.5372          | 0.6960 | 0.6138    | 0.8036 | None    |
-| 0.3952        | 2.22  | 3000 | 0.5873          | 0.6892 | 0.6155    | 0.7829 | None    |
-| 0.4008        | 2.59  | 3500 | 0.5679          | 0.6829 | 0.6482    | 0.7216 | None    |
+| 0.5719        | 0.37  | 500  | 0.5269          | 0.6734 | 0.6553    | 0.6926 | None    |
+| 0.546         | 0.74  | 1000 | 0.5745          | 0.6825 | 0.5624    | 0.8679 | None    |
+| 0.5263        | 1.11  | 1500 | 0.5165          | 0.6978 | 0.6526    | 0.7497 | None    |
+| 0.4871        | 1.48  | 2000 | 0.5864          | 0.6893 | 0.5803    | 0.8489 | None    |
+| 0.4888        | 1.85  | 2500 | 0.5491          | 0.7030 | 0.6296    | 0.7957 | None    |
+| 0.4133        | 2.22  | 3000 | 0.5504          | 0.6946 | 0.6394    | 0.7602 | None    |
+| 0.4146        | 2.58  | 3500 | 0.5977          | 0.6898 | 0.6076    | 0.7977 | None    |
+| 0.4132        | 2.95  | 4000 | 0.6111          | 0.6906 | 0.5994    | 0.8145 | None    |
+| 0.353         | 3.32  | 4500 | 0.6571          | 0.6853 | 0.6047    | 0.7907 | None    |
+| 0.3524        | 3.69  | 5000 | 0.6703          | 0.6843 | 0.6129    | 0.7746 | None    |
 
 
 ### Framework versions

config.json

@@ -23,7 +23,7 @@
   "max_position_embeddings": 512,
   "model_type": "bert",
   "num_attention_heads": 12,
-  "num_hidden_layers": 4,
+  "num_hidden_layers": 12,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "problem_type": "single_label_classification",

config.toml

@@ -1,5 +1,5 @@
 [experiment]
-name = "binary-33"
+name = "binary-34"
 type = "binary"
 
 
@@ -12,19 +12,19 @@ undersampling_strategy = "majority"
 
 [model]
 pretrained_model_name_or_path = "thejosango/nuha-mlm"
-revision = "2caf9ebc5b275737c95f8bb16953288107a7131c"
+revision = "main"
 
 
 [training]
-num_train_epochs = 30
-warmup_steps = 0 
+num_train_epochs = 30 
+warmup_steps = 0
 lr_scheduler_type = "linear"
 learning_rate = 5e-5
-per_device_train_batch_size = 32 
-per_device_eval_batch_size = 32
-gradient_accumulation_steps = 2
+per_device_train_batch_size = 64 
+per_device_eval_batch_size = 64
+gradient_accumulation_steps = 1 
 weight_decay = 0.01
 label_smoothing_factor = 0.1
-weighted_loss = false 
+weighted_loss = false
 early_stopping_patience = 5
 early_stopping_threshold = 0.005

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9220848386970aac9b2da72bbb5578de2bb93048fe6f3526b02d7e634df83bca
-size 313992076
+oid sha256:124dde1bf6384539fd41ca2c0139a1f484f4ceab637056c26ad7e72c20d7e148
+size 540847921

tokenizer.json

@@ -6,14 +6,7 @@
     "strategy": "LongestFirst",
     "stride": 0
   },
-  "padding": {
-    "strategy": "BatchLongest",
-    "direction": "Right",
-    "pad_to_multiple_of": null,
-    "pad_id": 0,
-    "pad_type_id": 0,
-    "pad_token": "[PAD]"
-  },
+  "padding": null,
   "added_tokens": [
     {
       "id": 0,

tokenizer_config.json

@@ -5,24 +5,17 @@
   "do_lower_case": false,
   "mask_token": "[MASK]",
   "max_len": 512,
-  "max_length": 512,
   "model_max_length": 512,
   "never_split": [
     "[بريد]",
     "[مستخدم]",
     "[رابط]"
   ],
-  "pad_to_multiple_of": null,
   "pad_token": "[PAD]",
-  "pad_token_type_id": 0,
-  "padding_side": "right",
   "sep_token": "[SEP]",
-  "stride": 0,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "[UNK]",
   "use_fast": true
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75405e81cc63a82a054f4fcfa9912126203b652f08c0eabc3a7dfac51214aa66
+oid sha256:c00b37a67e5dfc7f98a1ae7c6cc19fa8db64609170e8d2528fe98804c957eec8
 size 4091