binary-18

README.md

@@ -23,13 +23,13 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6488855020003811
+      value: 0.5023543990086741
     - name: Precision
       type: precision
-      value: 0.530860349127182
+      value: 0.33620832642229226
     - name: Recall
       type: recall
-      value: 0.8343949044585988
+      value: 0.993140617344439
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -39,10 +39,10 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [thejosango/nuha-mlm](https://huggingface.co/thejosango/nuha-mlm) on the nuha-dataset dataset.
 It achieves the following results on the evaluation set:
-- F1: 0.6489
-- Loss: 0.6212
-- Precision: 0.5309
-- Recall: 0.8344
+- Loss: 7.8973
+- F1: 0.5024
+- Precision: 0.3362
+- Recall: 0.9931
 - Support: None
 
 ## Model description
@@ -62,10 +62,12 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-05
-- train_batch_size: 16
+- learning_rate: 3e-05
+- train_batch_size: 32
 - eval_batch_size: 32
 - seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 128
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: constant
 - lr_scheduler_warmup_steps: 1000.0
@@ -74,28 +76,10 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step  | F1     | Validation Loss | Precision | Recall | Support |
-|:-------------:|:-----:|:-----:|:------:|:---------------:|:---------:|:------:|:-------:|
-| 3.0309        | 0.24  | 500   | 0.5091 | 0.8040          | 0.4004    | 0.6987 | None    |
-| 1.9804        | 0.49  | 1000  | 0.5306 | 0.9799          | 0.3681    | 0.9500 | None    |
-| 1.7294        | 0.73  | 1500  | 0.5238 | 1.9662          | 0.3589    | 0.9691 | None    |
-| 1.2761        | 0.98  | 2000  | 0.4732 | 0.6147          | 0.6136    | 0.3851 | None    |
-| 1.0954        | 1.22  | 2500  | 0.5614 | 0.6456          | 0.5693    | 0.5537 | None    |
-| 0.9203        | 1.47  | 3000  | 0.5684 | 0.6862          | 0.4105    | 0.9236 | None    |
-| 0.8422        | 1.71  | 3500  | 0.0    | 0.6237          | 1.0       | 0.0    | None    |
-| 0.7892        | 1.96  | 4000  | 0.3473 | 0.6111          | 0.7300    | 0.2278 | None    |
-| 0.7834        | 2.2   | 4500  | 0.4296 | 0.5738          | 0.7014    | 0.3097 | None    |
-| 0.7446        | 2.45  | 5000  | 0.5189 | 0.6224          | 0.6777    | 0.4204 | None    |
-| 0.7392        | 2.69  | 5500  | 0.6231 | 0.6188          | 0.6007    | 0.6472 | None    |
-| 0.7007        | 2.94  | 6000  | 0.6252 | 0.5837          | 0.6153    | 0.6355 | None    |
-| 0.7127        | 3.18  | 6500  | 0.6041 | 0.6996          | 0.4461    | 0.9353 | None    |
-| 0.7023        | 3.43  | 7000  | 0.5909 | 0.7383          | 0.4291    | 0.9486 | None    |
-| 0.6853        | 3.67  | 7500  | 0.5861 | 0.7998          | 0.4239    | 0.9490 | None    |
-| 0.6708        | 3.92  | 8000  | 0.6406 | 0.7828          | 0.6127    | 0.6712 | None    |
-| 0.6463        | 4.16  | 8500  | 0.5856 | 0.7472          | 0.4226    | 0.9530 | None    |
-| 0.6517        | 4.41  | 9000  | 0.6540 | 0.8006          | 0.5701    | 0.7668 | None    |
-| 0.6325        | 4.65  | 9500  | 0.6432 | 0.7416          | 0.5150    | 0.8564 | None    |
-| 0.6434        | 4.9   | 10000 | 0.6489 | 0.6212          | 0.5309    | 0.8344 | None    |
+| Training Loss | Epoch | Step | Validation Loss | F1     | Precision | Recall | Support |
+|:-------------:|:-----:|:----:|:---------------:|:------:|:---------:|:------:|:-------:|
+| 6.0423        | 1.96  | 500  | 5.0312          | 0.5020 | 0.3358    | 0.9946 | None    |
+| 4.545         | 3.92  | 1000 | 7.8973          | 0.5024 | 0.3362    | 0.9931 | None    |
 
 
 ### Framework versions

config.json

@@ -23,7 +23,7 @@
   "max_position_embeddings": 512,
   "model_type": "bert",
   "num_attention_heads": 12,
-  "num_hidden_layers": 8,
+  "num_hidden_layers": 12,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "problem_type": "single_label_classification",

config.toml

@@ -1,5 +1,5 @@
 [experiment]
-name = "binary-17"
+name = "binary-18"
 type = "binary"
 
 
@@ -12,7 +12,7 @@ augment_ratio = 0.0
 
 [model]
 pretrained_model_name_or_path = "thejosango/nuha-mlm"
-revision = "1e1d7fecb3f448e0152c39725c43f55969393cfb"
+revision = "984ac09880b24959f6767fdbea8757d2c312df46"
 hidden_dropout_prob = 0.5
 attention_probs_dropout_prob = 0.5
 classifier_dropout = 0.5
@@ -26,10 +26,10 @@ classifier_dropout = 0.5
 num_train_epochs = 5
 warmup_steps = 1e3
 lr_scheduler_type = "constant"
-learning_rate = 5e-5
-per_device_train_batch_size = 16
+learning_rate = 3e-5
+per_device_train_batch_size = 32 
 per_device_eval_batch_size = 32
-gradient_accumulation_steps = 1
+gradient_accumulation_steps = 4
 weight_decay = 0.01
 label_smoothing_factor = 0.1
 weighted_loss = false

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f7ab0988aeafc4d1040d62ea16ec7830a483ebf721884a93851f20b164979a3
-size 427419953
+oid sha256:0f2120e451b3207731f3ea2482404e5e855e15e0cdc2ec9d891e3cbc362bc96c
+size 540847921

tokenizer.json

@@ -6,14 +6,7 @@
     "strategy": "LongestFirst",
     "stride": 0
   },
-  "padding": {
-    "strategy": "BatchLongest",
-    "direction": "Right",
-    "pad_to_multiple_of": null,
-    "pad_id": 0,
-    "pad_type_id": 0,
-    "pad_token": "[PAD]"
-  },
+  "padding": null,
   "added_tokens": [
     {
       "id": 0,

tokenizer_config.json

@@ -5,24 +5,17 @@
   "do_lower_case": false,
   "mask_token": "[MASK]",
   "max_len": 512,
-  "max_length": 512,
   "model_max_length": 512,
   "never_split": [
     "[بريد]",
     "[مستخدم]",
     "[رابط]"
   ],
-  "pad_to_multiple_of": null,
   "pad_token": "[PAD]",
-  "pad_token_type_id": 0,
-  "padding_side": "right",
   "sep_token": "[SEP]",
-  "stride": 0,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "[UNK]",
   "use_fast": true
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0bbcaae74c9fa5e9d632c695d7ac53efdc5da8d29fd14f36321d8376ec0e5ba
+oid sha256:0ad45f0d683378bbfdda4cdb0210bb80501d2ea5059e9e98d9f3d786101f9476
 size 4091