Mariaaaaa/best_model_with_bitfit

README.md

@@ -0,0 +1,64 @@
+---
+license: mit
+base_model: prajjwal1/bert-tiny
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: checkpoints
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# checkpoints
+
+This model is a fine-tuned version of [prajjwal1/bert-tiny](https://huggingface.co/prajjwal1/bert-tiny) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6530
+- Accuracy: 0.6392
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0003
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 500
+- num_epochs: 4
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Accuracy |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|
+| 0.6751        | 1.0   | 1250 | 0.6668          | 0.626    |
+| 0.6634        | 2.0   | 2500 | 0.6576          | 0.634    |
+| 0.6584        | 3.0   | 3750 | 0.6556          | 0.6286   |
+| 0.6548        | 4.0   | 5000 | 0.6530          | 0.6392   |
+
+
+### Framework versions
+
+- Transformers 4.37.2
+- Pytorch 2.1.0+cu121
+- Datasets 2.17.1
+- Tokenizers 0.15.2

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "prajjwal1/bert-tiny",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c087f92c89fb34ecd55ac65bd9d9d391513e79a359c7cc035c159f654476c25c
+size 17549312

run-0/checkpoint-1250/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "prajjwal1/bert-tiny",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

run-0/checkpoint-1250/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cb0ac9475dee574184a31476daf56392c2590cac155d3ff2fe8224036d5cac7
+size 17549312

run-0/checkpoint-1250/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31f67dc4ad932fd15a74f16fa3a55e515f7a9e96794ff0fb4daee51a3e4db2c9
+size 40805

run-0/checkpoint-1250/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7b1c3ccb4d2d5a9a8b3bdae1c75223fdd55a7bd761dd089194d28f49f825aa1
+size 14244

run-0/checkpoint-1250/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85b8b77870406f2bacaaaa993ed168a3f9a9f933ef6201d2c8a4a8a690384667
+size 1064

run-0/checkpoint-1250/trainer_state.json

@@ -0,0 +1,51 @@
+{
+  "best_metric": 0.626,
+  "best_model_checkpoint": "./checkpoints/run-0/checkpoint-1250",
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1250,
+  "is_hyper_param_search": true,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00018779999999999998,
+      "loss": 0.6899,
+      "step": 313
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002916,
+      "loss": 0.6815,
+      "step": 626
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002707333333333333,
+      "loss": 0.6751,
+      "step": 939
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.626,
+      "eval_loss": 0.6668456792831421,
+      "eval_runtime": 7.6525,
+      "eval_samples_per_second": 653.38,
+      "eval_steps_per_second": 81.672,
+      "step": 1250
+    }
+  ],
+  "logging_steps": 313,
+  "max_steps": 5000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "total_flos": 19087797288960.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": {
+    "learning_rate": 0.0003,
+    "per_device_train_batch_size": 16
+  }
+}

run-0/checkpoint-1250/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3236f86e0b159c9991b34d915361fd9b07b129777aa98cc5e19134aa7282032c
+size 4664

run-0/checkpoint-2500/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "prajjwal1/bert-tiny",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

run-0/checkpoint-2500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0629337c2173551a7e99b0fc8b7c97988eb9fdf4362409a8f46a0eeb28bba97d
+size 17549312

run-0/checkpoint-2500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f54192d7c77fd2eb6116a2fda633c238187f364f86c8dab47bfd3f0a43b5e811
+size 40805

run-0/checkpoint-2500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d57ee2f69dbb1c189c7399630a4a96ea91cac95e08c010611169b72e42e1495f
+size 14244

run-0/checkpoint-2500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b27237992201a74abf308a5fe5eecd5e38f8498240f4f2ef7d630676fc7d3a09
+size 1064

run-0/checkpoint-2500/trainer_state.json

@@ -0,0 +1,84 @@
+{
+  "best_metric": 0.634,
+  "best_model_checkpoint": "./checkpoints/run-0/checkpoint-2500",
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 2500,
+  "is_hyper_param_search": true,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00018779999999999998,
+      "loss": 0.6899,
+      "step": 313
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002916,
+      "loss": 0.6815,
+      "step": 626
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002707333333333333,
+      "loss": 0.6751,
+      "step": 939
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.626,
+      "eval_loss": 0.6668456792831421,
+      "eval_runtime": 7.6525,
+      "eval_samples_per_second": 653.38,
+      "eval_steps_per_second": 81.672,
+      "step": 1250
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 0.00024986666666666665,
+      "loss": 0.6697,
+      "step": 1252
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 0.00022899999999999998,
+      "loss": 0.6678,
+      "step": 1565
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 0.00020813333333333334,
+      "loss": 0.6671,
+      "step": 1878
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 0.00018726666666666665,
+      "loss": 0.6634,
+      "step": 2191
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.634,
+      "eval_loss": 0.6575816869735718,
+      "eval_runtime": 7.6029,
+      "eval_samples_per_second": 657.642,
+      "eval_steps_per_second": 82.205,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 313,
+  "max_steps": 5000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "total_flos": 44538193674240.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": {
+    "learning_rate": 0.0003,
+    "per_device_train_batch_size": 16
+  }
+}

run-0/checkpoint-2500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3236f86e0b159c9991b34d915361fd9b07b129777aa98cc5e19134aa7282032c
+size 4664

run-0/checkpoint-3750/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "prajjwal1/bert-tiny",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

run-0/checkpoint-3750/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8addbedaf442e1bf9f5fea405f726b19c2734f37bb807cbdb61480a632858b8
+size 17549312

run-0/checkpoint-3750/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffaeb45c8708f9f8b578462ae2801235386a569b9c51336b6995daff0741d3c1
+size 40805

run-0/checkpoint-3750/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3098b823e87fe48df3b74feeaa69c5bd28d4f5b615ec3f26585fbe5696b01048
+size 14244

run-0/checkpoint-3750/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca4494508bf2ee18104465611a37a19c2ea8f26b1e37e181499ab7525c1aae8e
+size 1064

run-0/checkpoint-3750/trainer_state.json

@@ -0,0 +1,117 @@
+{
+  "best_metric": 0.634,
+  "best_model_checkpoint": "./checkpoints/run-0/checkpoint-2500",
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 3750,
+  "is_hyper_param_search": true,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00018779999999999998,
+      "loss": 0.6899,
+      "step": 313
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002916,
+      "loss": 0.6815,
+      "step": 626
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002707333333333333,
+      "loss": 0.6751,
+      "step": 939
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.626,
+      "eval_loss": 0.6668456792831421,
+      "eval_runtime": 7.6525,
+      "eval_samples_per_second": 653.38,
+      "eval_steps_per_second": 81.672,
+      "step": 1250
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 0.00024986666666666665,
+      "loss": 0.6697,
+      "step": 1252
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 0.00022899999999999998,
+      "loss": 0.6678,
+      "step": 1565
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 0.00020813333333333334,
+      "loss": 0.6671,
+      "step": 1878
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 0.00018726666666666665,
+      "loss": 0.6634,
+      "step": 2191
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.634,
+      "eval_loss": 0.6575816869735718,
+      "eval_runtime": 7.6029,
+      "eval_samples_per_second": 657.642,
+      "eval_steps_per_second": 82.205,
+      "step": 2500
+    },
+    {
+      "epoch": 2.0,
+      "learning_rate": 0.00016639999999999998,
+      "loss": 0.6587,
+      "step": 2504
+    },
+    {
+      "epoch": 2.25,
+      "learning_rate": 0.0001455333333333333,
+      "loss": 0.6592,
+      "step": 2817
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 0.00012466666666666667,
+      "loss": 0.6576,
+      "step": 3130
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 0.00010379999999999999,
+      "loss": 0.6584,
+      "step": 3443
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.6286,
+      "eval_loss": 0.6556090712547302,
+      "eval_runtime": 7.6842,
+      "eval_samples_per_second": 650.684,
+      "eval_steps_per_second": 81.335,
+      "step": 3750
+    }
+  ],
+  "logging_steps": 313,
+  "max_steps": 5000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "total_flos": 69988590059520.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": {
+    "learning_rate": 0.0003,
+    "per_device_train_batch_size": 16
+  }
+}

run-0/checkpoint-3750/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3236f86e0b159c9991b34d915361fd9b07b129777aa98cc5e19134aa7282032c
+size 4664

run-0/checkpoint-5000/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "prajjwal1/bert-tiny",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

run-0/checkpoint-5000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c087f92c89fb34ecd55ac65bd9d9d391513e79a359c7cc035c159f654476c25c
+size 17549312

run-0/checkpoint-5000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a36d74fcc2b4a08025b97cbc8cb387d95eb4aa0aa07af7fc4f6410b0c1c0be
+size 40805

run-0/checkpoint-5000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7d540daf0a7410db08f8a7673a817be2bc75893e9826b486974ca3ff5dc6c74
+size 14244

run-0/checkpoint-5000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5ea909a9a1709835fb587fc769c92ff497ebd1ac747a4345abd483b61f725e8
+size 1064

run-0/checkpoint-5000/trainer_state.json

@@ -0,0 +1,150 @@
+{
+  "best_metric": 0.6392,
+  "best_model_checkpoint": "./checkpoints/run-0/checkpoint-5000",
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 5000,
+  "is_hyper_param_search": true,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00018779999999999998,
+      "loss": 0.6899,
+      "step": 313
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002916,
+      "loss": 0.6815,
+      "step": 626
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0002707333333333333,
+      "loss": 0.6751,
+      "step": 939
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.626,
+      "eval_loss": 0.6668456792831421,
+      "eval_runtime": 7.6525,
+      "eval_samples_per_second": 653.38,
+      "eval_steps_per_second": 81.672,
+      "step": 1250
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 0.00024986666666666665,
+      "loss": 0.6697,
+      "step": 1252
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 0.00022899999999999998,
+      "loss": 0.6678,
+      "step": 1565
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 0.00020813333333333334,
+      "loss": 0.6671,
+      "step": 1878
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 0.00018726666666666665,
+      "loss": 0.6634,
+      "step": 2191
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.634,
+      "eval_loss": 0.6575816869735718,
+      "eval_runtime": 7.6029,
+      "eval_samples_per_second": 657.642,
+      "eval_steps_per_second": 82.205,
+      "step": 2500
+    },
+    {
+      "epoch": 2.0,
+      "learning_rate": 0.00016639999999999998,
+      "loss": 0.6587,
+      "step": 2504
+    },
+    {
+      "epoch": 2.25,
+      "learning_rate": 0.0001455333333333333,
+      "loss": 0.6592,
+      "step": 2817
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 0.00012466666666666667,
+      "loss": 0.6576,
+      "step": 3130
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 0.00010379999999999999,
+      "loss": 0.6584,
+      "step": 3443
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.6286,
+      "eval_loss": 0.6556090712547302,
+      "eval_runtime": 7.6842,
+      "eval_samples_per_second": 650.684,
+      "eval_steps_per_second": 81.335,
+      "step": 3750
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 8.293333333333332e-05,
+      "loss": 0.6575,
+      "step": 3756
+    },
+    {
+      "epoch": 3.26,
+      "learning_rate": 6.206666666666667e-05,
+      "loss": 0.6566,
+      "step": 4069
+    },
+    {
+      "epoch": 3.51,
+      "learning_rate": 4.12e-05,
+      "loss": 0.6554,
+      "step": 4382
+    },
+    {
+      "epoch": 3.76,
+      "learning_rate": 2.0333333333333334e-05,
+      "loss": 0.6548,
+      "step": 4695
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.6392,
+      "eval_loss": 0.6530249714851379,
+      "eval_runtime": 6.6707,
+      "eval_samples_per_second": 749.551,
+      "eval_steps_per_second": 93.694,
+      "step": 5000
+    }
+  ],
+  "logging_steps": 313,
+  "max_steps": 5000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "total_flos": 95438986444800.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": {
+    "learning_rate": 0.0003,
+    "per_device_train_batch_size": 16
+  }
+}

run-0/checkpoint-5000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3236f86e0b159c9991b34d915361fd9b07b129777aa98cc5e19134aa7282032c
+size 4664

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3236f86e0b159c9991b34d915361fd9b07b129777aa98cc5e19134aa7282032c
+size 4664