Pushing deberta-v3-large-irony to hub

README.md

@@ -0,0 +1,70 @@
+---
+license: mit
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: deberta-v3-large-irony-lr8e-6-gas2-ls0.1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# deberta-v3-large-irony-lr8e-6-gas2-ls0.1
+
+This model is a fine-tuned version of [microsoft/deberta-v3-large](https://huggingface.co/microsoft/deberta-v3-large) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.7673
+- Accuracy: 0.7675
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 8e-06
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 42
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 50
+- num_epochs: 10.0
+- label_smoothing_factor: 0.1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Accuracy |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|
+| 0.6478        | 1.12  | 100  | 0.5890          | 0.7529   |
+| 0.5013        | 2.25  | 200  | 0.5873          | 0.7707   |
+| 0.388         | 3.37  | 300  | 0.6993          | 0.7602   |
+| 0.3169        | 4.49  | 400  | 0.6773          | 0.7874   |
+| 0.2693        | 5.61  | 500  | 0.7172          | 0.7707   |
+| 0.2396        | 6.74  | 600  | 0.7397          | 0.7801   |
+| 0.2284        | 7.86  | 700  | 0.8096          | 0.7550   |
+| 0.2207        | 8.98  | 800  | 0.7827          | 0.7654   |
+
+
+### Framework versions
+
+- Transformers 4.20.0.dev0
+- Pytorch 1.9.0
+- Datasets 2.2.2
+- Tokenizers 0.11.6

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 9.99,
+    "eval_accuracy": 0.7675392627716064,
+    "eval_loss": 0.7672834396362305,
+    "eval_runtime": 6.3297,
+    "eval_samples": 955,
+    "eval_samples_per_second": 150.876,
+    "eval_steps_per_second": 9.479,
+    "train_loss": 0.33834649632486063,
+    "train_runtime": 677.9414,
+    "train_samples": 2862,
+    "train_samples_per_second": 42.216,
+    "train_steps_per_second": 1.313
+}

config.json

@@ -0,0 +1,43 @@
+{
+  "_name_or_path": "microsoft/deberta-v3-large",
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": 0,
+    "1": 1
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "label2id": {
+    "0": 0,
+    "1": 1
+  },
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1024,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.20.0.dev0",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "eval_accuracy": 0.7874345779418945,
+    "eval_loss": 0.5925345420837402,
+    "eval_runtime": 7.2146,
+    "eval_samples": 955,
+    "eval_samples_per_second": 132.371,
+    "eval_steps_per_second": 8.316
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c4b4cd8e258c97f6ddd4ec624d9ece030473071f026ea6d805c823274f2b7d1
+size 1740393387

run_test.sh

@@ -0,0 +1 @@
+jbsub -queue x86_1h -cores 4+1 -mem 30g -require a100 -o outputs/train/tweet_eval2/irony/deberta-v3-large-irony-lr8e-6-gas2-ls0.1/test.log /dccstor/tslm/envs/anaconda3/envs/tslm-gen/bin/python train_clf.py --model_name_or_path outputs/train/tweet_eval2/irony/deberta-v3-large-irony-lr8e-6-gas2-ls0.1/best_checkpoint --train_file data/tweet_eval/irony/train.csv --validation_file data/tweet_eval/irony/validation.csv --test_file data/tweet_eval/irony/test.csv --do_eval --do_predict --report_to none --per_device_eval_batch_size 16 --max_seq_length 256 --output_dir outputs/train/tweet_eval2/irony/deberta-v3-large-irony-lr8e-6-gas2-ls0.1/best_checkpoint

run_train.sh

@@ -0,0 +1 @@
+jbsub -queue x86_6h -cores 4+1 -mem 30g -require a100 -o outputs/train/tweet_eval2/irony/deberta-v3-large-irony-lr8e-6-gas2-ls0.1/train.log /dccstor/tslm/envs/anaconda3/envs/tslm-gen/bin/python train_clf.py --model_name_or_path microsoft/deberta-v3-large --train_file data/tweet_eval/irony/train.csv --validation_file data/tweet_eval/irony/validation.csv --do_train --do_eval --per_device_train_batch_size 16 --per_device_eval_batch_size 16 --max_seq_length 256 --learning_rate 8e-6 --output_dir outputs/train/tweet_eval2/irony/deberta-v3-large-irony-lr8e-6-gas2-ls0.1 --evaluation_strategy steps --save_strategy no --warmup_steps 50 --num_train_epochs 10 --overwrite_output_dir --logging_steps 100 --gradient_accumulation_steps 2 --label_smoothing_factor 0.1 --report_to clearml --metric_for_best_model accuracy --logging_dir outputs/train/tweet_eval2/irony/deberta-v3-large-irony-lr8e-6-gas2-ls0.1/tb \; rm -rf outputs/train/tweet_eval2/irony/deberta-v3-large-irony-lr8e-6-gas2-ls0.1/tb \; rm -rf outputs/train/tweet_eval2/irony/deberta-v3-large-irony-lr8e-6-gas2-ls0.1/checkpoint-* \; . outputs/train/tweet_eval2/irony/deberta-v3-large-irony-lr8e-6-gas2-ls0.1/run_test.sh

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

test_results.json

@@ -0,0 +1,8 @@
+{
+    "eval_accuracy": 0.8711734414100647,
+    "eval_loss": 0.37051811814308167,
+    "eval_runtime": 4.8769,
+    "eval_samples_per_second": 160.759,
+    "eval_steps_per_second": 10.047,
+    "test_samples": 784
+}

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "name_or_path": "microsoft/deberta-v3-large",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": null,
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

trainer_state.json

@@ -0,0 +1,145 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 9.994413407821229,
+  "global_step": 890,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.12,
+      "learning_rate": 7.5238095238095236e-06,
+      "loss": 0.6478,
+      "step": 100
+    },
+    {
+      "epoch": 1.12,
+      "eval_accuracy": 0.7528795599937439,
+      "eval_loss": 0.5890260338783264,
+      "eval_runtime": 6.3154,
+      "eval_samples_per_second": 151.217,
+      "eval_steps_per_second": 9.501,
+      "step": 100
+    },
+    {
+      "epoch": 2.25,
+      "learning_rate": 6.571428571428571e-06,
+      "loss": 0.5013,
+      "step": 200
+    },
+    {
+      "epoch": 2.25,
+      "eval_accuracy": 0.7706806063652039,
+      "eval_loss": 0.5872902870178223,
+      "eval_runtime": 6.2684,
+      "eval_samples_per_second": 152.353,
+      "eval_steps_per_second": 9.572,
+      "step": 200
+    },
+    {
+      "epoch": 3.37,
+      "learning_rate": 5.6190476190476185e-06,
+      "loss": 0.388,
+      "step": 300
+    },
+    {
+      "epoch": 3.37,
+      "eval_accuracy": 0.7602094411849976,
+      "eval_loss": 0.6993213891983032,
+      "eval_runtime": 6.3121,
+      "eval_samples_per_second": 151.298,
+      "eval_steps_per_second": 9.506,
+      "step": 300
+    },
+    {
+      "epoch": 4.49,
+      "learning_rate": 4.666666666666667e-06,
+      "loss": 0.3169,
+      "step": 400
+    },
+    {
+      "epoch": 4.49,
+      "eval_accuracy": 0.7874345779418945,
+      "eval_loss": 0.6773470640182495,
+      "eval_runtime": 6.2982,
+      "eval_samples_per_second": 151.63,
+      "eval_steps_per_second": 9.527,
+      "step": 400
+    },
+    {
+      "epoch": 5.61,
+      "learning_rate": 3.714285714285714e-06,
+      "loss": 0.2693,
+      "step": 500
+    },
+    {
+      "epoch": 5.61,
+      "eval_accuracy": 0.7706806063652039,
+      "eval_loss": 0.717186450958252,
+      "eval_runtime": 6.3196,
+      "eval_samples_per_second": 151.116,
+      "eval_steps_per_second": 9.494,
+      "step": 500
+    },
+    {
+      "epoch": 6.74,
+      "learning_rate": 2.7619047619047616e-06,
+      "loss": 0.2396,
+      "step": 600
+    },
+    {
+      "epoch": 6.74,
+      "eval_accuracy": 0.7801046967506409,
+      "eval_loss": 0.7397065758705139,
+      "eval_runtime": 6.2974,
+      "eval_samples_per_second": 151.649,
+      "eval_steps_per_second": 9.528,
+      "step": 600
+    },
+    {
+      "epoch": 7.86,
+      "learning_rate": 1.8095238095238095e-06,
+      "loss": 0.2284,
+      "step": 700
+    },
+    {
+      "epoch": 7.86,
+      "eval_accuracy": 0.754973828792572,
+      "eval_loss": 0.8096243143081665,
+      "eval_runtime": 6.3109,
+      "eval_samples_per_second": 151.325,
+      "eval_steps_per_second": 9.507,
+      "step": 700
+    },
+    {
+      "epoch": 8.98,
+      "learning_rate": 8.57142857142857e-07,
+      "loss": 0.2207,
+      "step": 800
+    },
+    {
+      "epoch": 8.98,
+      "eval_accuracy": 0.7654450535774231,
+      "eval_loss": 0.7827323079109192,
+      "eval_runtime": 6.3145,
+      "eval_samples_per_second": 151.239,
+      "eval_steps_per_second": 9.502,
+      "step": 800
+    },
+    {
+      "epoch": 9.99,
+      "step": 890,
+      "total_flos": 1.3329504276375552e+16,
+      "train_loss": 0.33834649632486063,
+      "train_runtime": 677.9414,
+      "train_samples_per_second": 42.216,
+      "train_steps_per_second": 1.313
+    }
+  ],
+  "max_steps": 890,
+  "num_train_epochs": 10,
+  "total_flos": 1.3329504276375552e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a579f34da76ac1f097f1e3de8367214d02e4123e699bb75515a4919f6c59ae39
+size 3311