Upload batch 5/24: cola/mBERT/mean/seed_43 ... cola/mBERT/mha_attention/seed_43

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.33258184790611267,
+    "eval_matthews_correlation": 0.7664668619596934,
+    "eval_runtime": 1.3847,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 753.253,
+    "eval_steps_per_second": 7.944,
+    "total_flos": 392514650311680.0,
+    "train_loss": 0.23003221512728786,
+    "train_runtime": 70.1961,
+    "train_samples": 8551,
+    "train_samples_per_second": 365.447,
+    "train_steps_per_second": 7.65
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.33258184790611267,
+    "eval_matthews_correlation": 0.7664668619596934,
+    "eval_runtime": 1.3847,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 753.253,
+    "eval_steps_per_second": 7.944
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 43 --output_dir result_ablation_mbert/cola/mBERT/mean/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mean --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 392514650311680.0,
+    "train_loss": 0.23003221512728786,
+    "train_runtime": 70.1961,
+    "train_samples": 8551,
+    "train_samples_per_second": 365.447,
+    "train_steps_per_second": 7.65
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_43/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 6.389020919799805,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5248,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 4.873429775238037,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4064,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 5.82171630859375,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.347,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 5.291533470153809,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3355,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 3.822481870651245,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.318,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 9.306968688964844,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3281,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.138879776000977,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.306,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.24677596986293793,
+      "eval_matthews_correlation": 0.7618086846092919,
+      "eval_runtime": 1.7037,
+      "eval_samples_per_second": 612.186,
+      "eval_steps_per_second": 6.456,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 8.063065528869629,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.262,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 6.4467244148254395,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2082,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 9.68303394317627,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2286,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 6.11402702331543,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.2177,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 7.346621036529541,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1853,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 8.611226081848145,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.1785,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 5.996860027313232,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1857,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.26176193356513977,
+      "eval_matthews_correlation": 0.7706000995690468,
+      "eval_runtime": 1.431,
+      "eval_samples_per_second": 728.881,
+      "eval_steps_per_second": 7.687,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 5.248822212219238,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1366,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 3.4434709548950195,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1295,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 6.465655326843262,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1264,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 3.4134044647216797,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1174,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 7.440462112426758,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1277,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 3.309462070465088,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1119,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 6.255798816680908,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1145,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.33258184790611267,
+      "eval_matthews_correlation": 0.7664668619596934,
+      "eval_runtime": 1.7,
+      "eval_samples_per_second": 613.519,
+      "eval_steps_per_second": 6.47,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 392514650311680.0,
+      "train_loss": 0.23003221512728786,
+      "train_runtime": 70.1961,
+      "train_samples_per_second": 365.447,
+      "train_steps_per_second": 7.65
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 392514650311680.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.32358258962631226,
+    "eval_matthews_correlation": 0.7776844819969452,
+    "eval_runtime": 1.5743,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 662.499,
+    "eval_steps_per_second": 6.987,
+    "total_flos": 393644211158016.0,
+    "train_loss": 0.21521307658438799,
+    "train_runtime": 70.6413,
+    "train_samples": 8551,
+    "train_samples_per_second": 363.144,
+    "train_steps_per_second": 7.602
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.32358258962631226,
+    "eval_matthews_correlation": 0.7776844819969452,
+    "eval_runtime": 1.5743,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 662.499,
+    "eval_steps_per_second": 6.987
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 44 --output_dir result_ablation_mbert/cola/mBERT/mean/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mean --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 393644211158016.0,
+    "train_loss": 0.21521307658438799,
+    "train_runtime": 70.6413,
+    "train_samples": 8551,
+    "train_samples_per_second": 363.144,
+    "train_steps_per_second": 7.602
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_44/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 7.130160331726074,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5072,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 4.389698505401611,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4105,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 3.935558795928955,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3695,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 4.967840671539307,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3194,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 4.188772201538086,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.296,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 6.362126350402832,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3032,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.475619792938232,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.2797,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2606765627861023,
+      "eval_matthews_correlation": 0.7445988544944172,
+      "eval_runtime": 1.846,
+      "eval_samples_per_second": 565.017,
+      "eval_steps_per_second": 5.959,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 10.69326400756836,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.1881,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 5.7043867111206055,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2191,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 5.313024044036865,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.1986,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 6.501938343048096,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1792,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 6.583113670349121,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1803,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 8.288426399230957,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.1872,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 6.221704483032227,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1797,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.29681745171546936,
+      "eval_matthews_correlation": 0.7571474526874054,
+      "eval_runtime": 1.6242,
+      "eval_samples_per_second": 642.146,
+      "eval_steps_per_second": 6.772,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 12.89474105834961,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1285,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 9.707332611083984,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 10.674256324768066,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1063,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 9.124011039733887,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1222,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 7.6302595138549805,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1036,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 0.5679661631584167,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.0942,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 7.487257957458496,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1015,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.32358258962631226,
+      "eval_matthews_correlation": 0.7776844819969452,
+      "eval_runtime": 1.6248,
+      "eval_samples_per_second": 641.907,
+      "eval_steps_per_second": 6.77,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 393644211158016.0,
+      "train_loss": 0.21521307658438799,
+      "train_runtime": 70.6413,
+      "train_samples_per_second": 363.144,
+      "train_steps_per_second": 7.602
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 393644211158016.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.30496320128440857,
+    "eval_matthews_correlation": 0.7589772289071167,
+    "eval_runtime": 2.128,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 490.137,
+    "eval_steps_per_second": 5.169,
+    "total_flos": 396844371313236.0,
+    "train_loss": 0.24128270149230957,
+    "train_runtime": 64.4446,
+    "train_samples": 8551,
+    "train_samples_per_second": 398.063,
+    "train_steps_per_second": 8.333
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.30496320128440857,
+    "eval_matthews_correlation": 0.7589772289071167,
+    "eval_runtime": 2.128,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 490.137,
+    "eval_steps_per_second": 5.169
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 42 --output_dir result_ablation_mbert/cola/mBERT/mha_attention/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mha_attention --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 396844371313236.0,
+    "train_loss": 0.24128270149230957,
+    "train_runtime": 64.4446,
+    "train_samples": 8551,
+    "train_samples_per_second": 398.063,
+    "train_steps_per_second": 8.333
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_42/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 6.6887288093566895,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5424,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 8.120587348937988,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4329,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 4.903838634490967,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3568,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 4.808609485626221,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3385,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 5.117950916290283,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3354,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 6.087044715881348,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.2953,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 7.076235771179199,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3056,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2507948577404022,
+      "eval_matthews_correlation": 0.7615859559856272,
+      "eval_runtime": 2.0756,
+      "eval_samples_per_second": 502.514,
+      "eval_steps_per_second": 5.3,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 9.16220760345459,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2553,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 6.5086565017700195,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2282,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 7.519230365753174,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2014,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 9.912128448486328,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1998,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 4.397462368011475,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.2236,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 5.064901828765869,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.1968,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 8.0729341506958,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.2082,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.28933051228523254,
+      "eval_matthews_correlation": 0.7357366013907761,
+      "eval_runtime": 2.0038,
+      "eval_samples_per_second": 520.513,
+      "eval_steps_per_second": 5.49,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 5.93152379989624,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.2038,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 16.0588436126709,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1223,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 11.691866874694824,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1613,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 4.330988883972168,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1363,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 4.87614631652832,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1387,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 17.215696334838867,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1144,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 7.220628261566162,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1354,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.30496320128440857,
+      "eval_matthews_correlation": 0.7589772289071167,
+      "eval_runtime": 2.0172,
+      "eval_samples_per_second": 517.045,
+      "eval_steps_per_second": 5.453,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 396844371313236.0,
+      "train_loss": 0.24128270149230957,
+      "train_runtime": 64.4446,
+      "train_samples_per_second": 398.063,
+      "train_steps_per_second": 8.333
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 396844371313236.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.30815455317497253,
+    "eval_matthews_correlation": 0.768437400905162,
+    "eval_runtime": 2.0242,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 515.271,
+    "eval_steps_per_second": 5.434,
+    "total_flos": 394311966543330.0,
+    "train_loss": 0.23670564372668276,
+    "train_runtime": 66.176,
+    "train_samples": 8551,
+    "train_samples_per_second": 387.648,
+    "train_steps_per_second": 8.115
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.30815455317497253,
+    "eval_matthews_correlation": 0.768437400905162,
+    "eval_runtime": 2.0242,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 515.271,
+    "eval_steps_per_second": 5.434
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 43 --output_dir result_ablation_mbert/cola/mBERT/mha_attention/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mha_attention --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 394311966543330.0,
+    "train_loss": 0.23670564372668276,
+    "train_runtime": 66.176,
+    "train_samples": 8551,
+    "train_samples_per_second": 387.648,
+    "train_steps_per_second": 8.115
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_43/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 6.425873756408691,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.52,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 8.044377326965332,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.439,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 4.339436054229736,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3839,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 4.6755757331848145,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.325,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 4.038980960845947,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3414,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 7.957472801208496,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3272,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.100985527038574,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3112,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2582744359970093,
+      "eval_matthews_correlation": 0.7593801302015138,
+      "eval_runtime": 2.1291,
+      "eval_samples_per_second": 489.884,
+      "eval_steps_per_second": 5.167,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 6.082333564758301,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2594,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 5.3019700050354,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2345,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 10.235492706298828,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2284,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 5.929510116577148,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.228,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 5.407842636108398,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1859,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 12.29301643371582,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.1937,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 7.683435916900635,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1844,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.253669410943985,
+      "eval_matthews_correlation": 0.7797203752611466,
+      "eval_runtime": 2.2197,
+      "eval_samples_per_second": 469.889,
+      "eval_steps_per_second": 4.956,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 9.869709968566895,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1427,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 5.643942356109619,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1357,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 3.2232820987701416,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1251,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 8.573506355285645,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1262,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 6.6308417320251465,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.132,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 2.5399162769317627,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1085,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 3.2188637256622314,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1019,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.30815455317497253,
+      "eval_matthews_correlation": 0.768437400905162,
+      "eval_runtime": 2.2115,
+      "eval_samples_per_second": 471.635,
+      "eval_steps_per_second": 4.974,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 394311966543330.0,
+      "train_loss": 0.23670564372668276,
+      "train_runtime": 66.176,
+      "train_samples_per_second": 387.648,
+      "train_steps_per_second": 8.115
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 394311966543330.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}