Upload batch 2/24: cola/mBERT/cls/seed_43 ... cola/mBERT/gated_multi_branch/seed_43

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3098720908164978,
+    "eval_matthews_correlation": 0.7612536086874112,
+    "eval_runtime": 1.4715,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 708.777,
+    "eval_steps_per_second": 7.475,
+    "total_flos": 392514650311680.0,
+    "train_loss": 0.247887350549467,
+    "train_runtime": 70.2797,
+    "train_samples": 8551,
+    "train_samples_per_second": 365.013,
+    "train_steps_per_second": 7.641
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3098720908164978,
+    "eval_matthews_correlation": 0.7612536086874112,
+    "eval_runtime": 1.4715,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 708.777,
+    "eval_steps_per_second": 7.475
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 43 --output_dir result_ablation_mbert/cola/mBERT/cls/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy cls --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 392514650311680.0,
+    "train_loss": 0.247887350549467,
+    "train_runtime": 70.2797,
+    "train_samples": 8551,
+    "train_samples_per_second": 365.013,
+    "train_steps_per_second": 7.641
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_43/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 7.828573703765869,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5501,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 5.213064193725586,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4357,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 4.780999660491943,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3656,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 4.77189826965332,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3394,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 3.0435750484466553,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3262,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 7.160293102264404,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3459,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.691946506500244,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3067,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.24901342391967773,
+      "eval_matthews_correlation": 0.7673294192440293,
+      "eval_runtime": 1.7938,
+      "eval_samples_per_second": 581.442,
+      "eval_steps_per_second": 6.132,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 7.130148887634277,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2721,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 12.120050430297852,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2384,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 12.451245307922363,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2386,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 7.135454177856445,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.2408,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 8.131049156188965,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.2087,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 8.45036506652832,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.2019,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 6.865825176239014,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1971,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2513435184955597,
+      "eval_matthews_correlation": 0.7850269505565478,
+      "eval_runtime": 1.5188,
+      "eval_samples_per_second": 686.715,
+      "eval_steps_per_second": 7.242,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 7.068305015563965,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1486,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 5.299299716949463,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.139,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 4.0898895263671875,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1432,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 4.907407760620117,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1333,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 5.126960754394531,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1489,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 7.767978668212891,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1377,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 13.47618579864502,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1416,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.3098720908164978,
+      "eval_matthews_correlation": 0.7612536086874112,
+      "eval_runtime": 1.6475,
+      "eval_samples_per_second": 633.093,
+      "eval_steps_per_second": 6.677,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 392514650311680.0,
+      "train_loss": 0.247887350549467,
+      "train_runtime": 70.2797,
+      "train_samples_per_second": 365.013,
+      "train_steps_per_second": 7.641
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 392514650311680.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.32316330075263977,
+    "eval_matthews_correlation": 0.7800284946221248,
+    "eval_runtime": 1.6195,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 644.018,
+    "eval_steps_per_second": 6.792,
+    "total_flos": 393644211158016.0,
+    "train_loss": 0.2231286038233581,
+    "train_runtime": 70.7229,
+    "train_samples": 8551,
+    "train_samples_per_second": 362.726,
+    "train_steps_per_second": 7.593
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.32316330075263977,
+    "eval_matthews_correlation": 0.7800284946221248,
+    "eval_runtime": 1.6195,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 644.018,
+    "eval_steps_per_second": 6.792
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 44 --output_dir result_ablation_mbert/cola/mBERT/cls/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy cls --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 393644211158016.0,
+    "train_loss": 0.2231286038233581,
+    "train_runtime": 70.7229,
+    "train_samples": 8551,
+    "train_samples_per_second": 362.726,
+    "train_steps_per_second": 7.593
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_44/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 9.166629791259766,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.4978,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 5.044792175292969,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4099,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 2.871767044067383,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3861,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 5.388686656951904,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.341,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 3.8670451641082764,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.301,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 6.388020992279053,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3056,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.2678303718566895,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.2876,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2798306345939636,
+      "eval_matthews_correlation": 0.7474766309951999,
+      "eval_runtime": 1.8093,
+      "eval_samples_per_second": 576.454,
+      "eval_steps_per_second": 6.08,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 8.470736503601074,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.196,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 5.648648262023926,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2181,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 6.1873602867126465,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.208,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 8.917803764343262,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1985,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 7.913937091827393,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1881,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 7.182322978973389,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.2089,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 5.53671407699585,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1859,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2769400179386139,
+      "eval_matthews_correlation": 0.7659279295510801,
+      "eval_runtime": 1.5804,
+      "eval_samples_per_second": 659.959,
+      "eval_steps_per_second": 6.96,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 16.43154525756836,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1308,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 12.334074020385742,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1145,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 10.784822463989258,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1238,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 9.777654647827148,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1237,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 7.081629753112793,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1137,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 4.2817182540893555,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.093,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 6.487602710723877,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1074,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.32316330075263977,
+      "eval_matthews_correlation": 0.7800284946221248,
+      "eval_runtime": 1.5681,
+      "eval_samples_per_second": 665.123,
+      "eval_steps_per_second": 7.015,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 393644211158016.0,
+      "train_loss": 0.2231286038233581,
+      "train_runtime": 70.7229,
+      "train_samples_per_second": 362.726,
+      "train_steps_per_second": 7.593
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 393644211158016.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3221026062965393,
+    "eval_matthews_correlation": 0.7613241357624281,
+    "eval_runtime": 2.2157,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 470.734,
+    "eval_steps_per_second": 4.965,
+    "total_flos": 407693995914744.0,
+    "train_loss": 0.2219125554104313,
+    "train_runtime": 64.8806,
+    "train_samples": 8551,
+    "train_samples_per_second": 395.388,
+    "train_steps_per_second": 8.277
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3221026062965393,
+    "eval_matthews_correlation": 0.7613241357624281,
+    "eval_runtime": 2.2157,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 470.734,
+    "eval_steps_per_second": 4.965
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 42 --output_dir result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy gated_multi_branch --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 407693995914744.0,
+    "train_loss": 0.2219125554104313,
+    "train_runtime": 64.8806,
+    "train_samples": 8551,
+    "train_samples_per_second": 395.388,
+    "train_steps_per_second": 8.277
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_42/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 5.284165382385254,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.4957,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 4.862030029296875,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.3957,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 7.777950286865234,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3345,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 4.530188083648682,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3351,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 6.847299575805664,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3309,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 9.399067878723145,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.2898,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.898521423339844,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3038,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.29004859924316406,
+      "eval_matthews_correlation": 0.7331437713515573,
+      "eval_runtime": 1.9678,
+      "eval_samples_per_second": 530.04,
+      "eval_steps_per_second": 5.59,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 7.823760509490967,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2258,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 7.346217155456543,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2124,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 2.401973247528076,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.1859,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 2.681915283203125,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1914,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 11.01183795928955,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.222,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 5.245687007904053,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.189,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 8.480674743652344,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1961,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.26085805892944336,
+      "eval_matthews_correlation": 0.7637565208987979,
+      "eval_runtime": 2.5589,
+      "eval_samples_per_second": 407.594,
+      "eval_steps_per_second": 4.299,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 2.7569196224212646,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1628,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 8.515286445617676,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1016,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 13.688443183898926,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1341,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 6.613780975341797,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1171,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 7.413669586181641,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.0987,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 10.65579891204834,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 4.584552764892578,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1073,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.3221026062965393,
+      "eval_matthews_correlation": 0.7613241357624281,
+      "eval_runtime": 2.1188,
+      "eval_samples_per_second": 492.265,
+      "eval_steps_per_second": 5.192,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 407693995914744.0,
+      "train_loss": 0.2219125554104313,
+      "train_runtime": 64.8806,
+      "train_samples_per_second": 395.388,
+      "train_steps_per_second": 8.277
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 407693995914744.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3194178640842438,
+    "eval_matthews_correlation": 0.7660150387812079,
+    "eval_runtime": 1.2856,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 811.266,
+    "eval_steps_per_second": 8.556,
+    "total_flos": 405092355839820.0,
+    "train_loss": 0.22317801773881113,
+    "train_runtime": 65.984,
+    "train_samples": 8551,
+    "train_samples_per_second": 388.776,
+    "train_steps_per_second": 8.138
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3194178640842438,
+    "eval_matthews_correlation": 0.7660150387812079,
+    "eval_runtime": 1.2856,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 811.266,
+    "eval_steps_per_second": 8.556
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 43 --output_dir result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy gated_multi_branch --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 405092355839820.0,
+    "train_loss": 0.22317801773881113,
+    "train_runtime": 65.984,
+    "train_samples": 8551,
+    "train_samples_per_second": 388.776,
+    "train_steps_per_second": 8.138
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/gated_multi_branch/seed_43/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 8.368849754333496,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5153,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 4.1223859786987305,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4083,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 7.473778247833252,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3534,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 5.652103424072266,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3318,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 5.1002092361450195,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.324,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 8.637200355529785,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3152,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.864533424377441,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.2919,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2664673924446106,
+      "eval_matthews_correlation": 0.7489819939590686,
+      "eval_runtime": 2.5435,
+      "eval_samples_per_second": 410.062,
+      "eval_steps_per_second": 4.325,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 8.43093204498291,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2571,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 8.061563491821289,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2118,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 7.198391437530518,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.1931,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 6.365251541137695,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.2204,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 8.19639778137207,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1723,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 9.565773010253906,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.2002,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 3.0346999168395996,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1776,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.26862624287605286,
+      "eval_matthews_correlation": 0.7659279295510801,
+      "eval_runtime": 1.8742,
+      "eval_samples_per_second": 556.494,
+      "eval_steps_per_second": 5.869,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 5.965743064880371,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.129,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 1.647736668586731,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1102,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 4.450601100921631,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1166,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 4.354552268981934,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1197,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 6.204374313354492,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1087,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 10.716383934020996,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.098,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 5.070870399475098,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.3194178640842438,
+      "eval_matthews_correlation": 0.7660150387812079,
+      "eval_runtime": 1.8161,
+      "eval_samples_per_second": 574.31,
+      "eval_steps_per_second": 6.057,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 405092355839820.0,
+      "train_loss": 0.22317801773881113,
+      "train_runtime": 65.984,
+      "train_samples_per_second": 388.776,
+      "train_steps_per_second": 8.138
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 405092355839820.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}