Upload batch 1/24: cola/mBERT/attention/seed_42 ... cola/mBERT/cls/seed_42

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.30340734124183655,
+    "eval_matthews_correlation": 0.7706218489153343,
+    "eval_runtime": 1.3797,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 755.971,
+    "eval_steps_per_second": 7.973,
+    "total_flos": 395039042732628.0,
+    "train_loss": 0.2244898515928613,
+    "train_runtime": 68.4246,
+    "train_samples": 8551,
+    "train_samples_per_second": 374.909,
+    "train_steps_per_second": 7.848
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.30340734124183655,
+    "eval_matthews_correlation": 0.7706218489153343,
+    "eval_runtime": 1.3797,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 755.971,
+    "eval_steps_per_second": 7.973
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 42 --output_dir result_ablation_mbert/cola/mBERT/attention/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy attention --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 395039042732628.0,
+    "train_loss": 0.2244898515928613,
+    "train_runtime": 68.4246,
+    "train_samples": 8551,
+    "train_samples_per_second": 374.909,
+    "train_steps_per_second": 7.848
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_42/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 8.298686981201172,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.4887,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 6.113591194152832,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4133,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 7.69141149520874,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3193,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 5.14018440246582,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3274,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 5.4107584953308105,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3402,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 9.844621658325195,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.2803,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 7.570620536804199,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3132,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.25700780749320984,
+      "eval_matthews_correlation": 0.7686079913089925,
+      "eval_runtime": 1.6531,
+      "eval_samples_per_second": 630.95,
+      "eval_steps_per_second": 6.654,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 6.16119909286499,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.256,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 6.379676818847656,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.226,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 2.9720115661621094,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.1879,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 3.687185287475586,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1705,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 6.894290447235107,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.2055,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 4.898176193237305,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.183,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 8.651183128356934,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1965,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.26676928997039795,
+      "eval_matthews_correlation": 0.7423025309306667,
+      "eval_runtime": 1.7705,
+      "eval_samples_per_second": 589.105,
+      "eval_steps_per_second": 6.213,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 8.692514419555664,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1734,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 11.364733695983887,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1104,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 10.94915771484375,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1541,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 5.912627696990967,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1249,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 9.138338088989258,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1053,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 10.012800216674805,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1039,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 7.307399749755859,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.0981,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.30340734124183655,
+      "eval_matthews_correlation": 0.7706218489153343,
+      "eval_runtime": 1.4603,
+      "eval_samples_per_second": 714.214,
+      "eval_steps_per_second": 7.532,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 395039042732628.0,
+      "train_loss": 0.2244898515928613,
+      "train_runtime": 68.4246,
+      "train_samples_per_second": 374.909,
+      "train_steps_per_second": 7.848
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 395039042732628.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.344061017036438,
+    "eval_matthews_correlation": 0.7546498935382564,
+    "eval_runtime": 1.3505,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 772.301,
+    "eval_steps_per_second": 8.145,
+    "total_flos": 392518158405090.0,
+    "train_loss": 0.23256953179947268,
+    "train_runtime": 68.2263,
+    "train_samples": 8551,
+    "train_samples_per_second": 375.999,
+    "train_steps_per_second": 7.871
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.344061017036438,
+    "eval_matthews_correlation": 0.7546498935382564,
+    "eval_runtime": 1.3505,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 772.301,
+    "eval_steps_per_second": 8.145
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 43 --output_dir result_ablation_mbert/cola/mBERT/attention/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy attention --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 392518158405090.0,
+    "train_loss": 0.23256953179947268,
+    "train_runtime": 68.2263,
+    "train_samples": 8551,
+    "train_samples_per_second": 375.999,
+    "train_steps_per_second": 7.871
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_43/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 5.432812213897705,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5208,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 4.507151126861572,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.41,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 5.634162902832031,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3583,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 6.021278381347656,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3426,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 3.303103446960449,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3307,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 6.159351825714111,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3318,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.0855712890625,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3191,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2599124014377594,
+      "eval_matthews_correlation": 0.7375376126207708,
+      "eval_runtime": 1.6317,
+      "eval_samples_per_second": 639.198,
+      "eval_steps_per_second": 6.741,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 7.605750560760498,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2701,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 6.322093963623047,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2025,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 10.499807357788086,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2198,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 5.4613189697265625,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.2271,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 6.360593795776367,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1812,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 7.707641124725342,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.1736,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 4.31608772277832,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.187,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.27209946513175964,
+      "eval_matthews_correlation": 0.7517811010300938,
+      "eval_runtime": 1.6132,
+      "eval_samples_per_second": 646.524,
+      "eval_steps_per_second": 6.819,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 7.300499439239502,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1443,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 4.6428961753845215,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1341,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 8.466405868530273,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1163,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 5.116489410400391,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1129,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 9.736248016357422,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1304,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 6.68882417678833,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.121,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 10.210979461669922,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1115,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.344061017036438,
+      "eval_matthews_correlation": 0.7546498935382564,
+      "eval_runtime": 1.4369,
+      "eval_samples_per_second": 725.866,
+      "eval_steps_per_second": 7.655,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 392518158405090.0,
+      "train_loss": 0.23256953179947268,
+      "train_runtime": 68.2263,
+      "train_samples_per_second": 375.999,
+      "train_steps_per_second": 7.871
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 392518158405090.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3155398666858673,
+    "eval_matthews_correlation": 0.777661541691299,
+    "eval_runtime": 1.3715,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 760.459,
+    "eval_steps_per_second": 8.02,
+    "total_flos": 393647729346858.0,
+    "train_loss": 0.22533691994970736,
+    "train_runtime": 68.4887,
+    "train_samples": 8551,
+    "train_samples_per_second": 374.558,
+    "train_steps_per_second": 7.841
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3155398666858673,
+    "eval_matthews_correlation": 0.777661541691299,
+    "eval_runtime": 1.3715,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 760.459,
+    "eval_steps_per_second": 8.02
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 44 --output_dir result_ablation_mbert/cola/mBERT/attention/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy attention --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 393647729346858.0,
+    "train_loss": 0.22533691994970736,
+    "train_runtime": 68.4887,
+    "train_samples": 8551,
+    "train_samples_per_second": 374.558,
+    "train_steps_per_second": 7.841
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/attention/seed_44/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 5.904765605926514,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5071,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 4.30718469619751,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4121,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 3.573841094970703,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3746,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 4.671476364135742,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3308,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 6.219083786010742,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3072,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 6.215616226196289,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3127,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.634036064147949,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.2991,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.253072053194046,
+      "eval_matthews_correlation": 0.7470182756696736,
+      "eval_runtime": 1.7331,
+      "eval_samples_per_second": 601.828,
+      "eval_steps_per_second": 6.347,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 7.406801223754883,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2149,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 7.621590614318848,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.222,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 6.2891645431518555,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2085,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 5.562518119812012,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1949,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 10.442362785339355,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1903,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 6.932034969329834,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.2095,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 8.042263984680176,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.184,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2911624610424042,
+      "eval_matthews_correlation": 0.7594788750343784,
+      "eval_runtime": 1.7138,
+      "eval_samples_per_second": 608.582,
+      "eval_steps_per_second": 6.418,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 13.051106452941895,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1355,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 10.083761215209961,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1081,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 7.988012790679932,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1217,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 7.611426830291748,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1295,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 9.671212196350098,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1123,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 0.7560170888900757,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1083,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 7.9756245613098145,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1087,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.3155398666858673,
+      "eval_matthews_correlation": 0.777661541691299,
+      "eval_runtime": 1.5308,
+      "eval_samples_per_second": 681.353,
+      "eval_steps_per_second": 7.186,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 393647729346858.0,
+      "train_loss": 0.22533691994970736,
+      "train_runtime": 68.4887,
+      "train_samples_per_second": 374.558,
+      "train_steps_per_second": 7.841
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 393647729346858.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3190879821777344,
+    "eval_matthews_correlation": 0.7493368300485673,
+    "eval_runtime": 1.517,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 687.525,
+    "eval_steps_per_second": 7.251,
+    "total_flos": 395035512109056.0,
+    "train_loss": 0.23319570001499168,
+    "train_runtime": 70.3238,
+    "train_samples": 8551,
+    "train_samples_per_second": 364.784,
+    "train_steps_per_second": 7.636
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3190879821777344,
+    "eval_matthews_correlation": 0.7493368300485673,
+    "eval_runtime": 1.517,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 687.525,
+    "eval_steps_per_second": 7.251
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 42 --output_dir result_ablation_mbert/cola/mBERT/cls/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy cls --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 395035512109056.0,
+    "train_loss": 0.23319570001499168,
+    "train_runtime": 70.3238,
+    "train_samples": 8551,
+    "train_samples_per_second": 364.784,
+    "train_steps_per_second": 7.636
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/cls/seed_42/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 5.4744391441345215,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5224,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 8.66765308380127,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4164,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 5.180931091308594,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3368,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 5.698576927185059,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3361,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 5.160890579223633,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3484,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 10.759689331054688,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.2852,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 6.528064250946045,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3177,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2563566565513611,
+      "eval_matthews_correlation": 0.7513222297930762,
+      "eval_runtime": 1.8148,
+      "eval_samples_per_second": 574.707,
+      "eval_steps_per_second": 6.061,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 4.215940952301025,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2616,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 5.068136215209961,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2396,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 3.3487465381622314,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2072,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 4.20163106918335,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1794,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 4.505293846130371,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.2279,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 5.202899932861328,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.183,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 9.486673355102539,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.2156,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2633655369281769,
+      "eval_matthews_correlation": 0.758860374064517,
+      "eval_runtime": 1.4162,
+      "eval_samples_per_second": 736.47,
+      "eval_steps_per_second": 7.767,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 5.227592468261719,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1774,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 7.751290321350098,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.103,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 12.556318283081055,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1537,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 12.484012603759766,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1185,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 8.79336929321289,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1141,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 11.68663215637207,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.112,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 10.570063591003418,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1086,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.3190879821777344,
+      "eval_matthews_correlation": 0.7493368300485673,
+      "eval_runtime": 1.5782,
+      "eval_samples_per_second": 660.878,
+      "eval_steps_per_second": 6.97,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 395035512109056.0,
+      "train_loss": 0.23319570001499168,
+      "train_runtime": 70.3238,
+      "train_samples_per_second": 364.784,
+      "train_steps_per_second": 7.636
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 395035512109056.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}