Upload batch 4/24: cola/mBERT/max/seed_42 ... cola/mBERT/mean/seed_42

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.28716450929641724,
+    "eval_matthews_correlation": 0.7823569443931843,
+    "eval_runtime": 1.6169,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 645.068,
+    "eval_steps_per_second": 6.803,
+    "total_flos": 395035512109056.0,
+    "train_loss": 0.227474433083774,
+    "train_runtime": 70.5688,
+    "train_samples": 8551,
+    "train_samples_per_second": 363.517,
+    "train_steps_per_second": 7.61
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.28716450929641724,
+    "eval_matthews_correlation": 0.7823569443931843,
+    "eval_runtime": 1.6169,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 645.068,
+    "eval_steps_per_second": 6.803
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 42 --output_dir result_ablation_mbert/cola/mBERT/max/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy max --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 395035512109056.0,
+    "train_loss": 0.227474433083774,
+    "train_runtime": 70.5688,
+    "train_samples": 8551,
+    "train_samples_per_second": 363.517,
+    "train_steps_per_second": 7.61
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_42/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 7.506533145904541,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5032,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 6.765139102935791,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4023,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 5.985041618347168,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3251,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 6.529898166656494,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3316,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 4.667145729064941,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3375,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 7.793707370758057,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.2753,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 7.965795516967773,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3299,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2594018280506134,
+      "eval_matthews_correlation": 0.7764429355875446,
+      "eval_runtime": 1.8218,
+      "eval_samples_per_second": 572.521,
+      "eval_steps_per_second": 6.038,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 5.244349002838135,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2611,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 4.337162971496582,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2335,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 5.328370571136475,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.1964,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 3.7711894512176514,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1792,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 5.171924114227295,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.2184,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 5.825191020965576,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.1842,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 8.417168617248535,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1959,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.25236082077026367,
+      "eval_matthews_correlation": 0.7779145608071428,
+      "eval_runtime": 1.5756,
+      "eval_samples_per_second": 661.985,
+      "eval_steps_per_second": 6.982,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 5.163099765777588,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1593,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 12.770713806152344,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.0989,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 14.984099388122559,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.15,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 6.552338600158691,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1361,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 8.521238327026367,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1159,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 14.332450866699219,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1141,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 6.633162021636963,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.0935,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.28716450929641724,
+      "eval_matthews_correlation": 0.7823569443931843,
+      "eval_runtime": 1.5348,
+      "eval_samples_per_second": 679.556,
+      "eval_steps_per_second": 7.167,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 395035512109056.0,
+      "train_loss": 0.227474433083774,
+      "train_runtime": 70.5688,
+      "train_samples_per_second": 363.517,
+      "train_steps_per_second": 7.61
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 395035512109056.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.32399871945381165,
+    "eval_matthews_correlation": 0.7493680462422366,
+    "eval_runtime": 2.0646,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 505.177,
+    "eval_steps_per_second": 5.328,
+    "total_flos": 392514650311680.0,
+    "train_loss": 0.23861054157633577,
+    "train_runtime": 65.9898,
+    "train_samples": 8551,
+    "train_samples_per_second": 388.742,
+    "train_steps_per_second": 8.138
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.32399871945381165,
+    "eval_matthews_correlation": 0.7493680462422366,
+    "eval_runtime": 2.0646,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 505.177,
+    "eval_steps_per_second": 5.328
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 43 --output_dir result_ablation_mbert/cola/mBERT/max/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy max --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 392514650311680.0,
+    "train_loss": 0.23861054157633577,
+    "train_runtime": 65.9898,
+    "train_samples": 8551,
+    "train_samples_per_second": 388.742,
+    "train_steps_per_second": 8.138
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_43/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 9.139832496643066,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5302,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 3.687873125076294,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4152,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 4.108961582183838,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3662,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 5.980782985687256,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3401,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 4.232458591461182,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3319,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 4.3962273597717285,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3485,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 3.4268150329589844,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3149,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.25328266620635986,
+      "eval_matthews_correlation": 0.7473565685735221,
+      "eval_runtime": 2.1528,
+      "eval_samples_per_second": 484.479,
+      "eval_steps_per_second": 5.11,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 5.443220138549805,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2778,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 6.192456245422363,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2271,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 8.065898895263672,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.226,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 7.433097839355469,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.2264,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 6.765533447265625,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1929,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 7.576679706573486,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.1834,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 5.533838272094727,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1701,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.26969248056411743,
+      "eval_matthews_correlation": 0.7638553055158107,
+      "eval_runtime": 1.9414,
+      "eval_samples_per_second": 537.232,
+      "eval_steps_per_second": 5.666,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 3.659409523010254,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.148,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 3.878593683242798,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1344,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 7.933662414550781,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1279,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 5.110512733459473,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1299,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 9.793325424194336,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1402,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 6.29339599609375,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1204,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 6.874462127685547,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1254,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.32399871945381165,
+      "eval_matthews_correlation": 0.7493680462422366,
+      "eval_runtime": 2.086,
+      "eval_samples_per_second": 500.01,
+      "eval_steps_per_second": 5.273,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 392514650311680.0,
+      "train_loss": 0.23861054157633577,
+      "train_runtime": 65.9898,
+      "train_samples_per_second": 388.742,
+      "train_steps_per_second": 8.138
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 392514650311680.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.32878631353378296,
+    "eval_matthews_correlation": 0.7731698517570625,
+    "eval_runtime": 1.3926,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 748.937,
+    "eval_steps_per_second": 7.899,
+    "total_flos": 393644211158016.0,
+    "train_loss": 0.22196905217801172,
+    "train_runtime": 67.8005,
+    "train_samples": 8551,
+    "train_samples_per_second": 378.36,
+    "train_steps_per_second": 7.92
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.32878631353378296,
+    "eval_matthews_correlation": 0.7731698517570625,
+    "eval_runtime": 1.3926,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 748.937,
+    "eval_steps_per_second": 7.899
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 44 --output_dir result_ablation_mbert/cola/mBERT/max/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy max --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 393644211158016.0,
+    "train_loss": 0.22196905217801172,
+    "train_runtime": 67.8005,
+    "train_samples": 8551,
+    "train_samples_per_second": 378.36,
+    "train_steps_per_second": 7.92
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/max/seed_44/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 6.849033355712891,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5163,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 5.070627212524414,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4104,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 3.0494155883789062,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3756,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 4.307983875274658,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3287,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 4.704650402069092,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3062,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 5.402581691741943,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3113,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 4.690877914428711,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.2858,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.27267423272132874,
+      "eval_matthews_correlation": 0.7517307484998179,
+      "eval_runtime": 1.754,
+      "eval_samples_per_second": 594.638,
+      "eval_steps_per_second": 6.271,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 6.862037181854248,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.1996,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 7.150571346282959,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2217,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 6.441170692443848,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2041,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 6.917917728424072,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1958,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 7.2949323654174805,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1812,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 7.782524585723877,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.2074,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 4.0520501136779785,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1757,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.28867119550704956,
+      "eval_matthews_correlation": 0.7847385093120407,
+      "eval_runtime": 1.7215,
+      "eval_samples_per_second": 605.882,
+      "eval_steps_per_second": 6.39,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 13.147331237792969,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1415,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 8.715579986572266,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1029,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 6.811680793762207,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1292,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 10.589837074279785,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1187,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 9.207088470458984,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1061,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 6.109454154968262,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.092,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 4.249560832977295,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1076,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.32878631353378296,
+      "eval_matthews_correlation": 0.7731698517570625,
+      "eval_runtime": 2.088,
+      "eval_samples_per_second": 499.518,
+      "eval_steps_per_second": 5.268,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 393644211158016.0,
+      "train_loss": 0.22196905217801172,
+      "train_runtime": 67.8005,
+      "train_samples_per_second": 378.36,
+      "train_steps_per_second": 7.92
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 393644211158016.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2967136800289154,
+    "eval_matthews_correlation": 0.7635290646756514,
+    "eval_runtime": 1.4434,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 722.592,
+    "eval_steps_per_second": 7.621,
+    "total_flos": 395035512109056.0,
+    "train_loss": 0.2256952981487111,
+    "train_runtime": 70.3903,
+    "train_samples": 8551,
+    "train_samples_per_second": 364.44,
+    "train_steps_per_second": 7.629
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2967136800289154,
+    "eval_matthews_correlation": 0.7635290646756514,
+    "eval_runtime": 1.4434,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 722.592,
+    "eval_steps_per_second": 7.621
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 42 --output_dir result_ablation_mbert/cola/mBERT/mean/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mean --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 395035512109056.0,
+    "train_loss": 0.2256952981487111,
+    "train_runtime": 70.3903,
+    "train_samples": 8551,
+    "train_samples_per_second": 364.44,
+    "train_steps_per_second": 7.629
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mean/seed_42/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 9.855199813842773,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.4923,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 6.724723815917969,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.3936,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 6.128042697906494,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3318,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 6.443691730499268,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3282,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 5.052712440490723,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3461,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 5.698122024536133,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.2792,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 7.067698955535889,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.314,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2573047876358032,
+      "eval_matthews_correlation": 0.7623108902059486,
+      "eval_runtime": 1.7886,
+      "eval_samples_per_second": 583.148,
+      "eval_steps_per_second": 6.15,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 6.118077754974365,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2532,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 4.651660919189453,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2332,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 5.305139064788818,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.1947,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 4.12755012512207,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1812,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 5.024567604064941,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.2205,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 5.94950532913208,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.1798,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 9.54784107208252,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1944,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.25511619448661804,
+      "eval_matthews_correlation": 0.7660475389427814,
+      "eval_runtime": 1.5492,
+      "eval_samples_per_second": 673.249,
+      "eval_steps_per_second": 7.1,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 5.767430782318115,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1658,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 6.832909107208252,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1006,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 12.019317626953125,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1479,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 7.294384956359863,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1211,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 10.134602546691895,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1114,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 10.2511625289917,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.112,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 7.9914631843566895,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1026,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.2967136800289154,
+      "eval_matthews_correlation": 0.7635290646756514,
+      "eval_runtime": 1.4008,
+      "eval_samples_per_second": 744.587,
+      "eval_steps_per_second": 7.853,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 395035512109056.0,
+      "train_loss": 0.2256952981487111,
+      "train_runtime": 70.3903,
+      "train_samples_per_second": 364.44,
+      "train_steps_per_second": 7.629
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 395035512109056.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}