Upload batch 6/24: cola/mBERT/mha_attention/seed_44 ... cola/mBERT/multi_branch_average/seed_44

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2813240885734558,
+    "eval_matthews_correlation": 0.7941238477427384,
+    "eval_runtime": 1.2582,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 828.978,
+    "eval_steps_per_second": 8.743,
+    "total_flos": 395446699624746.0,
+    "train_loss": 0.22841538486090007,
+    "train_runtime": 68.1473,
+    "train_samples": 8551,
+    "train_samples_per_second": 376.435,
+    "train_steps_per_second": 7.88
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2813240885734558,
+    "eval_matthews_correlation": 0.7941238477427384,
+    "eval_runtime": 1.2582,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 828.978,
+    "eval_steps_per_second": 8.743
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 44 --output_dir result_ablation_mbert/cola/mBERT/mha_attention/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mha_attention --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 395446699624746.0,
+    "train_loss": 0.22841538486090007,
+    "train_runtime": 68.1473,
+    "train_samples": 8551,
+    "train_samples_per_second": 376.435,
+    "train_steps_per_second": 7.88
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/mha_attention/seed_44/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 5.366695880889893,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5378,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 4.120193958282471,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4173,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 3.426011085510254,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3751,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 6.232273578643799,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3247,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 5.154740333557129,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.2946,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 4.993556022644043,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3136,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.428062915802002,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.2846,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.25077900290489197,
+      "eval_matthews_correlation": 0.7516948435200349,
+      "eval_runtime": 1.996,
+      "eval_samples_per_second": 522.557,
+      "eval_steps_per_second": 5.511,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 5.079008102416992,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.1994,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 4.522168159484863,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2241,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 5.346883773803711,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2295,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 6.479331016540527,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.2023,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 7.201396942138672,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1801,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 7.064659595489502,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.2112,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 7.188658714294434,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.2035,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2546181082725525,
+      "eval_matthews_correlation": 0.7919243799475026,
+      "eval_runtime": 1.76,
+      "eval_samples_per_second": 592.62,
+      "eval_steps_per_second": 6.25,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 5.307333946228027,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.142,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 2.6121551990509033,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1279,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 4.879550457000732,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1131,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 6.406044960021973,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.118,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 8.526052474975586,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1355,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 2.5646495819091797,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1105,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 4.7689361572265625,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1017,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.2813240885734558,
+      "eval_matthews_correlation": 0.7941238477427384,
+      "eval_runtime": 1.3959,
+      "eval_samples_per_second": 747.172,
+      "eval_steps_per_second": 7.88,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 395446699624746.0,
+      "train_loss": 0.22841538486090007,
+      "train_runtime": 68.1473,
+      "train_samples_per_second": 376.435,
+      "train_steps_per_second": 7.88
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 395446699624746.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2748015820980072,
+    "eval_matthews_correlation": 0.789889696708505,
+    "eval_runtime": 1.3619,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 765.836,
+    "eval_steps_per_second": 8.077,
+    "total_flos": 407683404044028.0,
+    "train_loss": 0.22175285980466136,
+    "train_runtime": 68.4151,
+    "train_samples": 8551,
+    "train_samples_per_second": 374.961,
+    "train_steps_per_second": 7.849
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2748015820980072,
+    "eval_matthews_correlation": 0.789889696708505,
+    "eval_runtime": 1.3619,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 765.836,
+    "eval_steps_per_second": 8.077
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 42 --output_dir result_ablation_mbert/cola/mBERT/multi_branch_average/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy multi_branch_average --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 407683404044028.0,
+    "train_loss": 0.22175285980466136,
+    "train_runtime": 68.4151,
+    "train_samples": 8551,
+    "train_samples_per_second": 374.961,
+    "train_steps_per_second": 7.849
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_42/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 6.961562633514404,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5346,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 5.958384037017822,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.398,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 5.4906134605407715,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.318,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 4.776484966278076,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3243,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 6.049837112426758,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3296,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 7.4059343338012695,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.2711,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 8.422361373901367,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.295,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.25132161378860474,
+      "eval_matthews_correlation": 0.7843496815080608,
+      "eval_runtime": 2.0361,
+      "eval_samples_per_second": 512.264,
+      "eval_steps_per_second": 5.403,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 9.339873313903809,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2607,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 7.047557353973389,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2019,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 4.100979328155518,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.186,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 6.741496562957764,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.185,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 7.608138084411621,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.2061,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 5.194343566894531,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.1838,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 7.62186861038208,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1914,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2854599058628082,
+      "eval_matthews_correlation": 0.7689548198090016,
+      "eval_runtime": 1.8181,
+      "eval_samples_per_second": 573.67,
+      "eval_steps_per_second": 6.05,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 9.80944538116455,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1873,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 12.422720909118652,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.0935,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 11.966200828552246,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1316,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 5.292356491088867,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1193,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 3.517305612564087,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.103,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 13.95299243927002,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.101,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 5.513173580169678,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1054,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.2748015820980072,
+      "eval_matthews_correlation": 0.789889696708505,
+      "eval_runtime": 1.4141,
+      "eval_samples_per_second": 737.576,
+      "eval_steps_per_second": 7.779,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 407683404044028.0,
+      "train_loss": 0.22175285980466136,
+      "train_runtime": 68.4151,
+      "train_samples_per_second": 374.961,
+      "train_steps_per_second": 7.849
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 407683404044028.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3285018801689148,
+    "eval_matthews_correlation": 0.7445812636597251,
+    "eval_runtime": 1.3497,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 772.777,
+    "eval_steps_per_second": 8.15,
+    "total_flos": 405081831559590.0,
+    "train_loss": 0.22625035480414024,
+    "train_runtime": 68.3403,
+    "train_samples": 8551,
+    "train_samples_per_second": 375.371,
+    "train_steps_per_second": 7.858
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3285018801689148,
+    "eval_matthews_correlation": 0.7445812636597251,
+    "eval_runtime": 1.3497,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 772.777,
+    "eval_steps_per_second": 8.15
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 43 --output_dir result_ablation_mbert/cola/mBERT/multi_branch_average/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy multi_branch_average --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 405081831559590.0,
+    "train_loss": 0.22625035480414024,
+    "train_runtime": 68.3403,
+    "train_samples": 8551,
+    "train_samples_per_second": 375.371,
+    "train_steps_per_second": 7.858
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_43/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 6.489190578460693,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5149,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 3.9423370361328125,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4139,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 6.0748138427734375,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3625,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 7.113004684448242,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.326,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 4.560152530670166,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.3248,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 7.792273044586182,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3199,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 5.581479072570801,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.3017,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2672134339809418,
+      "eval_matthews_correlation": 0.7482425793295719,
+      "eval_runtime": 2.0445,
+      "eval_samples_per_second": 510.139,
+      "eval_steps_per_second": 5.38,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 5.846633434295654,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2645,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 8.551812171936035,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.22,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 6.874224662780762,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.1996,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 8.078627586364746,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.2168,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 7.300671100616455,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1765,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 9.31351375579834,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.2132,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 3.0243983268737793,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.1746,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2810530960559845,
+      "eval_matthews_correlation": 0.7540639595850344,
+      "eval_runtime": 1.8428,
+      "eval_samples_per_second": 565.982,
+      "eval_steps_per_second": 5.969,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 6.306654453277588,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1233,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 1.8831703662872314,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1074,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 4.937216281890869,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1204,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 2.8739070892333984,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1237,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 6.478601932525635,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1087,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 6.211483478546143,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.1091,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 7.3320112228393555,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1035,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.3285018801689148,
+      "eval_matthews_correlation": 0.7445812636597251,
+      "eval_runtime": 1.347,
+      "eval_samples_per_second": 774.318,
+      "eval_steps_per_second": 8.166,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 405081831559590.0,
+      "train_loss": 0.22625035480414024,
+      "train_runtime": 68.3403,
+      "train_samples_per_second": 375.371,
+      "train_steps_per_second": 7.858
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 405081831559590.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2956400215625763,
+    "eval_matthews_correlation": 0.7729583162617174,
+    "eval_runtime": 1.8363,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 567.982,
+    "eval_steps_per_second": 5.99,
+    "total_flos": 406247557669758.0,
+    "train_loss": 0.22685242720378399,
+    "train_runtime": 61.7389,
+    "train_samples": 8551,
+    "train_samples_per_second": 415.508,
+    "train_steps_per_second": 8.698
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2956400215625763,
+    "eval_matthews_correlation": 0.7729583162617174,
+    "eval_runtime": 1.8363,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 567.982,
+    "eval_steps_per_second": 5.99
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 44 --output_dir result_ablation_mbert/cola/mBERT/multi_branch_average/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy multi_branch_average --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 406247557669758.0,
+    "train_loss": 0.22685242720378399,
+    "train_runtime": 61.7389,
+    "train_samples": 8551,
+    "train_samples_per_second": 415.508,
+    "train_steps_per_second": 8.698
+}

mbert_rtx6000_metrics/result_ablation_mbert/cola/mBERT/multi_branch_average/seed_44/trainer_state.json

@@ -0,0 +1,216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 537,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13966480446927373,
+      "grad_norm": 7.106081962585449,
+      "learning_rate": 2.8603351955307262e-05,
+      "loss": 0.5234,
+      "step": 25
+    },
+    {
+      "epoch": 0.27932960893854747,
+      "grad_norm": 4.174944877624512,
+      "learning_rate": 2.7206703910614527e-05,
+      "loss": 0.4147,
+      "step": 50
+    },
+    {
+      "epoch": 0.41899441340782123,
+      "grad_norm": 4.04843282699585,
+      "learning_rate": 2.5810055865921788e-05,
+      "loss": 0.3594,
+      "step": 75
+    },
+    {
+      "epoch": 0.5586592178770949,
+      "grad_norm": 4.837288856506348,
+      "learning_rate": 2.441340782122905e-05,
+      "loss": 0.3182,
+      "step": 100
+    },
+    {
+      "epoch": 0.6983240223463687,
+      "grad_norm": 3.896435022354126,
+      "learning_rate": 2.3016759776536314e-05,
+      "loss": 0.2923,
+      "step": 125
+    },
+    {
+      "epoch": 0.8379888268156425,
+      "grad_norm": 5.405098915100098,
+      "learning_rate": 2.1620111731843575e-05,
+      "loss": 0.3102,
+      "step": 150
+    },
+    {
+      "epoch": 0.9776536312849162,
+      "grad_norm": 7.0171051025390625,
+      "learning_rate": 2.0223463687150836e-05,
+      "loss": 0.2955,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.24399316310882568,
+      "eval_matthews_correlation": 0.7650319046010935,
+      "eval_runtime": 2.611,
+      "eval_samples_per_second": 399.458,
+      "eval_steps_per_second": 4.213,
+      "step": 179
+    },
+    {
+      "epoch": 1.1173184357541899,
+      "grad_norm": 8.45476245880127,
+      "learning_rate": 1.88268156424581e-05,
+      "loss": 0.2121,
+      "step": 200
+    },
+    {
+      "epoch": 1.2569832402234637,
+      "grad_norm": 4.377360820770264,
+      "learning_rate": 1.7430167597765365e-05,
+      "loss": 0.2034,
+      "step": 225
+    },
+    {
+      "epoch": 1.3966480446927374,
+      "grad_norm": 7.181662559509277,
+      "learning_rate": 1.6033519553072626e-05,
+      "loss": 0.2136,
+      "step": 250
+    },
+    {
+      "epoch": 1.536312849162011,
+      "grad_norm": 7.380345821380615,
+      "learning_rate": 1.4636871508379887e-05,
+      "loss": 0.1891,
+      "step": 275
+    },
+    {
+      "epoch": 1.675977653631285,
+      "grad_norm": 5.790942668914795,
+      "learning_rate": 1.324022346368715e-05,
+      "loss": 0.1732,
+      "step": 300
+    },
+    {
+      "epoch": 1.8156424581005588,
+      "grad_norm": 6.188945770263672,
+      "learning_rate": 1.1843575418994415e-05,
+      "loss": 0.2111,
+      "step": 325
+    },
+    {
+      "epoch": 1.9553072625698324,
+      "grad_norm": 7.357444763183594,
+      "learning_rate": 1.0446927374301676e-05,
+      "loss": 0.2373,
+      "step": 350
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.29736000299453735,
+      "eval_matthews_correlation": 0.7631323571672735,
+      "eval_runtime": 2.586,
+      "eval_samples_per_second": 403.328,
+      "eval_steps_per_second": 4.254,
+      "step": 358
+    },
+    {
+      "epoch": 2.094972067039106,
+      "grad_norm": 10.879669189453125,
+      "learning_rate": 9.050279329608939e-06,
+      "loss": 0.1459,
+      "step": 375
+    },
+    {
+      "epoch": 2.2346368715083798,
+      "grad_norm": 7.827243804931641,
+      "learning_rate": 7.653631284916202e-06,
+      "loss": 0.1221,
+      "step": 400
+    },
+    {
+      "epoch": 2.3743016759776534,
+      "grad_norm": 7.154805660247803,
+      "learning_rate": 6.256983240223464e-06,
+      "loss": 0.1115,
+      "step": 425
+    },
+    {
+      "epoch": 2.5139664804469275,
+      "grad_norm": 7.1307806968688965,
+      "learning_rate": 4.860335195530726e-06,
+      "loss": 0.1247,
+      "step": 450
+    },
+    {
+      "epoch": 2.653631284916201,
+      "grad_norm": 7.537153720855713,
+      "learning_rate": 3.463687150837989e-06,
+      "loss": 0.1187,
+      "step": 475
+    },
+    {
+      "epoch": 2.793296089385475,
+      "grad_norm": 1.762450933456421,
+      "learning_rate": 2.0670391061452514e-06,
+      "loss": 0.111,
+      "step": 500
+    },
+    {
+      "epoch": 2.9329608938547485,
+      "grad_norm": 9.45340347290039,
+      "learning_rate": 6.70391061452514e-07,
+      "loss": 0.1207,
+      "step": 525
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.2956400215625763,
+      "eval_matthews_correlation": 0.7729583162617174,
+      "eval_runtime": 1.782,
+      "eval_samples_per_second": 585.297,
+      "eval_steps_per_second": 6.173,
+      "step": 537
+    },
+    {
+      "epoch": 3.0,
+      "step": 537,
+      "total_flos": 406247557669758.0,
+      "train_loss": 0.22685242720378399,
+      "train_runtime": 61.7389,
+      "train_samples_per_second": 415.508,
+      "train_steps_per_second": 8.698
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 537,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 406247557669758.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}