Upload batch 2/14: cola/mBERT/gated_multi_branch/seed_43 ... cola/mBERT/max/seed_43

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.29004186391830444,
+    "eval_matthews_correlation": 0.7733151048821603,
+    "eval_runtime": 1.3984,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 745.833,
+    "eval_steps_per_second": 5.006,
+    "total_flos": 445723188802812.0,
+    "train_loss": 0.2568637455735251,
+    "train_runtime": 31.4305,
+    "train_samples": 8551,
+    "train_samples_per_second": 816.183,
+    "train_steps_per_second": 10.213
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.29004186391830444,
+    "eval_matthews_correlation": 0.7733151048821603,
+    "eval_runtime": 1.3984,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 745.833,
+    "eval_steps_per_second": 5.006
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 43 --output_dir result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy gated_multi_branch --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 445723188802812.0,
+    "train_loss": 0.2568637455735251,
+    "train_runtime": 31.4305,
+    "train_samples": 8551,
+    "train_samples_per_second": 816.183,
+    "train_steps_per_second": 10.213
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_43/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 5.017457008361816,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4265,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 5.256691932678223,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3242,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.29493391513824463,
+      "eval_matthews_correlation": 0.7000821033544287,
+      "eval_runtime": 1.4881,
+      "eval_samples_per_second": 700.888,
+      "eval_steps_per_second": 4.704,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 7.021495819091797,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2655,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 5.001773834228516,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2186,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.238822340965271,
+      "eval_matthews_correlation": 0.7808239264042255,
+      "eval_runtime": 1.3022,
+      "eval_samples_per_second": 800.951,
+      "eval_steps_per_second": 5.376,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 5.757174968719482,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1859,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 5.819880485534668,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1647,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.29004186391830444,
+      "eval_matthews_correlation": 0.7733151048821603,
+      "eval_runtime": 1.5837,
+      "eval_samples_per_second": 658.588,
+      "eval_steps_per_second": 4.42,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 445723188802812.0,
+      "train_loss": 0.2568637455735251,
+      "train_runtime": 31.4305,
+      "train_samples_per_second": 816.183,
+      "train_steps_per_second": 10.213
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 445723188802812.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2971993386745453,
+    "eval_matthews_correlation": 0.7612027683763856,
+    "eval_runtime": 1.442,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 723.28,
+    "eval_steps_per_second": 4.854,
+    "total_flos": 445789593220680.0,
+    "train_loss": 0.24624314412149684,
+    "train_runtime": 31.4852,
+    "train_samples": 8551,
+    "train_samples_per_second": 814.765,
+    "train_steps_per_second": 10.195
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2971993386745453,
+    "eval_matthews_correlation": 0.7612027683763856,
+    "eval_runtime": 1.442,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 723.28,
+    "eval_steps_per_second": 4.854
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 44 --output_dir result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy gated_multi_branch --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 445789593220680.0,
+    "train_loss": 0.24624314412149684,
+    "train_runtime": 31.4852,
+    "train_samples": 8551,
+    "train_samples_per_second": 814.765,
+    "train_steps_per_second": 10.195
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/gated_multi_branch/seed_44/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 6.518551349639893,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4293,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 5.744719982147217,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3058,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.25964823365211487,
+      "eval_matthews_correlation": 0.7445807376620748,
+      "eval_runtime": 1.5036,
+      "eval_samples_per_second": 693.649,
+      "eval_steps_per_second": 4.655,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 4.169808864593506,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2458,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 6.322945594787598,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2151,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2919430136680603,
+      "eval_matthews_correlation": 0.7310312837735538,
+      "eval_runtime": 1.3657,
+      "eval_samples_per_second": 763.721,
+      "eval_steps_per_second": 5.126,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 5.181381702423096,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1691,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 6.789882183074951,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1584,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.2971993386745453,
+      "eval_matthews_correlation": 0.7612027683763856,
+      "eval_runtime": 1.4424,
+      "eval_samples_per_second": 723.11,
+      "eval_steps_per_second": 4.853,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 445789593220680.0,
+      "train_loss": 0.24624314412149684,
+      "train_runtime": 31.4852,
+      "train_samples_per_second": 814.765,
+      "train_steps_per_second": 10.195
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 445789593220680.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.29159754514694214,
+    "eval_matthews_correlation": 0.7493368300485673,
+    "eval_runtime": 1.4191,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 734.973,
+    "eval_steps_per_second": 4.933,
+    "total_flos": 427139728936980.0,
+    "train_loss": 0.2649622825075904,
+    "train_runtime": 35.4989,
+    "train_samples": 8551,
+    "train_samples_per_second": 722.643,
+    "train_steps_per_second": 9.043
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.29159754514694214,
+    "eval_matthews_correlation": 0.7493368300485673,
+    "eval_runtime": 1.4191,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 734.973,
+    "eval_steps_per_second": 4.933
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 42 --output_dir result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 427139728936980.0,
+    "train_loss": 0.2649622825075904,
+    "train_runtime": 35.4989,
+    "train_samples": 8551,
+    "train_samples_per_second": 722.643,
+    "train_steps_per_second": 9.043
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_42/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 4.549562931060791,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4495,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 4.5690693855285645,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3183,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2829677164554596,
+      "eval_matthews_correlation": 0.7168798974635037,
+      "eval_runtime": 1.6118,
+      "eval_samples_per_second": 647.108,
+      "eval_steps_per_second": 4.343,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 4.647832870483398,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.269,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 5.157256603240967,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2311,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2866717576980591,
+      "eval_matthews_correlation": 0.7448734653696512,
+      "eval_runtime": 1.5508,
+      "eval_samples_per_second": 672.555,
+      "eval_steps_per_second": 4.514,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 6.212674140930176,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1985,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 5.383189678192139,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1705,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.29159754514694214,
+      "eval_matthews_correlation": 0.7493368300485673,
+      "eval_runtime": 1.4308,
+      "eval_samples_per_second": 728.943,
+      "eval_steps_per_second": 4.892,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 427139728936980.0,
+      "train_loss": 0.2649622825075904,
+      "train_runtime": 35.4989,
+      "train_samples_per_second": 722.643,
+      "train_steps_per_second": 9.043
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 427139728936980.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.29299455881118774,
+    "eval_matthews_correlation": 0.770612811219967,
+    "eval_runtime": 1.4686,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 710.195,
+    "eval_steps_per_second": 4.766,
+    "total_flos": 424270173989460.0,
+    "train_loss": 0.2762503274876009,
+    "train_runtime": 35.5792,
+    "train_samples": 8551,
+    "train_samples_per_second": 721.011,
+    "train_steps_per_second": 9.022
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.29299455881118774,
+    "eval_matthews_correlation": 0.770612811219967,
+    "eval_runtime": 1.4686,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 710.195,
+    "eval_steps_per_second": 4.766
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 43 --output_dir result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 424270173989460.0,
+    "train_loss": 0.2762503274876009,
+    "train_runtime": 35.5792,
+    "train_samples": 8551,
+    "train_samples_per_second": 721.011,
+    "train_steps_per_second": 9.022
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_43/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 6.983142375946045,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4794,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 5.011570930480957,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3407,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2928970158100128,
+      "eval_matthews_correlation": 0.7255589981043564,
+      "eval_runtime": 1.5391,
+      "eval_samples_per_second": 677.655,
+      "eval_steps_per_second": 4.548,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 5.626948356628418,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.271,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 4.365847110748291,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.231,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2618351876735687,
+      "eval_matthews_correlation": 0.7756438036656587,
+      "eval_runtime": 1.5241,
+      "eval_samples_per_second": 684.35,
+      "eval_steps_per_second": 4.593,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 7.09475564956665,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.196,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 4.770914554595947,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1815,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.29299455881118774,
+      "eval_matthews_correlation": 0.770612811219967,
+      "eval_runtime": 1.5086,
+      "eval_samples_per_second": 691.356,
+      "eval_steps_per_second": 4.64,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 424270173989460.0,
+      "train_loss": 0.2762503274876009,
+      "train_runtime": 35.5792,
+      "train_samples_per_second": 721.011,
+      "train_steps_per_second": 9.022
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 424270173989460.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_44/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2829234004020691,
+    "eval_matthews_correlation": 0.7729513152802419,
+    "eval_runtime": 1.3882,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 751.321,
+    "eval_steps_per_second": 5.042,
+    "total_flos": 424333382309400.0,
+    "train_loss": 0.2657259335027677,
+    "train_runtime": 35.0333,
+    "train_samples": 8551,
+    "train_samples_per_second": 732.247,
+    "train_steps_per_second": 9.163
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_44/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2829234004020691,
+    "eval_matthews_correlation": 0.7729513152802419,
+    "eval_runtime": 1.3882,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 751.321,
+    "eval_steps_per_second": 5.042
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 44 --output_dir result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_44/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 424333382309400.0,
+    "train_loss": 0.2657259335027677,
+    "train_runtime": 35.0333,
+    "train_samples": 8551,
+    "train_samples_per_second": 732.247,
+    "train_steps_per_second": 9.163
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/hf_sequence_classifier/seed_44/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 4.170408725738525,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4426,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 4.682508945465088,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.324,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2635350525379181,
+      "eval_matthews_correlation": 0.7542296587798193,
+      "eval_runtime": 1.513,
+      "eval_samples_per_second": 689.353,
+      "eval_steps_per_second": 4.627,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 6.913776397705078,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2693,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 6.106342792510986,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2331,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2619330585002899,
+      "eval_matthews_correlation": 0.7824250237671008,
+      "eval_runtime": 1.3738,
+      "eval_samples_per_second": 759.182,
+      "eval_steps_per_second": 5.095,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 8.283414840698242,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1894,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 10.368456840515137,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1812,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.2829234004020691,
+      "eval_matthews_correlation": 0.7729513152802419,
+      "eval_runtime": 1.4952,
+      "eval_samples_per_second": 697.546,
+      "eval_steps_per_second": 4.682,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 424333382309400.0,
+      "train_loss": 0.2657259335027677,
+      "train_runtime": 35.0333,
+      "train_samples_per_second": 732.247,
+      "train_steps_per_second": 9.163
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 424333382309400.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3104800581932068,
+    "eval_matthews_correlation": 0.7354753979999464,
+    "eval_runtime": 1.5803,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 659.999,
+    "eval_steps_per_second": 4.43,
+    "total_flos": 429104574692352.0,
+    "train_loss": 0.26581977386712285,
+    "train_runtime": 29.3437,
+    "train_samples": 8551,
+    "train_samples_per_second": 874.226,
+    "train_steps_per_second": 10.939
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.3104800581932068,
+    "eval_matthews_correlation": 0.7354753979999464,
+    "eval_runtime": 1.5803,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 659.999,
+    "eval_steps_per_second": 4.43
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 42 --output_dir result_ablation_mbert_paper/cola/mBERT/max/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy max --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 429104574692352.0,
+    "train_loss": 0.26581977386712285,
+    "train_runtime": 29.3437,
+    "train_samples": 8551,
+    "train_samples_per_second": 874.226,
+    "train_steps_per_second": 10.939
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_42/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 4.00053071975708,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4448,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 5.761870861053467,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3223,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.30257633328437805,
+      "eval_matthews_correlation": 0.7113064650024932,
+      "eval_runtime": 1.775,
+      "eval_samples_per_second": 587.608,
+      "eval_steps_per_second": 3.944,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 3.953287363052368,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2721,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 5.419213771820068,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2332,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.28807854652404785,
+      "eval_matthews_correlation": 0.7210288223868512,
+      "eval_runtime": 1.6387,
+      "eval_samples_per_second": 636.49,
+      "eval_steps_per_second": 4.272,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 6.456455230712891,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.2,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 5.79647159576416,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1687,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.3104800581932068,
+      "eval_matthews_correlation": 0.7354753979999464,
+      "eval_runtime": 1.5395,
+      "eval_samples_per_second": 677.48,
+      "eval_steps_per_second": 4.547,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 429104574692352.0,
+      "train_loss": 0.26581977386712285,
+      "train_runtime": 29.3437,
+      "train_samples_per_second": 874.226,
+      "train_steps_per_second": 10.939
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 429104574692352.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2829602062702179,
+    "eval_matthews_correlation": 0.7638553055158107,
+    "eval_runtime": 1.3595,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 767.182,
+    "eval_steps_per_second": 5.149,
+    "total_flos": 426221819771904.0,
+    "train_loss": 0.26830103241394615,
+    "train_runtime": 31.2693,
+    "train_samples": 8551,
+    "train_samples_per_second": 820.39,
+    "train_steps_per_second": 10.266
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.2829602062702179,
+    "eval_matthews_correlation": 0.7638553055158107,
+    "eval_runtime": 1.3595,
+    "eval_samples": 1043,
+    "eval_samples_per_second": 767.182,
+    "eval_steps_per_second": 5.149
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name cola --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 43 --output_dir result_ablation_mbert_paper/cola/mBERT/max/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy max --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 426221819771904.0,
+    "train_loss": 0.26830103241394615,
+    "train_runtime": 31.2693,
+    "train_samples": 8551,
+    "train_samples_per_second": 820.39,
+    "train_steps_per_second": 10.266
+}

mbert_paper_metrics/result_ablation_mbert_paper/cola/mBERT/max/seed_43/trainer_state.json

@@ -0,0 +1,111 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 321,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 3.332101345062256,
+      "learning_rate": 1.688473520249221e-05,
+      "loss": 0.4634,
+      "step": 50
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 5.116937160491943,
+      "learning_rate": 1.3769470404984425e-05,
+      "loss": 0.3316,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.2942867577075958,
+      "eval_matthews_correlation": 0.7116335016864657,
+      "eval_runtime": 1.6351,
+      "eval_samples_per_second": 637.886,
+      "eval_steps_per_second": 4.281,
+      "step": 107
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 4.77727746963501,
+      "learning_rate": 1.0654205607476635e-05,
+      "loss": 0.2673,
+      "step": 150
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 3.866352081298828,
+      "learning_rate": 7.538940809968847e-06,
+      "loss": 0.2258,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.2581494450569153,
+      "eval_matthews_correlation": 0.762410162302737,
+      "eval_runtime": 1.6224,
+      "eval_samples_per_second": 642.892,
+      "eval_steps_per_second": 4.315,
+      "step": 214
+    },
+    {
+      "epoch": 2.336448598130841,
+      "grad_norm": 7.299858570098877,
+      "learning_rate": 4.42367601246106e-06,
+      "loss": 0.1864,
+      "step": 250
+    },
+    {
+      "epoch": 2.803738317757009,
+      "grad_norm": 3.356229305267334,
+      "learning_rate": 1.308411214953271e-06,
+      "loss": 0.1754,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.2829602062702179,
+      "eval_matthews_correlation": 0.7638553055158107,
+      "eval_runtime": 1.328,
+      "eval_samples_per_second": 785.412,
+      "eval_steps_per_second": 5.271,
+      "step": 321
+    },
+    {
+      "epoch": 3.0,
+      "step": 321,
+      "total_flos": 426221819771904.0,
+      "train_loss": 0.26830103241394615,
+      "train_runtime": 31.2693,
+      "train_samples_per_second": 820.39,
+      "train_steps_per_second": 10.266
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 321,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 426221819771904.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}