Upload batch 24/24: vsfc/mBERT/mha_attention/seed_44 ... vsfc/mBERT/multi_branch_average/seed_44

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/mha_attention/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 44 --output_dir result_ablation_mbert/vsfc/mBERT/mha_attention/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mha_attention --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9412507896399241,
+    "eval_loss": 0.21253333985805511,
+    "eval_runtime": 6.741,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 234.833,
+    "eval_steps_per_second": 2.522,
+    "total_flos": 1056957384788088.0,
+    "train_loss": 0.20452363604115975,
+    "train_runtime": 98.7034,
+    "train_samples": 11426,
+    "train_samples_per_second": 347.283,
+    "train_steps_per_second": 7.264
+}

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9412507896399241,
+    "eval_loss": 0.21253333985805511,
+    "eval_runtime": 6.741,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 234.833,
+    "eval_steps_per_second": 2.522
+}

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 42 --output_dir result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy multi_branch_average --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1056957384788088.0,
+    "train_loss": 0.20452363604115975,
+    "train_runtime": 98.7034,
+    "train_samples": 11426,
+    "train_samples_per_second": 347.283,
+    "train_steps_per_second": 7.264
+}

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_42/trainer_state.json

@@ -0,0 +1,265 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 717,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.10460251046025104,
+      "grad_norm": 14.201335906982422,
+      "learning_rate": 2.8953974895397488e-05,
+      "loss": 0.5405,
+      "step": 25
+    },
+    {
+      "epoch": 0.20920502092050208,
+      "grad_norm": 5.348233699798584,
+      "learning_rate": 2.7907949790794982e-05,
+      "loss": 0.3294,
+      "step": 50
+    },
+    {
+      "epoch": 0.3138075313807531,
+      "grad_norm": 7.623049259185791,
+      "learning_rate": 2.686192468619247e-05,
+      "loss": 0.3422,
+      "step": 75
+    },
+    {
+      "epoch": 0.41841004184100417,
+      "grad_norm": 5.9439311027526855,
+      "learning_rate": 2.5815899581589957e-05,
+      "loss": 0.2934,
+      "step": 100
+    },
+    {
+      "epoch": 0.5230125523012552,
+      "grad_norm": 4.006845951080322,
+      "learning_rate": 2.4769874476987448e-05,
+      "loss": 0.2685,
+      "step": 125
+    },
+    {
+      "epoch": 0.6276150627615062,
+      "grad_norm": 5.4574055671691895,
+      "learning_rate": 2.372384937238494e-05,
+      "loss": 0.263,
+      "step": 150
+    },
+    {
+      "epoch": 0.7322175732217573,
+      "grad_norm": 6.177022457122803,
+      "learning_rate": 2.2677824267782426e-05,
+      "loss": 0.2551,
+      "step": 175
+    },
+    {
+      "epoch": 0.8368200836820083,
+      "grad_norm": 5.705365180969238,
+      "learning_rate": 2.1631799163179917e-05,
+      "loss": 0.2929,
+      "step": 200
+    },
+    {
+      "epoch": 0.9414225941422594,
+      "grad_norm": 5.65452766418457,
+      "learning_rate": 2.0585774058577408e-05,
+      "loss": 0.2615,
+      "step": 225
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9311433986102338,
+      "eval_loss": 0.22236785292625427,
+      "eval_runtime": 5.7488,
+      "eval_samples_per_second": 275.361,
+      "eval_steps_per_second": 2.957,
+      "step": 239
+    },
+    {
+      "epoch": 1.0460251046025104,
+      "grad_norm": 4.936914920806885,
+      "learning_rate": 1.9539748953974895e-05,
+      "loss": 0.1962,
+      "step": 250
+    },
+    {
+      "epoch": 1.1506276150627615,
+      "grad_norm": 8.016222953796387,
+      "learning_rate": 1.8493723849372386e-05,
+      "loss": 0.192,
+      "step": 275
+    },
+    {
+      "epoch": 1.2552301255230125,
+      "grad_norm": 3.698970079421997,
+      "learning_rate": 1.7447698744769877e-05,
+      "loss": 0.1453,
+      "step": 300
+    },
+    {
+      "epoch": 1.3598326359832635,
+      "grad_norm": 3.36641001701355,
+      "learning_rate": 1.6401673640167365e-05,
+      "loss": 0.2161,
+      "step": 325
+    },
+    {
+      "epoch": 1.4644351464435146,
+      "grad_norm": 2.944875955581665,
+      "learning_rate": 1.5355648535564852e-05,
+      "loss": 0.2049,
+      "step": 350
+    },
+    {
+      "epoch": 1.5690376569037658,
+      "grad_norm": 6.417823314666748,
+      "learning_rate": 1.4309623430962344e-05,
+      "loss": 0.192,
+      "step": 375
+    },
+    {
+      "epoch": 1.6736401673640167,
+      "grad_norm": 3.3958539962768555,
+      "learning_rate": 1.3263598326359832e-05,
+      "loss": 0.1681,
+      "step": 400
+    },
+    {
+      "epoch": 1.778242677824268,
+      "grad_norm": 11.595094680786133,
+      "learning_rate": 1.2217573221757323e-05,
+      "loss": 0.1696,
+      "step": 425
+    },
+    {
+      "epoch": 1.8828451882845187,
+      "grad_norm": 6.112818241119385,
+      "learning_rate": 1.1171548117154812e-05,
+      "loss": 0.1422,
+      "step": 450
+    },
+    {
+      "epoch": 1.98744769874477,
+      "grad_norm": 8.460183143615723,
+      "learning_rate": 1.0125523012552301e-05,
+      "loss": 0.1938,
+      "step": 475
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9374605180037903,
+      "eval_loss": 0.20332568883895874,
+      "eval_runtime": 4.7706,
+      "eval_samples_per_second": 331.821,
+      "eval_steps_per_second": 3.563,
+      "step": 478
+    },
+    {
+      "epoch": 2.092050209205021,
+      "grad_norm": 6.223247051239014,
+      "learning_rate": 9.079497907949792e-06,
+      "loss": 0.1168,
+      "step": 500
+    },
+    {
+      "epoch": 2.196652719665272,
+      "grad_norm": 6.7170090675354,
+      "learning_rate": 8.03347280334728e-06,
+      "loss": 0.1232,
+      "step": 525
+    },
+    {
+      "epoch": 2.301255230125523,
+      "grad_norm": 7.830327033996582,
+      "learning_rate": 6.98744769874477e-06,
+      "loss": 0.155,
+      "step": 550
+    },
+    {
+      "epoch": 2.405857740585774,
+      "grad_norm": 8.447519302368164,
+      "learning_rate": 5.941422594142259e-06,
+      "loss": 0.1328,
+      "step": 575
+    },
+    {
+      "epoch": 2.510460251046025,
+      "grad_norm": 5.235635280609131,
+      "learning_rate": 4.895397489539749e-06,
+      "loss": 0.1044,
+      "step": 600
+    },
+    {
+      "epoch": 2.6150627615062763,
+      "grad_norm": 6.357935905456543,
+      "learning_rate": 3.849372384937238e-06,
+      "loss": 0.1431,
+      "step": 625
+    },
+    {
+      "epoch": 2.719665271966527,
+      "grad_norm": 4.409952640533447,
+      "learning_rate": 2.803347280334728e-06,
+      "loss": 0.1192,
+      "step": 650
+    },
+    {
+      "epoch": 2.8242677824267783,
+      "grad_norm": 2.4843714237213135,
+      "learning_rate": 1.7573221757322176e-06,
+      "loss": 0.1143,
+      "step": 675
+    },
+    {
+      "epoch": 2.928870292887029,
+      "grad_norm": 5.9728851318359375,
+      "learning_rate": 7.112970711297071e-07,
+      "loss": 0.111,
+      "step": 700
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.9412507896399241,
+      "eval_loss": 0.21253333985805511,
+      "eval_runtime": 5.2439,
+      "eval_samples_per_second": 301.876,
+      "eval_steps_per_second": 3.242,
+      "step": 717
+    },
+    {
+      "epoch": 3.0,
+      "step": 717,
+      "total_flos": 1056957384788088.0,
+      "train_loss": 0.20452363604115975,
+      "train_runtime": 98.7034,
+      "train_samples_per_second": 347.283,
+      "train_steps_per_second": 7.264
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 717,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1056957384788088.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9330385344283006,
+    "eval_loss": 0.22735266387462616,
+    "eval_runtime": 4.952,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 319.67,
+    "eval_steps_per_second": 3.433,
+    "total_flos": 1055495423686536.0,
+    "train_loss": 0.20476110576752007,
+    "train_runtime": 98.8837,
+    "train_samples": 11426,
+    "train_samples_per_second": 346.65,
+    "train_steps_per_second": 7.251
+}

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9330385344283006,
+    "eval_loss": 0.22735266387462616,
+    "eval_runtime": 4.952,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 319.67,
+    "eval_steps_per_second": 3.433
+}

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 43 --output_dir result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy multi_branch_average --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1055495423686536.0,
+    "train_loss": 0.20476110576752007,
+    "train_runtime": 98.8837,
+    "train_samples": 11426,
+    "train_samples_per_second": 346.65,
+    "train_steps_per_second": 7.251
+}

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_43/trainer_state.json

@@ -0,0 +1,265 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 717,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.10460251046025104,
+      "grad_norm": 5.623245716094971,
+      "learning_rate": 2.8953974895397488e-05,
+      "loss": 0.5208,
+      "step": 25
+    },
+    {
+      "epoch": 0.20920502092050208,
+      "grad_norm": 7.216926097869873,
+      "learning_rate": 2.7907949790794982e-05,
+      "loss": 0.3094,
+      "step": 50
+    },
+    {
+      "epoch": 0.3138075313807531,
+      "grad_norm": 7.644581317901611,
+      "learning_rate": 2.686192468619247e-05,
+      "loss": 0.2967,
+      "step": 75
+    },
+    {
+      "epoch": 0.41841004184100417,
+      "grad_norm": 4.29410982131958,
+      "learning_rate": 2.5815899581589957e-05,
+      "loss": 0.2768,
+      "step": 100
+    },
+    {
+      "epoch": 0.5230125523012552,
+      "grad_norm": 5.988767147064209,
+      "learning_rate": 2.4769874476987448e-05,
+      "loss": 0.3272,
+      "step": 125
+    },
+    {
+      "epoch": 0.6276150627615062,
+      "grad_norm": 6.748810768127441,
+      "learning_rate": 2.372384937238494e-05,
+      "loss": 0.2716,
+      "step": 150
+    },
+    {
+      "epoch": 0.7322175732217573,
+      "grad_norm": 4.940288066864014,
+      "learning_rate": 2.2677824267782426e-05,
+      "loss": 0.2694,
+      "step": 175
+    },
+    {
+      "epoch": 0.8368200836820083,
+      "grad_norm": 8.23596477508545,
+      "learning_rate": 2.1631799163179917e-05,
+      "loss": 0.2408,
+      "step": 200
+    },
+    {
+      "epoch": 0.9414225941422594,
+      "grad_norm": 5.696824073791504,
+      "learning_rate": 2.0585774058577408e-05,
+      "loss": 0.2343,
+      "step": 225
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9222994314592545,
+      "eval_loss": 0.24964654445648193,
+      "eval_runtime": 6.5874,
+      "eval_samples_per_second": 240.306,
+      "eval_steps_per_second": 2.581,
+      "step": 239
+    },
+    {
+      "epoch": 1.0460251046025104,
+      "grad_norm": 3.607553720474243,
+      "learning_rate": 1.9539748953974895e-05,
+      "loss": 0.2256,
+      "step": 250
+    },
+    {
+      "epoch": 1.1506276150627615,
+      "grad_norm": 3.8270370960235596,
+      "learning_rate": 1.8493723849372386e-05,
+      "loss": 0.1854,
+      "step": 275
+    },
+    {
+      "epoch": 1.2552301255230125,
+      "grad_norm": 5.27299165725708,
+      "learning_rate": 1.7447698744769877e-05,
+      "loss": 0.2133,
+      "step": 300
+    },
+    {
+      "epoch": 1.3598326359832635,
+      "grad_norm": 4.560113906860352,
+      "learning_rate": 1.6401673640167365e-05,
+      "loss": 0.1894,
+      "step": 325
+    },
+    {
+      "epoch": 1.4644351464435146,
+      "grad_norm": 2.413438320159912,
+      "learning_rate": 1.5355648535564852e-05,
+      "loss": 0.1656,
+      "step": 350
+    },
+    {
+      "epoch": 1.5690376569037658,
+      "grad_norm": 3.1509647369384766,
+      "learning_rate": 1.4309623430962344e-05,
+      "loss": 0.1924,
+      "step": 375
+    },
+    {
+      "epoch": 1.6736401673640167,
+      "grad_norm": 4.149273872375488,
+      "learning_rate": 1.3263598326359832e-05,
+      "loss": 0.1712,
+      "step": 400
+    },
+    {
+      "epoch": 1.778242677824268,
+      "grad_norm": 2.754230499267578,
+      "learning_rate": 1.2217573221757323e-05,
+      "loss": 0.2081,
+      "step": 425
+    },
+    {
+      "epoch": 1.8828451882845187,
+      "grad_norm": 5.342536449432373,
+      "learning_rate": 1.1171548117154812e-05,
+      "loss": 0.1723,
+      "step": 450
+    },
+    {
+      "epoch": 1.98744769874477,
+      "grad_norm": 4.396164894104004,
+      "learning_rate": 1.0125523012552301e-05,
+      "loss": 0.1659,
+      "step": 475
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9336702463676564,
+      "eval_loss": 0.2127257138490677,
+      "eval_runtime": 6.0575,
+      "eval_samples_per_second": 261.33,
+      "eval_steps_per_second": 2.806,
+      "step": 478
+    },
+    {
+      "epoch": 2.092050209205021,
+      "grad_norm": 7.640400409698486,
+      "learning_rate": 9.079497907949792e-06,
+      "loss": 0.1083,
+      "step": 500
+    },
+    {
+      "epoch": 2.196652719665272,
+      "grad_norm": 7.712110996246338,
+      "learning_rate": 8.03347280334728e-06,
+      "loss": 0.1285,
+      "step": 525
+    },
+    {
+      "epoch": 2.301255230125523,
+      "grad_norm": 0.2726390063762665,
+      "learning_rate": 6.98744769874477e-06,
+      "loss": 0.1304,
+      "step": 550
+    },
+    {
+      "epoch": 2.405857740585774,
+      "grad_norm": 3.730720043182373,
+      "learning_rate": 5.941422594142259e-06,
+      "loss": 0.1384,
+      "step": 575
+    },
+    {
+      "epoch": 2.510460251046025,
+      "grad_norm": 3.225276470184326,
+      "learning_rate": 4.895397489539749e-06,
+      "loss": 0.1357,
+      "step": 600
+    },
+    {
+      "epoch": 2.6150627615062763,
+      "grad_norm": 4.274138450622559,
+      "learning_rate": 3.849372384937238e-06,
+      "loss": 0.126,
+      "step": 625
+    },
+    {
+      "epoch": 2.719665271966527,
+      "grad_norm": 0.4543670117855072,
+      "learning_rate": 2.803347280334728e-06,
+      "loss": 0.1263,
+      "step": 650
+    },
+    {
+      "epoch": 2.8242677824267783,
+      "grad_norm": 3.624878406524658,
+      "learning_rate": 1.7573221757322176e-06,
+      "loss": 0.137,
+      "step": 675
+    },
+    {
+      "epoch": 2.928870292887029,
+      "grad_norm": 4.657623291015625,
+      "learning_rate": 7.112970711297071e-07,
+      "loss": 0.1329,
+      "step": 700
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.9330385344283006,
+      "eval_loss": 0.22735266387462616,
+      "eval_runtime": 5.5479,
+      "eval_samples_per_second": 285.334,
+      "eval_steps_per_second": 3.064,
+      "step": 717
+    },
+    {
+      "epoch": 3.0,
+      "step": 717,
+      "total_flos": 1055495423686536.0,
+      "train_loss": 0.20476110576752007,
+      "train_runtime": 98.8837,
+      "train_samples_per_second": 346.65,
+      "train_steps_per_second": 7.251
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 717,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1055495423686536.0,
+  "train_batch_size": 48,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_rtx6000_metrics/result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 128 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 25 --report_to none --num_train_epochs 3 --learning_rate 3e-05 --per_device_train_batch_size 48 --per_device_eval_batch_size 96 --gradient_accumulation_steps 1 --dataloader_num_workers 1 --seed 44 --output_dir result_ablation_mbert/vsfc/mBERT/multi_branch_average/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy multi_branch_average --n_heads 4 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05