Upload batch 13/14: vsfc/mBERT/max/seed_42 ... vsfc/mBERT/mha_attention/seed_42

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9336702463676564,
+    "eval_loss": 0.2267763763666153,
+    "eval_runtime": 5.5932,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 283.021,
+    "eval_steps_per_second": 1.788,
+    "total_flos": 1153398965526528.0,
+    "train_loss": 0.23415064422678558,
+    "train_runtime": 47.1045,
+    "train_samples": 11426,
+    "train_samples_per_second": 727.701,
+    "train_steps_per_second": 9.107
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9336702463676564,
+    "eval_loss": 0.2267763763666153,
+    "eval_runtime": 5.5932,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 283.021,
+    "eval_steps_per_second": 1.788
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 42 --output_dir result_ablation_mbert_paper/vsfc/mBERT/max/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy max --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1153398965526528.0,
+    "train_loss": 0.23415064422678558,
+    "train_runtime": 47.1045,
+    "train_samples": 11426,
+    "train_samples_per_second": 727.701,
+    "train_steps_per_second": 9.107
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_42/trainer_state.json

@@ -0,0 +1,125 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 429,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.34965034965034963,
+      "grad_norm": 2.3628718852996826,
+      "learning_rate": 1.766899766899767e-05,
+      "loss": 0.4727,
+      "step": 50
+    },
+    {
+      "epoch": 0.6993006993006993,
+      "grad_norm": 5.217874050140381,
+      "learning_rate": 1.533799533799534e-05,
+      "loss": 0.2792,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9222994314592545,
+      "eval_loss": 0.24157020449638367,
+      "eval_runtime": 6.1353,
+      "eval_samples_per_second": 258.016,
+      "eval_steps_per_second": 1.63,
+      "step": 143
+    },
+    {
+      "epoch": 1.048951048951049,
+      "grad_norm": 4.534977436065674,
+      "learning_rate": 1.3006993006993008e-05,
+      "loss": 0.2609,
+      "step": 150
+    },
+    {
+      "epoch": 1.3986013986013985,
+      "grad_norm": 6.660038948059082,
+      "learning_rate": 1.0675990675990678e-05,
+      "loss": 0.1958,
+      "step": 200
+    },
+    {
+      "epoch": 1.7482517482517483,
+      "grad_norm": 3.937138080596924,
+      "learning_rate": 8.344988344988347e-06,
+      "loss": 0.2138,
+      "step": 250
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9267214150347441,
+      "eval_loss": 0.2210218608379364,
+      "eval_runtime": 6.0659,
+      "eval_samples_per_second": 260.968,
+      "eval_steps_per_second": 1.649,
+      "step": 286
+    },
+    {
+      "epoch": 2.097902097902098,
+      "grad_norm": 5.04971981048584,
+      "learning_rate": 6.013986013986014e-06,
+      "loss": 0.1759,
+      "step": 300
+    },
+    {
+      "epoch": 2.4475524475524475,
+      "grad_norm": 3.143134117126465,
+      "learning_rate": 3.6829836829836835e-06,
+      "loss": 0.1653,
+      "step": 350
+    },
+    {
+      "epoch": 2.797202797202797,
+      "grad_norm": 4.743638038635254,
+      "learning_rate": 1.3519813519813522e-06,
+      "loss": 0.154,
+      "step": 400
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.9336702463676564,
+      "eval_loss": 0.2267763763666153,
+      "eval_runtime": 6.5397,
+      "eval_samples_per_second": 242.062,
+      "eval_steps_per_second": 1.529,
+      "step": 429
+    },
+    {
+      "epoch": 3.0,
+      "step": 429,
+      "total_flos": 1153398965526528.0,
+      "train_loss": 0.23415064422678558,
+      "train_runtime": 47.1045,
+      "train_samples_per_second": 727.701,
+      "train_steps_per_second": 9.107
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 429,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1153398965526528.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_43/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9349336702463676,
+    "eval_loss": 0.22572965919971466,
+    "eval_runtime": 5.6655,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 279.408,
+    "eval_steps_per_second": 1.765,
+    "total_flos": 1153669483536384.0,
+    "train_loss": 0.23497871141055804,
+    "train_runtime": 48.4075,
+    "train_samples": 11426,
+    "train_samples_per_second": 708.113,
+    "train_steps_per_second": 8.862
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_43/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9349336702463676,
+    "eval_loss": 0.22572965919971466,
+    "eval_runtime": 5.6655,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 279.408,
+    "eval_steps_per_second": 1.765
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 43 --output_dir result_ablation_mbert_paper/vsfc/mBERT/max/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy max --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_43/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1153669483536384.0,
+    "train_loss": 0.23497871141055804,
+    "train_runtime": 48.4075,
+    "train_samples": 11426,
+    "train_samples_per_second": 708.113,
+    "train_steps_per_second": 8.862
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_43/trainer_state.json

@@ -0,0 +1,125 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 429,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.34965034965034963,
+      "grad_norm": 3.059544324874878,
+      "learning_rate": 1.766899766899767e-05,
+      "loss": 0.4643,
+      "step": 50
+    },
+    {
+      "epoch": 0.6993006993006993,
+      "grad_norm": 3.559900999069214,
+      "learning_rate": 1.533799533799534e-05,
+      "loss": 0.2919,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9191408717624763,
+      "eval_loss": 0.24622738361358643,
+      "eval_runtime": 6.0755,
+      "eval_samples_per_second": 260.553,
+      "eval_steps_per_second": 1.646,
+      "step": 143
+    },
+    {
+      "epoch": 1.048951048951049,
+      "grad_norm": 2.815946578979492,
+      "learning_rate": 1.3006993006993008e-05,
+      "loss": 0.2539,
+      "step": 150
+    },
+    {
+      "epoch": 1.3986013986013985,
+      "grad_norm": 4.200994968414307,
+      "learning_rate": 1.0675990675990678e-05,
+      "loss": 0.2107,
+      "step": 200
+    },
+    {
+      "epoch": 1.7482517482517483,
+      "grad_norm": 2.537310838699341,
+      "learning_rate": 8.344988344988347e-06,
+      "loss": 0.2111,
+      "step": 250
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.932406822488945,
+      "eval_loss": 0.2130560725927353,
+      "eval_runtime": 5.8672,
+      "eval_samples_per_second": 269.806,
+      "eval_steps_per_second": 1.704,
+      "step": 286
+    },
+    {
+      "epoch": 2.097902097902098,
+      "grad_norm": 2.2481701374053955,
+      "learning_rate": 6.013986013986014e-06,
+      "loss": 0.1788,
+      "step": 300
+    },
+    {
+      "epoch": 2.4475524475524475,
+      "grad_norm": 3.530592679977417,
+      "learning_rate": 3.6829836829836835e-06,
+      "loss": 0.1615,
+      "step": 350
+    },
+    {
+      "epoch": 2.797202797202797,
+      "grad_norm": 3.9793636798858643,
+      "learning_rate": 1.3519813519813522e-06,
+      "loss": 0.1543,
+      "step": 400
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.9349336702463676,
+      "eval_loss": 0.22572965919971466,
+      "eval_runtime": 5.7121,
+      "eval_samples_per_second": 277.131,
+      "eval_steps_per_second": 1.751,
+      "step": 429
+    },
+    {
+      "epoch": 3.0,
+      "step": 429,
+      "total_flos": 1153669483536384.0,
+      "train_loss": 0.23497871141055804,
+      "train_runtime": 48.4075,
+      "train_samples_per_second": 708.113,
+      "train_steps_per_second": 8.862
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 429,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1153669483536384.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/max/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 44 --output_dir result_ablation_mbert_paper/vsfc/mBERT/max/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy max --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9330385344283006,
+    "eval_loss": 0.2111007124185562,
+    "eval_runtime": 5.9493,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 266.082,
+    "eval_steps_per_second": 1.681,
+    "total_flos": 1153398965526528.0,
+    "train_loss": 0.2191808607194807,
+    "train_runtime": 45.739,
+    "train_samples": 11426,
+    "train_samples_per_second": 749.426,
+    "train_steps_per_second": 9.379
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9330385344283006,
+    "eval_loss": 0.2111007124185562,
+    "eval_runtime": 5.9493,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 266.082,
+    "eval_steps_per_second": 1.681
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 42 --output_dir result_ablation_mbert_paper/vsfc/mBERT/mean/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mean --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1153398965526528.0,
+    "train_loss": 0.2191808607194807,
+    "train_runtime": 45.739,
+    "train_samples": 11426,
+    "train_samples_per_second": 749.426,
+    "train_steps_per_second": 9.379
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_42/trainer_state.json

@@ -0,0 +1,125 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 429,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.34965034965034963,
+      "grad_norm": 3.4379467964172363,
+      "learning_rate": 1.766899766899767e-05,
+      "loss": 0.4115,
+      "step": 50
+    },
+    {
+      "epoch": 0.6993006993006993,
+      "grad_norm": 4.643185138702393,
+      "learning_rate": 1.533799533799534e-05,
+      "loss": 0.2678,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9317751105495894,
+      "eval_loss": 0.2103436142206192,
+      "eval_runtime": 6.2172,
+      "eval_samples_per_second": 254.615,
+      "eval_steps_per_second": 1.608,
+      "step": 143
+    },
+    {
+      "epoch": 1.048951048951049,
+      "grad_norm": 5.319267749786377,
+      "learning_rate": 1.3006993006993008e-05,
+      "loss": 0.2493,
+      "step": 150
+    },
+    {
+      "epoch": 1.3986013986013985,
+      "grad_norm": 6.126291751861572,
+      "learning_rate": 1.0675990675990678e-05,
+      "loss": 0.1854,
+      "step": 200
+    },
+    {
+      "epoch": 1.7482517482517483,
+      "grad_norm": 2.6652402877807617,
+      "learning_rate": 8.344988344988347e-06,
+      "loss": 0.206,
+      "step": 250
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9330385344283006,
+      "eval_loss": 0.2055298537015915,
+      "eval_runtime": 5.6993,
+      "eval_samples_per_second": 277.751,
+      "eval_steps_per_second": 1.755,
+      "step": 286
+    },
+    {
+      "epoch": 2.097902097902098,
+      "grad_norm": 3.662348747253418,
+      "learning_rate": 6.013986013986014e-06,
+      "loss": 0.1714,
+      "step": 300
+    },
+    {
+      "epoch": 2.4475524475524475,
+      "grad_norm": 5.0589799880981445,
+      "learning_rate": 3.6829836829836835e-06,
+      "loss": 0.1555,
+      "step": 350
+    },
+    {
+      "epoch": 2.797202797202797,
+      "grad_norm": 5.420856475830078,
+      "learning_rate": 1.3519813519813522e-06,
+      "loss": 0.1479,
+      "step": 400
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.9330385344283006,
+      "eval_loss": 0.2111007124185562,
+      "eval_runtime": 5.4999,
+      "eval_samples_per_second": 287.823,
+      "eval_steps_per_second": 1.818,
+      "step": 429
+    },
+    {
+      "epoch": 3.0,
+      "step": 429,
+      "total_flos": 1153398965526528.0,
+      "train_loss": 0.2191808607194807,
+      "train_runtime": 45.739,
+      "train_samples_per_second": 749.426,
+      "train_steps_per_second": 9.379
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 429,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1153398965526528.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_43/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 43 --output_dir result_ablation_mbert_paper/vsfc/mBERT/mean/seed_43 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mean --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_44/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 44 --output_dir result_ablation_mbert_paper/vsfc/mBERT/mean/seed_44 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mean --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_42/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9330385344283006,
+    "eval_loss": 0.21529895067214966,
+    "eval_runtime": 6.107,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 259.209,
+    "eval_steps_per_second": 1.637,
+    "total_flos": 1163951372889900.0,
+    "train_loss": 0.22224652350365698,
+    "train_runtime": 49.0001,
+    "train_samples": 11426,
+    "train_samples_per_second": 699.549,
+    "train_steps_per_second": 8.755
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_42/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.9330385344283006,
+    "eval_loss": 0.21529895067214966,
+    "eval_runtime": 6.107,
+    "eval_samples": 1583,
+    "eval_samples_per_second": 259.209,
+    "eval_steps_per_second": 1.637
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_42/run_command.txt

@@ -0,0 +1 @@
+/usr/local/bin/python /workspace/run_glue_MHA_gated.py --model_name_or_path bert-base-multilingual-cased --dataset_name tmnam20/ViGLUE --task_name vsfc --max_seq_length 256 --do_train --do_eval --evaluation_strategy epoch --save_strategy no --logging_strategy steps --logging_steps 50 --report_to none --num_train_epochs 3 --learning_rate 2e-05 --per_device_train_batch_size 80 --per_device_eval_batch_size 168 --gradient_accumulation_steps 1 --dataloader_num_workers 2 --seed 42 --output_dir result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_42 --overwrite_output_dir --trust_remote_code true --bf16 --tf32 true --skip_model_save --use_mha true --pooling_strategy mha_attention --n_heads 8 --head_dim 32 --classifier_hidden_dim 512 --head_dropout 0.05

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_42/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1163951372889900.0,
+    "train_loss": 0.22224652350365698,
+    "train_runtime": 49.0001,
+    "train_samples": 11426,
+    "train_samples_per_second": 699.549,
+    "train_steps_per_second": 8.755
+}

mbert_paper_metrics/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_42/trainer_state.json

@@ -0,0 +1,125 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 429,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.34965034965034963,
+      "grad_norm": 3.088902473449707,
+      "learning_rate": 1.766899766899767e-05,
+      "loss": 0.4348,
+      "step": 50
+    },
+    {
+      "epoch": 0.6993006993006993,
+      "grad_norm": 3.8165502548217773,
+      "learning_rate": 1.533799533799534e-05,
+      "loss": 0.2746,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9279848389134555,
+      "eval_loss": 0.21321365237236023,
+      "eval_runtime": 6.7525,
+      "eval_samples_per_second": 234.43,
+      "eval_steps_per_second": 1.481,
+      "step": 143
+    },
+    {
+      "epoch": 1.048951048951049,
+      "grad_norm": 7.52340030670166,
+      "learning_rate": 1.3006993006993008e-05,
+      "loss": 0.2525,
+      "step": 150
+    },
+    {
+      "epoch": 1.3986013986013985,
+      "grad_norm": 5.6060051918029785,
+      "learning_rate": 1.0675990675990678e-05,
+      "loss": 0.1901,
+      "step": 200
+    },
+    {
+      "epoch": 1.7482517482517483,
+      "grad_norm": 3.5827577114105225,
+      "learning_rate": 8.344988344988347e-06,
+      "loss": 0.2054,
+      "step": 250
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9330385344283006,
+      "eval_loss": 0.21123935282230377,
+      "eval_runtime": 6.0771,
+      "eval_samples_per_second": 260.484,
+      "eval_steps_per_second": 1.646,
+      "step": 286
+    },
+    {
+      "epoch": 2.097902097902098,
+      "grad_norm": 2.805939197540283,
+      "learning_rate": 6.013986013986014e-06,
+      "loss": 0.1663,
+      "step": 300
+    },
+    {
+      "epoch": 2.4475524475524475,
+      "grad_norm": 4.410253047943115,
+      "learning_rate": 3.6829836829836835e-06,
+      "loss": 0.153,
+      "step": 350
+    },
+    {
+      "epoch": 2.797202797202797,
+      "grad_norm": 6.72662353515625,
+      "learning_rate": 1.3519813519813522e-06,
+      "loss": 0.1434,
+      "step": 400
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.9330385344283006,
+      "eval_loss": 0.21529895067214966,
+      "eval_runtime": 6.1496,
+      "eval_samples_per_second": 257.416,
+      "eval_steps_per_second": 1.626,
+      "step": 429
+    },
+    {
+      "epoch": 3.0,
+      "step": 429,
+      "total_flos": 1163951372889900.0,
+      "train_loss": 0.22224652350365698,
+      "train_runtime": 49.0001,
+      "train_samples_per_second": 699.549,
+      "train_steps_per_second": 8.755
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 429,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1163951372889900.0,
+  "train_batch_size": 80,
+  "trial_name": null,
+  "trial_params": null
+}