Upload ablation summaries

mbert_paper_metrics/docs/ablation_results.csv

@@ -73,14 +73,20 @@ result_ablation_mbert_paper,sst2,mBERT,gated_multi_branch,43,eval_accuracy,0.879
 result_ablation_mbert_paper,sst2,mBERT,gated_multi_branch,44,eval_accuracy,0.875,0.3715299665927887,0.2534543433457448,3.0,872,/workspace/result_ablation_mbert_paper/sst2/mBERT/gated_multi_branch/seed_44/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,hf_sequence_classifier,42,eval_accuracy,0.932406822488945,0.22124937176704407,0.2343874844637784,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/hf_sequence_classifier/seed_42/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,hf_sequence_classifier,43,eval_accuracy,0.9317751105495894,0.2234826385974884,0.2347770626450474,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/hf_sequence_classifier/seed_43/all_results.json
+result_ablation_mbert_paper,vsfc,mBERT,hf_sequence_classifier,44,eval_accuracy,0.9311433986102338,0.22898335754871368,0.23928687622497133,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/hf_sequence_classifier/seed_44/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,cls,42,eval_accuracy,0.932406822488945,0.21786975860595703,0.22804758987782442,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/cls/seed_42/all_results.json
+result_ablation_mbert_paper,vsfc,mBERT,cls,43,eval_accuracy,0.9374605180037903,0.22002610564231873,0.22090962772324924,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/cls/seed_43/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,cls,44,eval_accuracy,0.934301958307012,0.21943630278110504,0.232700464608786,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/cls/seed_44/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,mean,42,eval_accuracy,0.9330385344283006,0.2111007124185562,0.2191808607194807,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_42/all_results.json
+result_ablation_mbert_paper,vsfc,mBERT,mean,44,eval_accuracy,0.932406822488945,0.21627604961395264,0.22658519144658443,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_44/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,max,42,eval_accuracy,0.9336702463676564,0.2267763763666153,0.23415064422678558,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/max/seed_42/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,max,43,eval_accuracy,0.9349336702463676,0.22572965919971466,0.23497871141055804,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/max/seed_43/all_results.json
+result_ablation_mbert_paper,vsfc,mBERT,max,44,eval_accuracy,0.9336702463676564,0.2205778807401657,0.23737809152314157,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/max/seed_44/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,attention,44,eval_accuracy,0.9368288060644346,0.20899568498134613,0.22297021336766668,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/attention/seed_44/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,mha_attention,42,eval_accuracy,0.9330385344283006,0.21529895067214966,0.22224652350365698,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_42/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,mha_attention,43,eval_accuracy,0.9330385344283006,0.21815043687820435,0.22466682554124953,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_43/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,mha_attention,44,eval_accuracy,0.934301958307012,0.21530689299106598,0.22445858664168067,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_44/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,multi_branch_average,42,eval_accuracy,0.9355653821857233,0.2141973078250885,0.21831525011218234,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/multi_branch_average/seed_42/all_results.json
+result_ablation_mbert_paper,vsfc,mBERT,gated_multi_branch,42,eval_accuracy,0.9374605180037903,0.22037982940673828,0.2221188189544322,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/gated_multi_branch/seed_42/all_results.json
+result_ablation_mbert_paper,vsfc,mBERT,gated_multi_branch,43,eval_accuracy,0.932406822488945,0.2144102156162262,0.22194261373061955,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/gated_multi_branch/seed_43/all_results.json
 result_ablation_mbert_paper,vsfc,mBERT,gated_multi_branch,44,eval_accuracy,0.9279848389134555,0.21946457028388977,0.2245002004094335,3.0,1583,/workspace/result_ablation_mbert_paper/vsfc/mBERT/gated_multi_branch/seed_44/all_results.json

mbert_paper_metrics/docs/ablation_results_aggregate.csv

@@ -24,10 +24,10 @@ result_ablation_mbert_paper,sst2,mBERT,mean,eval_accuracy,3,0.8807339449541284,0
 result_ablation_mbert_paper,sst2,mBERT,mha_attention,eval_accuracy,3,0.8761467889908258,0.001986296797670735,0.8738532110091743,0.8772935779816514
 result_ablation_mbert_paper,sst2,mBERT,multi_branch_average,eval_accuracy,3,0.8814984709480123,0.01288969059639708,0.8704128440366973,0.8956422018348624
 result_ablation_mbert_paper,vsfc,mBERT,attention,eval_accuracy,1,0.9368288060644346,0.0,0.9368288060644346,0.9368288060644346
-result_ablation_mbert_paper,vsfc,mBERT,cls,eval_accuracy,2,0.9333543903979785,0.0013400633882246875,0.932406822488945,0.934301958307012
-result_ablation_mbert_paper,vsfc,mBERT,gated_multi_branch,eval_accuracy,1,0.9279848389134555,0.0,0.9279848389134555,0.9279848389134555
-result_ablation_mbert_paper,vsfc,mBERT,hf_sequence_classifier,eval_accuracy,2,0.9320909665192672,0.0004466877960748697,0.9317751105495894,0.932406822488945
-result_ablation_mbert_paper,vsfc,mBERT,max,eval_accuracy,2,0.934301958307012,0.0008933755921497394,0.9336702463676564,0.9349336702463676
-result_ablation_mbert_paper,vsfc,mBERT,mean,eval_accuracy,1,0.9330385344283006,0.0,0.9330385344283006,0.9330385344283006
+result_ablation_mbert_paper,vsfc,mBERT,cls,eval_accuracy,3,0.9347230995999157,0.0025530334076610166,0.932406822488945,0.9374605180037903
+result_ablation_mbert_paper,vsfc,mBERT,gated_multi_branch,eval_accuracy,3,0.9326173931353969,0.004741347757084709,0.9279848389134555,0.9374605180037903
+result_ablation_mbert_paper,vsfc,mBERT,hf_sequence_classifier,eval_accuracy,3,0.9317751105495894,0.0006317119393556281,0.9311433986102338,0.932406822488945
+result_ablation_mbert_paper,vsfc,mBERT,max,eval_accuracy,3,0.9340913876605601,0.0007294381164745449,0.9336702463676564,0.9349336702463676
+result_ablation_mbert_paper,vsfc,mBERT,mean,eval_accuracy,2,0.9327226784586229,0.0004466877960748697,0.932406822488945,0.9330385344283006
 result_ablation_mbert_paper,vsfc,mBERT,mha_attention,eval_accuracy,3,0.9334596757212045,0.0007294381164746089,0.9330385344283006,0.934301958307012
 result_ablation_mbert_paper,vsfc,mBERT,multi_branch_average,eval_accuracy,1,0.9355653821857233,0.0,0.9355653821857233,0.9355653821857233

mbert_paper_metrics/docs/ablation_summary.md

@@ -30,11 +30,11 @@ Main metric is selected per task: CoLA uses Matthews correlation; MRPC/QQP/STSB
 | result_ablation_mbert_paper | sst2 | mBERT | mha_attention | eval_accuracy | 3 | 0.8761 | 0.0020 | 0.8739 | 0.8773 |
 | result_ablation_mbert_paper | sst2 | mBERT | multi_branch_average | eval_accuracy | 3 | 0.8815 | 0.0129 | 0.8704 | 0.8956 |
 | result_ablation_mbert_paper | vsfc | mBERT | attention | eval_accuracy | 1 | 0.9368 | 0.0000 | 0.9368 | 0.9368 |
-| result_ablation_mbert_paper | vsfc | mBERT | cls | eval_accuracy | 2 | 0.9334 | 0.0013 | 0.9324 | 0.9343 |
-| result_ablation_mbert_paper | vsfc | mBERT | gated_multi_branch | eval_accuracy | 1 | 0.9280 | 0.0000 | 0.9280 | 0.9280 |
-| result_ablation_mbert_paper | vsfc | mBERT | hf_sequence_classifier | eval_accuracy | 2 | 0.9321 | 0.0004 | 0.9318 | 0.9324 |
-| result_ablation_mbert_paper | vsfc | mBERT | max | eval_accuracy | 2 | 0.9343 | 0.0009 | 0.9337 | 0.9349 |
-| result_ablation_mbert_paper | vsfc | mBERT | mean | eval_accuracy | 1 | 0.9330 | 0.0000 | 0.9330 | 0.9330 |
+| result_ablation_mbert_paper | vsfc | mBERT | cls | eval_accuracy | 3 | 0.9347 | 0.0026 | 0.9324 | 0.9375 |
+| result_ablation_mbert_paper | vsfc | mBERT | gated_multi_branch | eval_accuracy | 3 | 0.9326 | 0.0047 | 0.9280 | 0.9375 |
+| result_ablation_mbert_paper | vsfc | mBERT | hf_sequence_classifier | eval_accuracy | 3 | 0.9318 | 0.0006 | 0.9311 | 0.9324 |
+| result_ablation_mbert_paper | vsfc | mBERT | max | eval_accuracy | 3 | 0.9341 | 0.0007 | 0.9337 | 0.9349 |
+| result_ablation_mbert_paper | vsfc | mBERT | mean | eval_accuracy | 2 | 0.9327 | 0.0004 | 0.9324 | 0.9330 |
 | result_ablation_mbert_paper | vsfc | mBERT | mha_attention | eval_accuracy | 3 | 0.9335 | 0.0007 | 0.9330 | 0.9343 |
 | result_ablation_mbert_paper | vsfc | mBERT | multi_branch_average | eval_accuracy | 1 | 0.9356 | 0.0000 | 0.9356 | 0.9356 |
 
@@ -53,10 +53,10 @@ Main metric is selected per task: CoLA uses Matthews correlation; MRPC/QQP/STSB
 | result_ablation_mbert_paper | sst2 | mBERT | mha_attention | 0.8758 | 0.8761 | -0.0004 |
 | result_ablation_mbert_paper | sst2 | mBERT | multi_branch_average | 0.8758 | 0.8815 | -0.0057 |
 | result_ablation_mbert_paper | sst2 | mBERT | hf_sequence_classifier | 0.8758 | 0.8788 | -0.0031 |
-| result_ablation_mbert_paper | vsfc | mBERT | attention | 0.9280 | 0.9368 | -0.0088 |
-| result_ablation_mbert_paper | vsfc | mBERT | mha_attention | 0.9280 | 0.9335 | -0.0055 |
-| result_ablation_mbert_paper | vsfc | mBERT | multi_branch_average | 0.9280 | 0.9356 | -0.0076 |
-| result_ablation_mbert_paper | vsfc | mBERT | hf_sequence_classifier | 0.9280 | 0.9321 | -0.0041 |
+| result_ablation_mbert_paper | vsfc | mBERT | attention | 0.9326 | 0.9368 | -0.0042 |
+| result_ablation_mbert_paper | vsfc | mBERT | mha_attention | 0.9326 | 0.9335 | -0.0008 |
+| result_ablation_mbert_paper | vsfc | mBERT | multi_branch_average | 0.9326 | 0.9356 | -0.0029 |
+| result_ablation_mbert_paper | vsfc | mBERT | hf_sequence_classifier | 0.9326 | 0.9318 | 0.0008 |
 
 ## Raw Runs
 | source | task | model | strategy | seed | metric | score | eval_loss | train_loss | epoch | eval_samples | path |
@@ -135,14 +135,20 @@ Main metric is selected per task: CoLA uses Matthews correlation; MRPC/QQP/STSB
 | result_ablation_mbert_paper | sst2 | mBERT | gated_multi_branch | 44.0000 | eval_accuracy | 0.8750 | 0.3715 | 0.2535 | 3.0000 | 872 | /workspace/result_ablation_mbert_paper/sst2/mBERT/gated_multi_branch/seed_44/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | hf_sequence_classifier | 42.0000 | eval_accuracy | 0.9324 | 0.2212 | 0.2344 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/hf_sequence_classifier/seed_42/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | hf_sequence_classifier | 43.0000 | eval_accuracy | 0.9318 | 0.2235 | 0.2348 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/hf_sequence_classifier/seed_43/all_results.json |
+| result_ablation_mbert_paper | vsfc | mBERT | hf_sequence_classifier | 44.0000 | eval_accuracy | 0.9311 | 0.2290 | 0.2393 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/hf_sequence_classifier/seed_44/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | cls | 42.0000 | eval_accuracy | 0.9324 | 0.2179 | 0.2280 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/cls/seed_42/all_results.json |
+| result_ablation_mbert_paper | vsfc | mBERT | cls | 43.0000 | eval_accuracy | 0.9375 | 0.2200 | 0.2209 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/cls/seed_43/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | cls | 44.0000 | eval_accuracy | 0.9343 | 0.2194 | 0.2327 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/cls/seed_44/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | mean | 42.0000 | eval_accuracy | 0.9330 | 0.2111 | 0.2192 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_42/all_results.json |
+| result_ablation_mbert_paper | vsfc | mBERT | mean | 44.0000 | eval_accuracy | 0.9324 | 0.2163 | 0.2266 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/mean/seed_44/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | max | 42.0000 | eval_accuracy | 0.9337 | 0.2268 | 0.2342 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/max/seed_42/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | max | 43.0000 | eval_accuracy | 0.9349 | 0.2257 | 0.2350 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/max/seed_43/all_results.json |
+| result_ablation_mbert_paper | vsfc | mBERT | max | 44.0000 | eval_accuracy | 0.9337 | 0.2206 | 0.2374 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/max/seed_44/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | attention | 44.0000 | eval_accuracy | 0.9368 | 0.2090 | 0.2230 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/attention/seed_44/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | mha_attention | 42.0000 | eval_accuracy | 0.9330 | 0.2153 | 0.2222 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_42/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | mha_attention | 43.0000 | eval_accuracy | 0.9330 | 0.2182 | 0.2247 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_43/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | mha_attention | 44.0000 | eval_accuracy | 0.9343 | 0.2153 | 0.2245 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/mha_attention/seed_44/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | multi_branch_average | 42.0000 | eval_accuracy | 0.9356 | 0.2142 | 0.2183 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/multi_branch_average/seed_42/all_results.json |
+| result_ablation_mbert_paper | vsfc | mBERT | gated_multi_branch | 42.0000 | eval_accuracy | 0.9375 | 0.2204 | 0.2221 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/gated_multi_branch/seed_42/all_results.json |
+| result_ablation_mbert_paper | vsfc | mBERT | gated_multi_branch | 43.0000 | eval_accuracy | 0.9324 | 0.2144 | 0.2219 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/gated_multi_branch/seed_43/all_results.json |
 | result_ablation_mbert_paper | vsfc | mBERT | gated_multi_branch | 44.0000 | eval_accuracy | 0.9280 | 0.2195 | 0.2245 | 3.0000 | 1583 | /workspace/result_ablation_mbert_paper/vsfc/mBERT/gated_multi_branch/seed_44/all_results.json |