Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1338", "n_embd": 128}

attention_kindself_n_heads2_seed1338/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7416
 0 val perplexity 125697.3672
-0 train 11.753315 (lr=2.5000e-07) (hash(x)=150327452)
-100 val loss 10.3420
-100 val perplexity 31008.8418
-100 train 10.328853 (lr=2.5250e-05) (hash(x)=166441190)
-200 val loss 9.3460
-200 val perplexity 11452.9824
-200 train 9.453581 (lr=5.0000e-05) (hash(x)=166780046)
-300 val loss 8.4134
-300 val perplexity 4507.2549
-300 train 8.403763 (lr=4.9988e-05) (hash(x)=159835303)
-400 val loss 7.9846
-400 val perplexity 2935.4409
-400 train 7.832768 (lr=4.9954e-05) (hash(x)=155040610)
-500 val loss 7.7774
-500 val perplexity 2386.0586
-500 train 7.562558 (lr=4.9896e-05) (hash(x)=130190460)
-600 val loss 7.6696
-600 val perplexity 2142.2061
-600 train 7.707840 (lr=4.9815e-05) (hash(x)=155504036)
-700 val loss 7.6034
-700 val perplexity 2005.0209
-700 train 7.450253 (lr=4.9712e-05) (hash(x)=137347213)
-800 val loss 7.5625
-800 val perplexity 1924.6025
-800 train 7.433071 (lr=4.9585e-05) (hash(x)=143823248)
-900 val loss 7.5305
-900 val perplexity 1864.0834
-900 train 7.644044 (lr=4.9436e-05) (hash(x)=156260416)
-1000 val loss 7.5082
-1000 val perplexity 1822.9496
-1000 train 7.477904 (lr=4.9264e-05) (hash(x)=143734685)
-1100 val loss 7.4909
-1100 val perplexity 1791.7383
-1100 train 7.440104 (lr=4.9070e-05) (hash(x)=160013925)
-1200 val loss 7.4746
-1200 val perplexity 1762.7366
-1200 train 7.243001 (lr=4.8854e-05) (hash(x)=150678249)
-1300 val loss 7.4591
-1300 val perplexity 1735.6089
-1300 train 7.257017 (lr=4.8616e-05) (hash(x)=149073315)
-1400 val loss 7.4199
-1400 val perplexity 1668.8872
-1400 train 7.819283 (lr=4.8356e-05) (hash(x)=175802021)
-1500 val loss 7.3796
-1500 val perplexity 1603.0197
-1500 train 7.631110 (lr=4.8074e-05) (hash(x)=171034639)
-1600 val loss 7.3561
-1600 val perplexity 1565.6826
-1600 train 7.582740 (lr=4.7772e-05) (hash(x)=158681215)
-1700 val loss 7.3446
-1700 val perplexity 1547.8124
-1700 train 7.297821 (lr=4.7448e-05) (hash(x)=152116061)
-1800 val loss 7.3153
-1800 val perplexity 1503.1793
-1800 train 7.247635 (lr=4.7105e-05) (hash(x)=146108145)
-1900 val loss 7.3141
-1900 val perplexity 1501.3834
-1900 train 7.144433 (lr=4.6741e-05) (hash(x)=147598108)
-2000 val loss 7.2828
-2000 val perplexity 1455.0540
-2000 train 7.159031 (lr=4.6357e-05) (hash(x)=154996086)
-2100 val loss 7.2758
-2100 val perplexity 1444.8994
-2100 train 7.088547 (lr=4.5954e-05) (hash(x)=153396183)
-2200 val loss 7.2541
-2200 val perplexity 1413.9039
-2200 train 7.251993 (lr=4.5532e-05) (hash(x)=153885445)
-2300 val loss 7.2741
-2300 val perplexity 1442.4404
-2300 train 7.256250 (lr=4.5091e-05) (hash(x)=159666385)
-2400 val loss 7.2112
-2400 val perplexity 1354.4802
-2400 train 7.179773 (lr=4.4633e-05) (hash(x)=142353087)
-2500 val loss 7.2195
-2500 val perplexity 1365.7396
-2500 train 7.236165 (lr=4.4156e-05) (hash(x)=146491718)
-2600 val loss 7.1954
-2600 val perplexity 1333.2642
-2600 train 7.147064 (lr=4.3663e-05) (hash(x)=150750353)
-2700 val loss 7.1977
-2700 val perplexity 1336.3307
-2700 train 6.934301 (lr=4.3153e-05) (hash(x)=129849193)
-2800 val loss 7.1808
-2800 val perplexity 1313.9878
-2800 train 7.054011 (lr=4.2627e-05) (hash(x)=152767913)
-2900 val loss 7.1768
-2900 val perplexity 1308.7028
-2900 train 6.983736 (lr=4.2085e-05) (hash(x)=146531140)
-3000 val loss 7.1609
-3000 val perplexity 1288.1282
-3000 train 7.214781 (lr=4.1529e-05) (hash(x)=151562048)
-3100 val loss 7.1527
-3100 val perplexity 1277.4952
-3100 train 7.144102 (lr=4.0957e-05) (hash(x)=146001424)
-3200 val loss 7.1358
-3200 val perplexity 1256.1335
-3200 train 7.131488 (lr=4.0373e-05) (hash(x)=166486165)
-3300 val loss 7.1202
-3300 val perplexity 1236.6389
-3300 train 7.046004 (lr=3.9775e-05) (hash(x)=150866680)
-3400 val loss 7.1299
-3400 val perplexity 1248.7705
-3400 train 7.053288 (lr=3.9164e-05) (hash(x)=143900419)
-3500 val loss 7.1033
-3500 val perplexity 1215.9182
-3500 train 6.957945 (lr=3.8541e-05) (hash(x)=148845794)
-3600 val loss 7.1090
-3600 val perplexity 1222.8630
-3600 train 6.980226 (lr=3.7907e-05) (hash(x)=145667796)
-3700 val loss 7.1050
-3700 val perplexity 1218.0392
-3700 train 7.114458 (lr=3.7262e-05) (hash(x)=163563851)
-3800 val loss 7.1221
-3800 val perplexity 1239.0602
-3800 train 7.084368 (lr=3.6608e-05) (hash(x)=147488689)
-3900 val loss 7.1000
-3900 val perplexity 1211.9652
-3900 train 7.077944 (lr=3.5944e-05) (hash(x)=148186608)
-4000 val loss 7.1020
-4000 val perplexity 1214.4360
-4000 train 6.946519 (lr=3.5271e-05) (hash(x)=142970187)
-4100 val loss 7.0786
-4100 val perplexity 1186.2617
-4100 train 7.081099 (lr=3.4590e-05) (hash(x)=141584883)
-4200 val loss 7.0843
-4200 val perplexity 1193.0679
-4200 train 6.903859 (lr=3.3902e-05) (hash(x)=145664585)
-4300 val loss 7.0775
-4300 val perplexity 1184.9812
-4300 train 6.921177 (lr=3.3207e-05) (hash(x)=143736499)
-4400 val loss 7.0813
-4400 val perplexity 1189.5664
-4400 train 6.910584 (lr=3.2507e-05) (hash(x)=151883322)
-4500 val loss 7.0931
-4500 val perplexity 1203.6604
-4500 train 7.027113 (lr=3.1801e-05) (hash(x)=153904871)
-4600 val loss 7.0660
-4600 val perplexity 1171.4612
-4600 train 7.100923 (lr=3.1091e-05) (hash(x)=154893521)
-4700 val loss 7.0486
-4700 val perplexity 1151.2043
-4700 train 7.120266 (lr=3.0377e-05) (hash(x)=152323949)
-4800 val loss 7.0417
-4800 val perplexity 1143.3562
-4800 train 6.989632 (lr=2.9661e-05) (hash(x)=154104619)
-4900 val loss 7.0359
-4900 val perplexity 1136.7014
-4900 train 7.077444 (lr=2.8942e-05) (hash(x)=146311426)
-5000 val loss 7.0312
-5000 val perplexity 1131.3464
-5000 train 7.074374 (lr=2.8221e-05) (hash(x)=156741847)
-5100 val loss 7.0339
-5100 val perplexity 1134.4683
-5100 train 6.836224 (lr=2.7500e-05) (hash(x)=142086346)
-5200 val loss 7.0282
-5200 val perplexity 1127.9674
-5200 train 6.841303 (lr=2.6779e-05) (hash(x)=150265428)
-5300 val loss 7.0328
-5300 val perplexity 1133.2178
-5300 train 6.949815 (lr=2.6058e-05) (hash(x)=151339108)
-5400 val loss 7.0233
-5400 val perplexity 1122.4384
-5400 train 7.012791 (lr=2.5339e-05) (hash(x)=154654372)
-5500 val loss 7.0070
-5500 val perplexity 1104.3318
-5500 train 6.992548 (lr=2.4623e-05) (hash(x)=150575051)
-5600 val loss 7.0122
-5600 val perplexity 1110.1151
-5600 train 6.907769 (lr=2.3909e-05) (hash(x)=140396423)
-5700 val loss 6.9910
-5700 val perplexity 1086.8059
-5700 train 6.885193 (lr=2.3199e-05) (hash(x)=144678758)
-5800 val loss 6.9951
-5800 val perplexity 1091.2894
-5800 train 7.114344 (lr=2.2493e-05) (hash(x)=151992743)
-5900 val loss 6.9942
-5900 val perplexity 1090.2992
-5900 train 6.799440 (lr=2.1793e-05) (hash(x)=144396927)
-6000 val loss 6.9799
-6000 val perplexity 1074.8026
-6000 train 6.948314 (lr=2.1098e-05) (hash(x)=165478625)
-6100 val loss 6.9931
-6100 val perplexity 1089.1113
-6100 train 6.789269 (lr=2.0410e-05) (hash(x)=147088621)
-6200 val loss 6.9827
-6200 val perplexity 1077.7745
-6200 train 6.811790 (lr=1.9729e-05) (hash(x)=140794994)
-6300 val loss 6.9733
-6300 val perplexity 1067.7700
-6300 train 6.887860 (lr=1.9056e-05) (hash(x)=134780906)
-6400 val loss 6.9637
-6400 val perplexity 1057.5562
-6400 train 6.974144 (lr=1.8392e-05) (hash(x)=149023655)
-6500 val loss 6.9550
-6500 val perplexity 1048.3354
-6500 train 6.833264 (lr=1.7738e-05) (hash(x)=147497796)
-6600 val loss 6.9554
-6600 val perplexity 1048.7729
-6600 train 6.844741 (lr=1.7093e-05) (hash(x)=152902689)
-6700 val loss 6.9507
-6700 val perplexity 1043.9287
-6700 train 7.012224 (lr=1.6459e-05) (hash(x)=153846046)
-6800 val loss 6.9446
-6800 val perplexity 1037.5162
-6800 train 7.072952 (lr=1.5836e-05) (hash(x)=158512738)
-6900 val loss 6.9373
-6900 val perplexity 1029.9656
-6900 train 7.254258 (lr=1.5225e-05) (hash(x)=156849968)
-7000 val loss 6.9460
-7000 val perplexity 1038.9885
-7000 train 6.769375 (lr=1.4627e-05) (hash(x)=142395855)
-7100 val loss 6.9355
-7100 val perplexity 1028.1099
-7100 train 6.835872 (lr=1.4043e-05) (hash(x)=147114884)
-7200 val loss 6.9281
-7200 val perplexity 1020.5371
-7200 train 6.944231 (lr=1.3471e-05) (hash(x)=156979839)
-7300 val loss 6.9315
-7300 val perplexity 1023.9854
-7300 train 6.715333 (lr=1.2915e-05) (hash(x)=145584373)
-7400 val loss 6.9387
-7400 val perplexity 1031.4105
-7400 train 6.686437 (lr=1.2373e-05) (hash(x)=141508204)
-7500 val loss 6.9232
-7500 val perplexity 1015.5560
-7500 train 6.973086 (lr=1.1847e-05) (hash(x)=148803965)
-7600 val loss 6.9176
-7600 val perplexity 1009.8733
-7600 train 6.958951 (lr=1.1337e-05) (hash(x)=151019676)
-7700 val loss 6.9107
-7700 val perplexity 1002.9551
-7700 train 6.883573 (lr=1.0844e-05) (hash(x)=143155750)
-7800 val loss 6.9094
-7800 val perplexity 1001.6260
-7800 train 6.919430 (lr=1.0367e-05) (hash(x)=152569653)
-7900 val loss 6.9066
-7900 val perplexity 998.8763
-7900 train 6.761757 (lr=9.9088e-06) (hash(x)=143519455)
-8000 val loss 6.9058
-8000 val perplexity 998.0589
-8000 train 7.089988 (lr=9.4682e-06) (hash(x)=161180944)
-8100 val loss 6.9017
-8100 val perplexity 993.9507
-8100 train 6.942193 (lr=9.0461e-06) (hash(x)=154107345)
-8200 val loss 6.9020
-8200 val perplexity 994.2878
-8200 train 6.913141 (lr=8.6430e-06) (hash(x)=152486517)
-8300 val loss 6.8937
-8300 val perplexity 986.0807
-8300 train 6.929050 (lr=8.2593e-06) (hash(x)=156167749)
-8400 val loss 6.8926
-8400 val perplexity 984.9684
-8400 train 6.948983 (lr=7.8953e-06) (hash(x)=149155006)
-8500 val loss 6.8884
-8500 val perplexity 980.8519
-8500 train 6.867157 (lr=7.5515e-06) (hash(x)=147844390)
-8600 val loss 6.8855
-8600 val perplexity 978.0273
-8600 train 7.149722 (lr=7.2282e-06) (hash(x)=165753320)
-8700 val loss 6.8860
-8700 val perplexity 978.4308
-8700 train 6.695945 (lr=6.9257e-06) (hash(x)=146079979)
-8800 val loss 6.8852
-8800 val perplexity 977.6963
-8800 train 7.105085 (lr=6.6444e-06) (hash(x)=172259509)
-8900 val loss 6.8825
-8900 val perplexity 975.0267
-8900 train 6.578733 (lr=6.3845e-06) (hash(x)=145148314)
-9000 val loss 6.8882
-9000 val perplexity 980.6425
-9000 train 6.826063 (lr=6.1462e-06) (hash(x)=144250633)
-9100 val loss 6.8776
-9100 val perplexity 970.2793
-9100 train 6.986150 (lr=5.9300e-06) (hash(x)=157219797)
-9200 val loss 6.8739
-9200 val perplexity 966.7206
-9200 train 6.840438 (lr=5.7359e-06) (hash(x)=142743778)
-9300 val loss 6.8724
-9300 val perplexity 965.2374
-9300 train 6.752581 (lr=5.5641e-06) (hash(x)=139669771)
-9400 val loss 6.8700
-9400 val perplexity 962.9398
-9400 train 6.804702 (lr=5.4149e-06) (hash(x)=145916843)
-9500 val loss 6.8689
-9500 val perplexity 961.9329
-9500 train 6.787594 (lr=5.2884e-06) (hash(x)=150196125)
-9600 val loss 6.8722
-9600 val perplexity 965.0754
-9600 train 7.153524 (lr=5.1847e-06) (hash(x)=160041419)
-9700 val loss 6.8670
-9700 val perplexity 960.0651
-9700 train 6.683146 (lr=5.1040e-06) (hash(x)=139931627)
-9800 val loss 6.8644
-9800 val perplexity 957.5345
-9800 train 6.816484 (lr=5.0462e-06) (hash(x)=150370792)
-9900 val loss 6.8633
-9900 val perplexity 956.5634
-9900 train 6.752453 (lr=5.0116e-06) (hash(x)=153014886)
-9999 val loss 6.8622
-9999 val perplexity 955.4310
+0 train 11.753315 (lr=1.5000e-07) (hash(x)=150327452)
+100 val loss 10.4564
+100 val perplexity 34767.2266
+100 train 10.447220 (lr=1.5150e-05) (hash(x)=166441190)
+200 val loss 9.8938
+200 val perplexity 19806.2168
+200 train 9.974361 (lr=3.0000e-05) (hash(x)=166780046)
+300 val loss 8.9664
+300 val perplexity 7835.0234
+300 train 8.965942 (lr=2.9993e-05) (hash(x)=159835303)
+400 val loss 8.4649
+400 val perplexity 4745.4341
+400 train 8.350051 (lr=2.9972e-05) (hash(x)=155040610)
+500 val loss 8.1630
+500 val perplexity 3508.5935
+500 train 8.007442 (lr=2.9938e-05) (hash(x)=130190460)
+600 val loss 7.9528
+600 val perplexity 2843.6174
+600 train 8.010097 (lr=2.9889e-05) (hash(x)=155504036)
+700 val loss 7.8085
+700 val perplexity 2461.4197
+700 train 7.697314 (lr=2.9827e-05) (hash(x)=137347213)
+800 val loss 7.7243
+800 val perplexity 2262.5676
+800 train 7.606063 (lr=2.9751e-05) (hash(x)=143823248)
+900 val loss 7.6723
+900 val perplexity 2147.9226
+900 train 7.780948 (lr=2.9662e-05) (hash(x)=156260416)
+1000 val loss 7.6323
+1000 val perplexity 2063.8354
+1000 train 7.610696 (lr=2.9558e-05) (hash(x)=143734685)
+1100 val loss 7.6053
+1100 val perplexity 2008.8862
+1100 train 7.565187 (lr=2.9442e-05) (hash(x)=160013925)
+1200 val loss 7.5891
+1200 val perplexity 1976.4996
+1200 train 7.392840 (lr=2.9312e-05) (hash(x)=150678249)
+1300 val loss 7.5770
+1300 val perplexity 1952.6896
+1300 train 7.409501 (lr=2.9169e-05) (hash(x)=149073315)
+1400 val loss 7.5394
+1400 val perplexity 1880.6127
+1400 train 7.974899 (lr=2.9013e-05) (hash(x)=175802021)
+1500 val loss 7.5057
+1500 val perplexity 1818.3042
+1500 train 7.747274 (lr=2.8845e-05) (hash(x)=171034639)
+1600 val loss 7.4841
+1600 val perplexity 1779.5776
+1600 train 7.699660 (lr=2.8663e-05) (hash(x)=158681215)
+1700 val loss 7.4604
+1700 val perplexity 1737.9037
+1700 train 7.422320 (lr=2.8469e-05) (hash(x)=152116061)
+1800 val loss 7.4415
+1800 val perplexity 1705.2765
+1800 train 7.385502 (lr=2.8263e-05) (hash(x)=146108145)
+1900 val loss 7.4210
+1900 val perplexity 1670.7145
+1900 train 7.272210 (lr=2.8044e-05) (hash(x)=147598108)
+2000 val loss 7.4019
+2000 val perplexity 1639.0876
+2000 train 7.276218 (lr=2.7814e-05) (hash(x)=154996086)
+2100 val loss 7.3911
+2100 val perplexity 1621.4485
+2100 train 7.192940 (lr=2.7572e-05) (hash(x)=153396183)
+2200 val loss 7.3393
+2200 val perplexity 1539.6034
+2200 train 7.329708 (lr=2.7319e-05) (hash(x)=153885445)
+2300 val loss 7.3191
+2300 val perplexity 1508.8842
+2300 train 7.300901 (lr=2.7055e-05) (hash(x)=159666385)
+2400 val loss 7.2834
+2400 val perplexity 1455.9938
+2400 train 7.237995 (lr=2.6780e-05) (hash(x)=142353087)
+2500 val loss 7.2604
+2500 val perplexity 1422.8267
+2500 train 7.265779 (lr=2.6494e-05) (hash(x)=146491718)
+2600 val loss 7.2472
+2600 val perplexity 1404.1697
+2600 train 7.196903 (lr=2.6198e-05) (hash(x)=150750353)
+2700 val loss 7.2248
+2700 val perplexity 1373.0660
+2700 train 6.951437 (lr=2.5892e-05) (hash(x)=129849193)
+2800 val loss 7.2092
+2800 val perplexity 1351.7651
+2800 train 7.081229 (lr=2.5576e-05) (hash(x)=152767913)
+2900 val loss 7.1980
+2900 val perplexity 1336.7666
+2900 train 6.993177 (lr=2.5251e-05) (hash(x)=146531140)
+3000 val loss 7.1672
+3000 val perplexity 1296.2114
+3000 train 7.227105 (lr=2.4917e-05) (hash(x)=151562048)
+3100 val loss 7.1410
+3100 val perplexity 1262.7070
+3100 train 7.134817 (lr=2.4574e-05) (hash(x)=146001424)
+3200 val loss 7.1243
+3200 val perplexity 1241.7686
+3200 train 7.119705 (lr=2.4224e-05) (hash(x)=166486165)
+3300 val loss 7.1074
+3300 val perplexity 1220.9246
+3300 train 7.034734 (lr=2.3865e-05) (hash(x)=150866680)
+3400 val loss 7.1016
+3400 val perplexity 1213.9016
+3400 train 7.019073 (lr=2.3498e-05) (hash(x)=143900419)
+3500 val loss 7.0881
+3500 val perplexity 1197.5924
+3500 train 6.940794 (lr=2.3125e-05) (hash(x)=148845794)
+3600 val loss 7.0806
+3600 val perplexity 1188.7073
+3600 train 6.947712 (lr=2.2744e-05) (hash(x)=145667796)
+3700 val loss 7.0681
+3700 val perplexity 1173.8868
+3700 train 7.084729 (lr=2.2357e-05) (hash(x)=163563851)
+3800 val loss 7.0660
+3800 val perplexity 1171.4529
+3800 train 7.021557 (lr=2.1965e-05) (hash(x)=147488689)
+3900 val loss 7.0489
+3900 val perplexity 1151.5602
+3900 train 7.024464 (lr=2.1566e-05) (hash(x)=148186608)
+4000 val loss 7.0363
+4000 val perplexity 1137.1958
+4000 train 6.876589 (lr=2.1162e-05) (hash(x)=142970187)
+4100 val loss 7.0211
+4100 val perplexity 1120.0314
+4100 train 7.025841 (lr=2.0754e-05) (hash(x)=141584883)
+4200 val loss 7.0113
+4200 val perplexity 1109.0807
+4200 train 6.820885 (lr=2.0341e-05) (hash(x)=145664585)
+4300 val loss 7.0026
+4300 val perplexity 1099.4919
+4300 train 6.850004 (lr=1.9924e-05) (hash(x)=143736499)
+4400 val loss 7.0038
+4400 val perplexity 1100.8297
+4400 train 6.842593 (lr=1.9504e-05) (hash(x)=151883322)
+4500 val loss 6.9983
+4500 val perplexity 1094.7209
+4500 train 6.933634 (lr=1.9081e-05) (hash(x)=153904871)
+4600 val loss 6.9911
+4600 val perplexity 1086.9075
+4600 train 7.039299 (lr=1.8655e-05) (hash(x)=154893521)
+4700 val loss 6.9738
+4700 val perplexity 1068.2222
+4700 train 7.047583 (lr=1.8226e-05) (hash(x)=152323949)
+4800 val loss 6.9590
+4800 val perplexity 1052.5825
+4800 train 6.904230 (lr=1.7796e-05) (hash(x)=154104619)
+4900 val loss 6.9583
+4900 val perplexity 1051.8029
+4900 train 6.992964 (lr=1.7365e-05) (hash(x)=146311426)
+5000 val loss 6.9458
+5000 val perplexity 1038.7706
+5000 train 6.989071 (lr=1.6933e-05) (hash(x)=156741847)
+5100 val loss 6.9606
+5100 val perplexity 1054.3060
+5100 train 6.759447 (lr=1.6500e-05) (hash(x)=142086346)
+5200 val loss 6.9412
+5200 val perplexity 1033.9725
+5200 train 6.749950 (lr=1.6067e-05) (hash(x)=150265428)
+5300 val loss 6.9325
+5300 val perplexity 1025.0934
+5300 train 6.848131 (lr=1.5635e-05) (hash(x)=151339108)
+5400 val loss 6.9238
+5400 val perplexity 1016.1281
+5400 train 6.924018 (lr=1.5204e-05) (hash(x)=154654372)
+5500 val loss 6.9107
+5500 val perplexity 1002.9852
+5500 train 6.898966 (lr=1.4774e-05) (hash(x)=150575051)
+5600 val loss 6.9009
+5600 val perplexity 993.1709
+5600 train 6.794309 (lr=1.4345e-05) (hash(x)=140396423)
+5700 val loss 6.8927
+5700 val perplexity 985.0370
+5700 train 6.787779 (lr=1.3919e-05) (hash(x)=144678758)
+5800 val loss 6.8807
+5800 val perplexity 973.2927
+5800 train 7.008542 (lr=1.3496e-05) (hash(x)=151992743)
+5900 val loss 6.8796
+5900 val perplexity 972.2138
+5900 train 6.681162 (lr=1.3076e-05) (hash(x)=144396927)
+6000 val loss 6.8718
+6000 val perplexity 964.6475
+6000 train 6.838612 (lr=1.2659e-05) (hash(x)=165478625)
+6100 val loss 6.8823
+6100 val perplexity 974.8538
+6100 train 6.666516 (lr=1.2246e-05) (hash(x)=147088621)
+6200 val loss 6.8625
+6200 val perplexity 955.7655
+6200 train 6.680235 (lr=1.1838e-05) (hash(x)=140794994)
+6300 val loss 6.8502
+6300 val perplexity 944.0838
+6300 train 6.760979 (lr=1.1434e-05) (hash(x)=134780906)
+6400 val loss 6.8387
+6400 val perplexity 933.2811
+6400 train 6.842179 (lr=1.1035e-05) (hash(x)=149023655)
+6500 val loss 6.8310
+6500 val perplexity 926.1088
+6500 train 6.720614 (lr=1.0643e-05) (hash(x)=147497796)
+6600 val loss 6.8290
+6600 val perplexity 924.2899
+6600 train 6.706747 (lr=1.0256e-05) (hash(x)=152902689)
+6700 val loss 6.8244
+6700 val perplexity 920.0207
+6700 train 6.887218 (lr=9.8753e-06) (hash(x)=153846046)
+6800 val loss 6.8080
+6800 val perplexity 905.0447
+6800 train 6.937702 (lr=9.5017e-06) (hash(x)=158512738)
+6900 val loss 6.8018
+6900 val perplexity 899.4612
+6900 train 7.154380 (lr=9.1353e-06) (hash(x)=156849968)
+7000 val loss 6.7952
+7000 val perplexity 893.5900
+7000 train 6.618436 (lr=8.7764e-06) (hash(x)=142395855)
+7100 val loss 6.7909
+7100 val perplexity 889.7031
+7100 train 6.678675 (lr=8.4255e-06) (hash(x)=147114884)
+7200 val loss 6.7953
+7200 val perplexity 893.6625
+7200 train 6.817767 (lr=8.0829e-06) (hash(x)=156979839)
+7300 val loss 6.7830
+7300 val perplexity 882.7558
+7300 train 6.563062 (lr=7.7489e-06) (hash(x)=145584373)
+7400 val loss 6.7786
+7400 val perplexity 878.8247
+7400 train 6.524684 (lr=7.4239e-06) (hash(x)=141508204)
+7500 val loss 6.7729
+7500 val perplexity 873.8824
+7500 train 6.826341 (lr=7.1083e-06) (hash(x)=148803965)
+7600 val loss 6.7658
+7600 val perplexity 867.6932
+7600 train 6.819306 (lr=6.8023e-06) (hash(x)=151019676)
+7700 val loss 6.7569
+7700 val perplexity 859.9560
+7700 train 6.733743 (lr=6.5062e-06) (hash(x)=143155750)
+7800 val loss 6.7511
+7800 val perplexity 855.0184
+7800 train 6.764259 (lr=6.2205e-06) (hash(x)=152569653)
+7900 val loss 6.7497
+7900 val perplexity 853.8071
+7900 train 6.591384 (lr=5.9453e-06) (hash(x)=143519455)
+8000 val loss 6.7448
+8000 val perplexity 849.6033
+8000 train 6.939550 (lr=5.6809e-06) (hash(x)=161180944)
+8100 val loss 6.7418
+8100 val perplexity 847.0533
+8100 train 6.779875 (lr=5.4277e-06) (hash(x)=154107345)
+8200 val loss 6.7415
+8200 val perplexity 846.8283
+8200 train 6.754670 (lr=5.1858e-06) (hash(x)=152486517)
+8300 val loss 6.7322
+8300 val perplexity 838.9739
+8300 train 6.769043 (lr=4.9556e-06) (hash(x)=156167749)
+8400 val loss 6.7278
+8400 val perplexity 835.3394
+8400 train 6.786001 (lr=4.7372e-06) (hash(x)=149155006)
+8500 val loss 6.7232
+8500 val perplexity 831.4517
+8500 train 6.695972 (lr=4.5309e-06) (hash(x)=147844390)
+8600 val loss 6.7186
+8600 val perplexity 827.6779
+8600 train 6.966806 (lr=4.3369e-06) (hash(x)=165753320)
+8700 val loss 6.7181
+8700 val perplexity 827.2810
+8700 train 6.516498 (lr=4.1554e-06) (hash(x)=146079979)
+8800 val loss 6.7177
+8800 val perplexity 826.8961
+8800 train 6.937664 (lr=3.9866e-06) (hash(x)=172259509)
+8900 val loss 6.7141
+8900 val perplexity 823.9076
+8900 train 6.411192 (lr=3.8307e-06) (hash(x)=145148314)
+9000 val loss 6.7114
+9000 val perplexity 821.6909
+9000 train 6.640437 (lr=3.6877e-06) (hash(x)=144250633)
+9100 val loss 6.7066
+9100 val perplexity 817.8230
+9100 train 6.797017 (lr=3.5580e-06) (hash(x)=157219797)
+9200 val loss 6.7057
+9200 val perplexity 817.0325
+9200 train 6.665264 (lr=3.4415e-06) (hash(x)=142743778)
+9300 val loss 6.6999
+9300 val perplexity 812.3007
+9300 train 6.579364 (lr=3.3385e-06) (hash(x)=139669771)
+9400 val loss 6.6991
+9400 val perplexity 811.6924
+9400 train 6.639892 (lr=3.2490e-06) (hash(x)=145916843)
+9500 val loss 6.6944
+9500 val perplexity 807.8905
+9500 train 6.609858 (lr=3.1730e-06) (hash(x)=150196125)
+9600 val loss 6.6925
+9600 val perplexity 806.3307
+9600 train 6.956125 (lr=3.1108e-06) (hash(x)=160041419)
+9700 val loss 6.6914
+9700 val perplexity 805.4365
+9700 train 6.495003 (lr=3.0624e-06) (hash(x)=139931627)
+9800 val loss 6.6895
+9800 val perplexity 803.9136
+9800 train 6.644150 (lr=3.0277e-06) (hash(x)=150370792)
+9900 val loss 6.6886
+9900 val perplexity 803.1917
+9900 train 6.569376 (lr=3.0069e-06) (hash(x)=153014886)
+9999 val loss 6.6859
+9999 val perplexity 801.0265

attention_kindself_n_heads2_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:162ddaf18e89e228ba6652b011f18b3fbe6ea4dbd15b1927a464f0b1261d1618
+oid sha256:93c616f73004d6b6d9c51119e3770122d4a6c8058946bd88b2fa4aa58af0c30d
 size 38587970

attention_kindself_n_heads2_seed1338/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:081e193042470f48a4364138916cd15f9e7a8a14ac5cd03770c2edee904f9123
+oid sha256:930852f791917058581874324175321f714ac954226c837bb6c1732ac9f94e42
 size 38587970

attention_kindself_n_heads2_seed1338/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e2cf7e7d8da54c074e0e2d65d09569394851c94956d6a02075c9763d6c8e36a
+oid sha256:9d161df8aafa2022a262df60c7ca1ae3647ca9ac6fcdae8d82fc70478c36e7b2
 size 38587970

attention_kindself_n_heads2_seed1338/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1492e55ab8c618c621a1011bdaab52c625084b440cd4698cbc66d65a782b10af
+oid sha256:bef4a08d067cf787e7977f6ff4fa65f8f51289f98a74e912ca829b8e73dbc117
 size 38587970

attention_kindself_n_heads2_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b138f1a8c51d246cb28f518fe899ca98d36455537a633967c69559581e6a74e5
+oid sha256:07e95cffe880959fe21d01be91c2d672b0a8961a34e57fe4fa12e277a9134c32
 size 70895430

attention_kindself_n_heads2_seed1338/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd574b2deede7ffd710dde6911c704c358d5dcb400314372a424fcf3cc9a56fd
+oid sha256:e582155475d08ef8246148d6999df45b3eaed03c25adcb02dae2b4d81b8feaf5
 size 70895430

attention_kindself_n_heads2_seed1338/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5ffea7152403ad721e061dfae3afd353afaa0a69ab1a376e22731e6c936c489
+oid sha256:edacfe1eb0281a18605dca0a126c99c3e22ff76bc2fa7fd24aa0dcee9308af2d
 size 70895430

attention_kindself_n_heads2_seed1338/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1db75df11392c74683e75465555e72bb02c2978e80a3a78d67f0a44440e1b501
+oid sha256:9cf7d5506d8fa88301e8f268a344f26260fd87e811990ac25dcb63f2d10b8745
 size 70895430