Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1341", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1341", "n_embd": 128}

attention_kindself_n_heads2_seed1341/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7485
 0 val perplexity 126564.2812
-0 train 11.765417 (lr=7.5000e-07) (hash(x)=145079536)
-100 val loss 9.4784
-100 val perplexity 13074.5156
-100 train 9.419608 (lr=7.5750e-05) (hash(x)=154745873)
-200 val loss 7.8020
-200 val perplexity 2445.3750
-200 train 7.989964 (lr=1.5000e-04) (hash(x)=155800595)
-300 val loss 7.6031
-300 val perplexity 2004.4130
-300 train 7.563851 (lr=1.4997e-04) (hash(x)=148595389)
-400 val loss 7.5158
-400 val perplexity 1836.9117
-400 train 7.429459 (lr=1.4986e-04) (hash(x)=145606733)
-500 val loss 7.4147
-500 val perplexity 1660.1486
-500 train 7.209660 (lr=1.4969e-04) (hash(x)=138221231)
-600 val loss 7.3171
-600 val perplexity 1505.7720
-600 train 7.316947 (lr=1.4945e-04) (hash(x)=150367139)
-700 val loss 7.2359
-700 val perplexity 1388.3401
-700 train 7.397117 (lr=1.4913e-04) (hash(x)=155579314)
-800 val loss 7.1930
-800 val perplexity 1330.1278
-800 train 7.265964 (lr=1.4876e-04) (hash(x)=155747374)
-900 val loss 7.1470
-900 val perplexity 1270.2546
-900 train 7.368423 (lr=1.4831e-04) (hash(x)=159334575)
-1000 val loss 7.1074
-1000 val perplexity 1220.9775
-1000 train 6.987814 (lr=1.4779e-04) (hash(x)=140604760)
-1100 val loss 7.0742
-1100 val perplexity 1181.1272
-1100 train 6.914634 (lr=1.4721e-04) (hash(x)=146208052)
-1200 val loss 7.0423
-1200 val perplexity 1143.9719
-1200 train 6.952001 (lr=1.4656e-04) (hash(x)=148404734)
-1300 val loss 7.0121
-1300 val perplexity 1109.9885
-1300 train 6.945094 (lr=1.4585e-04) (hash(x)=155681970)
-1400 val loss 6.9783
-1400 val perplexity 1073.1346
-1400 train 6.930018 (lr=1.4507e-04) (hash(x)=148115934)
-1500 val loss 6.9871
-1500 val perplexity 1082.5601
-1500 train 7.123736 (lr=1.4422e-04) (hash(x)=157074034)
-1600 val loss 6.9358
-1600 val perplexity 1028.4413
-1600 train 6.715860 (lr=1.4332e-04) (hash(x)=137464699)
-1700 val loss 6.9239
-1700 val perplexity 1016.2919
-1700 train 7.034598 (lr=1.4235e-04) (hash(x)=166955614)
-1800 val loss 6.9125
-1800 val perplexity 1004.7353
-1800 train 6.741632 (lr=1.4131e-04) (hash(x)=143886042)
-1900 val loss 6.8980
-1900 val perplexity 990.3217
-1900 train 7.446872 (lr=1.4022e-04) (hash(x)=193610391)
-2000 val loss 6.9033
-2000 val perplexity 995.5397
-2000 train 7.087366 (lr=1.3907e-04) (hash(x)=163799796)
-2100 val loss 6.8726
-2100 val perplexity 965.4910
-2100 train 6.842040 (lr=1.3786e-04) (hash(x)=154107339)
-2200 val loss 6.8655
-2200 val perplexity 958.5820
-2200 train 6.853259 (lr=1.3660e-04) (hash(x)=144592844)
-2300 val loss 6.8478
-2300 val perplexity 941.7633
-2300 train 6.925510 (lr=1.3527e-04) (hash(x)=154535861)
-2400 val loss 6.8561
-2400 val perplexity 949.6703
-2400 train 7.553051 (lr=1.3390e-04) (hash(x)=204706354)
-2500 val loss 6.8432
-2500 val perplexity 937.5167
-2500 train 6.673442 (lr=1.3247e-04) (hash(x)=143783202)
-2600 val loss 6.8431
-2600 val perplexity 937.3611
-2600 train 6.760664 (lr=1.3099e-04) (hash(x)=150862210)
-2700 val loss 6.8435
-2700 val perplexity 937.7397
-2700 train 6.881901 (lr=1.2946e-04) (hash(x)=150753426)
-2800 val loss 6.8609
-2800 val perplexity 954.1826
-2800 train 6.834103 (lr=1.2788e-04) (hash(x)=147766811)
-2900 val loss 6.8580
-2900 val perplexity 951.4316
-2900 train 6.800954 (lr=1.2626e-04) (hash(x)=151177814)
-3000 val loss 6.8636
-3000 val perplexity 956.8106
-3000 train 6.742915 (lr=1.2459e-04) (hash(x)=135925327)
-3100 val loss 6.8565
-3100 val perplexity 950.0213
-3100 train 6.527127 (lr=1.2287e-04) (hash(x)=124842353)
-3200 val loss 6.8793
-3200 val perplexity 971.9348
-3200 train 6.991030 (lr=1.2112e-04) (hash(x)=160440642)
-3300 val loss 6.8749
-3300 val perplexity 967.7066
-3300 train 6.796642 (lr=1.1932e-04) (hash(x)=143017131)
-3400 val loss 6.8772
-3400 val perplexity 969.8584
-3400 train 7.140552 (lr=1.1749e-04) (hash(x)=148143474)
-3500 val loss 6.8898
-3500 val perplexity 982.2486
-3500 train 6.721855 (lr=1.1562e-04) (hash(x)=148732639)
-3600 val loss 6.9119
-3600 val perplexity 1004.1658
-3600 train 6.894956 (lr=1.1372e-04) (hash(x)=146333803)
-3700 val loss 6.9203
-3700 val perplexity 1012.6199
-3700 train 6.769926 (lr=1.1179e-04) (hash(x)=143611445)
-3800 val loss 6.9329
-3800 val perplexity 1025.4385
-3800 train 6.760955 (lr=1.0982e-04) (hash(x)=141272115)
-3900 val loss 6.9786
-3900 val perplexity 1073.4069
-3900 train 6.905924 (lr=1.0783e-04) (hash(x)=149161848)
-4000 val loss 6.9697
-4000 val perplexity 1063.8679
-4000 train 7.017169 (lr=1.0581e-04) (hash(x)=152036955)
-4100 val loss 6.9576
-4100 val perplexity 1051.1597
-4100 train 7.015798 (lr=1.0377e-04) (hash(x)=153260968)
-4200 val loss 6.9468
-4200 val perplexity 1039.8118
-4200 train 6.828518 (lr=1.0171e-04) (hash(x)=147629478)
-4300 val loss 6.9483
-4300 val perplexity 1041.4170
-4300 train 6.986649 (lr=9.9622e-05) (hash(x)=161701080)
-4400 val loss 6.9648
-4400 val perplexity 1058.7490
-4400 train 7.014188 (lr=9.7520e-05) (hash(x)=155290524)
-4500 val loss 6.9779
-4500 val perplexity 1072.6814
-4500 train 6.799006 (lr=9.5403e-05) (hash(x)=144726593)
-4600 val loss 6.9727
-4600 val perplexity 1067.1399
-4600 train 6.923756 (lr=9.3273e-05) (hash(x)=149517202)
-4700 val loss 6.9861
-4700 val perplexity 1081.5153
-4700 train 6.972135 (lr=9.1132e-05) (hash(x)=139403008)
-4800 val loss 6.9899
-4800 val perplexity 1085.6399
-4800 train 7.032064 (lr=8.8982e-05) (hash(x)=153387238)
-4900 val loss 6.9891
-4900 val perplexity 1084.7205
-4900 train 6.754529 (lr=8.6825e-05) (hash(x)=139881569)
-5000 val loss 7.0118
-5000 val perplexity 1109.6436
-5000 train 6.992263 (lr=8.4663e-05) (hash(x)=152602580)
-5100 val loss 7.0029
-5100 val perplexity 1099.8224
-5100 train 7.053523 (lr=8.2500e-05) (hash(x)=150598460)
-5200 val loss 7.0302
-5200 val perplexity 1130.2755
-5200 train 6.957996 (lr=8.0337e-05) (hash(x)=147660496)
-5300 val loss 7.0439
-5300 val perplexity 1145.8979
-5300 train 6.824490 (lr=7.8175e-05) (hash(x)=147693848)
-5400 val loss 7.0494
-5400 val perplexity 1152.1111
-5400 train 6.819526 (lr=7.6018e-05) (hash(x)=136048145)
-5500 val loss 7.0286
-5500 val perplexity 1128.4321
-5500 train 6.952648 (lr=7.3868e-05) (hash(x)=140313123)
-5600 val loss 7.0243
-5600 val perplexity 1123.5532
-5600 train 7.072820 (lr=7.1727e-05) (hash(x)=156997993)
-5700 val loss 7.0190
-5700 val perplexity 1117.7191
-5700 train 6.996371 (lr=6.9597e-05) (hash(x)=143513880)
-5800 val loss 7.0359
-5800 val perplexity 1136.7084
-5800 train 7.186930 (lr=6.7480e-05) (hash(x)=153736628)
-5900 val loss 7.0234
-5900 val perplexity 1122.5753
-5900 train 7.216059 (lr=6.5378e-05) (hash(x)=166385417)
-6000 val loss 7.0429
-6000 val perplexity 1144.7292
-6000 train 7.260878 (lr=6.3294e-05) (hash(x)=153216517)
-6100 val loss 7.0557
-6100 val perplexity 1159.4033
-6100 train 6.955714 (lr=6.1230e-05) (hash(x)=145387545)
-6200 val loss 7.0218
-6200 val perplexity 1120.8168
-6200 train 7.143466 (lr=5.9188e-05) (hash(x)=156306460)
-6300 val loss 7.0178
-6300 val perplexity 1116.2905
-6300 train 6.915298 (lr=5.7169e-05) (hash(x)=148610447)
-6400 val loss 7.0173
-6400 val perplexity 1115.7765
-6400 train 7.196865 (lr=5.5177e-05) (hash(x)=158343535)
-6500 val loss 7.0138
-6500 val perplexity 1111.9158
-6500 train 7.139856 (lr=5.3213e-05) (hash(x)=157742557)
-6600 val loss 7.0123
-6600 val perplexity 1110.1829
-6600 train 6.970760 (lr=5.1279e-05) (hash(x)=150457308)
-6700 val loss 7.0039
-6700 val perplexity 1100.9420
-6700 train 6.917010 (lr=4.9377e-05) (hash(x)=142426205)
-6800 val loss 6.9906
-6800 val perplexity 1086.3260
-6800 train 6.841436 (lr=4.7509e-05) (hash(x)=151522525)
-6900 val loss 6.9892
-6900 val perplexity 1084.8425
-6900 train 6.885376 (lr=4.5676e-05) (hash(x)=152029630)
-7000 val loss 6.9861
-7000 val perplexity 1081.5116
-7000 train 6.869016 (lr=4.3882e-05) (hash(x)=142978954)
-7100 val loss 6.9814
-7100 val perplexity 1076.4751
-7100 train 6.878259 (lr=4.2128e-05) (hash(x)=148064359)
-7200 val loss 6.9847
-7200 val perplexity 1080.0104
-7200 train 6.921792 (lr=4.0414e-05) (hash(x)=155771845)
-7300 val loss 6.9786
-7300 val perplexity 1073.4514
-7300 train 6.795961 (lr=3.8745e-05) (hash(x)=141639844)
-7400 val loss 6.9686
-7400 val perplexity 1062.6931
-7400 train 7.014091 (lr=3.7120e-05) (hash(x)=153466021)
-7500 val loss 6.9593
-7500 val perplexity 1052.8451
-7500 train 6.973252 (lr=3.5541e-05) (hash(x)=148843587)
-7600 val loss 6.9536
-7600 val perplexity 1046.9158
-7600 train 7.142570 (lr=3.4011e-05) (hash(x)=152053933)
-7700 val loss 6.9655
-7700 val perplexity 1059.4176
-7700 train 6.750750 (lr=3.2531e-05) (hash(x)=141564003)
-7800 val loss 6.9516
-7800 val perplexity 1044.7963
-7800 train 6.831956 (lr=3.1102e-05) (hash(x)=148617843)
-7900 val loss 6.9453
-7900 val perplexity 1038.2882
-7900 train 6.908850 (lr=2.9726e-05) (hash(x)=146417668)
-8000 val loss 6.9433
-8000 val perplexity 1036.2242
-8000 train 6.803242 (lr=2.8405e-05) (hash(x)=150165488)
-8100 val loss 6.9308
-8100 val perplexity 1023.3010
-8100 train 6.904036 (lr=2.7138e-05) (hash(x)=144818855)
-8200 val loss 6.9245
-8200 val perplexity 1016.8993
-8200 train 6.961853 (lr=2.5929e-05) (hash(x)=143074954)
-8300 val loss 6.9278
-8300 val perplexity 1020.2126
-8300 train 6.859095 (lr=2.4778e-05) (hash(x)=145172494)
-8400 val loss 6.9240
-8400 val perplexity 1016.4125
-8400 train 6.922443 (lr=2.3686e-05) (hash(x)=152284393)
-8500 val loss 6.9218
-8500 val perplexity 1014.1067
-8500 train 6.807056 (lr=2.2655e-05) (hash(x)=147044705)
-8600 val loss 6.9201
-8600 val perplexity 1012.4186
-8600 train 6.835523 (lr=2.1685e-05) (hash(x)=147762608)
-8700 val loss 6.9165
-8700 val perplexity 1008.7327
-8700 train 6.911353 (lr=2.0777e-05) (hash(x)=141994567)
-8800 val loss 6.9159
-8800 val perplexity 1008.1883
-8800 train 6.944641 (lr=1.9933e-05) (hash(x)=152552896)
-8900 val loss 6.9124
-8900 val perplexity 1004.6610
-8900 train 6.772640 (lr=1.9153e-05) (hash(x)=139714454)
-9000 val loss 6.9093
-9000 val perplexity 1001.4971
-9000 train 6.958855 (lr=1.8439e-05) (hash(x)=152274419)
-9100 val loss 6.9083
-9100 val perplexity 1000.5162
-9100 train 7.023602 (lr=1.7790e-05) (hash(x)=160536885)
-9200 val loss 6.9114
-9200 val perplexity 1003.6890
-9200 train 6.744764 (lr=1.7208e-05) (hash(x)=138386899)
-9300 val loss 6.9087
-9300 val perplexity 1000.9141
-9300 train 6.904637 (lr=1.6692e-05) (hash(x)=149957457)
-9400 val loss 6.9071
-9400 val perplexity 999.3447
-9400 train 7.003169 (lr=1.6245e-05) (hash(x)=161671548)
-9500 val loss 6.9127
-9500 val perplexity 1004.9370
-9500 train 7.034237 (lr=1.5865e-05) (hash(x)=171857400)
-9600 val loss 6.9064
-9600 val perplexity 998.6649
-9600 train 6.631417 (lr=1.5554e-05) (hash(x)=135378471)
-9700 val loss 6.9055
-9700 val perplexity 997.7429
-9700 train 6.824327 (lr=1.5312e-05) (hash(x)=149514591)
-9800 val loss 6.9051
-9800 val perplexity 997.3576
-9800 train 6.875938 (lr=1.5139e-05) (hash(x)=152679072)
-9900 val loss 6.9097
-9900 val perplexity 1001.9810
-9900 train 6.841558 (lr=1.5035e-05) (hash(x)=150415193)
-9999 val loss 6.9041
-9999 val perplexity 996.3181
+0 train 11.765417 (lr=2.5000e-07) (hash(x)=145079536)
+100 val loss 10.2826
+100 val perplexity 29218.3359
+100 train 10.229090 (lr=2.5250e-05) (hash(x)=154745873)
+200 val loss 9.1425
+200 val perplexity 9344.1045
+200 train 9.139270 (lr=5.0000e-05) (hash(x)=155800595)
+300 val loss 8.2396
+300 val perplexity 3788.2061
+300 train 8.208289 (lr=4.9988e-05) (hash(x)=148595389)
+400 val loss 7.8677
+400 val perplexity 2611.6658
+400 train 7.794479 (lr=4.9954e-05) (hash(x)=145606733)
+500 val loss 7.7057
+500 val perplexity 2221.0730
+500 train 7.568192 (lr=4.9896e-05) (hash(x)=138221231)
+600 val loss 7.6270
+600 val perplexity 2052.9622
+600 train 7.637253 (lr=4.9815e-05) (hash(x)=150367139)
+700 val loss 7.5933
+700 val perplexity 1984.8761
+700 train 7.732349 (lr=4.9712e-05) (hash(x)=155579314)
+800 val loss 7.5569
+800 val perplexity 1913.9260
+800 train 7.627458 (lr=4.9585e-05) (hash(x)=155747374)
+900 val loss 7.5306
+900 val perplexity 1864.1678
+900 train 7.728004 (lr=4.9436e-05) (hash(x)=159334575)
+1000 val loss 7.5001
+1000 val perplexity 1808.2632
+1000 train 7.395680 (lr=4.9264e-05) (hash(x)=140604760)
+1100 val loss 7.4848
+1100 val perplexity 1780.7771
+1100 train 7.329884 (lr=4.9070e-05) (hash(x)=146208052)
+1200 val loss 7.4578
+1200 val perplexity 1733.3807
+1200 train 7.383160 (lr=4.8854e-05) (hash(x)=148404734)
+1300 val loss 7.4405
+1300 val perplexity 1703.6826
+1300 train 7.380110 (lr=4.8616e-05) (hash(x)=155681970)
+1400 val loss 7.4114
+1400 val perplexity 1654.7939
+1400 train 7.363786 (lr=4.8356e-05) (hash(x)=148115934)
+1500 val loss 7.3769
+1500 val perplexity 1598.6610
+1500 train 7.503820 (lr=4.8074e-05) (hash(x)=157074034)
+1600 val loss 7.3488
+1600 val perplexity 1554.3053
+1600 train 7.120208 (lr=4.7772e-05) (hash(x)=137464699)
+1700 val loss 7.3234
+1700 val perplexity 1515.3431
+1700 train 7.531859 (lr=4.7448e-05) (hash(x)=166955614)
+1800 val loss 7.3060
+1800 val perplexity 1489.2195
+1800 train 7.140155 (lr=4.7105e-05) (hash(x)=143886042)
+1900 val loss 7.2840
+1900 val perplexity 1456.8646
+1900 train 7.898643 (lr=4.6741e-05) (hash(x)=193610391)
+2000 val loss 7.2716
+2000 val perplexity 1438.8134
+2000 train 7.425419 (lr=4.6357e-05) (hash(x)=163799796)
+2100 val loss 7.2522
+2100 val perplexity 1411.1982
+2100 train 7.212413 (lr=4.5954e-05) (hash(x)=154107339)
+2200 val loss 7.2275
+2200 val perplexity 1376.7487
+2200 train 7.202817 (lr=4.5532e-05) (hash(x)=144592844)
+2300 val loss 7.2104
+2300 val perplexity 1353.4292
+2300 train 7.279191 (lr=4.5091e-05) (hash(x)=154535861)
+2400 val loss 7.1937
+2400 val perplexity 1331.0225
+2400 train 7.922030 (lr=4.4633e-05) (hash(x)=204706354)
+2500 val loss 7.1846
+2500 val perplexity 1318.9752
+2500 train 7.007863 (lr=4.4156e-05) (hash(x)=143783202)
+2600 val loss 7.1982
+2600 val perplexity 1337.0598
+2600 train 7.098747 (lr=4.3663e-05) (hash(x)=150862210)
+2700 val loss 7.1884
+2700 val perplexity 1324.0164
+2700 train 7.256421 (lr=4.3153e-05) (hash(x)=150753426)
+2800 val loss 7.1686
+2800 val perplexity 1297.9996
+2800 train 7.134176 (lr=4.2627e-05) (hash(x)=147766811)
+2900 val loss 7.1589
+2900 val perplexity 1285.5081
+2900 train 7.091248 (lr=4.2085e-05) (hash(x)=151177814)
+3000 val loss 7.1545
+3000 val perplexity 1279.8695
+3000 train 7.002457 (lr=4.1529e-05) (hash(x)=135925327)
+3100 val loss 7.1629
+3100 val perplexity 1290.6305
+3100 train 6.864954 (lr=4.0957e-05) (hash(x)=124842353)
+3200 val loss 7.1606
+3200 val perplexity 1287.6792
+3200 train 7.264678 (lr=4.0373e-05) (hash(x)=160440642)
+3300 val loss 7.1483
+3300 val perplexity 1271.9863
+3300 train 7.056784 (lr=3.9775e-05) (hash(x)=143017131)
+3400 val loss 7.1487
+3400 val perplexity 1272.4194
+3400 train 7.358580 (lr=3.9164e-05) (hash(x)=148143474)
+3500 val loss 7.1200
+3500 val perplexity 1236.3984
+3500 train 6.953858 (lr=3.8541e-05) (hash(x)=148732639)
+3600 val loss 7.1114
+3600 val perplexity 1225.9130
+3600 train 7.110910 (lr=3.7907e-05) (hash(x)=146333803)
+3700 val loss 7.1119
+3700 val perplexity 1226.4486
+3700 train 6.949524 (lr=3.7262e-05) (hash(x)=143611445)
+3800 val loss 7.1171
+3800 val perplexity 1232.8684
+3800 train 6.939259 (lr=3.6608e-05) (hash(x)=141272115)
+3900 val loss 7.0960
+3900 val perplexity 1207.1256
+3900 train 7.020997 (lr=3.5944e-05) (hash(x)=149161848)
+4000 val loss 7.0935
+4000 val perplexity 1204.0765
+4000 train 7.162819 (lr=3.5271e-05) (hash(x)=152036955)
+4100 val loss 7.0942
+4100 val perplexity 1204.9524
+4100 train 7.145968 (lr=3.4590e-05) (hash(x)=153260968)
+4200 val loss 7.0696
+4200 val perplexity 1175.6902
+4200 train 6.957745 (lr=3.3902e-05) (hash(x)=147629478)
+4300 val loss 7.0848
+4300 val perplexity 1193.7286
+4300 train 7.111011 (lr=3.3207e-05) (hash(x)=161701080)
+4400 val loss 7.0667
+4400 val perplexity 1172.2228
+4400 train 7.119734 (lr=3.2507e-05) (hash(x)=155290524)
+4500 val loss 7.0701
+4500 val perplexity 1176.2117
+4500 train 6.902558 (lr=3.1801e-05) (hash(x)=144726593)
+4600 val loss 7.0655
+4600 val perplexity 1170.8447
+4600 train 7.015216 (lr=3.1091e-05) (hash(x)=149517202)
+4700 val loss 7.0668
+4700 val perplexity 1172.4336
+4700 train 7.049967 (lr=3.0377e-05) (hash(x)=139403008)
+4800 val loss 7.0334
+4800 val perplexity 1133.9307
+4800 train 7.076432 (lr=2.9661e-05) (hash(x)=153387238)
+4900 val loss 7.0237
+4900 val perplexity 1122.9238
+4900 train 6.798825 (lr=2.8942e-05) (hash(x)=139881569)
+5000 val loss 7.0251
+5000 val perplexity 1124.5347
+5000 train 6.988297 (lr=2.8221e-05) (hash(x)=152602580)
+5100 val loss 7.0178
+5100 val perplexity 1116.3683
+5100 train 7.075103 (lr=2.7500e-05) (hash(x)=150598460)
+5200 val loss 7.0146
+5200 val perplexity 1112.7893
+5200 train 6.948948 (lr=2.6779e-05) (hash(x)=147660496)
+5300 val loss 7.0149
+5300 val perplexity 1113.0590
+5300 train 6.783306 (lr=2.6058e-05) (hash(x)=147693848)
+5400 val loss 7.0324
+5400 val perplexity 1132.6899
+5400 train 6.810908 (lr=2.5339e-05) (hash(x)=136048145)
+5500 val loss 6.9941
+5500 val perplexity 1090.2325
+5500 train 6.928900 (lr=2.4623e-05) (hash(x)=140313123)
+5600 val loss 6.9882
+5600 val perplexity 1083.7391
+5600 train 7.023926 (lr=2.3909e-05) (hash(x)=156997993)
+5700 val loss 6.9746
+5700 val perplexity 1069.1527
+5700 train 6.957600 (lr=2.3199e-05) (hash(x)=143513880)
+5800 val loss 6.9747
+5800 val perplexity 1069.2542
+5800 train 7.135317 (lr=2.2493e-05) (hash(x)=153736628)
+5900 val loss 6.9736
+5900 val perplexity 1068.0149
+5900 train 7.152683 (lr=2.1793e-05) (hash(x)=166385417)
+6000 val loss 6.9625
+6000 val perplexity 1056.2238
+6000 train 7.161615 (lr=2.1098e-05) (hash(x)=153216517)
+6100 val loss 6.9613
+6100 val perplexity 1054.9874
+6100 train 6.849194 (lr=2.0410e-05) (hash(x)=145387545)
+6200 val loss 6.9644
+6200 val perplexity 1058.3038
+6200 train 7.072074 (lr=1.9729e-05) (hash(x)=156306460)
+6300 val loss 6.9596
+6300 val perplexity 1053.2604
+6300 train 6.850439 (lr=1.9056e-05) (hash(x)=148610447)
+6400 val loss 6.9467
+6400 val perplexity 1039.7458
+6400 train 7.140009 (lr=1.8392e-05) (hash(x)=158343535)
+6500 val loss 6.9415
+6500 val perplexity 1034.2822
+6500 train 7.083022 (lr=1.7738e-05) (hash(x)=157742557)
+6600 val loss 6.9320
+6600 val perplexity 1024.5250
+6600 train 6.891453 (lr=1.7093e-05) (hash(x)=150457308)
+6700 val loss 6.9216
+6700 val perplexity 1013.9897
+6700 train 6.822219 (lr=1.6459e-05) (hash(x)=142426205)
+6800 val loss 6.9205
+6800 val perplexity 1012.8657
+6800 train 6.774536 (lr=1.5836e-05) (hash(x)=151522525)
+6900 val loss 6.9227
+6900 val perplexity 1015.0526
+6900 train 6.847309 (lr=1.5225e-05) (hash(x)=152029630)
+7000 val loss 6.9094
+7000 val perplexity 1001.6866
+7000 train 6.780774 (lr=1.4627e-05) (hash(x)=142978954)
+7100 val loss 6.9035
+7100 val perplexity 995.7990
+7100 train 6.798730 (lr=1.4043e-05) (hash(x)=148064359)
+7200 val loss 6.8999
+7200 val perplexity 992.1963
+7200 train 6.835505 (lr=1.3471e-05) (hash(x)=155771845)
+7300 val loss 6.8986
+7300 val perplexity 990.8743
+7300 train 6.715855 (lr=1.2915e-05) (hash(x)=141639844)
+7400 val loss 6.8924
+7400 val perplexity 984.7956
+7400 train 6.928619 (lr=1.2373e-05) (hash(x)=153466021)
+7500 val loss 6.8917
+7500 val perplexity 984.0934
+7500 train 6.896158 (lr=1.1847e-05) (hash(x)=148843587)
+7600 val loss 6.8860
+7600 val perplexity 978.4490
+7600 train 7.061441 (lr=1.1337e-05) (hash(x)=152053933)
+7700 val loss 6.8777
+7700 val perplexity 970.3899
+7700 train 6.651704 (lr=1.0844e-05) (hash(x)=141564003)
+7800 val loss 6.8734
+7800 val perplexity 966.2131
+7800 train 6.761744 (lr=1.0367e-05) (hash(x)=148617843)
+7900 val loss 6.8699
+7900 val perplexity 962.8681
+7900 train 6.841489 (lr=9.9088e-06) (hash(x)=146417668)
+8000 val loss 6.8715
+8000 val perplexity 964.3954
+8000 train 6.710367 (lr=9.4682e-06) (hash(x)=150165488)
+8100 val loss 6.8669
+8100 val perplexity 959.9246
+8100 train 6.835996 (lr=9.0461e-06) (hash(x)=144818855)
+8200 val loss 6.8594
+8200 val perplexity 952.8196
+8200 train 6.893314 (lr=8.6430e-06) (hash(x)=143074954)
+8300 val loss 6.8578
+8300 val perplexity 951.2951
+8300 train 6.775650 (lr=8.2593e-06) (hash(x)=145172494)
+8400 val loss 6.8549
+8400 val perplexity 948.5271
+8400 train 6.863227 (lr=7.8953e-06) (hash(x)=152284393)
+8500 val loss 6.8517
+8500 val perplexity 945.4831
+8500 train 6.727867 (lr=7.5515e-06) (hash(x)=147044705)
+8600 val loss 6.8501
+8600 val perplexity 943.9672
+8600 train 6.775654 (lr=7.2282e-06) (hash(x)=147762608)
+8700 val loss 6.8476
+8700 val perplexity 941.6030
+8700 train 6.843468 (lr=6.9257e-06) (hash(x)=141994567)
+8800 val loss 6.8425
+8800 val perplexity 936.8553
+8800 train 6.876688 (lr=6.6444e-06) (hash(x)=152552896)
+8900 val loss 6.8413
+8900 val perplexity 935.6967
+8900 train 6.693859 (lr=6.3845e-06) (hash(x)=139714454)
+9000 val loss 6.8365
+9000 val perplexity 931.2620
+9000 train 6.874582 (lr=6.1462e-06) (hash(x)=152274419)
+9100 val loss 6.8367
+9100 val perplexity 931.3975
+9100 train 6.962732 (lr=5.9300e-06) (hash(x)=160536885)
+9200 val loss 6.8362
+9200 val perplexity 930.9202
+9200 train 6.664563 (lr=5.7359e-06) (hash(x)=138386899)
+9300 val loss 6.8327
+9300 val perplexity 927.6513
+9300 train 6.842355 (lr=5.5641e-06) (hash(x)=149957457)
+9400 val loss 6.8309
+9400 val perplexity 926.0545
+9400 train 6.910855 (lr=5.4149e-06) (hash(x)=161671548)
+9500 val loss 6.8306
+9500 val perplexity 925.7362
+9500 train 6.983879 (lr=5.2884e-06) (hash(x)=171857400)
+9600 val loss 6.8278
+9600 val perplexity 923.1623
+9600 train 6.536280 (lr=5.1847e-06) (hash(x)=135378471)
+9700 val loss 6.8259
+9700 val perplexity 921.4111
+9700 train 6.745991 (lr=5.1040e-06) (hash(x)=149514591)
+9800 val loss 6.8210
+9800 val perplexity 916.8771
+9800 train 6.784397 (lr=5.0462e-06) (hash(x)=152679072)
+9900 val loss 6.8200
+9900 val perplexity 915.9756
+9900 train 6.746526 (lr=5.0116e-06) (hash(x)=150415193)
+9999 val loss 6.8178
+9999 val perplexity 913.9669

attention_kindself_n_heads2_seed1341/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77ed0f7d4653690c071f763652be4c24339c49c1aaf59bbd59dde8f9bfd77b4d
+oid sha256:d9f37868ccb8a9a3b18ff2b594b248dd93b627e135bf3db4c7f1433ab9a4b58d
 size 38587970

attention_kindself_n_heads2_seed1341/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e676a70847e2cd0c45a6022f8fb780c6f363d2cc5aec763d9ad8b11925a5a278
+oid sha256:fe708cdb3a1ae1687fb840f5a7e8956cb9b9dced147881782a428f6f1784574e
 size 38587970

attention_kindself_n_heads2_seed1341/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2fbb92e6a73f27c026033df1c43df475c25ea1bace8e425bd876bc18fb0c1c14
+oid sha256:710739bd14fcc20618e82a2bbeb9ca427b9ce8836660c22d3447019556d4ddeb
 size 38587970

attention_kindself_n_heads2_seed1341/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab0bbb0a3d853852359e3c2a794b1008491b1776966827f857a7a4f837eb9a17
+oid sha256:09db754b0dacdd1829b4075398afe1e3433ba1a41853f8e571a6a4bdf83c5b66
 size 38587970

attention_kindself_n_heads2_seed1341/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c8775e66b7f5f93dc88fdfdc905853fa6e114da9e5dd4ef8343334bc32e20c1
+oid sha256:f3defe1d93fa733af9731e7281685a0b70e5e3008cda4cbc34e34b6f2ee15ac7
 size 70895430

attention_kindself_n_heads2_seed1341/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08ab1f75ddfa99ee6abfbae19643ac708f6b2b4e230359892d77635503e18718
+oid sha256:0b53829a19019ad5c51f06b8492adc622a4d0143f13a16b832e5702910682b00
 size 70895430

attention_kindself_n_heads2_seed1341/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a28f04aae3faf44fb13e2ae82ed23f947df716d02ff8dd0015876bc0966e48d4
+oid sha256:d1f9c0f5f930db421657db03ff19d8fd8d84625f6d114179ecf168a71bf32052
 size 70895430

attention_kindself_n_heads2_seed1341/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6531d46b69edf8d3eb3c505f69dd46829f338364b9bf2b90995a9da877f54548
+oid sha256:1d0b182a10b4f8654eafc4207c594cb7a55f1fb433528ec56ba720044f8becfe
 size 70895430