Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1341", "n_embd": 256}

attention_kindself_n_heads4_seed1341/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.3001
 0 val perplexity 80832.1328
-0 train 11.306698 (lr=7.5000e-07) (hash(x)=145079536)
-100 val loss 9.3955
-100 val perplexity 12033.8779
-100 train 9.328828 (lr=7.5750e-05) (hash(x)=154745873)
-200 val loss 7.8945
-200 val perplexity 2682.5410
-200 train 8.046517 (lr=1.5000e-04) (hash(x)=155800595)
-300 val loss 7.5646
-300 val perplexity 1928.6428
-300 train 7.518878 (lr=1.4997e-04) (hash(x)=148595389)
-400 val loss 7.4628
-400 val perplexity 1741.9973
-400 train 7.364924 (lr=1.4986e-04) (hash(x)=145606733)
-500 val loss 7.3461
-500 val perplexity 1550.1819
-500 train 7.149421 (lr=1.4969e-04) (hash(x)=138221231)
-600 val loss 7.2483
-600 val perplexity 1405.6442
-600 train 7.251230 (lr=1.4945e-04) (hash(x)=150367139)
-700 val loss 7.1820
-700 val perplexity 1315.5784
-700 train 7.341902 (lr=1.4913e-04) (hash(x)=155579314)
-800 val loss 7.1344
-800 val perplexity 1254.3445
-800 train 7.203811 (lr=1.4876e-04) (hash(x)=155747374)
-900 val loss 7.0829
-900 val perplexity 1191.3647
-900 train 7.300850 (lr=1.4831e-04) (hash(x)=159334575)
-1000 val loss 7.0469
-1000 val perplexity 1149.2343
-1000 train 6.926498 (lr=1.4779e-04) (hash(x)=140604760)
-1100 val loss 7.0195
-1100 val perplexity 1118.2042
-1100 train 6.859845 (lr=1.4721e-04) (hash(x)=146208052)
-1200 val loss 6.9986
-1200 val perplexity 1095.0535
-1200 train 6.893322 (lr=1.4656e-04) (hash(x)=148404734)
-1300 val loss 6.9836
-1300 val perplexity 1078.7654
-1300 train 6.927601 (lr=1.4585e-04) (hash(x)=155681970)
-1400 val loss 6.9791
-1400 val perplexity 1073.9409
-1400 train 6.923635 (lr=1.4507e-04) (hash(x)=148115934)
-1500 val loss 6.9565
-1500 val perplexity 1049.9689
-1500 train 7.091454 (lr=1.4422e-04) (hash(x)=157074034)
-1600 val loss 6.9471
-1600 val perplexity 1040.1470
-1600 train 6.729518 (lr=1.4332e-04) (hash(x)=137464699)
-1700 val loss 6.9357
-1700 val perplexity 1028.3800
-1700 train 7.057977 (lr=1.4235e-04) (hash(x)=166955614)
-1800 val loss 6.9376
-1800 val perplexity 1030.3394
-1800 train 6.764245 (lr=1.4131e-04) (hash(x)=143886042)
-1900 val loss 6.9336
-1900 val perplexity 1026.2212
-1900 train 7.465428 (lr=1.4022e-04) (hash(x)=193610391)
-2000 val loss 6.9376
-2000 val perplexity 1030.2677
-2000 train 7.123946 (lr=1.3907e-04) (hash(x)=163799796)
-2100 val loss 6.9394
-2100 val perplexity 1032.1489
-2100 train 6.917353 (lr=1.3786e-04) (hash(x)=154107339)
-2200 val loss 6.9471
-2200 val perplexity 1040.1465
-2200 train 6.929567 (lr=1.3660e-04) (hash(x)=144592844)
-2300 val loss 6.9317
-2300 val perplexity 1024.1855
-2300 train 7.003430 (lr=1.3527e-04) (hash(x)=154535861)
-2400 val loss 6.9294
-2400 val perplexity 1021.9094
-2400 train 7.594524 (lr=1.3390e-04) (hash(x)=204706354)
-2500 val loss 6.9464
-2500 val perplexity 1039.4346
-2500 train 6.789571 (lr=1.3247e-04) (hash(x)=143783202)
-2600 val loss 6.9908
-2600 val perplexity 1086.5525
-2600 train 6.898761 (lr=1.3099e-04) (hash(x)=150862210)
-2700 val loss 7.0879
-2700 val perplexity 1197.4342
-2700 train 7.115728 (lr=1.2946e-04) (hash(x)=150753426)
-2800 val loss 7.0544
-2800 val perplexity 1157.9237
-2800 train 7.033858 (lr=1.2788e-04) (hash(x)=147766811)
-2900 val loss 7.0444
-2900 val perplexity 1146.4587
-2900 train 6.995884 (lr=1.2626e-04) (hash(x)=151177814)
-3000 val loss 7.0758
-3000 val perplexity 1182.9889
-3000 train 6.961751 (lr=1.2459e-04) (hash(x)=135925327)
-3100 val loss 7.0452
-3100 val perplexity 1147.3901
-3100 train 6.727612 (lr=1.2287e-04) (hash(x)=124842353)
-3200 val loss 7.0620
-3200 val perplexity 1166.8268
-3200 train 7.186998 (lr=1.2112e-04) (hash(x)=160440642)
-3300 val loss 7.0383
-3300 val perplexity 1139.4587
-3300 train 6.945041 (lr=1.1932e-04) (hash(x)=143017131)
-3400 val loss 7.0683
-3400 val perplexity 1174.1327
-3400 train 7.301206 (lr=1.1749e-04) (hash(x)=148143474)
-3500 val loss 7.0685
-3500 val perplexity 1174.3734
-3500 train 6.894324 (lr=1.1562e-04) (hash(x)=148732639)
-3600 val loss 7.0478
-3600 val perplexity 1150.2919
-3600 train 7.020499 (lr=1.1372e-04) (hash(x)=146333803)
-3700 val loss 7.0632
-3700 val perplexity 1168.2002
-3700 train 6.920694 (lr=1.1179e-04) (hash(x)=143611445)
-3800 val loss 7.0754
-3800 val perplexity 1182.5382
-3800 train 6.906082 (lr=1.0982e-04) (hash(x)=141272115)
-3900 val loss 7.0808
-3900 val perplexity 1188.9431
-3900 train 7.021241 (lr=1.0783e-04) (hash(x)=149161848)
-4000 val loss 7.0733
-4000 val perplexity 1180.0232
-4000 train 7.144450 (lr=1.0581e-04) (hash(x)=152036955)
-4100 val loss 7.0870
-4100 val perplexity 1196.3624
-4100 train 7.139550 (lr=1.0377e-04) (hash(x)=153260968)
-4200 val loss 7.0995
-4200 val perplexity 1211.3430
-4200 train 7.001007 (lr=1.0171e-04) (hash(x)=147629478)
-4300 val loss 7.1032
-4300 val perplexity 1215.8655
-4300 train 7.152019 (lr=9.9622e-05) (hash(x)=161701080)
-4400 val loss 7.1204
-4400 val perplexity 1236.9551
-4400 train 7.195014 (lr=9.7520e-05) (hash(x)=155290524)
-4500 val loss 7.1122
-4500 val perplexity 1226.8528
-4500 train 6.941273 (lr=9.5403e-05) (hash(x)=144726593)
-4600 val loss 7.0994
-4600 val perplexity 1211.2708
-4600 train 7.047345 (lr=9.3273e-05) (hash(x)=149517202)
-4700 val loss 7.1178
-4700 val perplexity 1233.7041
-4700 train 7.075057 (lr=9.1132e-05) (hash(x)=139403008)
-4800 val loss 7.1471
-4800 val perplexity 1270.3921
-4800 train 7.197305 (lr=8.8982e-05) (hash(x)=153387238)
-4900 val loss 7.1528
-4900 val perplexity 1277.6281
-4900 train 6.937476 (lr=8.6825e-05) (hash(x)=139881569)
-5000 val loss 7.1791
-5000 val perplexity 1311.7185
-5000 train 7.149471 (lr=8.4663e-05) (hash(x)=152602580)
-5100 val loss 7.1850
-5100 val perplexity 1319.4634
-5100 train 7.225554 (lr=8.2500e-05) (hash(x)=150598460)
-5200 val loss 7.1622
-5200 val perplexity 1289.7594
-5200 train 7.097391 (lr=8.0337e-05) (hash(x)=147660496)
-5300 val loss 7.2029
-5300 val perplexity 1343.2982
-5300 train 7.020668 (lr=7.8175e-05) (hash(x)=147693848)
-5400 val loss 7.1715
-5400 val perplexity 1301.7905
-5400 train 6.968065 (lr=7.6018e-05) (hash(x)=136048145)
-5500 val loss 7.1732
-5500 val perplexity 1304.0588
-5500 train 7.087708 (lr=7.3868e-05) (hash(x)=140313123)
-5600 val loss 7.1686
-5600 val perplexity 1298.0120
-5600 train 7.184072 (lr=7.1727e-05) (hash(x)=156997993)
-5700 val loss 7.1618
-5700 val perplexity 1289.2294
-5700 train 7.135228 (lr=6.9597e-05) (hash(x)=143513880)
-5800 val loss 7.1865
-5800 val perplexity 1321.4089
-5800 train 7.319064 (lr=6.7480e-05) (hash(x)=153736628)
-5900 val loss 7.1542
-5900 val perplexity 1279.5192
-5900 train 7.347223 (lr=6.5378e-05) (hash(x)=166385417)
-6000 val loss 7.1738
-6000 val perplexity 1304.7692
-6000 train 7.386747 (lr=6.3294e-05) (hash(x)=153216517)
-6100 val loss 7.1957
-6100 val perplexity 1333.7188
-6100 train 7.121440 (lr=6.1230e-05) (hash(x)=145387545)
-6200 val loss 7.1819
-6200 val perplexity 1315.4391
-6200 train 7.279177 (lr=5.9188e-05) (hash(x)=156306460)
-6300 val loss 7.1514
-6300 val perplexity 1275.8540
-6300 train 7.050121 (lr=5.7169e-05) (hash(x)=148610447)
-6400 val loss 7.1558
-6400 val perplexity 1281.5262
-6400 train 7.306859 (lr=5.5177e-05) (hash(x)=158343535)
-6500 val loss 7.1446
-6500 val perplexity 1267.2605
-6500 train 7.276059 (lr=5.3213e-05) (hash(x)=157742557)
-6600 val loss 7.1413
-6600 val perplexity 1263.0376
-6600 train 7.115061 (lr=5.1279e-05) (hash(x)=150457308)
-6700 val loss 7.1304
-6700 val perplexity 1249.4382
-6700 train 7.057846 (lr=4.9377e-05) (hash(x)=142426205)
-6800 val loss 7.1562
-6800 val perplexity 1282.0842
-6800 train 7.020480 (lr=4.7509e-05) (hash(x)=151522525)
-6900 val loss 7.1463
-6900 val perplexity 1269.3373
-6900 train 7.045939 (lr=4.5676e-05) (hash(x)=152029630)
-7000 val loss 7.1282
-7000 val perplexity 1246.6816
-7000 train 7.010334 (lr=4.3882e-05) (hash(x)=142978954)
-7100 val loss 7.1316
-7100 val perplexity 1250.8677
-7100 train 7.048053 (lr=4.2128e-05) (hash(x)=148064359)
-7200 val loss 7.1285
-7200 val perplexity 1247.0454
-7200 train 7.076699 (lr=4.0414e-05) (hash(x)=155771845)
-7300 val loss 7.1311
-7300 val perplexity 1250.2863
-7300 train 6.963120 (lr=3.8745e-05) (hash(x)=141639844)
-7400 val loss 7.1365
-7400 val perplexity 1257.0677
-7400 train 7.203446 (lr=3.7120e-05) (hash(x)=153466021)
-7500 val loss 7.1197
-7500 val perplexity 1236.1337
-7500 train 7.136865 (lr=3.5541e-05) (hash(x)=148843587)
-7600 val loss 7.1116
-7600 val perplexity 1226.1165
-7600 train 7.277462 (lr=3.4011e-05) (hash(x)=152053933)
-7700 val loss 7.1060
-7700 val perplexity 1219.2164
-7700 train 6.897977 (lr=3.2531e-05) (hash(x)=141564003)
-7800 val loss 7.1045
-7800 val perplexity 1217.3755
-7800 train 6.994552 (lr=3.1102e-05) (hash(x)=148617843)
-7900 val loss 7.1060
-7900 val perplexity 1219.3043
-7900 train 7.082467 (lr=2.9726e-05) (hash(x)=146417668)
-8000 val loss 7.1018
-8000 val perplexity 1214.1541
-8000 train 6.974138 (lr=2.8405e-05) (hash(x)=150165488)
-8100 val loss 7.0963
-8100 val perplexity 1207.5344
-8100 train 7.060221 (lr=2.7138e-05) (hash(x)=144818855)
-8200 val loss 7.0909
-8200 val perplexity 1201.0133
-8200 train 7.119874 (lr=2.5929e-05) (hash(x)=143074954)
-8300 val loss 7.0967
-8300 val perplexity 1208.0228
-8300 train 7.036266 (lr=2.4778e-05) (hash(x)=145172494)
-8400 val loss 7.0913
-8400 val perplexity 1201.4298
-8400 train 7.098520 (lr=2.3686e-05) (hash(x)=152284393)
-8500 val loss 7.0969
-8500 val perplexity 1208.2601
-8500 train 6.983650 (lr=2.2655e-05) (hash(x)=147044705)
-8600 val loss 7.0955
-8600 val perplexity 1206.5703
-8600 train 7.039187 (lr=2.1685e-05) (hash(x)=147762608)
-8700 val loss 7.0910
-8700 val perplexity 1201.1680
-8700 train 7.076082 (lr=2.0777e-05) (hash(x)=141994567)
-8800 val loss 7.0858
-8800 val perplexity 1194.8208
-8800 train 7.110641 (lr=1.9933e-05) (hash(x)=152552896)
-8900 val loss 7.0899
-8900 val perplexity 1199.7546
-8900 train 6.966035 (lr=1.9153e-05) (hash(x)=139714454)
-9000 val loss 7.0826
-9000 val perplexity 1191.0267
-9000 train 7.120393 (lr=1.8439e-05) (hash(x)=152274419)
-9100 val loss 7.0831
-9100 val perplexity 1191.6499
-9100 train 7.232574 (lr=1.7790e-05) (hash(x)=160536885)
-9200 val loss 7.0775
-9200 val perplexity 1185.0089
-9200 train 6.920467 (lr=1.7208e-05) (hash(x)=138386899)
-9300 val loss 7.0761
-9300 val perplexity 1183.3505
-9300 train 7.090115 (lr=1.6692e-05) (hash(x)=149957457)
-9400 val loss 7.0705
-9400 val perplexity 1176.6985
-9400 train 7.159608 (lr=1.6245e-05) (hash(x)=161671548)
-9500 val loss 7.0700
-9500 val perplexity 1176.1343
-9500 train 7.194458 (lr=1.5865e-05) (hash(x)=171857400)
-9600 val loss 7.0682
-9600 val perplexity 1173.9771
-9600 train 6.827000 (lr=1.5554e-05) (hash(x)=135378471)
-9700 val loss 7.0748
-9700 val perplexity 1181.8060
-9700 train 7.002652 (lr=1.5312e-05) (hash(x)=149514591)
-9800 val loss 7.0707
-9800 val perplexity 1176.9982
-9800 train 7.059768 (lr=1.5139e-05) (hash(x)=152679072)
-9900 val loss 7.0689
-9900 val perplexity 1174.8854
-9900 train 6.991833 (lr=1.5035e-05) (hash(x)=150415193)
-9999 val loss 7.0655
-9999 val perplexity 1170.8290
+0 train 11.306698 (lr=2.5000e-07) (hash(x)=145079536)
+100 val loss 9.9349
+100 val perplexity 20638.1484
+100 train 9.874459 (lr=2.5250e-05) (hash(x)=154745873)
+200 val loss 9.1781
+200 val perplexity 9682.8174
+200 train 9.206763 (lr=5.0000e-05) (hash(x)=155800595)
+300 val loss 8.3602
+300 val perplexity 4273.7119
+300 train 8.323228 (lr=4.9988e-05) (hash(x)=148595389)
+400 val loss 8.0266
+400 val perplexity 3061.2158
+400 train 7.965455 (lr=4.9954e-05) (hash(x)=145606733)
+500 val loss 7.7974
+500 val perplexity 2434.1660
+500 train 7.668923 (lr=4.9896e-05) (hash(x)=138221231)
+600 val loss 7.6570
+600 val perplexity 2115.4138
+600 train 7.671975 (lr=4.9815e-05) (hash(x)=150367139)
+700 val loss 7.5883
+700 val perplexity 1974.9584
+700 train 7.727467 (lr=4.9712e-05) (hash(x)=155579314)
+800 val loss 7.5362
+800 val perplexity 1874.6230
+800 train 7.612894 (lr=4.9585e-05) (hash(x)=155747374)
+900 val loss 7.4908
+900 val perplexity 1791.4127
+900 train 7.689740 (lr=4.9436e-05) (hash(x)=159334575)
+1000 val loss 7.4582
+1000 val perplexity 1734.0844
+1000 train 7.335213 (lr=4.9264e-05) (hash(x)=140604760)
+1100 val loss 7.4387
+1100 val perplexity 1700.5392
+1100 train 7.287144 (lr=4.9070e-05) (hash(x)=146208052)
+1200 val loss 7.4228
+1200 val perplexity 1673.7517
+1200 train 7.334071 (lr=4.8854e-05) (hash(x)=148404734)
+1300 val loss 7.4098
+1300 val perplexity 1652.1472
+1300 train 7.351157 (lr=4.8616e-05) (hash(x)=155681970)
+1400 val loss 7.3949
+1400 val perplexity 1627.6387
+1400 train 7.345123 (lr=4.8356e-05) (hash(x)=148115934)
+1500 val loss 7.3776
+1500 val perplexity 1599.8215
+1500 train 7.515377 (lr=4.8074e-05) (hash(x)=157074034)
+1600 val loss 7.3625
+1600 val perplexity 1575.8352
+1600 train 7.143104 (lr=4.7772e-05) (hash(x)=137464699)
+1700 val loss 7.3544
+1700 val perplexity 1563.0792
+1700 train 7.558336 (lr=4.7448e-05) (hash(x)=166955614)
+1800 val loss 7.3451
+1800 val perplexity 1548.5469
+1800 train 7.180773 (lr=4.7105e-05) (hash(x)=143886042)
+1900 val loss 7.3411
+1900 val perplexity 1542.4170
+1900 train 8.015940 (lr=4.6741e-05) (hash(x)=193610391)
+2000 val loss 7.3353
+2000 val perplexity 1533.4343
+2000 train 7.497729 (lr=4.6357e-05) (hash(x)=163799796)
+2100 val loss 7.3237
+2100 val perplexity 1515.7485
+2100 train 7.289556 (lr=4.5954e-05) (hash(x)=154107339)
+2200 val loss 7.3198
+2200 val perplexity 1509.9255
+2200 train 7.293709 (lr=4.5532e-05) (hash(x)=144592844)
+2300 val loss 7.3144
+2300 val perplexity 1501.8445
+2300 train 7.388278 (lr=4.5091e-05) (hash(x)=154535861)
+2400 val loss 7.3082
+2400 val perplexity 1492.5593
+2400 train 8.053934 (lr=4.4633e-05) (hash(x)=204706354)
+2500 val loss 7.2990
+2500 val perplexity 1478.7517
+2500 train 7.128391 (lr=4.4156e-05) (hash(x)=143783202)
+2600 val loss 7.2942
+2600 val perplexity 1471.7782
+2600 train 7.203882 (lr=4.3663e-05) (hash(x)=150862210)
+2700 val loss 7.2886
+2700 val perplexity 1463.5914
+2700 train 7.364130 (lr=4.3153e-05) (hash(x)=150753426)
+2800 val loss 7.2890
+2800 val perplexity 1464.0759
+2800 train 7.250031 (lr=4.2627e-05) (hash(x)=147766811)
+2900 val loss 7.2737
+2900 val perplexity 1441.8539
+2900 train 7.208226 (lr=4.2085e-05) (hash(x)=151177814)
+3000 val loss 7.2692
+3000 val perplexity 1435.4247
+3000 train 7.112866 (lr=4.1529e-05) (hash(x)=135925327)
+3100 val loss 7.2730
+3100 val perplexity 1440.9088
+3100 train 6.978726 (lr=4.0957e-05) (hash(x)=124842353)
+3200 val loss 7.2753
+3200 val perplexity 1444.2098
+3200 train 7.387667 (lr=4.0373e-05) (hash(x)=160440642)
+3300 val loss 7.2647
+3300 val perplexity 1428.9902
+3300 train 7.161217 (lr=3.9775e-05) (hash(x)=143017131)
+3400 val loss 7.2511
+3400 val perplexity 1409.5935
+3400 train 7.469110 (lr=3.9164e-05) (hash(x)=148143474)
+3500 val loss 7.2516
+3500 val perplexity 1410.3889
+3500 train 7.080482 (lr=3.8541e-05) (hash(x)=148732639)
+3600 val loss 7.2410
+3600 val perplexity 1395.4398
+3600 train 7.236729 (lr=3.7907e-05) (hash(x)=146333803)
+3700 val loss 7.2483
+3700 val perplexity 1405.7367
+3700 train 7.086179 (lr=3.7262e-05) (hash(x)=143611445)
+3800 val loss 7.2378
+3800 val perplexity 1391.0444
+3800 train 7.065182 (lr=3.6608e-05) (hash(x)=141272115)
+3900 val loss 7.2511
+3900 val perplexity 1409.6836
+3900 train 7.180300 (lr=3.5944e-05) (hash(x)=149161848)
+4000 val loss 7.2304
+4000 val perplexity 1380.7662
+4000 train 7.299043 (lr=3.5271e-05) (hash(x)=152036955)
+4100 val loss 7.2417
+4100 val perplexity 1396.4901
+4100 train 7.309857 (lr=3.4590e-05) (hash(x)=153260968)
+4200 val loss 7.2438
+4200 val perplexity 1399.3445
+4200 train 7.125490 (lr=3.3902e-05) (hash(x)=147629478)
+4300 val loss 7.2442
+4300 val perplexity 1399.9458
+4300 train 7.276296 (lr=3.3207e-05) (hash(x)=161701080)
+4400 val loss 7.2339
+4400 val perplexity 1385.6603
+4400 train 7.305409 (lr=3.2507e-05) (hash(x)=155290524)
+4500 val loss 7.2393
+4500 val perplexity 1393.1593
+4500 train 7.063692 (lr=3.1801e-05) (hash(x)=144726593)
+4600 val loss 7.2351
+4600 val perplexity 1387.2800
+4600 train 7.197353 (lr=3.1091e-05) (hash(x)=149517202)
+4700 val loss 7.2225
+4700 val perplexity 1369.9283
+4700 train 7.208840 (lr=3.0377e-05) (hash(x)=139403008)
+4800 val loss 7.2164
+4800 val perplexity 1361.6372
+4800 train 7.258296 (lr=2.9661e-05) (hash(x)=153387238)
+4900 val loss 7.2094
+4900 val perplexity 1352.0862
+4900 train 6.986797 (lr=2.8942e-05) (hash(x)=139881569)
+5000 val loss 7.2091
+5000 val perplexity 1351.6555
+5000 train 7.152221 (lr=2.8221e-05) (hash(x)=152602580)
+5100 val loss 7.2024
+5100 val perplexity 1342.7072
+5100 train 7.261956 (lr=2.7500e-05) (hash(x)=150598460)
+5200 val loss 7.2082
+5200 val perplexity 1350.5076
+5200 train 7.145143 (lr=2.6779e-05) (hash(x)=147660496)
+5300 val loss 7.2034
+5300 val perplexity 1344.0082
+5300 train 6.975124 (lr=2.6058e-05) (hash(x)=147693848)
+5400 val loss 7.1950
+5400 val perplexity 1332.7544
+5400 train 6.972303 (lr=2.5339e-05) (hash(x)=136048145)
+5500 val loss 7.1836
+5500 val perplexity 1317.5847
+5500 train 7.096358 (lr=2.4623e-05) (hash(x)=140313123)
+5600 val loss 7.1728
+5600 val perplexity 1303.5310
+5600 train 7.204102 (lr=2.3909e-05) (hash(x)=156997993)
+5700 val loss 7.1696
+5700 val perplexity 1299.3303
+5700 train 7.145248 (lr=2.3199e-05) (hash(x)=143513880)
+5800 val loss 7.1668
+5800 val perplexity 1295.6720
+5800 train 7.339622 (lr=2.2493e-05) (hash(x)=153736628)
+5900 val loss 7.1585
+5900 val perplexity 1285.0411
+5900 train 7.393508 (lr=2.1793e-05) (hash(x)=166385417)
+6000 val loss 7.1613
+6000 val perplexity 1288.5822
+6000 train 7.372916 (lr=2.1098e-05) (hash(x)=153216517)
+6100 val loss 7.1614
+6100 val perplexity 1288.6780
+6100 train 7.048747 (lr=2.0410e-05) (hash(x)=145387545)
+6200 val loss 7.1572
+6200 val perplexity 1283.2543
+6200 train 7.264054 (lr=1.9729e-05) (hash(x)=156306460)
+6300 val loss 7.1461
+6300 val perplexity 1269.2065
+6300 train 7.032260 (lr=1.9056e-05) (hash(x)=148610447)
+6400 val loss 7.1415
+6400 val perplexity 1263.2659
+6400 train 7.327859 (lr=1.8392e-05) (hash(x)=158343535)
+6500 val loss 7.1378
+6500 val perplexity 1258.6140
+6500 train 7.277085 (lr=1.7738e-05) (hash(x)=157742557)
+6600 val loss 7.1429
+6600 val perplexity 1265.0973
+6600 train 7.098745 (lr=1.7093e-05) (hash(x)=150457308)
+6700 val loss 7.1355
+6700 val perplexity 1255.7797
+6700 train 7.037126 (lr=1.6459e-05) (hash(x)=142426205)
+6800 val loss 7.1432
+6800 val perplexity 1265.4321
+6800 train 6.993477 (lr=1.5836e-05) (hash(x)=151522525)
+6900 val loss 7.1385
+6900 val perplexity 1259.5896
+6900 train 7.049893 (lr=1.5225e-05) (hash(x)=152029630)
+7000 val loss 7.1301
+7000 val perplexity 1249.0457
+7000 train 7.005527 (lr=1.4627e-05) (hash(x)=142978954)
+7100 val loss 7.1271
+7100 val perplexity 1245.2415
+7100 train 7.016007 (lr=1.4043e-05) (hash(x)=148064359)
+7200 val loss 7.1236
+7200 val perplexity 1240.9304
+7200 train 7.060221 (lr=1.3471e-05) (hash(x)=155771845)
+7300 val loss 7.1249
+7300 val perplexity 1242.5178
+7300 train 6.948674 (lr=1.2915e-05) (hash(x)=141639844)
+7400 val loss 7.1217
+7400 val perplexity 1238.5835
+7400 train 7.149739 (lr=1.2373e-05) (hash(x)=153466021)
+7500 val loss 7.1157
+7500 val perplexity 1231.1477
+7500 train 7.134110 (lr=1.1847e-05) (hash(x)=148843587)
+7600 val loss 7.1180
+7600 val perplexity 1233.9883
+7600 train 7.286146 (lr=1.1337e-05) (hash(x)=152053933)
+7700 val loss 7.1133
+7700 val perplexity 1228.1533
+7700 train 6.874091 (lr=1.0844e-05) (hash(x)=141564003)
+7800 val loss 7.1118
+7800 val perplexity 1226.3030
+7800 train 7.004002 (lr=1.0367e-05) (hash(x)=148617843)
+7900 val loss 7.1108
+7900 val perplexity 1225.1328
+7900 train 7.076463 (lr=9.9088e-06) (hash(x)=146417668)
+8000 val loss 7.1130
+8000 val perplexity 1227.8202
+8000 train 6.954848 (lr=9.4682e-06) (hash(x)=150165488)
+8100 val loss 7.1062
+8100 val perplexity 1219.5206
+8100 train 7.087504 (lr=9.0461e-06) (hash(x)=144818855)
+8200 val loss 7.1025
+8200 val perplexity 1214.9938
+8200 train 7.132918 (lr=8.6430e-06) (hash(x)=143074954)
+8300 val loss 7.1034
+8300 val perplexity 1216.0747
+8300 train 7.034755 (lr=8.2593e-06) (hash(x)=145172494)
+8400 val loss 7.1012
+8400 val perplexity 1213.4415
+8400 train 7.108291 (lr=7.8953e-06) (hash(x)=152284393)
+8500 val loss 7.1001
+8500 val perplexity 1212.0831
+8500 train 6.977907 (lr=7.5515e-06) (hash(x)=147044705)
+8600 val loss 7.0980
+8600 val perplexity 1209.4879
+8600 train 7.043639 (lr=7.2282e-06) (hash(x)=147762608)
+8700 val loss 7.1013
+8700 val perplexity 1213.5769
+8700 train 7.067952 (lr=6.9257e-06) (hash(x)=141994567)
+8800 val loss 7.0941
+8800 val perplexity 1204.8772
+8800 train 7.125029 (lr=6.6444e-06) (hash(x)=152552896)
+8900 val loss 7.0935
+8900 val perplexity 1204.1541
+8900 train 6.956048 (lr=6.3845e-06) (hash(x)=139714454)
+9000 val loss 7.0907
+9000 val perplexity 1200.7579
+9000 train 7.112830 (lr=6.1462e-06) (hash(x)=152274419)
+9100 val loss 7.0909
+9100 val perplexity 1200.9858
+9100 train 7.268093 (lr=5.9300e-06) (hash(x)=160536885)
+9200 val loss 7.0907
+9200 val perplexity 1200.7115
+9200 train 6.918797 (lr=5.7359e-06) (hash(x)=138386899)
+9300 val loss 7.0895
+9300 val perplexity 1199.2856
+9300 train 7.092598 (lr=5.5641e-06) (hash(x)=149957457)
+9400 val loss 7.0853
+9400 val perplexity 1194.3065
+9400 train 7.210474 (lr=5.4149e-06) (hash(x)=161671548)
+9500 val loss 7.0858
+9500 val perplexity 1194.8624
+9500 train 7.251560 (lr=5.2884e-06) (hash(x)=171857400)
+9600 val loss 7.0832
+9600 val perplexity 1191.8232
+9600 train 6.773832 (lr=5.1847e-06) (hash(x)=135378471)
+9700 val loss 7.0826
+9700 val perplexity 1191.0563
+9700 train 6.997484 (lr=5.1040e-06) (hash(x)=149514591)
+9800 val loss 7.0818
+9800 val perplexity 1190.0991
+9800 train 7.044656 (lr=5.0462e-06) (hash(x)=152679072)
+9900 val loss 7.0810
+9900 val perplexity 1189.1620
+9900 train 7.002032 (lr=5.0116e-06) (hash(x)=150415193)
+9999 val loss 7.0800
+9999 val perplexity 1188.0104

attention_kindself_n_heads4_seed1341/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c33d853b824a618231b03a5099efc279b9d9e5b64406a41441614ca3495a496
+oid sha256:0ac1d5aa92ba4713050144037ebcf63fd3e9a1a0ef11fb75b70a385d1aa5ca4b
 size 92843394

attention_kindself_n_heads4_seed1341/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7dfce5e2e2490fbc54169a381133a6a7f50c5bd6315114f93ec0d34ab57035b
+oid sha256:825d3da37092151cfb6fab5159d8442c013f4a85019f5356a30d2c828bb55c82
 size 92843394

attention_kindself_n_heads4_seed1341/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f022e1f318a9ff5f71f8bf03ca4a0be59c7e19aec7499228c3cb84c234b02fbe
+oid sha256:67c1d697cb0414321c4a54e6b2141ec020c319a790f62e712fa3ba3a56888c71
 size 92843394

attention_kindself_n_heads4_seed1341/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41a5c62783abbee1a7e70f418aafd4affde384eb53f7607404494b593f2a22c2
+oid sha256:2cf6289db5de1e15682ac425c55d0f01dc704a9b78e758572e7252f3a5bb6c93
 size 92843394

attention_kindself_n_heads4_seed1341/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:767fa84c5f70a8e53dbf1419e39982a08fbe6e6fc5ae1b5c775d0feb0f1e7d9a
+oid sha256:5c01df2ef9425fd5dccebc1cf0b91577579f972cdf4f0907b953e6d13ac7933e
 size 179406214

attention_kindself_n_heads4_seed1341/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42bd6d830de914e5aabdd77f01551e171e6027880554a54365e1ed34959782d1
+oid sha256:58ee5955f79793b20bca681e1d28a5fc28cd98cf1864eaa9f06d0fe569d7829a
 size 179406214

attention_kindself_n_heads4_seed1341/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f52907cfa74b5974aa1b2136db3f96d064bc659e38dd36c475a0c72d2119681f
+oid sha256:d8dd27a1fa4a6b7d8f3233476db26df0e11831f7d04f7eca75548086a5eb9e47
 size 179406214

attention_kindself_n_heads4_seed1341/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e77d542f2c6b2c551056a5b7ea5c142d14614182cc2e0410bc1f0990e310db03
+oid sha256:da6de5394f11bd1e581cd639bc807a8c4d18610b171933496284661c8c7149f3
 size 179406214